diff options
Diffstat (limited to 'Lib')
528 files changed, 45862 insertions, 4771 deletions
diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py index ed50520ab3..13b1f26965 100644 --- a/Lib/_dummy_thread.py +++ b/Lib/_dummy_thread.py @@ -24,11 +24,7 @@ TIMEOUT_MAX = 2**31 # imports are done when needed on a function-by-function basis. Since threads # are disabled, the import lock should not be an issue anyway (??). -class error(Exception): - """Dummy implementation of _thread.error.""" - - def __init__(self, *args): - self.args = args +error = RuntimeError def start_new_thread(function, args, kwargs={}): """Dummy implementation of _thread.start_new_thread(). diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 78c6d95c23..a9c31d59ee 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -944,6 +944,12 @@ class BufferedReader(_BufferedIOMixin): # Special case for when the number of bytes to read is unspecified. if n is None or n == -1: self._reset_read_buf() + if hasattr(self.raw, 'readall'): + chunk = self.raw.readall() + if chunk is None: + return buf[pos:] or None + else: + return buf[pos:] + chunk chunks = [buf[pos:]] # Strip the consumed bytes. current_size = 0 while True: @@ -1516,6 +1522,7 @@ class TextIOWrapper(TextIOBase): self._snapshot = None # info for reconstructing decoder state self._seekable = self._telling = self.buffer.seekable() self._has_read1 = hasattr(self.buffer, 'read1') + self._b2cratio = 0.0 if self._seekable and self.writable(): position = self.buffer.tell() @@ -1686,7 +1693,12 @@ class TextIOWrapper(TextIOBase): else: input_chunk = self.buffer.read(self._CHUNK_SIZE) eof = not input_chunk - self._set_decoded_chars(self._decoder.decode(input_chunk, eof)) + decoded_chars = self._decoder.decode(input_chunk, eof) + self._set_decoded_chars(decoded_chars) + if decoded_chars: + self._b2cratio = len(input_chunk) / len(self._decoded_chars) + else: + self._b2cratio = 0.0 if self._telling: # At the snapshot point, len(dec_buffer) bytes before the read, @@ -1740,20 +1752,56 @@ class TextIOWrapper(TextIOBase): # forward until it gives us enough decoded characters. saved_state = decoder.getstate() try: + # Fast search for an acceptable start point, close to our + # current pos. + # Rationale: calling decoder.decode() has a large overhead + # regardless of chunk size; we want the number of such calls to + # be O(1) in most situations (common decoders, non-crazy input). + # Actually, it will be exactly 1 for fixed-size codecs (all + # 8-bit codecs, also UTF-16 and UTF-32). + skip_bytes = int(self._b2cratio * chars_to_skip) + skip_back = 1 + assert skip_bytes <= len(next_input) + while skip_bytes > 0: + decoder.setstate((b'', dec_flags)) + # Decode up to temptative start point + n = len(decoder.decode(next_input[:skip_bytes])) + if n <= chars_to_skip: + b, d = decoder.getstate() + if not b: + # Before pos and no bytes buffered in decoder => OK + dec_flags = d + chars_to_skip -= n + break + # Skip back by buffered amount and reset heuristic + skip_bytes -= len(b) + skip_back = 1 + else: + # We're too far ahead, skip back a bit + skip_bytes -= skip_back + skip_back = skip_back * 2 + else: + skip_bytes = 0 + decoder.setstate((b'', dec_flags)) + # Note our initial start point. - decoder.setstate((b'', dec_flags)) - start_pos = position - start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0 - need_eof = 0 + start_pos = position + skip_bytes + start_flags = dec_flags + if chars_to_skip == 0: + # We haven't moved from the start point. + return self._pack_cookie(start_pos, start_flags) # Feed the decoder one byte at a time. As we go, note the # nearest "safe start point" before the current location # (a point where the decoder has nothing buffered, so seek() # can safely start from there and advance to this location). - next_byte = bytearray(1) - for next_byte[0] in next_input: + bytes_fed = 0 + need_eof = 0 + # Chars decoded since `start_pos` + chars_decoded = 0 + for i in range(skip_bytes, len(next_input)): bytes_fed += 1 - chars_decoded += len(decoder.decode(next_byte)) + chars_decoded += len(decoder.decode(next_input[i:i+1])) dec_buffer, dec_flags = decoder.getstate() if not dec_buffer and chars_decoded <= chars_to_skip: # Decoder buffer is empty, so this is a safe start point. diff --git a/Lib/abc.py b/Lib/abc.py index a6c2dc4877..40f88b92c0 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -133,11 +133,14 @@ class ABCMeta(type): return cls def register(cls, subclass): - """Register a virtual subclass of an ABC.""" + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ if not isinstance(subclass, type): raise TypeError("Can only register classes") if issubclass(subclass, cls): - return # Already a subclass + return subclass # Already a subclass # Subtle: test for cycles *after* testing for "already a subclass"; # this means we allow X.register(X) and interpret it as a no-op. if issubclass(cls, subclass): @@ -145,6 +148,7 @@ class ABCMeta(type): raise RuntimeError("Refusing to create an inheritance cycle") cls._abc_registry.add(subclass) ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache + return subclass def _dump_registry(cls, file=None): """Debug helper to print the ABC registry.""" diff --git a/Lib/argparse.py b/Lib/argparse.py index 63561f7cb1..f0cfe27eb8 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -71,6 +71,7 @@ __all__ = [ 'ArgumentDefaultsHelpFormatter', 'RawDescriptionHelpFormatter', 'RawTextHelpFormatter', + 'MetavarTypeHelpFormatter', 'Namespace', 'Action', 'ONE_OR_MORE', @@ -423,7 +424,8 @@ class HelpFormatter(object): # produce all arg strings elif not action.option_strings: - part = self._format_args(action, action.dest) + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) # if it's in a group, strip the outer [] if action in group_actions: @@ -445,7 +447,7 @@ class HelpFormatter(object): # if the Optional takes a value, format is: # -s ARGS or --long ARGS else: - default = action.dest.upper() + default = self._get_default_metavar_for_optional(action) args_string = self._format_args(action, default) part = '%s %s' % (option_string, args_string) @@ -531,7 +533,8 @@ class HelpFormatter(object): def _format_action_invocation(self, action): if not action.option_strings: - metavar, = self._metavar_formatter(action, action.dest)(1) + default = self._get_default_metavar_for_positional(action) + metavar, = self._metavar_formatter(action, default)(1) return metavar else: @@ -545,7 +548,7 @@ class HelpFormatter(object): # if the Optional takes a value, format is: # -s ARGS, --long ARGS else: - default = action.dest.upper() + default = self._get_default_metavar_for_optional(action) args_string = self._format_args(action, default) for option_string in action.option_strings: parts.append('%s %s' % (option_string, args_string)) @@ -623,6 +626,12 @@ class HelpFormatter(object): def _get_help_string(self, action): return action.help + def _get_default_metavar_for_optional(self, action): + return action.dest.upper() + + def _get_default_metavar_for_positional(self, action): + return action.dest + class RawDescriptionHelpFormatter(HelpFormatter): """Help message formatter which retains any formatting in descriptions. @@ -663,6 +672,22 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter): return help +class MetavarTypeHelpFormatter(HelpFormatter): + """Help message formatter which uses the argument 'type' as the default + metavar value (instead of the argument 'dest') + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _get_default_metavar_for_optional(self, action): + return action.type.__name__ + + def _get_default_metavar_for_positional(self, action): + return action.type.__name__ + + + # ===================== # Options and Arguments # ===================== @@ -1944,17 +1969,12 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): # if we didn't consume all the argument strings, there were extras extras.extend(arg_strings[stop_index:]) - # if we didn't use all the Positional objects, there were too few - # arg strings supplied. - if positionals: - self.error(_('too few arguments')) - # make sure all required actions were present - for action in self._actions: - if action.required: - if action not in seen_actions: - name = _get_action_name(action) - self.error(_('argument %s is required') % name) + required_actions = [_get_action_name(action) for action in self._actions + if action.required and action not in seen_actions] + if required_actions: + self.error(_('the following arguments are required: %s') % + ', '.join(required_actions)) # make sure all required groups had one option present for group in self._mutually_exclusive_groups: diff --git a/Lib/ast.py b/Lib/ast.py index fb5adac8fd..13f59f9dfa 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -25,7 +25,6 @@ :license: Python License. """ from _ast import * -from _ast import __version__ def parse(source, filename='<unknown>', mode='exec'): diff --git a/Lib/asynchat.py b/Lib/asynchat.py index 65585129ce..2199d1b3ef 100644 --- a/Lib/asynchat.py +++ b/Lib/asynchat.py @@ -75,7 +75,7 @@ class async_chat (asyncore.dispatcher): # sign of an application bug that we don't want to pass silently use_encoding = 0 - encoding = 'latin1' + encoding = 'latin-1' def __init__ (self, sock=None, map=None): # for string terminator matching diff --git a/Lib/asyncore.py b/Lib/asyncore.py index 7f42d39f33..e699815749 100644 --- a/Lib/asyncore.py +++ b/Lib/asyncore.py @@ -291,7 +291,7 @@ class dispatcher: del map[fd] self._fileno = None - def create_socket(self, family, type): + def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM): self.family_and_type = family, type sock = socket.socket(family, type) sock.setblocking(0) diff --git a/Lib/binhex.py b/Lib/binhex.py index 999a675513..7bf9278430 100644 --- a/Lib/binhex.py +++ b/Lib/binhex.py @@ -23,7 +23,6 @@ hexbin(inputfilename, outputfilename) # import io import os -import sys import struct import binascii diff --git a/Lib/bz2.py b/Lib/bz2.py new file mode 100644 index 0000000000..cc71ae025a --- /dev/null +++ b/Lib/bz2.py @@ -0,0 +1,413 @@ +"""Interface to the libbzip2 compression library. + +This module provides a file interface, classes for incremental +(de)compression, and functions for one-shot (de)compression. +""" + +__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor", "compress", + "decompress"] + +__author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>" + +import io +import threading +import warnings + +from _bz2 import BZ2Compressor, BZ2Decompressor + + +_MODE_CLOSED = 0 +_MODE_READ = 1 +_MODE_READ_EOF = 2 +_MODE_WRITE = 3 + +_BUFFER_SIZE = 8192 + + +class BZ2File(io.BufferedIOBase): + + """A file object providing transparent bzip2 (de)compression. + + A BZ2File can act as a wrapper for an existing file object, or refer + directly to a named file on disk. + + Note that BZ2File provides a *binary* file interface - data read is + returned as bytes, and data to be written should be given as bytes. + """ + + def __init__(self, filename=None, mode="r", buffering=None, + compresslevel=9, fileobj=None): + """Open a bzip2-compressed file. + + If filename is given, open the named file. Otherwise, operate on + the file object given by fileobj. Exactly one of these two + parameters should be provided. + + mode can be 'r' for reading (default), or 'w' for writing. + + buffering is ignored. Its use is deprecated. + + If mode is 'w', compresslevel can be a number between 1 and 9 + specifying the level of compression: 1 produces the least + compression, and 9 (default) produces the most compression. + """ + # This lock must be recursive, so that BufferedIOBase's + # readline(), readlines() and writelines() don't deadlock. + self._lock = threading.RLock() + self._fp = None + self._closefp = False + self._mode = _MODE_CLOSED + self._pos = 0 + self._size = -1 + + if buffering is not None: + warnings.warn("Use of 'buffering' argument is deprecated", + DeprecationWarning) + + if not (1 <= compresslevel <= 9): + raise ValueError("compresslevel must be between 1 and 9") + + if mode in ("", "r", "rb"): + mode = "rb" + mode_code = _MODE_READ + self._decompressor = BZ2Decompressor() + self._buffer = None + elif mode in ("w", "wb"): + mode = "wb" + mode_code = _MODE_WRITE + self._compressor = BZ2Compressor() + elif mode in ("a", "ab"): + mode = "ab" + mode_code = _MODE_WRITE + self._compressor = BZ2Compressor() + else: + raise ValueError("Invalid mode: {!r}".format(mode)) + + if filename is not None and fileobj is None: + self._fp = open(filename, mode) + self._closefp = True + self._mode = mode_code + elif fileobj is not None and filename is None: + self._fp = fileobj + self._mode = mode_code + else: + raise ValueError("Must give exactly one of filename and fileobj") + + def close(self): + """Flush and close the file. + + May be called more than once without error. Once the file is + closed, any other operation on it will raise a ValueError. + """ + with self._lock: + if self._mode == _MODE_CLOSED: + return + try: + if self._mode in (_MODE_READ, _MODE_READ_EOF): + self._decompressor = None + elif self._mode == _MODE_WRITE: + self._fp.write(self._compressor.flush()) + self._compressor = None + finally: + try: + if self._closefp: + self._fp.close() + finally: + self._fp = None + self._closefp = False + self._mode = _MODE_CLOSED + self._buffer = None + + @property + def closed(self): + """True if this file is closed.""" + return self._mode == _MODE_CLOSED + + def fileno(self): + """Return the file descriptor for the underlying file.""" + return self._fp.fileno() + + def seekable(self): + """Return whether the file supports seeking.""" + return self.readable() + + def readable(self): + """Return whether the file was opened for reading.""" + return self._mode in (_MODE_READ, _MODE_READ_EOF) + + def writable(self): + """Return whether the file was opened for writing.""" + return self._mode == _MODE_WRITE + + # Mode-checking helper functions. + + def _check_not_closed(self): + if self.closed: + raise ValueError("I/O operation on closed file") + + def _check_can_read(self): + if not self.readable(): + self._check_not_closed() + raise io.UnsupportedOperation("File not open for reading") + + def _check_can_write(self): + if not self.writable(): + self._check_not_closed() + raise io.UnsupportedOperation("File not open for writing") + + def _check_can_seek(self): + if not self.seekable(): + self._check_not_closed() + raise io.UnsupportedOperation("Seeking is only supported " + "on files open for reading") + + # Fill the readahead buffer if it is empty. Returns False on EOF. + def _fill_buffer(self): + if self._buffer: + return True + + if self._decompressor.unused_data: + rawblock = self._decompressor.unused_data + else: + rawblock = self._fp.read(_BUFFER_SIZE) + + if not rawblock: + if self._decompressor.eof: + self._mode = _MODE_READ_EOF + self._size = self._pos + return False + else: + raise EOFError("Compressed file ended before the " + "end-of-stream marker was reached") + + # Continue to next stream. + if self._decompressor.eof: + self._decompressor = BZ2Decompressor() + + self._buffer = self._decompressor.decompress(rawblock) + return True + + # Read data until EOF. + # If return_data is false, consume the data without returning it. + def _read_all(self, return_data=True): + blocks = [] + while self._fill_buffer(): + if return_data: + blocks.append(self._buffer) + self._pos += len(self._buffer) + self._buffer = None + if return_data: + return b"".join(blocks) + + # Read a block of up to n bytes. + # If return_data is false, consume the data without returning it. + def _read_block(self, n, return_data=True): + blocks = [] + while n > 0 and self._fill_buffer(): + if n < len(self._buffer): + data = self._buffer[:n] + self._buffer = self._buffer[n:] + else: + data = self._buffer + self._buffer = None + if return_data: + blocks.append(data) + self._pos += len(data) + n -= len(data) + if return_data: + return b"".join(blocks) + + def peek(self, n=0): + """Return buffered data without advancing the file position. + + Always returns at least one byte of data, unless at EOF. + The exact number of bytes returned is unspecified. + """ + with self._lock: + self._check_can_read() + if self._mode == _MODE_READ_EOF or not self._fill_buffer(): + return b"" + return self._buffer + + def read(self, size=-1): + """Read up to size uncompressed bytes from the file. + + If size is negative or omitted, read until EOF is reached. + Returns b'' if the file is already at EOF. + """ + with self._lock: + self._check_can_read() + if self._mode == _MODE_READ_EOF or size == 0: + return b"" + elif size < 0: + return self._read_all() + else: + return self._read_block(size) + + def read1(self, size=-1): + """Read up to size uncompressed bytes with at most one read + from the underlying stream. + + Returns b'' if the file is at EOF. + """ + with self._lock: + self._check_can_read() + if (size == 0 or self._mode == _MODE_READ_EOF or + not self._fill_buffer()): + return b"" + if 0 < size < len(self._buffer): + data = self._buffer[:size] + self._buffer = self._buffer[size:] + else: + data = self._buffer + self._buffer = None + self._pos += len(data) + return data + + def readinto(self, b): + """Read up to len(b) bytes into b. + + Returns the number of bytes read (0 for EOF). + """ + with self._lock: + return io.BufferedIOBase.readinto(self, b) + + def readline(self, size=-1): + """Read a line of uncompressed bytes from the file. + + The terminating newline (if present) is retained. If size is + non-negative, no more than size bytes will be read (in which + case the line may be incomplete). Returns b'' if already at EOF. + """ + if not hasattr(size, "__index__"): + raise TypeError("Integer argument expected") + size = size.__index__() + with self._lock: + return io.BufferedIOBase.readline(self, size) + + def readlines(self, size=-1): + """Read a list of lines of uncompressed bytes from the file. + + size can be specified to control the number of lines read: no + further lines will be read once the total size of the lines read + so far equals or exceeds size. + """ + if not hasattr(size, "__index__"): + raise TypeError("Integer argument expected") + size = size.__index__() + with self._lock: + return io.BufferedIOBase.readlines(self, size) + + def write(self, data): + """Write a byte string to the file. + + Returns the number of uncompressed bytes written, which is + always len(data). Note that due to buffering, the file on disk + may not reflect the data written until close() is called. + """ + with self._lock: + self._check_can_write() + compressed = self._compressor.compress(data) + self._fp.write(compressed) + self._pos += len(data) + return len(data) + + def writelines(self, seq): + """Write a sequence of byte strings to the file. + + Returns the number of uncompressed bytes written. + seq can be any iterable yielding byte strings. + + Line separators are not added between the written byte strings. + """ + with self._lock: + return io.BufferedIOBase.writelines(self, seq) + + # Rewind the file to the beginning of the data stream. + def _rewind(self): + self._fp.seek(0, 0) + self._mode = _MODE_READ + self._pos = 0 + self._decompressor = BZ2Decompressor() + self._buffer = None + + def seek(self, offset, whence=0): + """Change the file position. + + The new position is specified by offset, relative to the + position indicated by whence. Values for whence are: + + 0: start of stream (default); offset must not be negative + 1: current stream position + 2: end of stream; offset must not be positive + + Returns the new file position. + + Note that seeking is emulated, so depending on the parameters, + this operation may be extremely slow. + """ + with self._lock: + self._check_can_seek() + + # Recalculate offset as an absolute file position. + if whence == 0: + pass + elif whence == 1: + offset = self._pos + offset + elif whence == 2: + # Seeking relative to EOF - we need to know the file's size. + if self._size < 0: + self._read_all(return_data=False) + offset = self._size + offset + else: + raise ValueError("Invalid value for whence: {}".format(whence)) + + # Make it so that offset is the number of bytes to skip forward. + if offset < self._pos: + self._rewind() + else: + offset -= self._pos + + # Read and discard data until we reach the desired position. + if self._mode != _MODE_READ_EOF: + self._read_block(offset, return_data=False) + + return self._pos + + def tell(self): + """Return the current file position.""" + with self._lock: + self._check_not_closed() + return self._pos + + +def compress(data, compresslevel=9): + """Compress a block of data. + + compresslevel, if given, must be a number between 1 and 9. + + For incremental compression, use a BZ2Compressor object instead. + """ + comp = BZ2Compressor(compresslevel) + return comp.compress(data) + comp.flush() + + +def decompress(data): + """Decompress a block of data. + + For incremental decompression, use a BZ2Decompressor object instead. + """ + if len(data) == 0: + return b"" + + results = [] + while True: + decomp = BZ2Decompressor() + results.append(decomp.decompress(data)) + if not decomp.eof: + raise ValueError("Compressed data ended before the " + "end-of-stream marker was reached") + if not decomp.unused_data: + return b"".join(results) + # There is unused data left over. Proceed to next stream. + data = decomp.unused_data diff --git a/Lib/cgi.py b/Lib/cgi.py index e198ed8653..63771fc9c6 100755 --- a/Lib/cgi.py +++ b/Lib/cgi.py @@ -76,7 +76,7 @@ def initlog(*allargs): send an error message). """ - global logfp, log + global log, logfile, logfp if logfile and not logfp: try: logfp = open(logfile, "a") @@ -96,6 +96,15 @@ def nolog(*allargs): """Dummy function, assigned to log when logging is disabled.""" pass +def closelog(): + """Close the log file.""" + global log, logfile, logfp + logfile = '' + if logfp: + logfp.close() + logfp = None + log = initlog + log = initlog # The current logging function diff --git a/Lib/cgitb.py b/Lib/cgitb.py index 7b52c8e74c..e3ce2cb9d6 100644 --- a/Lib/cgitb.py +++ b/Lib/cgitb.py @@ -31,7 +31,6 @@ import tempfile import time import tokenize import traceback -import types def reset(): """Return a string that resets the CGI and browser to a known state.""" diff --git a/Lib/collections.py b/Lib/collections/__init__.py index 2b6abd879f..2007e68816 100644 --- a/Lib/collections.py +++ b/Lib/collections/__init__.py @@ -1,10 +1,11 @@ __all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList', 'UserString', 'Counter', 'OrderedDict'] -# For bootstrapping reasons, the collection ABCs are defined in _abcoll.py. -# They should however be considered an integral part of collections.py. -from _abcoll import * -import _abcoll -__all__ += _abcoll.__all__ + +# For backwards compatibility, continue to make the collections ABCs +# available through the collections module. +from collections.abc import * +import collections.abc +__all__ += collections.abc.__all__ from _collections import deque, defaultdict from operator import itemgetter as _itemgetter @@ -364,8 +365,9 @@ def namedtuple(typename, field_names, verbose=False, rename=False): except SyntaxError as e: raise SyntaxError(e.msg + ':\n\n' + class_definition) result = namespace[typename] + result._source = class_definition if verbose: - print(class_definition) + print(result._source) # For pickling to work, the __module__ variable needs to be set to the frame # where the named tuple is created. Bypass this step in enviroments where @@ -670,12 +672,23 @@ class Counter(dict): result[elem] = newcount return result + def __pos__(self): + 'Adds an empty counter, effectively stripping negative and zero counts' + return self + Counter() + + def __neg__(self): + '''Subtracts from an empty counter. Strips positive and zero counts, + and flips the sign on negative counts. + + ''' + return Counter() - self + ######################################################################## -### ChainMap (helper for configparser) +### ChainMap (helper for configparser and string.Template) ######################################################################## -class _ChainMap(MutableMapping): +class ChainMap(MutableMapping): ''' A ChainMap groups multiple dicts (or other mappings) together to create a single, updateable view. @@ -886,6 +899,8 @@ class UserList(MutableSequence): def insert(self, i, item): self.data.insert(i, item) def pop(self, i=-1): return self.data.pop(i) def remove(self, item): self.data.remove(item) + def clear(self): self.data.clear() + def copy(self): return self.__class__(self) def count(self, item): return self.data.count(item) def index(self, item, *args): return self.data.index(item, *args) def reverse(self): self.data.reverse() diff --git a/Lib/_abcoll.py b/Lib/collections/abc.py index 2417d187cd..7fbe84dfe9 100644 --- a/Lib/_abcoll.py +++ b/Lib/collections/abc.py @@ -3,9 +3,7 @@ """Abstract Base Classes (ABCs) for collections, according to PEP 3119. -DON'T USE THIS MODULE DIRECTLY! The classes here should be imported -via collections; they are defined here only to alleviate certain -bootstrapping issues. Unit tests are in test_collections. +Unit tests are in test_collections. """ from abc import ABCMeta, abstractmethod @@ -48,6 +46,8 @@ dict_proxy = type(type.__dict__) class Hashable(metaclass=ABCMeta): + __slots__ = () + @abstractmethod def __hash__(self): return 0 @@ -65,6 +65,8 @@ class Hashable(metaclass=ABCMeta): class Iterable(metaclass=ABCMeta): + __slots__ = () + @abstractmethod def __iter__(self): while False: @@ -80,6 +82,8 @@ class Iterable(metaclass=ABCMeta): class Iterator(Iterable): + __slots__ = () + @abstractmethod def __next__(self): raise StopIteration @@ -111,6 +115,8 @@ Iterator.register(zip_iterator) class Sized(metaclass=ABCMeta): + __slots__ = () + @abstractmethod def __len__(self): return 0 @@ -125,6 +131,8 @@ class Sized(metaclass=ABCMeta): class Container(metaclass=ABCMeta): + __slots__ = () + @abstractmethod def __contains__(self, x): return False @@ -139,6 +147,8 @@ class Container(metaclass=ABCMeta): class Callable(metaclass=ABCMeta): + __slots__ = () + @abstractmethod def __call__(self, *args, **kwds): return False @@ -166,6 +176,8 @@ class Set(Sized, Iterable, Container): then the other operations will automatically follow suit. """ + __slots__ = () + def __le__(self, other): if not isinstance(other, Set): return NotImplemented @@ -277,6 +289,8 @@ Set.register(frozenset) class MutableSet(Set): + __slots__ = () + @abstractmethod def add(self, value): """Add an element.""" @@ -350,6 +364,8 @@ MutableSet.register(set) class Mapping(Sized, Iterable, Container): + __slots__ = () + @abstractmethod def __getitem__(self, key): raise KeyError @@ -453,6 +469,8 @@ ValuesView.register(dict_values) class MutableMapping(Mapping): + __slots__ = () + @abstractmethod def __setitem__(self, key, value): raise KeyError @@ -532,6 +550,8 @@ class Sequence(Sized, Iterable, Container): __getitem__, and __len__. """ + __slots__ = () + @abstractmethod def __getitem__(self, index): raise IndexError @@ -577,12 +597,16 @@ class ByteString(Sequence): XXX Should add all their methods. """ + __slots__ = () + ByteString.register(bytes) ByteString.register(bytearray) class MutableSequence(Sequence): + __slots__ = () + @abstractmethod def __setitem__(self, index, value): raise IndexError @@ -598,6 +622,13 @@ class MutableSequence(Sequence): def append(self, value): self.insert(len(self), value) + def clear(self): + try: + while True: + self.pop() + except IndexError: + pass + def reverse(self): n = len(self) for i in range(n//2): diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index 79b91d495f..6cfded3075 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -536,15 +536,19 @@ class Executor(object): fs = [self.submit(fn, *args) for args in zip(*iterables)] - try: - for future in fs: - if timeout is None: - yield future.result() - else: - yield future.result(end_time - time.time()) - finally: - for future in fs: - future.cancel() + # Yield must be hidden in closure so that the futures are submitted + # before the first iterator value is required. + def result_iterator(): + try: + for future in fs: + if timeout is None: + yield future.result() + else: + yield future.result(end_time - time.time()) + finally: + for future in fs: + future.cancel() + return result_iterator() def shutdown(self, wait=True): """Clean-up the resources associated with the Executor. diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 8082940677..9b2e0f3ab7 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -46,9 +46,11 @@ Process #1..n: __author__ = 'Brian Quinlan (brian@sweetapp.com)' import atexit +import os from concurrent.futures import _base import queue import multiprocessing +from multiprocessing.queues import SimpleQueue, SentinelReady, Full import threading import weakref @@ -121,7 +123,7 @@ def _process_worker(call_queue, result_queue): call_item = call_queue.get(block=True) if call_item is None: # Wake up queue management thread - result_queue.put(None) + result_queue.put(os.getpid()) return try: r = call_item.fn(*call_item.args, **call_item.kwargs) @@ -193,51 +195,92 @@ def _queue_management_worker(executor_reference, result_queue: A multiprocessing.Queue of _ResultItems generated by the process workers. """ - nb_shutdown_processes = 0 - def shutdown_one_process(): - """Tell a worker to terminate, which will in turn wake us again""" - nonlocal nb_shutdown_processes - call_queue.put(None) - nb_shutdown_processes += 1 + executor = None + + def shutting_down(): + return _shutdown or executor is None or executor._shutdown_thread + + def shutdown_worker(): + # This is an upper bound + nb_children_alive = sum(p.is_alive() for p in processes.values()) + for i in range(0, nb_children_alive): + call_queue.put_nowait(None) + # Release the queue's resources as soon as possible. + call_queue.close() + # If .join() is not called on the created processes then + # some multiprocessing.Queue methods may deadlock on Mac OS X. + for p in processes.values(): + p.join() + while True: _add_call_item_to_queue(pending_work_items, work_ids_queue, call_queue) - result_item = result_queue.get(block=True) - if result_item is not None: - work_item = pending_work_items[result_item.work_id] - del pending_work_items[result_item.work_id] - - if result_item.exception: - work_item.future.set_exception(result_item.exception) - else: - work_item.future.set_result(result_item.result) - continue - # If we come here, we either got a timeout or were explicitly woken up. - # In either case, check whether we should start shutting down. + sentinels = [p.sentinel for p in processes.values()] + assert sentinels + try: + result_item = result_queue.get(sentinels=sentinels) + except SentinelReady as e: + # Mark the process pool broken so that submits fail right now. + executor = executor_reference() + if executor is not None: + executor._broken = True + executor._shutdown_thread = True + executor = None + # All futures in flight must be marked failed + for work_id, work_item in pending_work_items.items(): + work_item.future.set_exception( + BrokenProcessPool( + "A process in the process pool was " + "terminated abruptly while the future was " + "running or pending." + )) + pending_work_items.clear() + # Terminate remaining workers forcibly: the queues or their + # locks may be in a dirty state and block forever. + for p in processes.values(): + p.terminate() + shutdown_worker() + return + if isinstance(result_item, int): + # Clean shutdown of a worker using its PID + # (avoids marking the executor broken) + assert shutting_down() + p = processes.pop(result_item) + p.join() + if not processes: + shutdown_worker() + return + elif result_item is not None: + work_item = pending_work_items.pop(result_item.work_id, None) + # work_item can be None if another process terminated (see above) + if work_item is not None: + if result_item.exception: + work_item.future.set_exception(result_item.exception) + else: + work_item.future.set_result(result_item.result) + # Check whether we should start shutting down. executor = executor_reference() # No more work items can be added if: # - The interpreter is shutting down OR # - The executor that owns this worker has been collected OR # - The executor that owns this worker has been shutdown. - if _shutdown or executor is None or executor._shutdown_thread: - # Since no new work items can be added, it is safe to shutdown - # this thread if there are no pending work items. - if not pending_work_items: - while nb_shutdown_processes < len(processes): - shutdown_one_process() - # If .join() is not called on the created processes then - # some multiprocessing.Queue methods may deadlock on Mac OS - # X. - for p in processes: - p.join() - call_queue.close() - return - else: - # Start shutting down by telling a process it can exit. - shutdown_one_process() - del executor + if shutting_down(): + try: + # Since no new work items can be added, it is safe to shutdown + # this thread if there are no pending work items. + if not pending_work_items: + shutdown_worker() + return + else: + # Start shutting down by telling a process it can exit. + call_queue.put_nowait(None) + except Full: + # This is not a problem: we will eventually be woken up (in + # result_queue.get()) and be able to send a sentinel again. + pass + executor = None _system_limits_checked = False _system_limited = None @@ -264,6 +307,14 @@ def _check_system_limits(): _system_limited = "system provides too few semaphores (%d available, 256 necessary)" % nsems_max raise NotImplementedError(_system_limited) + +class BrokenProcessPool(RuntimeError): + """ + Raised when a process in a ProcessPoolExecutor terminated abruptly + while a future was in the running state. + """ + + class ProcessPoolExecutor(_base.Executor): def __init__(self, max_workers=None): """Initializes a new ProcessPoolExecutor instance. @@ -285,14 +336,20 @@ class ProcessPoolExecutor(_base.Executor): # because futures in the call queue cannot be cancelled. self._call_queue = multiprocessing.Queue(self._max_workers + EXTRA_QUEUED_CALLS) - self._result_queue = multiprocessing.Queue() + # Killed worker processes can produce spurious "broken pipe" + # tracebacks in the queue's own worker thread. But we detect killed + # processes anyway, so silence the tracebacks. + self._call_queue._ignore_epipe = True + self._result_queue = SimpleQueue() self._work_ids = queue.Queue() self._queue_management_thread = None - self._processes = set() + # Map of pids to processes + self._processes = {} # Shutdown is a two-step process. self._shutdown_thread = False self._shutdown_lock = threading.Lock() + self._broken = False self._queue_count = 0 self._pending_work_items = {} @@ -302,6 +359,8 @@ class ProcessPoolExecutor(_base.Executor): def weakref_cb(_, q=self._result_queue): q.put(None) if self._queue_management_thread is None: + # Start the processes so that their sentinels are known. + self._adjust_process_count() self._queue_management_thread = threading.Thread( target=_queue_management_worker, args=(weakref.ref(self, weakref_cb), @@ -321,10 +380,13 @@ class ProcessPoolExecutor(_base.Executor): args=(self._call_queue, self._result_queue)) p.start() - self._processes.add(p) + self._processes[p.pid] = p def submit(self, fn, *args, **kwargs): with self._shutdown_lock: + if self._broken: + raise BrokenProcessPool('A child process terminated ' + 'abruptly, the process pool is not usable anymore') if self._shutdown_thread: raise RuntimeError('cannot schedule new futures after shutdown') @@ -338,7 +400,6 @@ class ProcessPoolExecutor(_base.Executor): self._result_queue.put(None) self._start_queue_management_thread() - self._adjust_process_count() return f submit.__doc__ = _base.Executor.submit.__doc__ diff --git a/Lib/configparser.py b/Lib/configparser.py index 12ba5ad81c..b843c00cee 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -119,7 +119,8 @@ ConfigParser -- responsible for parsing a list of between keys and values are surrounded by spaces. """ -from collections import MutableMapping, OrderedDict as _default_dict, _ChainMap +from collections.abc import MutableMapping +from collections import OrderedDict as _default_dict, ChainMap as _ChainMap import functools import io import itertools diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 5ebbbc6583..2f8f00d296 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -2,7 +2,6 @@ import sys from functools import wraps -from warnings import warn __all__ = ["contextmanager", "closing", "ContextDecorator"] diff --git a/Lib/copy.py b/Lib/copy.py index 089d101c7c..d96201ea98 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -173,8 +173,10 @@ def deepcopy(x, memo=None, _nil=[]): "un(deep)copyable object of type %s" % cls) y = _reconstruct(x, rv, 1, memo) - memo[d] = y - _keep_alive(x, memo) # Make sure x lives at least as long as d + # If is its own copy, don't memoize. + if y is not x: + memo[d] = y + _keep_alive(x, memo) # Make sure x lives at least as long as d return y _deepcopy_dispatch = d = {} @@ -214,9 +216,10 @@ def _deepcopy_tuple(x, memo): y = [] for a in x: y.append(deepcopy(a, memo)) - d = id(x) + # We're not going to put the tuple in the memo, but it's still important we + # check for it, in case the tuple contains recursive mutable structures. try: - return memo[d] + return memo[id(x)] except KeyError: pass for i in range(len(x)): @@ -225,7 +228,6 @@ def _deepcopy_tuple(x, memo): break else: y = x - memo[d] = y return y d[tuple] = _deepcopy_tuple @@ -321,68 +323,3 @@ del types # Helper for instance creation without calling __init__ class _EmptyClass: pass - -def _test(): - l = [None, 1, 2, 3.14, 'xyzzy', (1, 2), [3.14, 'abc'], - {'abc': 'ABC'}, (), [], {}] - l1 = copy(l) - print(l1==l) - l1 = map(copy, l) - print(l1==l) - l1 = deepcopy(l) - print(l1==l) - class C: - def __init__(self, arg=None): - self.a = 1 - self.arg = arg - if __name__ == '__main__': - import sys - file = sys.argv[0] - else: - file = __file__ - self.fp = open(file) - self.fp.close() - def __getstate__(self): - return {'a': self.a, 'arg': self.arg} - def __setstate__(self, state): - for key, value in state.items(): - setattr(self, key, value) - def __deepcopy__(self, memo=None): - new = self.__class__(deepcopy(self.arg, memo)) - new.a = self.a - return new - c = C('argument sketch') - l.append(c) - l2 = copy(l) - print(l == l2) - print(l) - print(l2) - l2 = deepcopy(l) - print(l == l2) - print(l) - print(l2) - l.append({l[1]: l, 'xyz': l[2]}) - l3 = copy(l) - import reprlib - print(map(reprlib.repr, l)) - print(map(reprlib.repr, l1)) - print(map(reprlib.repr, l2)) - print(map(reprlib.repr, l3)) - l3 = deepcopy(l) - print(map(reprlib.repr, l)) - print(map(reprlib.repr, l1)) - print(map(reprlib.repr, l2)) - print(map(reprlib.repr, l3)) - class odict(dict): - def __init__(self, d = {}): - self.a = 99 - dict.__init__(self, d) - def __setitem__(self, k, i): - dict.__setitem__(self, k, i) - self.a - o = odict({"A" : "B"}) - x = deepcopy(o) - print(o, x) - -if __name__ == '__main__': - _test() diff --git a/Lib/crypt.py b/Lib/crypt.py new file mode 100644 index 0000000000..e65b0cbe4d --- /dev/null +++ b/Lib/crypt.py @@ -0,0 +1,62 @@ +"""Wrapper to the POSIX crypt library call and associated functionality.""" + +import _crypt +import string +from random import choice +from collections import namedtuple + + +_saltchars = string.ascii_letters + string.digits + './' + + +class _Method(namedtuple('_Method', 'name ident salt_chars total_size')): + + """Class representing a salt method per the Modular Crypt Format or the + legacy 2-character crypt method.""" + + def __repr__(self): + return '<crypt.METHOD_{}>'.format(self.name) + + + +def mksalt(method=None): + """Generate a salt for the specified method. + + If not specified, the strongest available method will be used. + + """ + if method is None: + method = methods[0] + s = '${}$'.format(method.ident) if method.ident else '' + s += ''.join(choice(_saltchars) for _ in range(method.salt_chars)) + return s + + +def crypt(word, salt=None): + """Return a string representing the one-way hash of a password, with a salt + prepended. + + If ``salt`` is not specified or is ``None``, the strongest + available method will be selected and a salt generated. Otherwise, + ``salt`` may be one of the ``crypt.METHOD_*`` values, or a string as + returned by ``crypt.mksalt()``. + + """ + if salt is None or isinstance(salt, _Method): + salt = mksalt(salt) + return _crypt.crypt(word, salt) + + +# available salting/crypto methods +METHOD_CRYPT = _Method('CRYPT', None, 2, 13) +METHOD_MD5 = _Method('MD5', '1', 8, 34) +METHOD_SHA256 = _Method('SHA256', '5', 16, 63) +METHOD_SHA512 = _Method('SHA512', '6', 16, 106) + +methods = [] +for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5): + _result = crypt('', _method) + if _result and len(_result) == _method.total_size: + methods.append(_method) +methods.append(METHOD_CRYPT) +del _result, _method diff --git a/Lib/ctypes/test/test_memfunctions.py b/Lib/ctypes/test/test_memfunctions.py index aa2113b885..aec4aaadac 100644 --- a/Lib/ctypes/test/test_memfunctions.py +++ b/Lib/ctypes/test/test_memfunctions.py @@ -1,4 +1,5 @@ import sys +from test import support import unittest from ctypes import * @@ -49,6 +50,7 @@ class MemFunctionsTest(unittest.TestCase): self.assertEqual(cast(a, POINTER(c_byte))[:7:7], [97]) + @support.refcount_test def test_string_at(self): s = string_at(b"foo bar") # XXX The following may be wrong, depending on how Python diff --git a/Lib/ctypes/test/test_python_api.py b/Lib/ctypes/test/test_python_api.py index 1f4c6039dc..9de3980ed4 100644 --- a/Lib/ctypes/test/test_python_api.py +++ b/Lib/ctypes/test/test_python_api.py @@ -1,5 +1,6 @@ from ctypes import * import unittest, sys +from test import support from ctypes.test import is_resource_enabled ################################################################ @@ -25,6 +26,7 @@ class PythonAPITestCase(unittest.TestCase): self.assertEqual(PyBytes_FromStringAndSize(b"abcdefghi", 3), b"abc") + @support.refcount_test def test_PyString_FromString(self): pythonapi.PyBytes_FromString.restype = py_object pythonapi.PyBytes_FromString.argtypes = (c_char_p,) @@ -56,6 +58,7 @@ class PythonAPITestCase(unittest.TestCase): del res self.assertEqual(grc(42), ref42) + @support.refcount_test def test_PyObj_FromPtr(self): s = "abc def ghi jkl" ref = grc(s) diff --git a/Lib/ctypes/test/test_refcounts.py b/Lib/ctypes/test/test_refcounts.py index 35a81aa40b..5613e7a387 100644 --- a/Lib/ctypes/test/test_refcounts.py +++ b/Lib/ctypes/test/test_refcounts.py @@ -1,4 +1,5 @@ import unittest +from test import support import ctypes import gc @@ -10,6 +11,7 @@ dll = ctypes.CDLL(_ctypes_test.__file__) class RefcountTestCase(unittest.TestCase): + @support.refcount_test def test_1(self): from sys import getrefcount as grc @@ -34,6 +36,7 @@ class RefcountTestCase(unittest.TestCase): self.assertEqual(grc(callback), 2) + @support.refcount_test def test_refcount(self): from sys import getrefcount as grc def func(*args): diff --git a/Lib/ctypes/test/test_stringptr.py b/Lib/ctypes/test/test_stringptr.py index 3d25fa5360..95cd1614c6 100644 --- a/Lib/ctypes/test/test_stringptr.py +++ b/Lib/ctypes/test/test_stringptr.py @@ -1,4 +1,5 @@ import unittest +from test import support from ctypes import * import _ctypes_test @@ -7,6 +8,7 @@ lib = CDLL(_ctypes_test.__file__) class StringPtrTestCase(unittest.TestCase): + @support.refcount_test def test__POINTER_c_char(self): class X(Structure): _fields_ = [("str", POINTER(c_char))] diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 1881e89688..97d0c2f3fa 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -1,5 +1,6 @@ import sys, os import contextlib +import subprocess # find_library(name) returns the pathname of a library, or None. if os.name == "nt": @@ -136,16 +137,12 @@ elif os.name == "posix": rv = f.close() if rv == 10: raise OSError('objdump command not found') - with contextlib.closing(os.popen(cmd)) as f: - data = f.read() - res = re.search(r'\sSONAME\s+([^\s]+)', data) + res = re.search(r'\sSONAME\s+([^\s]+)', dump) if not res: return None return res.group(1) - if (sys.platform.startswith("freebsd") - or sys.platform.startswith("openbsd") - or sys.platform.startswith("dragonfly")): + if sys.platform.startswith(("freebsd", "openbsd", "dragonfly")): def _num_version(libname): # "libxyz.so.MAJOR.MINOR" => [ MAJOR, MINOR ] @@ -171,22 +168,6 @@ elif os.name == "posix": else: - def _findLib_ldconfig(name): - # XXX assuming GLIBC's ldconfig (with option -p) - expr = r'/[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name) - with contextlib.closing(os.popen('/sbin/ldconfig -p 2>/dev/null')) as f: - data = f.read() - res = re.search(expr, data) - if not res: - # Hm, this works only for libs needed by the python executable. - cmd = 'ldd %s 2>/dev/null' % sys.executable - with contextlib.closing(os.popen(cmd)) as f: - data = f.read() - res = re.search(expr, data) - if not res: - return None - return res.group(0) - def _findSoname_ldconfig(name): import struct if struct.calcsize('l') == 4: @@ -203,14 +184,19 @@ elif os.name == "posix": abi_type = mach_map.get(machine, 'libc6') # XXX assuming GLIBC's ldconfig (with option -p) - expr = r'(\S+)\s+\((%s(?:, OS ABI:[^\)]*)?)\)[^/]*(/[^\(\)\s]*lib%s\.[^\(\)\s]*)' \ - % (abi_type, re.escape(name)) - with contextlib.closing(os.popen('LC_ALL=C LANG=C /sbin/ldconfig -p 2>/dev/null')) as f: - data = f.read() - res = re.search(expr, data) - if not res: - return None - return res.group(1) + regex = os.fsencode( + '\s+(lib%s\.[^\s]+)\s+\(%s' % (re.escape(name), abi_type)) + try: + with subprocess.Popen(['/sbin/ldconfig', '-p'], + stdin=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdout=subprocess.PIPE, + env={'LC_ALL': 'C', 'LANG': 'C'}) as p: + res = re.search(regex, p.stdout.read()) + if res: + return os.fsdecode(res.group(1)) + except OSError: + pass def find_library(name): return _findSoname_ldconfig(name) or _get_soname(_findLib_gcc(name)) diff --git a/Lib/curses/__init__.py b/Lib/curses/__init__.py index 303ea3e791..61ce443a68 100644 --- a/Lib/curses/__init__.py +++ b/Lib/curses/__init__.py @@ -11,7 +11,6 @@ the package, and perhaps a particular module inside it. """ from _curses import * -from curses.wrapper import wrapper import os as _os import sys as _sys @@ -55,3 +54,48 @@ try: has_key except NameError: from has_key import has_key + +# Wrapper for the entire curses-based application. Runs a function which +# should be the rest of your curses-based application. If the application +# raises an exception, wrapper() will restore the terminal to a sane state so +# you can read the resulting traceback. + +def wrapper(func, *args, **kwds): + """Wrapper function that initializes curses and calls another function, + restoring normal keyboard/screen behavior on error. + The callable object 'func' is then passed the main window 'stdscr' + as its first argument, followed by any other arguments passed to + wrapper(). + """ + + try: + # Initialize curses + stdscr = initscr() + + # Turn off echoing of keys, and enter cbreak mode, + # where no buffering is performed on keyboard input + noecho() + cbreak() + + # In keypad mode, escape sequences for special keys + # (like the cursor keys) will be interpreted and + # a special value like curses.KEY_LEFT will be returned + stdscr.keypad(1) + + # Start color, too. Harmless if the terminal doesn't have + # color; user can test with has_color() later on. The try/catch + # works around a minor bit of over-conscientiousness in the curses + # module -- the error return from C start_color() is ignorable. + try: + start_color() + except: + pass + + return func(stdscr, *args, **kwds) + finally: + # Set everything back to normal + if 'stdscr' in locals(): + stdscr.keypad(0) + echo() + nocbreak() + endwin() diff --git a/Lib/curses/wrapper.py b/Lib/curses/wrapper.py deleted file mode 100644 index 5183ce741f..0000000000 --- a/Lib/curses/wrapper.py +++ /dev/null @@ -1,50 +0,0 @@ -"""curses.wrapper - -Contains one function, wrapper(), which runs another function which -should be the rest of your curses-based application. If the -application raises an exception, wrapper() will restore the terminal -to a sane state so you can read the resulting traceback. - -""" - -import curses - -def wrapper(func, *args, **kwds): - """Wrapper function that initializes curses and calls another function, - restoring normal keyboard/screen behavior on error. - The callable object 'func' is then passed the main window 'stdscr' - as its first argument, followed by any other arguments passed to - wrapper(). - """ - - try: - # Initialize curses - stdscr = curses.initscr() - - # Turn off echoing of keys, and enter cbreak mode, - # where no buffering is performed on keyboard input - curses.noecho() - curses.cbreak() - - # In keypad mode, escape sequences for special keys - # (like the cursor keys) will be interpreted and - # a special value like curses.KEY_LEFT will be returned - stdscr.keypad(1) - - # Start color, too. Harmless if the terminal doesn't have - # color; user can test with has_color() later on. The try/catch - # works around a minor bit of over-conscientiousness in the curses - # module -- the error return from C start_color() is ignorable. - try: - curses.start_color() - except: - pass - - return func(stdscr, *args, **kwds) - finally: - # Set everything back to normal - if 'stdscr' in locals(): - stdscr.keypad(0) - curses.echo() - curses.nocbreak() - curses.endwin() diff --git a/Lib/datetime.py b/Lib/datetime.py index 1ae7cb5305..1f8c8f79bd 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -172,10 +172,6 @@ def _format_time(hh, mm, ss, us): # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): - year = timetuple[0] - if year < 1000: - raise ValueError("year=%d is before 1000; the datetime strftime() " - "methods require year >= 1000" % year) # Don't call utcoffset() or tzname() unless actually needed. freplace = None # the string to use for %f zreplace = None # the string to use for %z diff --git a/Lib/decimal.py b/Lib/decimal.py index f5277c597c..8acb4ad863 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -1871,6 +1871,7 @@ class Decimal(object): """ other = _convert_other(other, raiseit=True) + third = _convert_other(third, raiseit=True) # compute product; raise InvalidOperation if either operand is # a signaling NaN or if the product is zero times infinity. @@ -1900,7 +1901,6 @@ class Decimal(object): str(int(self._int) * int(other._int)), self._exp + other._exp) - third = _convert_other(third, raiseit=True) return product.__add__(third, context) def _power_modulo(self, other, modulo, context=None): @@ -2001,9 +2001,9 @@ class Decimal(object): nonzero. For efficiency, other._exp should not be too large, so that 10**abs(other._exp) is a feasible calculation.""" - # In the comments below, we write x for the value of self and - # y for the value of other. Write x = xc*10**xe and y = - # yc*10**ye. + # In the comments below, we write x for the value of self and y for the + # value of other. Write x = xc*10**xe and abs(y) = yc*10**ye, with xc + # and yc positive integers not divisible by 10. # The main purpose of this method is to identify the *failure* # of x**y to be exactly representable with as little effort as @@ -2011,13 +2011,12 @@ class Decimal(object): # eliminate the possibility of x**y being exact. Only if all # these tests are passed do we go on to actually compute x**y. - # Here's the main idea. First normalize both x and y. We - # express y as a rational m/n, with m and n relatively prime - # and n>0. Then for x**y to be exactly representable (at - # *any* precision), xc must be the nth power of a positive - # integer and xe must be divisible by n. If m is negative - # then additionally xc must be a power of either 2 or 5, hence - # a power of 2**n or 5**n. + # Here's the main idea. Express y as a rational number m/n, with m and + # n relatively prime and n>0. Then for x**y to be exactly + # representable (at *any* precision), xc must be the nth power of a + # positive integer and xe must be divisible by n. If y is negative + # then additionally xc must be a power of either 2 or 5, hence a power + # of 2**n or 5**n. # # There's a limit to how small |y| can be: if y=m/n as above # then: @@ -2089,21 +2088,43 @@ class Decimal(object): return None # now xc is a power of 2; e is its exponent e = _nbits(xc)-1 - # find e*y and xe*y; both must be integers - if ye >= 0: - y_as_int = yc*10**ye - e = e*y_as_int - xe = xe*y_as_int - else: - ten_pow = 10**-ye - e, remainder = divmod(e*yc, ten_pow) - if remainder: - return None - xe, remainder = divmod(xe*yc, ten_pow) - if remainder: - return None - - if e*65 >= p*93: # 93/65 > log(10)/log(5) + + # We now have: + # + # x = 2**e * 10**xe, e > 0, and y < 0. + # + # The exact result is: + # + # x**y = 5**(-e*y) * 10**(e*y + xe*y) + # + # provided that both e*y and xe*y are integers. Note that if + # 5**(-e*y) >= 10**p, then the result can't be expressed + # exactly with p digits of precision. + # + # Using the above, we can guard against large values of ye. + # 93/65 is an upper bound for log(10)/log(5), so if + # + # ye >= len(str(93*p//65)) + # + # then + # + # -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5), + # + # so 5**(-e*y) >= 10**p, and the coefficient of the result + # can't be expressed in p digits. + + # emax >= largest e such that 5**e < 10**p. + emax = p*93//65 + if ye >= len(str(emax)): + return None + + # Find -e*y and -xe*y; both must be integers + e = _decimal_lshift_exact(e * yc, ye) + xe = _decimal_lshift_exact(xe * yc, ye) + if e is None or xe is None: + return None + + if e > emax: return None xc = 5**e @@ -2117,19 +2138,20 @@ class Decimal(object): while xc % 5 == 0: xc //= 5 e -= 1 - if ye >= 0: - y_as_integer = yc*10**ye - e = e*y_as_integer - xe = xe*y_as_integer - else: - ten_pow = 10**-ye - e, remainder = divmod(e*yc, ten_pow) - if remainder: - return None - xe, remainder = divmod(xe*yc, ten_pow) - if remainder: - return None - if e*3 >= p*10: # 10/3 > log(10)/log(2) + + # Guard against large values of ye, using the same logic as in + # the 'xc is a power of 2' branch. 10/3 is an upper bound for + # log(10)/log(2). + emax = p*10//3 + if ye >= len(str(emax)): + return None + + e = _decimal_lshift_exact(e * yc, ye) + xe = _decimal_lshift_exact(xe * yc, ye) + if e is None or xe is None: + return None + + if e > emax: return None xc = 2**e else: @@ -5529,6 +5551,27 @@ def _normalize(op1, op2, prec = 0): _nbits = int.bit_length +def _decimal_lshift_exact(n, e): + """ Given integers n and e, return n * 10**e if it's an integer, else None. + + The computation is designed to avoid computing large powers of 10 + unnecessarily. + + >>> _decimal_lshift_exact(3, 4) + 30000 + >>> _decimal_lshift_exact(300, -999999999) # returns None + + """ + if n == 0: + return 0 + elif e >= 0: + return n * 10**e + else: + # val_n = largest power of 10 dividing n. + str_n = str(abs(n)) + val_n = len(str_n) - len(str_n.rstrip('0')) + return None if val_n < -e else n // 10**-e + def _sqrt_nearest(n, a): """Closest integer to the square root of the positive integer n. a is an initial approximation to the square root. Any positive integer diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py index c06002eab3..f883916f10 100644 --- a/Lib/distutils/__init__.py +++ b/Lib/distutils/__init__.py @@ -13,5 +13,5 @@ used from a setup script as # Updated automatically by the Python release process. # #--start constants-- -__version__ = "3.2.1" +__version__ = "3.3a0" #--end constants-- diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py index e3ed3ad82c..959a8bf62e 100644 --- a/Lib/distutils/command/bdist_wininst.py +++ b/Lib/distutils/command/bdist_wininst.py @@ -265,11 +265,11 @@ class bdist_wininst(Command): cfgdata = cfgdata + b"\0" if self.pre_install_script: # We need to normalize newlines, so we open in text mode and - # convert back to bytes. "latin1" simply avoids any possible + # convert back to bytes. "latin-1" simply avoids any possible # failures. with open(self.pre_install_script, "r", - encoding="latin1") as script: - script_data = script.read().encode("latin1") + encoding="latin-1") as script: + script_data = script.read().encode("latin-1") cfgdata = cfgdata + script_data + b"\n\0" else: # empty pre-install script diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py index 8d843d689f..8baf538f2a 100644 --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -240,8 +240,7 @@ class build_ext(Command): # for extensions under Linux or Solaris with a shared Python library, # Python's library directory must be appended to library_dirs sysconfig.get_config_var('Py_ENABLE_SHARED') - if ((sys.platform.startswith('linux') or sys.platform.startswith('gnu') - or sys.platform.startswith('sunos')) + if (sys.platform.startswith(('linux', 'gnu', 'sunos')) and sysconfig.get_config_var('Py_ENABLE_SHARED')): if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): # building third party extensions diff --git a/Lib/distutils/command/build_scripts.py b/Lib/distutils/command/build_scripts.py index ec43477061..4b5b22ec20 100644 --- a/Lib/distutils/command/build_scripts.py +++ b/Lib/distutils/command/build_scripts.py @@ -126,10 +126,9 @@ class build_scripts(Command): "The shebang ({!r}) is not decodable " "from the script encoding ({})" .format(shebang, encoding)) - outf = open(outfile, "wb") - outf.write(shebang) - outf.writelines(f.readlines()) - outf.close() + with open(outfile, "wb") as outf: + outf.write(shebang) + outf.writelines(f.readlines()) if f: f.close() else: diff --git a/Lib/distutils/tests/test_bdist_rpm.py b/Lib/distutils/tests/test_bdist_rpm.py index 804fb1355f..9b0639a549 100644 --- a/Lib/distutils/tests/test_bdist_rpm.py +++ b/Lib/distutils/tests/test_bdist_rpm.py @@ -28,6 +28,11 @@ class BuildRpmTestCase(support.TempdirManager, unittest.TestCase): def setUp(self): + try: + sys.executable.encode("UTF-8") + except UnicodeEncodeError: + raise unittest.SkipTest("sys.executable is not encodable to UTF-8") + super(BuildRpmTestCase, self).setUp() self.old_location = os.getcwd() self.old_sys_argv = sys.argv, sys.argv[:] @@ -42,7 +47,7 @@ class BuildRpmTestCase(support.TempdirManager, # XXX I am unable yet to make this test work without # spurious sdtout/stderr output under Mac OS X - if sys.platform != 'linux2': + if not sys.platform.startswith('linux'): return # this test will run only if the rpm commands are found @@ -82,7 +87,7 @@ class BuildRpmTestCase(support.TempdirManager, # XXX I am unable yet to make this test work without # spurious sdtout/stderr output under Mac OS X - if sys.platform != 'linux2': + if not sys.platform.startswith('linux'): return # http://bugs.python.org/issue1533164 diff --git a/Lib/doctest.py b/Lib/doctest.py index f60b06d7c9..be8cc23eb5 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1378,6 +1378,7 @@ class DocTestRunner: # Note that the interactive output will go to *our* # save_stdout, even if that's not the real sys.stdout; this # allows us to write test cases for the set_trace behavior. + save_trace = sys.gettrace() save_set_trace = pdb.set_trace self.debugger = _OutputRedirectingPdb(save_stdout) self.debugger.reset() @@ -1397,6 +1398,7 @@ class DocTestRunner: finally: sys.stdout = save_stdout pdb.set_trace = save_set_trace + sys.settrace(save_trace) linecache.getlines = self.save_linecache_getlines sys.displayhook = save_displayhook if clear_globs: diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 41694f9b1a..c455e056d0 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -47,6 +47,21 @@ def parsedate_tz(data): Accounts for military timezones. """ + res = _parsedate_tz(data) + if res[9] is None: + res[9] = 0 + return tuple(res) + +def _parsedate_tz(data): + """Convert date to extended time tuple. + + The last (additional) element is the time zone offset in seconds, except if + the timezone was specified as -0000. In that case the last element is + None. This indicates a UTC timestamp that explicitly declaims knowledge of + the source timezone, as opposed to a +0000 timestamp that indicates the + source timezone really was UTC. + + """ data = data.split() # The FWS after the comma after the day-of-week is optional, so search and # adjust for this. @@ -99,6 +114,14 @@ def parsedate_tz(data): tss = '0' elif len(tm) == 3: [thh, tmm, tss] = tm + elif len(tm) == 1 and '.' in tm[0]: + # Some non-compliant MUAs use '.' to separate time elements. + tm = tm[0].split('.') + if len(tm) == 2: + [thh, tmm] = tm + tss = 0 + elif len(tm) == 3: + [thh, tmm, tss] = tm else: return None try: @@ -130,6 +153,8 @@ def parsedate_tz(data): tzoffset = int(tz) except ValueError: pass + if tzoffset==0 and tz.startswith('-'): + tzoffset = None # Convert a timezone offset into seconds ; -0500 -> -18000 if tzoffset: if tzoffset < 0: @@ -139,7 +164,7 @@ def parsedate_tz(data): tzsign = 1 tzoffset = tzsign * ( (tzoffset//100)*3600 + (tzoffset % 100)*60) # Daylight Saving Time flag is set to -1, since DST is unknown. - return yy, mm, dd, thh, tmm, tss, 0, 1, -1, tzoffset + return [yy, mm, dd, thh, tmm, tss, 0, 1, -1, tzoffset] def parsedate(data): diff --git a/Lib/email/errors.py b/Lib/email/errors.py index d52a624601..c04deb40ed 100644 --- a/Lib/email/errors.py +++ b/Lib/email/errors.py @@ -32,7 +32,7 @@ class CharsetError(MessageError): # These are parsing defects which the parser was able to work around. -class MessageDefect: +class MessageDefect(Exception): """Base class for a message defect.""" def __init__(self, line=None): @@ -55,3 +55,6 @@ class MalformedHeaderDefect(MessageDefect): class MultipartInvariantViolationDefect(MessageDefect): """A message claimed to be a multipart but no subparts were found.""" + +class InvalidMultipartContentTransferEncodingDefect(MessageDefect): + """An invalid content transfer encoding was set on the multipart itself.""" diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 60a83255c0..e754d89cb6 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -25,6 +25,7 @@ import re from email import errors from email import message +from email import policy NLCRE = re.compile('\r\n|\r|\n') NLCRE_bol = re.compile('(\r\n|\r|\n)') @@ -120,9 +121,6 @@ class BufferedSubFile(object): # Reverse and insert at the front of the lines. self._lines[:0] = lines[::-1] - def is_closed(self): - return self._closed - def __iter__(self): return self @@ -137,9 +135,16 @@ class BufferedSubFile(object): class FeedParser: """A feed-style parser of email.""" - def __init__(self, _factory=message.Message): - """_factory is called with no arguments to create a new message obj""" + def __init__(self, _factory=message.Message, *, policy=policy.default): + """_factory is called with no arguments to create a new message obj + + The policy keyword specifies a policy object that controls a number of + aspects of the parser's operation. The default policy maintains + backward compatibility. + + """ self._factory = _factory + self.policy = policy self._input = BufferedSubFile() self._msgstack = [] self._parse = self._parsegen().__next__ @@ -171,7 +176,8 @@ class FeedParser: # Look for final set of defects if root.get_content_maintype() == 'multipart' \ and not root.is_multipart(): - root.defects.append(errors.MultipartInvariantViolationDefect()) + defect = errors.MultipartInvariantViolationDefect() + self.policy.handle_defect(root, defect) return root def _new_message(self): @@ -284,7 +290,8 @@ class FeedParser: # defined a boundary. That's a problem which we'll handle by # reading everything until the EOF and marking the message as # defective. - self._cur.defects.append(errors.NoBoundaryInMultipartDefect()) + defect = errors.NoBoundaryInMultipartDefect() + self.policy.handle_defect(self._cur, defect) lines = [] for line in self._input: if line is NeedMoreData: @@ -293,6 +300,11 @@ class FeedParser: lines.append(line) self._cur.set_payload(EMPTYSTRING.join(lines)) return + # Make sure a valid content type was specified per RFC 2045:6.4. + if (self._cur.get('content-transfer-encoding', '8bit').lower() + not in ('7bit', '8bit', 'binary')): + defect = errors.InvalidMultipartContentTransferEncodingDefect() + self.policy.handle_defect(self._cur, defect) # Create a line match predicate which matches the inter-part # boundary as well as the end-of-multipart boundary. Don't push # this onto the input stream until we've scanned past the @@ -388,7 +400,8 @@ class FeedParser: # that as a defect and store the captured text as the payload. # Everything from here to the EOF is epilogue. if capturing_preamble: - self._cur.defects.append(errors.StartBoundaryNotFoundDefect()) + defect = errors.StartBoundaryNotFoundDefect() + self.policy.handle_defect(self._cur, defect) self._cur.set_payload(EMPTYSTRING.join(preamble)) epilogue = [] for line in self._input: @@ -440,7 +453,7 @@ class FeedParser: # is illegal, so let's note the defect, store the illegal # line, and ignore it for purposes of headers. defect = errors.FirstHeaderLineIsContinuationDefect(line) - self._cur.defects.append(defect) + self.policy.handle_defect(self._cur, defect) continue lastvalue.append(line) continue diff --git a/Lib/email/generator.py b/Lib/email/generator.py index f0e7a95477..d8b8fa960b 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -13,8 +13,10 @@ import random import warnings from io import StringIO, BytesIO +from email import policy from email.header import Header from email.message import _has_surrogates +import email.charset as _charset UNDERSCORE = '_' NL = '\n' # XXX: no longer used by the code below. @@ -33,7 +35,8 @@ class Generator: # Public interface # - def __init__(self, outfp, mangle_from_=True, maxheaderlen=78): + def __init__(self, outfp, mangle_from_=True, maxheaderlen=None, *, + policy=policy.default): """Create the generator for message flattening. outfp is the output file-like object for writing the message to. It @@ -49,16 +52,23 @@ class Generator: defined in the Header class. Set maxheaderlen to zero to disable header wrapping. The default is 78, as recommended (but not required) by RFC 2822. + + The policy keyword specifies a policy object that controls a number of + aspects of the generator's operation. The default policy maintains + backward compatibility. + """ self._fp = outfp self._mangle_from_ = mangle_from_ - self._maxheaderlen = maxheaderlen + self._maxheaderlen = (maxheaderlen if maxheaderlen is not None else + policy.max_line_length) + self.policy = policy def write(self, s): # Just delegate to the file object self._fp.write(s) - def flatten(self, msg, unixfrom=False, linesep='\n'): + def flatten(self, msg, unixfrom=False, linesep=None): r"""Print the message object tree rooted at msg to the output file specified when the Generator instance was created. @@ -70,17 +80,15 @@ class Generator: Note that for subobjects, no From_ line is printed. linesep specifies the characters used to indicate a new line in - the output. The default value is the most useful for typical - Python applications, but it can be set to \r\n to produce RFC-compliant - line separators when needed. + the output. The default value is determined by the policy. """ # We use the _XXX constants for operating on data that comes directly # from the msg, and _encoded_XXX constants for operating on data that # has already been converted (to bytes in the BytesGenerator) and # inserted into a temporary buffer. - self._NL = linesep - self._encoded_NL = self._encode(linesep) + self._NL = linesep if linesep is not None else self.policy.linesep + self._encoded_NL = self._encode(self._NL) self._EMPTY = '' self._encoded_EMTPY = self._encode('') if unixfrom: @@ -297,10 +305,12 @@ class Generator: # message/rfc822. Such messages are generated by, for example, # Groupwise when forwarding unadorned messages. (Issue 7970.) So # in that case we just emit the string body. - payload = msg.get_payload() + payload = msg._payload if isinstance(payload, list): g.flatten(msg.get_payload(0), unixfrom=False, linesep=self._NL) payload = s.getvalue() + else: + payload = self._encode(payload) self._fp.write(payload) # This used to be a module level function; we use a classmethod for this @@ -336,7 +346,10 @@ class BytesGenerator(Generator): Functionally identical to the base Generator except that the output is bytes and not string. When surrogates were used in the input to encode - bytes, these are decoded back to bytes for output. + bytes, these are decoded back to bytes for output. If the policy has + must_be_7bit set true, then the message is transformed such that the + non-ASCII bytes are properly content transfer encoded, using the + charset unknown-8bit. The outfp object must accept bytes in its write method. """ @@ -359,21 +372,22 @@ class BytesGenerator(Generator): # strings with 8bit bytes. for h, v in msg._headers: self.write('%s: ' % h) - if isinstance(v, Header): - self.write(v.encode(maxlinelen=self._maxheaderlen)+NL) - elif _has_surrogates(v): - # If we have raw 8bit data in a byte string, we have no idea - # what the encoding is. There is no safe way to split this - # string. If it's ascii-subset, then we could do a normal - # ascii split, but if it's multibyte then we could break the - # string. There's no way to know so the least harm seems to - # be to not split the string and risk it being too long. - self.write(v+NL) - else: - # Header's got lots of smarts and this string is safe... - header = Header(v, maxlinelen=self._maxheaderlen, - header_name=h) - self.write(header.encode(linesep=self._NL)+self._NL) + if isinstance(v, str): + if _has_surrogates(v): + if not self.policy.must_be_7bit: + # If we have raw 8bit data in a byte string, we have no idea + # what the encoding is. There is no safe way to split this + # string. If it's ascii-subset, then we could do a normal + # ascii split, but if it's multibyte then we could break the + # string. There's no way to know so the least harm seems to + # be to not split the string and risk it being too long. + self.write(v+NL) + continue + h = Header(v, charset=_charset.UNKNOWN8BIT, header_name=h) + else: + h = Header(v, header_name=h) + self.write(h.encode(linesep=self._NL, + maxlinelen=self._maxheaderlen)+self._NL) # A blank line always separates headers from body self.write(self._NL) @@ -382,7 +396,7 @@ class BytesGenerator(Generator): # just write it back out. if msg._payload is None: return - if _has_surrogates(msg._payload): + if _has_surrogates(msg._payload) and not self.policy.must_be_7bit: self.write(msg._payload) else: super(BytesGenerator,self)._handle_text(msg) diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 6caaff53ad..0f921609dd 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -4,18 +4,19 @@ """A parser of RFC 2822 and MIME email messages.""" -__all__ = ['Parser', 'HeaderParser'] +__all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser'] import warnings from io import StringIO, TextIOWrapper from email.feedparser import FeedParser from email.message import Message +from email import policy class Parser: - def __init__(self, *args, **kws): + def __init__(self, _class=Message, *, policy=policy.default): """Parser of RFC 2822 and MIME email messages. Creates an in-memory object tree representing the email message, which @@ -30,28 +31,14 @@ class Parser: _class is the class to instantiate for new message objects when they must be created. This class must have a constructor that can take zero arguments. Default is Message.Message. + + The policy keyword specifies a policy object that controls a number of + aspects of the parser's operation. The default policy maintains + backward compatibility. + """ - if len(args) >= 1: - if '_class' in kws: - raise TypeError("Multiple values for keyword arg '_class'") - kws['_class'] = args[0] - if len(args) == 2: - if 'strict' in kws: - raise TypeError("Multiple values for keyword arg 'strict'") - kws['strict'] = args[1] - if len(args) > 2: - raise TypeError('Too many arguments') - if '_class' in kws: - self._class = kws['_class'] - del kws['_class'] - else: - self._class = Message - if 'strict' in kws: - warnings.warn("'strict' argument is deprecated (and ignored)", - DeprecationWarning, 2) - del kws['strict'] - if kws: - raise TypeError('Unexpected keyword arguments') + self._class = _class + self.policy = policy def parse(self, fp, headersonly=False): """Create a message structure from the data in a file. @@ -61,7 +48,7 @@ class Parser: parsing after reading the headers or not. The default is False, meaning it parses the entire contents of the file. """ - feedparser = FeedParser(self._class) + feedparser = FeedParser(self._class, policy=self.policy) if headersonly: feedparser._set_headersonly() while True: @@ -134,3 +121,11 @@ class BytesParser: """ text = text.decode('ASCII', errors='surrogateescape') return self.parser.parsestr(text, headersonly) + + +class BytesHeaderParser(BytesParser): + def parse(self, fp, headersonly=True): + return BytesParser.parse(self, fp, headersonly=True) + + def parsebytes(self, text, headersonly=True): + return BytesParser.parsebytes(self, text, headersonly=True) diff --git a/Lib/email/policy.py b/Lib/email/policy.py new file mode 100644 index 0000000000..88877a2f45 --- /dev/null +++ b/Lib/email/policy.py @@ -0,0 +1,174 @@ +"""Policy framework for the email package. + +Allows fine grained feature control of how the package parses and emits data. +""" + +__all__ = [ + 'Policy', + 'default', + 'strict', + 'SMTP', + 'HTTP', + ] + + +class _PolicyBase: + + """Policy Object basic framework. + + This class is useless unless subclassed. A subclass should define + class attributes with defaults for any values that are to be + managed by the Policy object. The constructor will then allow + non-default values to be set for these attributes at instance + creation time. The instance will be callable, taking these same + attributes keyword arguments, and returning a new instance + identical to the called instance except for those values changed + by the keyword arguments. Instances may be added, yielding new + instances with any non-default values from the right hand + operand overriding those in the left hand operand. That is, + + A + B == A(<non-default values of B>) + + The repr of an instance can be used to reconstruct the object + if and only if the repr of the values can be used to reconstruct + those values. + + """ + + def __init__(self, **kw): + """Create new Policy, possibly overriding some defaults. + + See class docstring for a list of overridable attributes. + + """ + for name, value in kw.items(): + if hasattr(self, name): + super(_PolicyBase,self).__setattr__(name, value) + else: + raise TypeError( + "{!r} is an invalid keyword argument for {}".format( + name, self.__class__.__name__)) + + def __repr__(self): + args = [ "{}={!r}".format(name, value) + for name, value in self.__dict__.items() ] + return "{}({})".format(self.__class__.__name__, args if args else '') + + def clone(self, **kw): + """Return a new instance with specified attributes changed. + + The new instance has the same attribute values as the current object, + except for the changes passed in as keyword arguments. + + """ + for attr, value in self.__dict__.items(): + if attr not in kw: + kw[attr] = value + return self.__class__(**kw) + + def __setattr__(self, name, value): + if hasattr(self, name): + msg = "{!r} object attribute {!r} is read-only" + else: + msg = "{!r} object has no attribute {!r}" + raise AttributeError(msg.format(self.__class__.__name__, name)) + + def __add__(self, other): + """Non-default values from right operand override those from left. + + The object returned is a new instance of the subclass. + + """ + return self.clone(**other.__dict__) + + +class Policy(_PolicyBase): + + """Controls for how messages are interpreted and formatted. + + Most of the classes and many of the methods in the email package + accept Policy objects as parameters. A Policy object contains a set + of values and functions that control how input is interpreted and how + output is rendered. For example, the parameter 'raise_on_defect' + controls whether or not an RFC violation throws an error or not, + while 'max_line_length' controls the maximum length of output lines + when a Message is serialized. + + Any valid attribute may be overridden when a Policy is created by + passing it as a keyword argument to the constructor. Policy + objects are immutable, but a new Policy object can be created + with only certain values changed by calling the Policy instance + with keyword arguments. Policy objects can also be added, + producing a new Policy object in which the non-default attributes + set in the right hand operand overwrite those specified in the + left operand. + + Settable attributes: + + raise_on_defect -- If true, then defects should be raised + as errors. Default False. + + linesep -- string containing the value to use as + separation between output lines. Default '\n'. + + must_be_7bit -- output must contain only 7bit clean data. + Default False. + + max_line_length -- maximum length of lines, excluding 'linesep', + during serialization. None means no line + wrapping is done. Default is 78. + + Methods: + + register_defect(obj, defect) + defect is a Defect instance. The default implementation appends defect + to the objs 'defects' attribute. + + handle_defect(obj, defect) + intended to be called by parser code that finds a defect. If + raise_on_defect is True, defect is raised as an error, otherwise + register_defect is called. + + """ + + raise_on_defect = False + linesep = '\n' + must_be_7bit = False + max_line_length = 78 + + def handle_defect(self, obj, defect): + """Based on policy, either raise defect or call register_defect. + + handle_defect(obj, defect) + + defect should be a Defect subclass, but in any case must be an + Exception subclass. obj is the object on which the defect should be + registered if it is not raised. If the raise_on_defect is True, the + defect is raised as an error, otherwise the object and the defect are + passed to register_defect. + + This class is intended to be called by parsers that discover defects, + and will not be called from code using the library unless that code is + implementing an alternate parser. + + """ + if self.raise_on_defect: + raise defect + self.register_defect(obj, defect) + + def register_defect(self, obj, defect): + """Record 'defect' on 'obj'. + + Called by handle_defect if raise_on_defect is False. This method is + part of the Policy API so that Policy subclasses can implement custom + defect handling. The default implementation calls the append method + of the defects attribute of obj. + + """ + obj.defects.append(defect) + + +default = Policy() +strict = default.clone(raise_on_defect=True) +SMTP = default.clone(linesep='\r\n') +HTTP = default.clone(linesep='\r\n', max_line_length=None) diff --git a/Lib/email/utils.py b/Lib/email/utils.py index ac4da3705f..aecea656e6 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -11,12 +11,14 @@ __all__ = [ 'encode_rfc2231', 'formataddr', 'formatdate', + 'format_datetime', 'getaddresses', 'make_msgid', 'mktime_tz', 'parseaddr', 'parsedate', 'parsedate_tz', + 'parsedate_to_datetime', 'unquote', ] @@ -26,6 +28,7 @@ import time import base64 import random import socket +import datetime import urllib.parse import warnings from io import StringIO @@ -37,11 +40,13 @@ from email._parseaddr import mktime_tz # We need wormarounds for bugs in these methods in older Pythons (see below) from email._parseaddr import parsedate as _parsedate from email._parseaddr import parsedate_tz as _parsedate_tz +from email._parseaddr import _parsedate_tz as __parsedate_tz from quopri import decodestring as _qdecode # Intrapackage imports from email.encoders import _bencode, _qencode +from email.charset import Charset COMMASPACE = ', ' EMPTYSTRING = '' @@ -56,21 +61,36 @@ escapesre = re.compile(r'[][\\()"]') # Helpers -def formataddr(pair): +def formataddr(pair, charset='utf-8'): """The inverse of parseaddr(), this takes a 2-tuple of the form (realname, email_address) and returns the string value suitable for an RFC 2822 From, To or Cc header. If the first element of pair is false, then the second element is returned unmodified. + + Optional charset if given is the character set that is used to encode + realname in case realname is not ASCII safe. Can be an instance of str or + a Charset-like object which has a header_encode method. Default is + 'utf-8'. """ name, address = pair + # The address MUST (per RFC) be ascii, so throw a UnicodeError if it isn't. + address.encode('ascii') if name: - quotes = '' - if specialsre.search(name): - quotes = '"' - name = escapesre.sub(r'\\\g<0>', name) - return '%s%s%s <%s>' % (quotes, name, quotes, address) + try: + name.encode('ascii') + except UnicodeEncodeError: + if isinstance(charset, str): + charset = Charset(charset) + encoded_name = charset.header_encode(name) + return "%s <%s>" % (encoded_name, address) + else: + quotes = '' + if specialsre.search(name): + quotes = '"' + name = escapesre.sub(r'\\\g<0>', name) + return '%s%s%s <%s>' % (quotes, name, quotes, address) return address @@ -94,6 +114,14 @@ ecre = re.compile(r''' ''', re.VERBOSE | re.IGNORECASE) +def _format_timetuple_and_zone(timetuple, zone): + return '%s, %02d %s %04d %02d:%02d:%02d %s' % ( + ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][timetuple[6]], + timetuple[2], + ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][timetuple[1] - 1], + timetuple[0], timetuple[3], timetuple[4], timetuple[5], + zone) def formatdate(timeval=None, localtime=False, usegmt=False): """Returns a date string as specified by RFC 2822, e.g.: @@ -138,14 +166,25 @@ def formatdate(timeval=None, localtime=False, usegmt=False): zone = 'GMT' else: zone = '-0000' - return '%s, %02d %s %04d %02d:%02d:%02d %s' % ( - ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][now[6]], - now[2], - ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', - 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][now[1] - 1], - now[0], now[3], now[4], now[5], - zone) + return _format_timetuple_and_zone(now, zone) + +def format_datetime(dt, usegmt=False): + """Turn a datetime into a date string as specified in RFC 2822. + If usegmt is True, dt must be an aware datetime with an offset of zero. In + this case 'GMT' will be rendered instead of the normal +0000 required by + RFC2822. This is to support HTTP headers involving date stamps. + """ + now = dt.timetuple() + if usegmt: + if dt.tzinfo is None or dt.tzinfo != datetime.timezone.utc: + raise ValueError("usegmt option requires a UTC datetime") + zone = 'GMT' + elif dt.tzinfo is None: + zone = '-0000' + else: + zone = dt.strftime("%z") + return _format_timetuple_and_zone(now, zone) def make_msgid(idstring=None, domain=None): @@ -187,6 +226,15 @@ def parsedate_tz(data): return None return _parsedate_tz(data) +def parsedate_to_datetime(data): + if not data: + return None + *dtuple, tz = __parsedate_tz(data) + if tz is None: + return datetime.datetime(*dtuple[:6]) + return datetime.datetime(*dtuple[:6], + tzinfo=datetime.timezone(datetime.timedelta(seconds=tz))) + def parseaddr(addr): addrs = _AddressList(addr).addresslist diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 8e53023270..f87f6909c6 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -100,14 +100,15 @@ class FTP: file = None welcome = None passiveserver = 1 - encoding = "latin1" + encoding = "latin-1" # Initialization method (called by class instantiation). # Initialize host to localhost, port to standard ftp port # Optional arguments are host (for connect()), # and user, passwd, acct (for login()) def __init__(self, host='', user='', passwd='', acct='', - timeout=_GLOBAL_DEFAULT_TIMEOUT): + timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None): + self.source_address = source_address self.timeout = timeout if host: self.connect(host) @@ -128,10 +129,12 @@ class FTP: if self.sock is not None: self.close() - def connect(self, host='', port=0, timeout=-999): + def connect(self, host='', port=0, timeout=-999, source_address=None): '''Connect to host. Arguments are: - host: hostname to connect to (string, default previous host) - port: port to connect to (integer, default previous port) + - source_address: a 2-tuple (host, port) for the socket to bind + to as its source address before connecting. ''' if host != '': self.host = host @@ -139,7 +142,10 @@ class FTP: self.port = port if timeout != -999: self.timeout = timeout - self.sock = socket.create_connection((self.host, self.port), self.timeout) + if source_address is not None: + self.source_address = source_address + self.sock = socket.create_connection((self.host, self.port), self.timeout, + source_address=self.source_address) self.af = self.sock.family self.file = self.sock.makefile('r', encoding=self.encoding) self.welcome = self.getresp() @@ -335,7 +341,8 @@ class FTP: size = None if self.passiveserver: host, port = self.makepasv() - conn = socket.create_connection((host, port), self.timeout) + conn = socket.create_connection((host, port), self.timeout, + source_address=self.source_address) try: if rest is not None: self.sendcmd("REST %s" % rest) @@ -426,7 +433,7 @@ class FTP: """Retrieve data in line mode. A new port is created for you. Args: - cmd: A RETR, LIST, NLST, or MLSD command. + cmd: A RETR, LIST, or NLST command. callback: An optional single parameter callable that is called for each line with the trailing CRLF stripped. [default: print_line()] @@ -527,6 +534,34 @@ class FTP: cmd = cmd + (' ' + arg) self.retrlines(cmd, func) + def mlsd(self, path="", facts=[]): + '''List a directory in a standardized format by using MLSD + command (RFC-3659). If path is omitted the current directory + is assumed. "facts" is a list of strings representing the type + of information desired (e.g. ["type", "size", "perm"]). + + Return a generator object yielding a tuple of two elements + for every file found in path. + First element is the file name, the second one is a dictionary + including a variable number of "facts" depending on the server + and whether "facts" argument has been provided. + ''' + if facts: + self.sendcmd("OPTS MLST " + ";".join(facts) + ";") + if path: + cmd = "MLSD %s" % path + else: + cmd = "MLSD" + lines = [] + self.retrlines(cmd, lines.append) + for line in lines: + facts_found, _, name = line.rstrip(CRLF).partition(' ') + entry = {} + for fact in facts_found[:-1].split(";"): + key, _, value = fact.partition("=") + entry[key.lower()] = value + yield (name, entry) + def rename(self, fromname, toname): '''Rename a file.''' resp = self.sendcmd('RNFR ' + fromname) @@ -596,11 +631,11 @@ class FTP: def close(self): '''Close the connection without assuming anything about it.''' - if self.file: + if self.file is not None: self.file.close() + if self.sock is not None: self.sock.close() - self.file = self.sock = None - + self.file = self.sock = None try: import ssl @@ -644,7 +679,7 @@ else: def __init__(self, host='', user='', passwd='', acct='', keyfile=None, certfile=None, context=None, - timeout=_GLOBAL_DEFAULT_TIMEOUT): + timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None): if context is not None and keyfile is not None: raise ValueError("context and keyfile arguments are mutually " "exclusive") @@ -655,7 +690,7 @@ else: self.certfile = certfile self.context = context self._prot_p = False - FTP.__init__(self, host, user, passwd, acct, timeout) + FTP.__init__(self, host, user, passwd, acct, timeout, source_address) def login(self, user='', passwd='', acct='', secure=True): if secure and not isinstance(self.sock, ssl.SSLSocket): @@ -679,6 +714,14 @@ else: self.file = self.sock.makefile(mode='r', encoding=self.encoding) return resp + def ccc(self): + '''Switch back to a clear-text control connection.''' + if not isinstance(self.sock, ssl.SSLSocket): + raise ValueError("not using TLS") + resp = self.voidcmd('CCC') + self.sock = self.sock.unwrap() + return resp + def prot_p(self): '''Set up secure data connection.''' # PROT defines whether or not the data channel is to be protected. diff --git a/Lib/functools.py b/Lib/functools.py index 03bcb1e69f..b2bcc21f48 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -114,6 +114,11 @@ def cmp_to_key(mycmp): __hash__ = None return K +try: + from _functools import cmp_to_key +except ImportError: + pass + _CacheInfo = namedtuple("CacheInfo", "hits misses maxsize currsize") def lru_cache(maxsize=100): diff --git a/Lib/getopt.py b/Lib/getopt.py index 980861d251..3d6ecbddb9 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -19,7 +19,7 @@ option involved with the exception. # Gerrit Holl <gerrit@nl.linux.org> moved the string-based exceptions # to class-based exceptions. # -# Peter Åstrand <astrand@lysator.liu.se> added gnu_getopt(). +# Peter Åstrand <astrand@lysator.liu.se> added gnu_getopt(). # # TODO for gnu_getopt(): # @@ -34,6 +34,11 @@ option involved with the exception. __all__ = ["GetoptError","error","getopt","gnu_getopt"] import os +try: + from gettext import gettext as _ +except ImportError: + # Bootstrapping Python: gettext's dependencies not built yet + def _(s): return s class GetoptError(Exception): opt = '' @@ -153,10 +158,10 @@ def do_longs(opts, opt, longopts, args): if has_arg: if optarg is None: if not args: - raise GetoptError('option --%s requires argument' % opt, opt) + raise GetoptError(_('option --%s requires argument') % opt, opt) optarg, args = args[0], args[1:] elif optarg is not None: - raise GetoptError('option --%s must not have an argument' % opt, opt) + raise GetoptError(_('option --%s must not have an argument') % opt, opt) opts.append(('--' + opt, optarg or '')) return opts, args @@ -166,7 +171,7 @@ def do_longs(opts, opt, longopts, args): def long_has_args(opt, longopts): possibilities = [o for o in longopts if o.startswith(opt)] if not possibilities: - raise GetoptError('option --%s not recognized' % opt, opt) + raise GetoptError(_('option --%s not recognized') % opt, opt) # Is there an exact match? if opt in possibilities: return False, opt @@ -176,7 +181,7 @@ def long_has_args(opt, longopts): if len(possibilities) > 1: # XXX since possibilities contains all valid continuations, might be # nice to work them into the error msg - raise GetoptError('option --%s not a unique prefix' % opt, opt) + raise GetoptError(_('option --%s not a unique prefix') % opt, opt) assert len(possibilities) == 1 unique_match = possibilities[0] has_arg = unique_match.endswith('=') @@ -190,7 +195,7 @@ def do_shorts(opts, optstring, shortopts, args): if short_has_arg(opt, shortopts): if optstring == '': if not args: - raise GetoptError('option -%s requires argument' % opt, + raise GetoptError(_('option -%s requires argument') % opt, opt) optstring, args = args[0], args[1:] optarg, optstring = optstring, '' @@ -203,7 +208,7 @@ def short_has_arg(opt, shortopts): for i in range(len(shortopts)): if opt == shortopts[i] != ':': return shortopts.startswith(':', i+1) - raise GetoptError('option -%s not recognized' % opt, opt) + raise GetoptError(_('option -%s not recognized') % opt, opt) if __name__ == '__main__': import sys diff --git a/Lib/gzip.py b/Lib/gzip.py index ba2149ebf9..e6b8193505 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -16,18 +16,6 @@ FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 READ, WRITE = 1, 2 -def U32(i): - """Return i as an unsigned integer, assuming it fits in 32 bits. - If it's >= 2GB when viewed as a 32-bit unsigned int, return a long. - """ - if i < 0: - i += 1 << 32 - return i - -def LOWU32(i): - """Return the low-order 32 bits, as a non-negative int""" - return i & 0xFFFFFFFF - def write32u(output, value): # The L format writes the bit pattern correctly whether signed # or unsigned. @@ -348,6 +336,28 @@ class GzipFile(io.BufferedIOBase): self.offset += size return chunk + def read1(self, size=-1): + self._check_closed() + if self.mode != READ: + import errno + raise IOError(errno.EBADF, "read1() on write-only GzipFile object") + + if self.extrasize <= 0 and self.fileobj is None: + return b'' + + try: + self._read() + except EOFError: + pass + if size < 0 or size > self.extrasize: + size = self.extrasize + + offset = self.offset - self.extrastart + chunk = self.extrabuf[offset: offset + size] + self.extrasize -= size + self.offset += size + return chunk + def peek(self, n): if self.mode != READ: import errno diff --git a/Lib/http/client.py b/Lib/http/client.py index 3f0272942e..57755730e9 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -697,7 +697,7 @@ class HTTPConnection: self.send(connect_bytes) for header, value in self._tunnel_headers.items(): header_str = "%s: %s\r\n" % (header, value) - header_bytes = header_str.encode("latin1") + header_bytes = header_str.encode("latin-1") self.send(header_bytes) self.send(b'\r\n') @@ -937,7 +937,7 @@ class HTTPConnection: values = list(values) for i, one_value in enumerate(values): if hasattr(one_value, 'encode'): - values[i] = one_value.encode('latin1') + values[i] = one_value.encode('latin-1') elif isinstance(one_value, int): values[i] = str(one_value).encode('ascii') value = b'\r\n\t'.join(values) diff --git a/Lib/http/server.py b/Lib/http/server.py index 86fa37fd5a..e571418cc0 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -355,6 +355,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): """ self.send_response_only(100) + self.flush_headers() return True def handle_one_request(self): @@ -432,7 +433,8 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): self.wfile.write(content.encode('UTF-8', 'replace')) def send_response(self, code, message=None): - """Send the response header and log the response code. + """Add the response header to the headers buffer and log the + response code. Also send two standard headers with the server software version and the current date. @@ -451,16 +453,19 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): else: message = '' if self.request_version != 'HTTP/0.9': - self.wfile.write(("%s %d %s\r\n" % - (self.protocol_version, code, message)).encode('latin1', 'strict')) + if not hasattr(self, '_headers_buffer'): + self._headers_buffer = [] + self._headers_buffer.append(("%s %d %s\r\n" % + (self.protocol_version, code, message)).encode( + 'latin-1', 'strict')) def send_header(self, keyword, value): - """Send a MIME header.""" + """Send a MIME header to the headers buffer.""" if self.request_version != 'HTTP/0.9': if not hasattr(self, '_headers_buffer'): self._headers_buffer = [] self._headers_buffer.append( - ("%s: %s\r\n" % (keyword, value)).encode('latin1', 'strict')) + ("%s: %s\r\n" % (keyword, value)).encode('latin-1', 'strict')) if keyword.lower() == 'connection': if value.lower() == 'close': @@ -472,6 +477,10 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): """Send the blank line ending the MIME headers.""" if self.request_version != 'HTTP/0.9': self._headers_buffer.append(b"\r\n") + self.flush_headers() + + def flush_headers(self): + if hasattr(self, '_headers_buffer'): self.wfile.write(b"".join(self._headers_buffer)) self._headers_buffer = [] @@ -888,11 +897,7 @@ def nobody_uid(): def executable(path): """Test for executable file.""" - try: - st = os.stat(path) - except os.error: - return False - return st.st_mode & 0o111 != 0 + return os.access(path, os.X_OK) class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): @@ -1006,7 +1011,7 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): scriptname) return ispy = self.is_python(scriptname) - if not ispy: + if self.have_fork or not ispy: if not self.is_executable(scriptfile): self.send_error(403, "CGI script is not executable (%r)" % scriptname) @@ -1081,6 +1086,7 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): env.setdefault(k, "") self.send_response(200, "Script output follows") + self.flush_headers() decoded_query = query.replace('+', ' ') diff --git a/Lib/idlelib/idlever.py b/Lib/idlelib/idlever.py index 3fc5e7f775..5e9afb17ae 100644 --- a/Lib/idlelib/idlever.py +++ b/Lib/idlelib/idlever.py @@ -1 +1 @@ -IDLE_VERSION = "3.2.1" +IDLE_VERSION = "3.3a0" diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 1022e77c4f..1b540655f2 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -249,15 +249,7 @@ class IMAP4: def read(self, size): """Read 'size' bytes from remote.""" - chunks = [] - read = 0 - while read < size: - data = self.file.read(min(size-read, 4096)) - if not data: - break - read += len(data) - chunks.append(data) - return b''.join(chunks) + return self.file.read(size) def readline(self): @@ -1177,25 +1169,40 @@ if HAVE_SSL: """IMAP4 client class over SSL connection - Instantiate with: IMAP4_SSL([host[, port[, keyfile[, certfile]]]]) + Instantiate with: IMAP4_SSL([host[, port[, keyfile[, certfile[, ssl_context]]]]]) host - host's name (default: localhost); - port - port number (default: standard IMAP4 SSL port). + port - port number (default: standard IMAP4 SSL port); keyfile - PEM formatted file that contains your private key (default: None); certfile - PEM formatted certificate chain file (default: None); + ssl_context - a SSLContext object that contains your certificate chain + and private key (default: None) + Note: if ssl_context is provided, then parameters keyfile or + certfile should not be set otherwise ValueError is thrown. for more documentation see the docstring of the parent class IMAP4. """ - def __init__(self, host = '', port = IMAP4_SSL_PORT, keyfile = None, certfile = None): + def __init__(self, host='', port=IMAP4_SSL_PORT, keyfile=None, certfile=None, ssl_context=None): + if ssl_context is not None and keyfile is not None: + raise ValueError("ssl_context and keyfile arguments are mutually " + "exclusive") + if ssl_context is not None and certfile is not None: + raise ValueError("ssl_context and certfile arguments are mutually " + "exclusive") + self.keyfile = keyfile self.certfile = certfile + self.ssl_context = ssl_context IMAP4.__init__(self, host, port) def _create_socket(self): sock = IMAP4._create_socket(self) - return ssl.wrap_socket(sock, self.keyfile, self.certfile) + if self.ssl_context: + return self.ssl_context.wrap_socket(sock) + else: + return ssl.wrap_socket(sock, self.keyfile, self.certfile) def open(self, host='', port=IMAP4_SSL_PORT): """Setup connection to remote server on "host:port". diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 425b8bf8c7..18ea85cb78 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -240,7 +240,7 @@ class BuiltinImporter: @classmethod @_requires_builtin def is_package(cls, fullname): - """Return None as built-in module are never packages.""" + """Return None as built-in modules are never packages.""" return False @@ -404,6 +404,7 @@ class SourceLoader(_LoaderBasics): else: found = marshal.loads(bytes_data) if isinstance(found, code_type): + imp._fix_co_filename(found, source_path) return found else: msg = "Non-code object in {}" @@ -758,14 +759,14 @@ class _ImportLockContext: _IMPLICIT_META_PATH = [BuiltinImporter, FrozenImporter, _DefaultPathFinder] -_ERR_MSG = 'No module named {}' +_ERR_MSG = 'No module named {!r}' def _gcd_import(name, package=None, level=0): """Import and return the module based on its name, the package the call is being made from, and the level adjustment. This function represents the greatest common denominator of functionality - between import_module and __import__. This includes settting __package__ if + between import_module and __import__. This includes setting __package__ if the loader did not. """ @@ -855,7 +856,7 @@ def __import__(name, globals={}, locals={}, fromlist=[], level=0): module = _gcd_import(name) else: # __package__ is not guaranteed to be defined or could be set to None - # to represent that it's proper value is unknown + # to represent that its proper value is unknown package = globals.get('__package__') if package is None: package = globals['__name__'] diff --git a/Lib/importlib/test/__main__.py b/Lib/importlib/test/__main__.py index decc53d8c5..a1990b1f01 100644 --- a/Lib/importlib/test/__main__.py +++ b/Lib/importlib/test/__main__.py @@ -4,7 +4,6 @@ Specifying the ``--builtin`` flag will run tests, where applicable, with builtins.__import__ instead of importlib.__import__. """ -import importlib from importlib.test.import_ import util import os.path from test.support import run_unittest @@ -13,11 +12,7 @@ import unittest def test_main(): - if '__pycache__' in __file__: - parts = __file__.split(os.path.sep) - start_dir = sep.join(parts[:-2]) - else: - start_dir = os.path.dirname(__file__) + start_dir = os.path.dirname(__file__) top_dir = os.path.dirname(os.path.dirname(start_dir)) test_loader = unittest.TestLoader() if '--builtin' in sys.argv: diff --git a/Lib/importlib/test/regrtest.py b/Lib/importlib/test/regrtest.py index b103ae7d0e..dc0eb97022 100644 --- a/Lib/importlib/test/regrtest.py +++ b/Lib/importlib/test/regrtest.py @@ -5,13 +5,6 @@ invalidates are automatically skipped if the entire test suite is run. Otherwise all command-line options valid for test.regrtest are also valid for this script. -XXX FAILING - * test_import - - test_incorrect_code_name - file name differing between __file__ and co_filename (r68360 on trunk) - - test_import_by_filename - exception for trying to import by file name does not match - """ import importlib import sys diff --git a/Lib/inspect.py b/Lib/inspect.py index bb46ea6f85..9898b114aa 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -33,7 +33,6 @@ import sys import os import types import itertools -import string import re import imp import tokenize @@ -918,6 +917,43 @@ def formatargvalues(args, varargs, varkw, locals, specs.append(formatvarkw(varkw) + formatvalue(locals[varkw])) return '(' + ', '.join(specs) + ')' +def _missing_arguments(f_name, argnames, pos, values): + names = [repr(name) for name in argnames if name not in values] + missing = len(names) + if missing == 1: + s = names[0] + elif missing == 2: + s = "{} and {}".format(*names) + else: + tail = ", {} and {}".format(names[-2:]) + del names[-2:] + s = ", ".join(names) + tail + raise TypeError("%s() missing %i required %s argument%s: %s" % + (f_name, missing, + "positional" if pos else "keyword-only", + "" if missing == 1 else "s", s)) + +def _too_many(f_name, args, kwonly, varargs, defcount, given, values): + atleast = len(args) - defcount + kwonly_given = len([arg for arg in kwonly if arg in values]) + if varargs: + plural = atleast != 1 + sig = "at least %d" % (atleast,) + elif defcount: + plural = True + sig = "from %d to %d" % (atleast, len(args)) + else: + plural = len(args) != 1 + sig = str(len(args)) + kwonly_sig = "" + if kwonly_given: + msg = " positional argument%s (and %d keyword-only argument%s)" + kwonly_sig = (msg % ("s" if given != 1 else "", kwonly_given, + "s" if kwonly_given != 1 else "")) + raise TypeError("%s() takes %s positional argument%s but %d%s %s given" % + (f_name, sig, "s" if plural else "", given, kwonly_sig, + "was" if given == 1 and not kwonly_given else "were")) + def getcallargs(func, *positional, **named): """Get the mapping of arguments to values. @@ -929,64 +965,53 @@ def getcallargs(func, *positional, **named): f_name = func.__name__ arg2value = {} + if ismethod(func) and func.__self__ is not None: # implicit 'self' (or 'cls' for classmethods) argument positional = (func.__self__,) + positional num_pos = len(positional) - num_total = num_pos + len(named) num_args = len(args) num_defaults = len(defaults) if defaults else 0 - for arg, value in zip(args, positional): - arg2value[arg] = value + + n = min(num_pos, num_args) + for i in range(n): + arg2value[args[i]] = positional[i] if varargs: - if num_pos > num_args: - arg2value[varargs] = positional[-(num_pos-num_args):] - else: - arg2value[varargs] = () - elif 0 < num_args < num_pos: - raise TypeError('%s() takes %s %d positional %s (%d given)' % ( - f_name, 'at most' if defaults else 'exactly', num_args, - 'arguments' if num_args > 1 else 'argument', num_total)) - elif num_args == 0 and num_total: - if varkw or kwonlyargs: - if num_pos: - # XXX: We should use num_pos, but Python also uses num_total: - raise TypeError('%s() takes exactly 0 positional arguments ' - '(%d given)' % (f_name, num_total)) - else: - raise TypeError('%s() takes no arguments (%d given)' % - (f_name, num_total)) - - for arg in itertools.chain(args, kwonlyargs): - if arg in named: - if arg in arg2value: - raise TypeError("%s() got multiple values for keyword " - "argument '%s'" % (f_name, arg)) - else: - arg2value[arg] = named.pop(arg) - for kwonlyarg in kwonlyargs: - if kwonlyarg not in arg2value: - try: - arg2value[kwonlyarg] = kwonlydefaults[kwonlyarg] - except KeyError: - raise TypeError("%s() needs keyword-only argument %s" % - (f_name, kwonlyarg)) - if defaults: # fill in any missing values with the defaults - for arg, value in zip(args[-num_defaults:], defaults): - if arg not in arg2value: - arg2value[arg] = value + arg2value[varargs] = tuple(positional[n:]) + possible_kwargs = set(args + kwonlyargs) if varkw: - arg2value[varkw] = named - elif named: - unexpected = next(iter(named)) - raise TypeError("%s() got an unexpected keyword argument '%s'" % - (f_name, unexpected)) - unassigned = num_args - len([arg for arg in args if arg in arg2value]) - if unassigned: - num_required = num_args - num_defaults - raise TypeError('%s() takes %s %d %s (%d given)' % ( - f_name, 'at least' if defaults else 'exactly', num_required, - 'arguments' if num_required > 1 else 'argument', num_total)) + arg2value[varkw] = {} + for kw, value in named.items(): + if kw not in possible_kwargs: + if not varkw: + raise TypeError("%s() got an unexpected keyword argument %r" % + (f_name, kw)) + arg2value[varkw][kw] = value + continue + if kw in arg2value: + raise TypeError("%s() got multiple values for argument %r" % + (f_name, kw)) + arg2value[kw] = value + if num_pos > num_args and not varargs: + _too_many(f_name, args, kwonlyargs, varargs, num_defaults, + num_pos, arg2value) + if num_pos < num_args: + req = args[:num_args - num_defaults] + for arg in req: + if arg not in arg2value: + _missing_arguments(f_name, req, True, arg2value) + for i, arg in enumerate(args[num_args - num_defaults:]): + if arg not in arg2value: + arg2value[arg] = defaults[i] + missing = 0 + for kwarg in kwonlyargs: + if kwarg not in arg2value: + if kwarg in kwonlydefaults: + arg2value[kwarg] = kwonlydefaults[kwarg] + else: + missing += 1 + if missing: + _missing_arguments(f_name, kwonlyargs, False, arg2value) return arg2value # -------------------------------------------------- stack frame extraction diff --git a/Lib/lib2to3/__main__.py b/Lib/lib2to3/__main__.py new file mode 100644 index 0000000000..80688baf27 --- /dev/null +++ b/Lib/lib2to3/__main__.py @@ -0,0 +1,4 @@ +import sys +from .main import main + +sys.exit(main("lib2to3.fixes")) diff --git a/Lib/lib2to3/fixer_base.py b/Lib/lib2to3/fixer_base.py index afc0467000..b176056199 100644 --- a/Lib/lib2to3/fixer_base.py +++ b/Lib/lib2to3/fixer_base.py @@ -27,7 +27,6 @@ class BaseFix(object): pattern_tree = None # Tree representation of the pattern options = None # Options object passed to initializer filename = None # The filename (set by set_filename) - logger = None # A logger (set by set_filename) numbers = itertools.count(1) # For new_name() used_names = set() # A set of all used NAMEs order = "post" # Does the fixer prefer pre- or post-order traversal @@ -70,12 +69,11 @@ class BaseFix(object): with_tree=True) def set_filename(self, filename): - """Set the filename, and a logger derived from it. + """Set the filename. The main refactoring tool should call this. """ self.filename = filename - self.logger = logging.getLogger(filename) def match(self, node): """Returns match for a given parse tree node. diff --git a/Lib/lib2to3/patcomp.py b/Lib/lib2to3/patcomp.py index bb538d5377..0a259e90af 100644 --- a/Lib/lib2to3/patcomp.py +++ b/Lib/lib2to3/patcomp.py @@ -11,6 +11,7 @@ The compiler compiles a pattern to a pytree.*Pattern instance. __author__ = "Guido van Rossum <guido@python.org>" # Python imports +import io import os # Fairly local imports @@ -32,7 +33,7 @@ class PatternSyntaxError(Exception): def tokenize_wrapper(input): """Tokenizes a string suppressing significant whitespace.""" skip = set((token.NEWLINE, token.INDENT, token.DEDENT)) - tokens = tokenize.generate_tokens(driver.generate_lines(input).__next__) + tokens = tokenize.generate_tokens(io.StringIO(input).readline) for quintuple in tokens: type, value, start, end, line_text = quintuple if type not in skip: diff --git a/Lib/lib2to3/pgen2/driver.py b/Lib/lib2to3/pgen2/driver.py index ee77a13899..e7828ffe16 100644 --- a/Lib/lib2to3/pgen2/driver.py +++ b/Lib/lib2to3/pgen2/driver.py @@ -17,6 +17,7 @@ __all__ = ["Driver", "load_grammar"] # Python imports import codecs +import io import os import logging import sys @@ -101,18 +102,10 @@ class Driver(object): def parse_string(self, text, debug=False): """Parse a string and return the syntax tree.""" - tokens = tokenize.generate_tokens(generate_lines(text).__next__) + tokens = tokenize.generate_tokens(io.StringIO(text).readline) return self.parse_tokens(tokens, debug) -def generate_lines(text): - """Generator that behaves like readline without using StringIO.""" - for line in text.splitlines(True): - yield line - while True: - yield "" - - def load_grammar(gt="Grammar.txt", gp=None, save=True, force=False, logger=None): """Load the grammar (maybe from a pickle).""" diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index ce39e41700..f3897955dc 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -18,6 +18,16 @@ import os # Local imports from lib2to3.pgen2 import tokenize from ..pgen2.parse import ParseError +from lib2to3.pygram import python_symbols as syms + + +class TestDriver(support.TestCase): + + def test_formfeed(self): + s = """print 1\n\x0Cprint 2\n""" + t = driver.parse_string(s) + self.assertEqual(t.children[0].children[0].type, syms.print_stmt) + self.assertEqual(t.children[1].children[0].type, syms.print_stmt) class GrammarTest(support.TestCase): diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index de5392beca..8406df3abd 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2001-2010 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2011 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -37,14 +37,13 @@ __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR', try: import codecs -except ImportError: +except ImportError: #pragma: no cover codecs = None try: - import _thread as thread import threading -except ImportError: - thread = None +except ImportError: #pragma: no cover + threading = None __author__ = "Vinay Sajip <vinay_sajip@red-dove.com>" __status__ = "production" @@ -65,16 +64,16 @@ else: _srcfile = __file__ _srcfile = os.path.normcase(_srcfile) -# next bit filched from 1.5.2's inspect.py -def currentframe(): - """Return the frame object for the caller's stack frame.""" - try: - raise Exception - except: - return sys.exc_info()[2].tb_frame.f_back -if hasattr(sys, '_getframe'): currentframe = lambda: sys._getframe(3) -# done filching +if hasattr(sys, '_getframe'): + currentframe = lambda: sys._getframe(3) +else: #pragma: no cover + def currentframe(): + """Return the frame object for the caller's stack frame.""" + try: + raise Exception + except: + return sys.exc_info()[2].tb_frame.f_back # _srcfile is only used in conjunction with sys._getframe(). # To provide compatibility with older versions of Python, set _srcfile @@ -92,22 +91,22 @@ _startTime = time.time() #raiseExceptions is used to see if exceptions during handling should be #propagated # -raiseExceptions = 1 +raiseExceptions = True # # If you don't want threading information in the log, set this to zero # -logThreads = 1 +logThreads = True # # If you don't want multiprocessing information in the log, set this to zero # -logMultiprocessing = 1 +logMultiprocessing = True # # If you don't want process information in the log, set this to zero # -logProcesses = 1 +logProcesses = True #--------------------------------------------------------------------------- # Level related stuff @@ -197,9 +196,9 @@ def _checkLevel(level): #the lock would already have been acquired - so we need an RLock. #The same argument applies to Loggers and Manager.loggerDict. # -if thread: +if threading: _lock = threading.RLock() -else: +else: #pragma: no cover _lock = None @@ -276,13 +275,13 @@ class LogRecord(object): self.created = ct self.msecs = (ct - int(ct)) * 1000 self.relativeCreated = (self.created - _startTime) * 1000 - if logThreads and thread: - self.thread = thread.get_ident() + if logThreads and threading: + self.thread = threading.get_ident() self.threadName = threading.current_thread().name - else: + else: # pragma: no cover self.thread = None self.threadName = None - if not logMultiprocessing: + if not logMultiprocessing: # pragma: no cover self.processName = None else: self.processName = 'MainProcess' @@ -294,7 +293,7 @@ class LogRecord(object): # for an example try: self.processName = mp.current_process().name - except StandardError: + except StandardError: #pragma: no cover pass if logProcesses and hasattr(os, 'getpid'): self.process = os.getpid() @@ -466,6 +465,9 @@ class Formatter(object): self._fmt = self._style._fmt self.datefmt = datefmt + default_time_format = '%Y-%m-%d %H:%M:%S' + default_msec_format = '%s,%03d' + def formatTime(self, record, datefmt=None): """ Return the creation time of the specified LogRecord as formatted text. @@ -488,8 +490,8 @@ class Formatter(object): if datefmt: s = time.strftime(datefmt, ct) else: - t = time.strftime("%Y-%m-%d %H:%M:%S", ct) - s = "%s,%03d" % (t, record.msecs) # the use of % here is internal + t = time.strftime(self.default_time_format, ct) + s = self.default_msec_format % (t, record.msecs) return s def formatException(self, ei): @@ -642,11 +644,11 @@ class Filter(object): yes. If deemed appropriate, the record may be modified in-place. """ if self.nlen == 0: - return 1 + return True elif self.name == record.name: - return 1 + return True elif record.name.find(self.name, 0, self.nlen) != 0: - return 0 + return False return (record.name[self.nlen] == ".") class Filterer(object): @@ -686,14 +688,14 @@ class Filterer(object): Allow filters to be just callables. """ - rv = 1 + rv = True for f in self.filters: if hasattr(f, 'filter'): result = f.filter(record) else: result = f(record) # assume callable - will raise if not if not result: - rv = 0 + rv = False break return rv @@ -771,9 +773,9 @@ class Handler(Filterer): """ Acquire a thread lock for serializing access to the underlying I/O. """ - if thread: + if threading: self.lock = threading.RLock() - else: + else: #pragma: no cover self.lock = None def acquire(self): @@ -888,7 +890,7 @@ class Handler(Filterer): None, sys.stderr) sys.stderr.write('Logged from file %s, line %s\n' % ( record.filename, record.lineno)) - except IOError: + except IOError: #pragma: no cover pass # see issue 5971 finally: del ei @@ -937,7 +939,7 @@ class StreamHandler(Handler): stream.write(msg) stream.write(self.terminator) self.flush() - except (KeyboardInterrupt, SystemExit): + except (KeyboardInterrupt, SystemExit): #pragma: no cover raise except: self.handleError(record) @@ -946,13 +948,13 @@ class FileHandler(StreamHandler): """ A handler class which writes formatted logging records to disk files. """ - def __init__(self, filename, mode='a', encoding=None, delay=0): + def __init__(self, filename, mode='a', encoding=None, delay=False): """ Open the specified file and use it as the stream for logging. """ #keep the absolute path, otherwise derived classes which use this #may come a cropper when the current directory changes - if codecs is None: + if codecs is None: #pragma: no cover encoding = None self.baseFilename = os.path.abspath(filename) self.mode = mode @@ -1195,9 +1197,9 @@ class Logger(Filterer): self.name = name self.level = _checkLevel(level) self.parent = None - self.propagate = 1 + self.propagate = True self.handlers = [] - self.disabled = 0 + self.disabled = False def setLevel(self, level): """ @@ -1350,9 +1352,9 @@ class Logger(Filterer): #IronPython can use logging. try: fn, lno, func, sinfo = self.findCaller(stack_info) - except ValueError: + except ValueError: # pragma: no cover fn, lno, func = "(unknown file)", 0, "(unknown function)" - else: + else: # pragma: no cover fn, lno, func = "(unknown file)", 0, "(unknown function)" if exc_info: if not isinstance(exc_info, tuple): @@ -1464,7 +1466,7 @@ class Logger(Filterer): Is this logger enabled for level 'level'? """ if self.manager.disable >= level: - return 0 + return False return level >= self.getEffectiveLevel() def getChild(self, suffix): @@ -1566,7 +1568,7 @@ class LoggerAdapter(object): """ Delegate an exception call to the underlying logger. """ - kwargs["exc_info"] = 1 + kwargs["exc_info"] = True self.log(ERROR, msg, *args, **kwargs) def critical(self, msg, *args, **kwargs): @@ -1649,6 +1651,10 @@ def basicConfig(**kwargs): stream Use the specified stream to initialize the StreamHandler. Note that this argument is incompatible with 'filename' - if both are present, 'stream' is ignored. + handlers If specified, this should be an iterable of already created + handlers, which will be added to the root handler. Any handler + in the list which does not have a formatter assigned will be + assigned the formatter created in this function. Note that you could specify a stream created using open(filename, mode) rather than passing the filename and mode in. However, it should be @@ -1656,27 +1662,47 @@ def basicConfig(**kwargs): using sys.stdout or sys.stderr), whereas FileHandler closes its stream when the handler is closed. - .. versionchanged: 3.2 + .. versionchanged:: 3.2 Added the ``style`` parameter. + + .. versionchanged:: 3.3 + Added the ``handlers`` parameter. A ``ValueError`` is now thrown for + incompatible arguments (e.g. ``handlers`` specified together with + ``filename``/``filemode``, or ``filename``/``filemode`` specified + together with ``stream``, or ``handlers`` specified together with + ``stream``. """ # Add thread safety in case someone mistakenly calls # basicConfig() from multiple threads _acquireLock() try: if len(root.handlers) == 0: - filename = kwargs.get("filename") - if filename: - mode = kwargs.get("filemode", 'a') - hdlr = FileHandler(filename, mode) + handlers = kwargs.get("handlers") + if handlers is None: + if "stream" in kwargs and "filename" in kwargs: + raise ValueError("'stream' and 'filename' should not be " + "specified together") else: - stream = kwargs.get("stream") - hdlr = StreamHandler(stream) + if "stream" in kwargs or "filename" in kwargs: + raise ValueError("'stream' or 'filename' should not be " + "specified together with 'handlers'") + if handlers is None: + filename = kwargs.get("filename") + if filename: + mode = kwargs.get("filemode", 'a') + h = FileHandler(filename, mode) + else: + stream = kwargs.get("stream") + h = StreamHandler(stream) + handlers = [h] fs = kwargs.get("format", BASIC_FORMAT) dfs = kwargs.get("datefmt", None) style = kwargs.get("style", '%') fmt = Formatter(fs, dfs, style) - hdlr.setFormatter(fmt) - root.addHandler(hdlr) + for h in handlers: + if h.formatter is None: + h.setFormatter(fmt) + root.addHandler(h) level = kwargs.get("level") if level is not None: root.setLevel(level) @@ -1825,10 +1851,10 @@ class NullHandler(Handler): package. """ def handle(self, record): - pass + """Stub.""" def emit(self, record): - pass + """Stub.""" def createLock(self): self.lock = None diff --git a/Lib/logging/config.py b/Lib/logging/config.py index c7359cabe9..7daa2df8f4 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -30,7 +30,7 @@ import types, io try: import _thread as thread import threading -except ImportError: +except ImportError: #pragma: no cover thread = None from socketserver import ThreadingTCPServer, StreamRequestHandler @@ -786,7 +786,7 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT): and which you can join() when appropriate. To stop the server, call stopListening(). """ - if not thread: + if not thread: #pragma: no cover raise NotImplementedError("listen() needs threading to work") class ConfigStreamHandler(StreamRequestHandler): @@ -825,7 +825,7 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT): file = io.StringIO(chunk) try: fileConfig(file) - except (KeyboardInterrupt, SystemExit): + except (KeyboardInterrupt, SystemExit): #pragma: no cover raise except: traceback.print_exc() diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 4a6b959946..ef17081908 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -29,12 +29,12 @@ from stat import ST_DEV, ST_INO, ST_MTIME import queue try: import threading -except ImportError: +except ImportError: #pragma: no cover threading = None try: import codecs -except ImportError: +except ImportError: #pragma: no cover codecs = None # @@ -60,7 +60,7 @@ class BaseRotatingHandler(logging.FileHandler): """ Use the specified filename for streamed logging """ - if codecs is None: + if codecs is None: #pragma: no cover encoding = None logging.FileHandler.__init__(self, filename, mode, encoding, delay) self.mode = mode @@ -77,7 +77,7 @@ class BaseRotatingHandler(logging.FileHandler): if self.shouldRollover(record): self.doRollover() logging.FileHandler.emit(self, record) - except (KeyboardInterrupt, SystemExit): + except (KeyboardInterrupt, SystemExit): #pragma: no cover raise except: self.handleError(record) @@ -391,7 +391,7 @@ class WatchedFileHandler(logging.FileHandler): """ if not os.path.exists(self.baseFilename): stat = None - changed = 1 + changed = True else: stat = os.stat(self.baseFilename) changed = (stat[ST_DEV] != self.dev) or (stat[ST_INO] != self.ino) @@ -421,15 +421,15 @@ class SocketHandler(logging.Handler): """ Initializes the handler with a specific host address and port. - The attribute 'closeOnError' is set to 1 - which means that if - a socket error occurs, the socket is silently closed and then - reopened on the next logging call. + When the attribute *closeOnError* is set to True - if a socket error + occurs, the socket is silently closed and then reopened on the next + logging call. """ logging.Handler.__init__(self) self.host = host self.port = port self.sock = None - self.closeOnError = 0 + self.closeOnError = False self.retryTime = None # # Exponential backoff parameters. @@ -446,8 +446,12 @@ class SocketHandler(logging.Handler): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if hasattr(s, 'settimeout'): s.settimeout(timeout) - s.connect((self.host, self.port)) - return s + try: + s.connect((self.host, self.port)) + return s + except socket.error: + s.close() + raise def createSocket(self): """ @@ -460,7 +464,7 @@ class SocketHandler(logging.Handler): # is the first time back after a disconnect, or # we've waited long enough. if self.retryTime is None: - attempt = 1 + attempt = True else: attempt = (now >= self.retryTime) if attempt: @@ -493,14 +497,14 @@ class SocketHandler(logging.Handler): try: if hasattr(self.sock, "sendall"): self.sock.sendall(s) - else: + else: #pragma: no cover sentsofar = 0 left = len(s) while left > 0: sent = self.sock.send(s[sentsofar:]) sentsofar = sentsofar + sent left = left - sent - except socket.error: + except socket.error: #pragma: no cover self.sock.close() self.sock = None # so we can call createSocket next time @@ -545,7 +549,7 @@ class SocketHandler(logging.Handler): try: s = self.makePickle(record) self.send(s) - except (KeyboardInterrupt, SystemExit): + except (KeyboardInterrupt, SystemExit): #pragma: no cover raise except: self.handleError(record) @@ -575,7 +579,7 @@ class DatagramHandler(SocketHandler): Initializes the handler with a specific host address and port. """ SocketHandler.__init__(self, host, port) - self.closeOnError = 0 + self.closeOnError = False def makeSocket(self): """ @@ -716,10 +720,10 @@ class SysLogHandler(logging.Handler): self.socktype = socktype if isinstance(address, str): - self.unixsocket = 1 + self.unixsocket = True self._connect_unixsocket(address) else: - self.unixsocket = 0 + self.unixsocket = False self.socket = socket.socket(socket.AF_INET, socktype) if socktype == socket.SOCK_STREAM: self.socket.connect(address) @@ -752,8 +756,7 @@ class SysLogHandler(logging.Handler): """ Closes the socket. """ - if self.unixsocket: - self.socket.close() + self.socket.close() logging.Handler.close(self) def mapPriority(self, levelName): @@ -766,6 +769,7 @@ class SysLogHandler(logging.Handler): """ return self.priority_map.get(levelName, "warning") + ident = '' # prepended to all messages append_nul = True # some old syslog daemons expect a NUL terminator def emit(self, record): @@ -776,6 +780,8 @@ class SysLogHandler(logging.Handler): exception information is present, it is NOT sent to the server. """ msg = self.format(record) + if self.ident: + msg = self.ident + msg if self.append_nul: msg += '\000' """ @@ -801,7 +807,7 @@ class SysLogHandler(logging.Handler): self.socket.sendto(msg, self.address) else: self.socket.sendall(msg) - except (KeyboardInterrupt, SystemExit): + except (KeyboardInterrupt, SystemExit): #pragma: no cover raise except: self.handleError(record) @@ -878,7 +884,7 @@ class SMTPHandler(logging.Handler): smtp.login(self.username, self.password) smtp.sendmail(self.fromaddr, self.toaddrs, msg) smtp.quit() - except (KeyboardInterrupt, SystemExit): + except (KeyboardInterrupt, SystemExit): #pragma: no cover raise except: self.handleError(record) @@ -965,7 +971,7 @@ class NTEventLogHandler(logging.Handler): type = self.getEventType(record) msg = self.format(record) self._welu.ReportEvent(self.appname, id, cat, type, [msg]) - except (KeyboardInterrupt, SystemExit): + except (KeyboardInterrupt, SystemExit): #pragma: no cover raise except: self.handleError(record) @@ -1048,9 +1054,11 @@ class HTTPHandler(logging.Handler): s = ('u%s:%s' % self.credentials).encode('utf-8') s = 'Basic ' + base64.b64encode(s).strip() h.putheader('Authorization', s) - h.endheaders(data if self.method == "POST" else None) + h.endheaders() + if self.method == "POST": + h.send(data.encode('utf-8')) h.getresponse() #can't do anything with the result - except (KeyboardInterrupt, SystemExit): + except (KeyboardInterrupt, SystemExit): #pragma: no cover raise except: self.handleError(record) @@ -1220,7 +1228,7 @@ class QueueHandler(logging.Handler): """ try: self.enqueue(self.prepare(record)) - except (KeyboardInterrupt, SystemExit): + except (KeyboardInterrupt, SystemExit): #pragma: no cover raise except: self.handleError(record) @@ -1317,6 +1325,16 @@ if threading: except queue.Empty: break + def enqueue_sentinel(self): + """ + This is used to enqueue the sentinel record. + + The base implementation uses put_nowait. You may want to override this + method if you want to use timeouts or work with custom queue + implementations. + """ + self.queue.put_nowait(self._sentinel) + def stop(self): """ Stop the listener. @@ -1326,6 +1344,6 @@ if threading: may be some records still left on the queue, which won't be processed. """ self._stop.set() - self.queue.put_nowait(self._sentinel) + self.enqueue_sentinel() self._thread.join() self._thread = None diff --git a/Lib/mailcap.py b/Lib/mailcap.py index 4ae13d7ed7..99f4958bf7 100644 --- a/Lib/mailcap.py +++ b/Lib/mailcap.py @@ -33,10 +33,10 @@ def getcaps(): def listmailcapfiles(): """Return a list of all mailcap files found on the system.""" - # XXX Actually, this is Unix-specific + # This is mostly a Unix thing, but we use the OS path separator anyway if 'MAILCAPS' in os.environ: - str = os.environ['MAILCAPS'] - mailcaps = str.split(':') + pathstr = os.environ['MAILCAPS'] + mailcaps = pathstr.split(os.pathsep) else: if 'HOME' in os.environ: home = os.environ['HOME'] diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 961e47fdf5..41af94bdd2 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -249,7 +249,6 @@ class MimeTypes: yield ctype i += 1 - default_encoding = sys.getdefaultencoding() with _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT, r'MIME\Database\Content Type') as mimedb: for ctype in enum_types(mimedb): diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index f033ba98ef..c0910aa877 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -1,6 +1,5 @@ """Find modules used by a script, using introspection.""" -from __future__ import generators import dis import imp import marshal @@ -9,8 +8,6 @@ import sys import types import struct -READ_MODE = "rU" - # XXX Clean up once str8's cstor matches bytes. LOAD_CONST = bytes([dis.opname.index('LOAD_CONST')]) IMPORT_NAME = bytes([dis.opname.index('IMPORT_NAME')]) @@ -29,9 +26,7 @@ packagePathMap = {} # A Public interface def AddPackagePath(packagename, path): - paths = packagePathMap.get(packagename, []) - paths.append(path) - packagePathMap[packagename] = paths + packagePathMap.setdefault(packagename, []).append(path) replacePackageMap = {} @@ -106,14 +101,14 @@ class ModuleFinder: def run_script(self, pathname): self.msg(2, "run_script", pathname) - with open(pathname, READ_MODE) as fp: + with open(pathname) as fp: stuff = ("", "r", imp.PY_SOURCE) self.load_module('__main__', fp, pathname, stuff) def load_file(self, pathname): dir, name = os.path.split(pathname) name, ext = os.path.splitext(name) - with open(pathname, READ_MODE) as fp: + with open(pathname) as fp: stuff = (ext, "r", imp.PY_SOURCE) self.load_module(name, fp, pathname, stuff) @@ -270,7 +265,8 @@ class ModuleFinder: try: m = self.load_module(fqname, fp, pathname, stuff) finally: - if fp: fp.close() + if fp: + fp.close() if parent: setattr(parent, partname, m) self.msgout(3, "import_module ->", m) @@ -662,4 +658,4 @@ if __name__ == '__main__': try: mf = test() except KeyboardInterrupt: - print("\n[interrupt]") + print("\n[interrupted]") diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index d6c23fb0ec..ae2d135e03 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -34,19 +34,31 @@ __all__ = [ 'Client', 'Listener', 'Pipe' ] +import io import os import sys +import pickle +import select import socket +import struct import errno import time import tempfile import itertools import _multiprocessing -from multiprocessing import current_process, AuthenticationError -from multiprocessing.util import get_temp_dir, Finalize, sub_debug, debug -from multiprocessing.forking import duplicate, close +from multiprocessing import current_process, AuthenticationError, BufferTooShort +from multiprocessing.util import ( + get_temp_dir, Finalize, sub_debug, debug, _eintr_retry) +try: + from _multiprocessing import win32 + from _subprocess import WAIT_OBJECT_0, WAIT_TIMEOUT, INFINITE +except ImportError: + if sys.platform == 'win32': + raise + win32 = None +_select = _eintr_retry(select.select) # # @@ -110,6 +122,326 @@ def address_type(address): else: raise ValueError('address type of %r unrecognized' % address) + +class SentinelReady(Exception): + """ + Raised when a sentinel is ready when polling. + """ + def __init__(self, *args): + Exception.__init__(self, *args) + self.sentinels = args[0] + +# +# Connection classes +# + +class _ConnectionBase: + _handle = None + + def __init__(self, handle, readable=True, writable=True): + handle = handle.__index__() + if handle < 0: + raise ValueError("invalid handle") + if not readable and not writable: + raise ValueError( + "at least one of `readable` and `writable` must be True") + self._handle = handle + self._readable = readable + self._writable = writable + + # XXX should we use util.Finalize instead of a __del__? + + def __del__(self): + if self._handle is not None: + self._close() + + def _check_closed(self): + if self._handle is None: + raise IOError("handle is closed") + + def _check_readable(self): + if not self._readable: + raise IOError("connection is write-only") + + def _check_writable(self): + if not self._writable: + raise IOError("connection is read-only") + + def _bad_message_length(self): + if self._writable: + self._readable = False + else: + self.close() + raise IOError("bad message length") + + @property + def closed(self): + """True if the connection is closed""" + return self._handle is None + + @property + def readable(self): + """True if the connection is readable""" + return self._readable + + @property + def writable(self): + """True if the connection is writable""" + return self._writable + + def fileno(self): + """File descriptor or handle of the connection""" + self._check_closed() + return self._handle + + def close(self): + """Close the connection""" + if self._handle is not None: + try: + self._close() + finally: + self._handle = None + + def send_bytes(self, buf, offset=0, size=None): + """Send the bytes data from a bytes-like object""" + self._check_closed() + self._check_writable() + m = memoryview(buf) + # HACK for byte-indexing of non-bytewise buffers (e.g. array.array) + if m.itemsize > 1: + m = memoryview(bytes(m)) + n = len(m) + if offset < 0: + raise ValueError("offset is negative") + if n < offset: + raise ValueError("buffer length < offset") + if size is None: + size = n - offset + elif size < 0: + raise ValueError("size is negative") + elif offset + size > n: + raise ValueError("buffer length < offset + size") + self._send_bytes(m[offset:offset + size]) + + def send(self, obj): + """Send a (picklable) object""" + self._check_closed() + self._check_writable() + buf = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL) + self._send_bytes(memoryview(buf)) + + def recv_bytes(self, maxlength=None): + """ + Receive bytes data as a bytes object. + """ + self._check_closed() + self._check_readable() + if maxlength is not None and maxlength < 0: + raise ValueError("negative maxlength") + buf = self._recv_bytes(maxlength) + if buf is None: + self._bad_message_length() + return buf.getvalue() + + def recv_bytes_into(self, buf, offset=0): + """ + Receive bytes data into a writeable buffer-like object. + Return the number of bytes read. + """ + self._check_closed() + self._check_readable() + with memoryview(buf) as m: + # Get bytesize of arbitrary buffer + itemsize = m.itemsize + bytesize = itemsize * len(m) + if offset < 0: + raise ValueError("negative offset") + elif offset > bytesize: + raise ValueError("offset too large") + result = self._recv_bytes() + size = result.tell() + if bytesize < offset + size: + raise BufferTooShort(result.getvalue()) + # Message can fit in dest + result.seek(0) + result.readinto(m[offset // itemsize : + (offset + size) // itemsize]) + return size + + def recv(self, sentinels=None): + """Receive a (picklable) object""" + self._check_closed() + self._check_readable() + buf = self._recv_bytes(sentinels=sentinels) + return pickle.loads(buf.getbuffer()) + + def poll(self, timeout=0.0): + """Whether there is any input available to be read""" + self._check_closed() + self._check_readable() + return self._poll(timeout) + + +if win32: + + class PipeConnection(_ConnectionBase): + """ + Connection class based on a Windows named pipe. + Overlapped I/O is used, so the handles must have been created + with FILE_FLAG_OVERLAPPED. + """ + _buffered = b'' + + def _close(self, _CloseHandle=win32.CloseHandle): + _CloseHandle(self._handle) + + def _send_bytes(self, buf): + overlapped = win32.WriteFile(self._handle, buf, overlapped=True) + nwritten, complete = overlapped.GetOverlappedResult(True) + assert complete + assert nwritten == len(buf) + + def _recv_bytes(self, maxsize=None, sentinels=()): + if sentinels: + self._poll(-1.0, sentinels) + buf = io.BytesIO() + firstchunk = self._buffered + if firstchunk: + lenfirstchunk = len(firstchunk) + buf.write(firstchunk) + self._buffered = b'' + else: + # A reasonable size for the first chunk transfer + bufsize = 128 + if maxsize is not None and maxsize < bufsize: + bufsize = maxsize + try: + overlapped = win32.ReadFile(self._handle, bufsize, overlapped=True) + lenfirstchunk, complete = overlapped.GetOverlappedResult(True) + firstchunk = overlapped.getbuffer() + assert lenfirstchunk == len(firstchunk) + except IOError as e: + if e.errno == win32.ERROR_BROKEN_PIPE: + raise EOFError + raise + buf.write(firstchunk) + if complete: + return buf + navail, nleft = win32.PeekNamedPipe(self._handle) + if maxsize is not None and lenfirstchunk + nleft > maxsize: + return None + if nleft > 0: + overlapped = win32.ReadFile(self._handle, nleft, overlapped=True) + res, complete = overlapped.GetOverlappedResult(True) + assert res == nleft + assert complete + buf.write(overlapped.getbuffer()) + return buf + + def _poll(self, timeout, sentinels=()): + # Fast non-blocking path + navail, nleft = win32.PeekNamedPipe(self._handle) + if navail > 0: + return True + elif timeout == 0.0: + return False + # Blocking: use overlapped I/O + if timeout < 0.0: + timeout = INFINITE + else: + timeout = int(timeout * 1000 + 0.5) + overlapped = win32.ReadFile(self._handle, 1, overlapped=True) + try: + handles = [overlapped.event] + handles += sentinels + res = win32.WaitForMultipleObjects(handles, False, timeout) + finally: + # Always cancel overlapped I/O in the same thread + # (because CancelIoEx() appears only in Vista) + overlapped.cancel() + if res == WAIT_TIMEOUT: + return False + idx = res - WAIT_OBJECT_0 + if idx == 0: + # I/O was successful, store received data + overlapped.GetOverlappedResult(True) + self._buffered += overlapped.getbuffer() + return True + assert 0 < idx < len(handles) + raise SentinelReady([handles[idx]]) + + +class Connection(_ConnectionBase): + """ + Connection class based on an arbitrary file descriptor (Unix only), or + a socket handle (Windows). + """ + + if win32: + def _close(self, _close=win32.closesocket): + _close(self._handle) + _write = win32.send + _read = win32.recv + else: + def _close(self, _close=os.close): + _close(self._handle) + _write = os.write + _read = os.read + + def _send(self, buf, write=_write): + remaining = len(buf) + while True: + n = write(self._handle, buf) + remaining -= n + if remaining == 0: + break + buf = buf[n:] + + def _recv(self, size, sentinels=(), read=_read): + buf = io.BytesIO() + handle = self._handle + if sentinels: + handles = [handle] + sentinels + remaining = size + while remaining > 0: + if sentinels: + r = _select(handles, [], [])[0] + if handle not in r: + raise SentinelReady(r) + chunk = read(handle, remaining) + n = len(chunk) + if n == 0: + if remaining == size: + raise EOFError + else: + raise IOError("got end of file during message") + buf.write(chunk) + remaining -= n + return buf + + def _send_bytes(self, buf): + # For wire compatibility with 3.2 and lower + n = len(buf) + self._send(struct.pack("=i", len(buf))) + # The condition is necessary to avoid "broken pipe" errors + # when sending a 0-length buffer if the other end closed the pipe. + if n > 0: + self._send(buf) + + def _recv_bytes(self, maxsize=None, sentinels=()): + buf = self._recv(4, sentinels) + size, = struct.unpack("=i", buf.getvalue()) + if maxsize is not None and size > maxsize: + return None + return self._recv(size, sentinels) + + def _poll(self, timeout): + if timeout < 0.0: + timeout = None + r = _select([self._handle], [], [], timeout)[0] + return bool(r) + + # # Public functions # @@ -186,21 +518,17 @@ if sys.platform != 'win32': ''' if duplex: s1, s2 = socket.socketpair() - c1 = _multiprocessing.Connection(os.dup(s1.fileno())) - c2 = _multiprocessing.Connection(os.dup(s2.fileno())) - s1.close() - s2.close() + c1 = Connection(s1.detach()) + c2 = Connection(s2.detach()) else: fd1, fd2 = os.pipe() - c1 = _multiprocessing.Connection(fd1, writable=False) - c2 = _multiprocessing.Connection(fd2, readable=False) + c1 = Connection(fd1, writable=False) + c2 = Connection(fd2, readable=False) return c1, c2 else: - from _multiprocessing import win32 - def Pipe(duplex=True): ''' Returns pair of connection objects at either end of a pipe @@ -216,26 +544,24 @@ else: obsize, ibsize = 0, BUFSIZE h1 = win32.CreateNamedPipe( - address, openmode, + address, openmode | win32.FILE_FLAG_OVERLAPPED, win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE | win32.PIPE_WAIT, 1, obsize, ibsize, win32.NMPWAIT_WAIT_FOREVER, win32.NULL ) h2 = win32.CreateFile( - address, access, 0, win32.NULL, win32.OPEN_EXISTING, 0, win32.NULL + address, access, 0, win32.NULL, win32.OPEN_EXISTING, + win32.FILE_FLAG_OVERLAPPED, win32.NULL ) win32.SetNamedPipeHandleState( h2, win32.PIPE_READMODE_MESSAGE, None, None ) - try: - win32.ConnectNamedPipe(h1, win32.NULL) - except WindowsError as e: - if e.args[0] != win32.ERROR_PIPE_CONNECTED: - raise + overlapped = win32.ConnectNamedPipe(h1, overlapped=True) + overlapped.GetOverlappedResult(True) - c1 = _multiprocessing.PipeConnection(h1, writable=duplex) - c2 = _multiprocessing.PipeConnection(h2, readable=duplex) + c1 = PipeConnection(h1, writable=duplex) + c2 = PipeConnection(h2, readable=duplex) return c1, c2 @@ -266,7 +592,7 @@ class SocketListener(object): def accept(self): s, self._last_accepted = self._socket.accept() fd = duplicate(s.fileno()) - conn = _multiprocessing.Connection(fd) + conn = Connection(fd) s.close() return conn @@ -298,7 +624,7 @@ def SocketClient(address): raise fd = duplicate(s.fileno()) - conn = _multiprocessing.Connection(fd) + conn = Connection(fd) return conn # @@ -345,7 +671,7 @@ if sys.platform == 'win32': except WindowsError as e: if e.args[0] != win32.ERROR_PIPE_CONNECTED: raise - return _multiprocessing.PipeConnection(handle) + return PipeConnection(handle) @staticmethod def _finalize_pipe_listener(queue, address): @@ -377,7 +703,7 @@ if sys.platform == 'win32': win32.SetNamedPipeHandleState( h, win32.PIPE_READMODE_MESSAGE, None, None ) - return _multiprocessing.PipeConnection(h) + return PipeConnection(h) # # Authentication stuff @@ -434,10 +760,10 @@ class ConnectionWrapper(object): return self._loads(s) def _xml_dumps(obj): - return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf8') + return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8') def _xml_loads(s): - (obj,), method = xmlrpclib.loads(s.decode('utf8')) + (obj,), method = xmlrpclib.loads(s.decode('utf-8')) return obj class XmlListener(Listener): @@ -451,3 +777,7 @@ def XmlClient(*args, **kwds): global xmlrpclib import xmlrpc.client as xmlrpclib return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads) + + +# Late import because of circular import +from multiprocessing.forking import duplicate, close diff --git a/Lib/multiprocessing/dummy/__init__.py b/Lib/multiprocessing/dummy/__init__.py index c4933d9ec5..60add922bf 100644 --- a/Lib/multiprocessing/dummy/__init__.py +++ b/Lib/multiprocessing/dummy/__init__.py @@ -51,7 +51,7 @@ import itertools from multiprocessing import TimeoutError, cpu_count from multiprocessing.dummy.connection import Pipe from threading import Lock, RLock, Semaphore, BoundedSemaphore -from threading import Event +from threading import Event, Condition from queue import Queue # @@ -84,17 +84,6 @@ class DummyProcess(threading.Thread): # # -class Condition(threading._Condition): - # XXX - if sys.version_info < (3, 0): - notify_all = threading._Condition.notify_all.__func__ - else: - notify_all = threading._Condition.notify_all - -# -# -# - Process = DummyProcess current_process = threading.current_thread current_process()._children = weakref.WeakKeyDictionary() diff --git a/Lib/multiprocessing/forking.py b/Lib/multiprocessing/forking.py index cc7c326c07..a2c61ef27e 100644 --- a/Lib/multiprocessing/forking.py +++ b/Lib/multiprocessing/forking.py @@ -35,6 +35,7 @@ import os import sys import signal +import select from multiprocessing import util, process @@ -101,10 +102,12 @@ else: if sys.platform != 'win32': import time + import select exit = os._exit duplicate = os.dup close = os.close + _select = util._eintr_retry(select.select) # # We define a Popen class similar to the one from subprocess, but @@ -118,8 +121,12 @@ if sys.platform != 'win32': sys.stderr.flush() self.returncode = None + r, w = os.pipe() + self.sentinel = r + self.pid = os.fork() if self.pid == 0: + os.close(r) if 'random' in sys.modules: import random random.seed() @@ -128,6 +135,11 @@ if sys.platform != 'win32': sys.stderr.flush() os._exit(code) + # `w` will be closed when the child exits, at which point `r` + # will become ready for reading (using e.g. select()). + os.close(w) + util.Finalize(self, os.close, (r,)) + def poll(self, flag=os.WNOHANG): if self.returncode is None: try: @@ -145,20 +157,14 @@ if sys.platform != 'win32': return self.returncode def wait(self, timeout=None): - if timeout is None: - return self.poll(0) - deadline = time.time() + timeout - delay = 0.0005 - while 1: - res = self.poll() - if res is not None: - break - remaining = deadline - time.time() - if remaining <= 0: - break - delay = min(delay * 2, remaining, 0.05) - time.sleep(delay) - return res + if self.returncode is None: + if timeout is not None: + r = _select([self.sentinel], [], [], timeout)[0] + if not r: + return None + # This shouldn't block if select() returned successfully. + return self.poll(os.WNOHANG if timeout == 0.0 else 0) + return self.returncode def terminate(self): if self.returncode is None: @@ -183,7 +189,7 @@ else: import time from pickle import dump, load, HIGHEST_PROTOCOL - from _multiprocessing import win32, Connection, PipeConnection + from _multiprocessing import win32 from .util import Finalize def dump(obj, file, protocol=None): @@ -258,6 +264,7 @@ else: self.pid = pid self.returncode = None self._handle = hp + self.sentinel = int(hp) # send information to child prep_data = get_preparation_data(process_obj._name) @@ -411,6 +418,9 @@ else: # Make (Pipe)Connection picklable # + # Late import because of circular import + from .connection import Connection, PipeConnection + def reduce_connection(conn): if not Popen.thread_is_spawning(): raise RuntimeError( diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 5987af9322..98ce0da6c0 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -92,12 +92,16 @@ class Process(object): ''' _Popen = None - def __init__(self, group=None, target=None, name=None, args=(), kwargs={}): + def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, + *, daemon=None): assert group is None, 'group argument must be None for now' count = next(_current_process._counter) self._identity = _current_process._identity + (count,) self._authkey = _current_process._authkey - self._daemonic = _current_process._daemonic + if daemon is not None: + self._daemonic = daemon + else: + self._daemonic = _current_process._daemonic self._tempdir = _current_process._tempdir self._parent_pid = os.getpid() self._popen = None @@ -130,6 +134,7 @@ class Process(object): else: from .forking import Popen self._popen = Popen(self) + self._sentinel = self._popen.sentinel _current_process._children.add(self) def terminate(self): @@ -216,6 +221,17 @@ class Process(object): pid = ident + @property + def sentinel(self): + ''' + Return a file descriptor (Unix) or handle (Windows) suitable for + waiting for process termination. + ''' + try: + return self._sentinel + except AttributeError: + raise ValueError("process not started") + def __repr__(self): if self is _current_process: status = 'started' diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 3280a2533d..bb4c7d874f 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -41,10 +41,11 @@ import collections import time import atexit import weakref +import errno from queue import Empty, Full import _multiprocessing -from multiprocessing import Pipe +from multiprocessing.connection import Pipe, SentinelReady from multiprocessing.synchronize import Lock, BoundedSemaphore, Semaphore, Condition from multiprocessing.util import debug, info, Finalize, register_after_fork from multiprocessing.forking import assert_spawning @@ -67,6 +68,8 @@ class Queue(object): else: self._wlock = Lock() self._sem = BoundedSemaphore(maxsize) + # For use by concurrent.futures + self._ignore_epipe = False self._after_fork() @@ -75,11 +78,11 @@ class Queue(object): def __getstate__(self): assert_spawning(self) - return (self._maxsize, self._reader, self._writer, + return (self._ignore_epipe, self._maxsize, self._reader, self._writer, self._rlock, self._wlock, self._sem, self._opid) def __setstate__(self, state): - (self._maxsize, self._reader, self._writer, + (self._ignore_epipe, self._maxsize, self._reader, self._writer, self._rlock, self._wlock, self._sem, self._opid) = state self._after_fork() @@ -178,7 +181,7 @@ class Queue(object): self._thread = threading.Thread( target=Queue._feed, args=(self._buffer, self._notempty, self._send, - self._wlock, self._writer.close), + self._wlock, self._writer.close, self._ignore_epipe), name='QueueFeederThread' ) self._thread.daemon = True @@ -229,7 +232,7 @@ class Queue(object): notempty.release() @staticmethod - def _feed(buffer, notempty, send, writelock, close): + def _feed(buffer, notempty, send, writelock, close, ignore_epipe): debug('starting thread to feed data to pipe') from .util import is_exiting @@ -271,6 +274,8 @@ class Queue(object): except IndexError: pass except Exception as e: + if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: + return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has @@ -372,10 +377,10 @@ class SimpleQueue(object): def _make_methods(self): recv = self._reader.recv racquire, rrelease = self._rlock.acquire, self._rlock.release - def get(): + def get(*, sentinels=None): racquire() try: - return recv() + return recv(sentinels) finally: rrelease() self.get = get diff --git a/Lib/multiprocessing/reduction.py b/Lib/multiprocessing/reduction.py index 6e5e5bc9de..b32c72590a 100644 --- a/Lib/multiprocessing/reduction.py +++ b/Lib/multiprocessing/reduction.py @@ -44,7 +44,7 @@ import _multiprocessing from multiprocessing import current_process from multiprocessing.forking import Popen, duplicate, close, ForkingPickler from multiprocessing.util import register_after_fork, debug, sub_debug -from multiprocessing.connection import Client, Listener +from multiprocessing.connection import Client, Listener, Connection # @@ -159,7 +159,7 @@ def rebuild_handle(pickled_data): return new_handle # -# Register `_multiprocessing.Connection` with `ForkingPickler` +# Register `Connection` with `ForkingPickler` # def reduce_connection(conn): @@ -168,11 +168,11 @@ def reduce_connection(conn): def rebuild_connection(reduced_handle, readable, writable): handle = rebuild_handle(reduced_handle) - return _multiprocessing.Connection( + return Connection( handle, readable=readable, writable=writable ) -ForkingPickler.register(_multiprocessing.Connection, reduce_connection) +ForkingPickler.register(Connection, reduce_connection) # # Register `socket.socket` with `ForkingPickler` @@ -201,6 +201,7 @@ ForkingPickler.register(socket.socket, reduce_socket) # if sys.platform == 'win32': + from multiprocessing.connection import PipeConnection def reduce_pipe_connection(conn): rh = reduce_handle(conn.fileno()) @@ -208,8 +209,8 @@ if sys.platform == 'win32': def rebuild_pipe_connection(reduced_handle, readable, writable): handle = rebuild_handle(reduced_handle) - return _multiprocessing.PipeConnection( + return PipeConnection( handle, readable=readable, writable=writable ) - ForkingPickler.register(_multiprocessing.PipeConnection, reduce_pipe_connection) + ForkingPickler.register(PipeConnection, reduce_pipe_connection) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 30b7a85fa2..c48718026b 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -32,9 +32,11 @@ # SUCH DAMAGE. # +import functools import itertools import weakref import atexit +import select import threading # we want threading to install it's # cleanup function before multiprocessing does @@ -186,7 +188,11 @@ class Finalize(object): _finalizer_registry[self._key] = self - def __call__(self, wr=None): + def __call__(self, wr=None, + # Need to bind these locally because the globals can have + # been cleared at shutdown + _finalizer_registry=_finalizer_registry, + sub_debug=sub_debug): ''' Run the callback unless it has already been called or cancelled ''' @@ -315,3 +321,21 @@ class ForkAwareLocal(threading.local): register_after_fork(self, lambda obj : obj.__dict__.clear()) def __reduce__(self): return type(self), () + + +# +# Automatic retry after EINTR +# + +def _eintr_retry(func, _errors=(EnvironmentError, select.error)): + @functools.wraps(func) + def wrapped(*args, **kwargs): + while True: + try: + return func(*args, **kwargs) + except _errors as e: + # select.error has no `errno` attribute + if e.args[0] == errno.EINTR: + continue + raise + return wrapped diff --git a/Lib/nntplib.py b/Lib/nntplib.py index bf66734a52..3e863dc740 100644 --- a/Lib/nntplib.py +++ b/Lib/nntplib.py @@ -346,6 +346,20 @@ class _NNTPBase: # Log in and encryption setup order is left to subclasses. self.authenticated = False + def __enter__(self): + return self + + def __exit__(self, *args): + is_connected = lambda: hasattr(self, "file") + if is_connected(): + try: + self.quit() + except (socket.error, EOFError): + pass + finally: + if is_connected(): + self._close() + def getwelcome(self): """Get the welcome message from the server (this is read and squirreled away by __init__()). diff --git a/Lib/opcode.py b/Lib/opcode.py index 8e15d13e98..b631b257c4 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -43,7 +43,6 @@ def jabs_op(name, op): # Instruction opcodes for compiled code # Blank lines correspond to available opcodes -def_op('STOP_CODE', 0) def_op('POP_TOP', 1) def_op('ROT_TWO', 2) def_op('ROT_THREE', 3) diff --git a/Lib/optparse.py b/Lib/optparse.py index cbd019afb7..b09a13cd34 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -86,10 +86,16 @@ def _repr(self): # Id: errors.py 509 2006-04-20 00:58:24Z gward try: - from gettext import gettext + from gettext import gettext, ngettext except ImportError: def gettext(message): return message + + def ngettext(singular, plural, n): + if n == 1: + return singular + return plural + _ = gettext @@ -1483,11 +1489,10 @@ class OptionParser (OptionContainer): if option.takes_value(): nargs = option.nargs if len(rargs) < nargs: - if nargs == 1: - self.error(_("%s option requires an argument") % opt) - else: - self.error(_("%s option requires %d arguments") - % (opt, nargs)) + self.error(ngettext( + "%(option)s option requires %(number)d argument", + "%(option)s option requires %(number)d arguments", + nargs) % {"option": opt, "number": nargs}) elif nargs == 1: value = rargs.pop(0) else: @@ -1522,11 +1527,10 @@ class OptionParser (OptionContainer): nargs = option.nargs if len(rargs) < nargs: - if nargs == 1: - self.error(_("%s option requires an argument") % opt) - else: - self.error(_("%s option requires %d arguments") - % (opt, nargs)) + self.error(ngettext( + "%(option)s option requires %(number)d argument", + "%(option)s option requires %(number)d arguments", + nargs) % {"option": opt, "number": nargs}) elif nargs == 1: value = rargs.pop(0) else: @@ -434,7 +434,7 @@ def get_exec_path(env=None): # Change environ to automatically call putenv(), unsetenv if they exist. -from _abcoll import MutableMapping # Can't use collections (bootstrap) +from collections.abc import MutableMapping class _Environ(MutableMapping): def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv): diff --git a/Lib/packaging/__init__.py b/Lib/packaging/__init__.py new file mode 100644 index 0000000000..93b611765c --- /dev/null +++ b/Lib/packaging/__init__.py @@ -0,0 +1,17 @@ +"""Support for packaging, distribution and installation of Python projects. + +Third-party tools can use parts of packaging as building blocks +without causing the other modules to be imported: + + import packaging.version + import packaging.metadata + import packaging.pypi.simple + import packaging.tests.pypi_server +""" + +from logging import getLogger + +__all__ = ['__version__', 'logger'] + +__version__ = "1.0a3" +logger = getLogger('packaging') diff --git a/Lib/packaging/_trove.py b/Lib/packaging/_trove.py new file mode 100644 index 0000000000..bc9bce8151 --- /dev/null +++ b/Lib/packaging/_trove.py @@ -0,0 +1,559 @@ +"""Temporary helper for create.""" + +# XXX get the list from PyPI and cache it instead of hardcoding + +# XXX see if it would be more useful to store it as another structure +# than a list of strings + +all_classifiers = [ +'Development Status :: 1 - Planning', +'Development Status :: 2 - Pre-Alpha', +'Development Status :: 3 - Alpha', +'Development Status :: 4 - Beta', +'Development Status :: 5 - Production/Stable', +'Development Status :: 6 - Mature', +'Development Status :: 7 - Inactive', +'Environment :: Console', +'Environment :: Console :: Curses', +'Environment :: Console :: Framebuffer', +'Environment :: Console :: Newt', +'Environment :: Console :: svgalib', +"Environment :: Handhelds/PDA's", +'Environment :: MacOS X', +'Environment :: MacOS X :: Aqua', +'Environment :: MacOS X :: Carbon', +'Environment :: MacOS X :: Cocoa', +'Environment :: No Input/Output (Daemon)', +'Environment :: Other Environment', +'Environment :: Plugins', +'Environment :: Web Environment', +'Environment :: Web Environment :: Buffet', +'Environment :: Web Environment :: Mozilla', +'Environment :: Web Environment :: ToscaWidgets', +'Environment :: Win32 (MS Windows)', +'Environment :: X11 Applications', +'Environment :: X11 Applications :: Gnome', +'Environment :: X11 Applications :: GTK', +'Environment :: X11 Applications :: KDE', +'Environment :: X11 Applications :: Qt', +'Framework :: BFG', +'Framework :: Buildout', +'Framework :: Buildout :: Extension', +'Framework :: Buildout :: Recipe', +'Framework :: Chandler', +'Framework :: CherryPy', +'Framework :: CubicWeb', +'Framework :: Django', +'Framework :: IDLE', +'Framework :: Paste', +'Framework :: Plone', +'Framework :: Pylons', +'Framework :: Setuptools Plugin', +'Framework :: Trac', +'Framework :: Tryton', +'Framework :: TurboGears', +'Framework :: TurboGears :: Applications', +'Framework :: TurboGears :: Widgets', +'Framework :: Twisted', +'Framework :: ZODB', +'Framework :: Zope2', +'Framework :: Zope3', +'Intended Audience :: Customer Service', +'Intended Audience :: Developers', +'Intended Audience :: Education', +'Intended Audience :: End Users/Desktop', +'Intended Audience :: Financial and Insurance Industry', +'Intended Audience :: Healthcare Industry', +'Intended Audience :: Information Technology', +'Intended Audience :: Legal Industry', +'Intended Audience :: Manufacturing', +'Intended Audience :: Other Audience', +'Intended Audience :: Religion', +'Intended Audience :: Science/Research', +'Intended Audience :: System Administrators', +'Intended Audience :: Telecommunications Industry', +'License :: Aladdin Free Public License (AFPL)', +'License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication', +'License :: DFSG approved', +'License :: Eiffel Forum License (EFL)', +'License :: Free For Educational Use', +'License :: Free For Home Use', +'License :: Free for non-commercial use', +'License :: Freely Distributable', +'License :: Free To Use But Restricted', +'License :: Freeware', +'License :: Netscape Public License (NPL)', +'License :: Nokia Open Source License (NOKOS)', +'License :: OSI Approved', +'License :: OSI Approved :: Academic Free License (AFL)', +'License :: OSI Approved :: Apache Software License', +'License :: OSI Approved :: Apple Public Source License', +'License :: OSI Approved :: Artistic License', +'License :: OSI Approved :: Attribution Assurance License', +'License :: OSI Approved :: BSD License', +'License :: OSI Approved :: Common Public License', +'License :: OSI Approved :: Eiffel Forum License', +'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)', +'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)', +'License :: OSI Approved :: GNU Affero General Public License v3', +'License :: OSI Approved :: GNU Free Documentation License (FDL)', +'License :: OSI Approved :: GNU General Public License (GPL)', +'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)', +'License :: OSI Approved :: IBM Public License', +'License :: OSI Approved :: Intel Open Source License', +'License :: OSI Approved :: ISC License (ISCL)', +'License :: OSI Approved :: Jabber Open Source License', +'License :: OSI Approved :: MIT License', +'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)', +'License :: OSI Approved :: Motosoto License', +'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)', +'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)', +'License :: OSI Approved :: Nethack General Public License', +'License :: OSI Approved :: Nokia Open Source License', +'License :: OSI Approved :: Open Group Test Suite License', +'License :: OSI Approved :: Python License (CNRI Python License)', +'License :: OSI Approved :: Python Software Foundation License', +'License :: OSI Approved :: Qt Public License (QPL)', +'License :: OSI Approved :: Ricoh Source Code Public License', +'License :: OSI Approved :: Sleepycat License', +'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)', +'License :: OSI Approved :: Sun Public License', +'License :: OSI Approved :: University of Illinois/NCSA Open Source License', +'License :: OSI Approved :: Vovida Software License 1.0', +'License :: OSI Approved :: W3C License', +'License :: OSI Approved :: X.Net License', +'License :: OSI Approved :: zlib/libpng License', +'License :: OSI Approved :: Zope Public License', +'License :: Other/Proprietary License', +'License :: Public Domain', +'License :: Repoze Public License', +'Natural Language :: Afrikaans', +'Natural Language :: Arabic', +'Natural Language :: Bengali', +'Natural Language :: Bosnian', +'Natural Language :: Bulgarian', +'Natural Language :: Catalan', +'Natural Language :: Chinese (Simplified)', +'Natural Language :: Chinese (Traditional)', +'Natural Language :: Croatian', +'Natural Language :: Czech', +'Natural Language :: Danish', +'Natural Language :: Dutch', +'Natural Language :: English', +'Natural Language :: Esperanto', +'Natural Language :: Finnish', +'Natural Language :: French', +'Natural Language :: German', +'Natural Language :: Greek', +'Natural Language :: Hebrew', +'Natural Language :: Hindi', +'Natural Language :: Hungarian', +'Natural Language :: Icelandic', +'Natural Language :: Indonesian', +'Natural Language :: Italian', +'Natural Language :: Japanese', +'Natural Language :: Javanese', +'Natural Language :: Korean', +'Natural Language :: Latin', +'Natural Language :: Latvian', +'Natural Language :: Macedonian', +'Natural Language :: Malay', +'Natural Language :: Marathi', +'Natural Language :: Norwegian', +'Natural Language :: Panjabi', +'Natural Language :: Persian', +'Natural Language :: Polish', +'Natural Language :: Portuguese', +'Natural Language :: Portuguese (Brazilian)', +'Natural Language :: Romanian', +'Natural Language :: Russian', +'Natural Language :: Serbian', +'Natural Language :: Slovak', +'Natural Language :: Slovenian', +'Natural Language :: Spanish', +'Natural Language :: Swedish', +'Natural Language :: Tamil', +'Natural Language :: Telugu', +'Natural Language :: Thai', +'Natural Language :: Turkish', +'Natural Language :: Ukranian', +'Natural Language :: Urdu', +'Natural Language :: Vietnamese', +'Operating System :: BeOS', +'Operating System :: MacOS', +'Operating System :: MacOS :: MacOS 9', +'Operating System :: MacOS :: MacOS X', +'Operating System :: Microsoft', +'Operating System :: Microsoft :: MS-DOS', +'Operating System :: Microsoft :: Windows', +'Operating System :: Microsoft :: Windows :: Windows 3.1 or Earlier', +'Operating System :: Microsoft :: Windows :: Windows 95/98/2000', +'Operating System :: Microsoft :: Windows :: Windows CE', +'Operating System :: Microsoft :: Windows :: Windows NT/2000', +'Operating System :: OS/2', +'Operating System :: OS Independent', +'Operating System :: Other OS', +'Operating System :: PalmOS', +'Operating System :: PDA Systems', +'Operating System :: POSIX', +'Operating System :: POSIX :: AIX', +'Operating System :: POSIX :: BSD', +'Operating System :: POSIX :: BSD :: BSD/OS', +'Operating System :: POSIX :: BSD :: FreeBSD', +'Operating System :: POSIX :: BSD :: NetBSD', +'Operating System :: POSIX :: BSD :: OpenBSD', +'Operating System :: POSIX :: GNU Hurd', +'Operating System :: POSIX :: HP-UX', +'Operating System :: POSIX :: IRIX', +'Operating System :: POSIX :: Linux', +'Operating System :: POSIX :: Other', +'Operating System :: POSIX :: SCO', +'Operating System :: POSIX :: SunOS/Solaris', +'Operating System :: Unix', +'Programming Language :: Ada', +'Programming Language :: APL', +'Programming Language :: ASP', +'Programming Language :: Assembly', +'Programming Language :: Awk', +'Programming Language :: Basic', +'Programming Language :: C', +'Programming Language :: C#', +'Programming Language :: C++', +'Programming Language :: Cold Fusion', +'Programming Language :: Cython', +'Programming Language :: Delphi/Kylix', +'Programming Language :: Dylan', +'Programming Language :: Eiffel', +'Programming Language :: Emacs-Lisp', +'Programming Language :: Erlang', +'Programming Language :: Euler', +'Programming Language :: Euphoria', +'Programming Language :: Forth', +'Programming Language :: Fortran', +'Programming Language :: Haskell', +'Programming Language :: Java', +'Programming Language :: JavaScript', +'Programming Language :: Lisp', +'Programming Language :: Logo', +'Programming Language :: ML', +'Programming Language :: Modula', +'Programming Language :: Objective C', +'Programming Language :: Object Pascal', +'Programming Language :: OCaml', +'Programming Language :: Other', +'Programming Language :: Other Scripting Engines', +'Programming Language :: Pascal', +'Programming Language :: Perl', +'Programming Language :: PHP', +'Programming Language :: Pike', +'Programming Language :: Pliant', +'Programming Language :: PL/SQL', +'Programming Language :: PROGRESS', +'Programming Language :: Prolog', +'Programming Language :: Python', +'Programming Language :: Python :: 2', +'Programming Language :: Python :: 2.3', +'Programming Language :: Python :: 2.4', +'Programming Language :: Python :: 2.5', +'Programming Language :: Python :: 2.6', +'Programming Language :: Python :: 2.7', +'Programming Language :: Python :: 3', +'Programming Language :: Python :: 3.0', +'Programming Language :: Python :: 3.1', +'Programming Language :: Python :: 3.2', +'Programming Language :: REBOL', +'Programming Language :: Rexx', +'Programming Language :: Ruby', +'Programming Language :: Scheme', +'Programming Language :: Simula', +'Programming Language :: Smalltalk', +'Programming Language :: SQL', +'Programming Language :: Tcl', +'Programming Language :: Unix Shell', +'Programming Language :: Visual Basic', +'Programming Language :: XBasic', +'Programming Language :: YACC', +'Programming Language :: Zope', +'Topic :: Adaptive Technologies', +'Topic :: Artistic Software', +'Topic :: Communications', +'Topic :: Communications :: BBS', +'Topic :: Communications :: Chat', +'Topic :: Communications :: Chat :: AOL Instant Messenger', +'Topic :: Communications :: Chat :: ICQ', +'Topic :: Communications :: Chat :: Internet Relay Chat', +'Topic :: Communications :: Chat :: Unix Talk', +'Topic :: Communications :: Conferencing', +'Topic :: Communications :: Email', +'Topic :: Communications :: Email :: Address Book', +'Topic :: Communications :: Email :: Email Clients (MUA)', +'Topic :: Communications :: Email :: Filters', +'Topic :: Communications :: Email :: Mailing List Servers', +'Topic :: Communications :: Email :: Mail Transport Agents', +'Topic :: Communications :: Email :: Post-Office', +'Topic :: Communications :: Email :: Post-Office :: IMAP', +'Topic :: Communications :: Email :: Post-Office :: POP3', +'Topic :: Communications :: Fax', +'Topic :: Communications :: FIDO', +'Topic :: Communications :: File Sharing', +'Topic :: Communications :: File Sharing :: Gnutella', +'Topic :: Communications :: File Sharing :: Napster', +'Topic :: Communications :: Ham Radio', +'Topic :: Communications :: Internet Phone', +'Topic :: Communications :: Telephony', +'Topic :: Communications :: Usenet News', +'Topic :: Database', +'Topic :: Database :: Database Engines/Servers', +'Topic :: Database :: Front-Ends', +'Topic :: Desktop Environment', +'Topic :: Desktop Environment :: File Managers', +'Topic :: Desktop Environment :: Gnome', +'Topic :: Desktop Environment :: GNUstep', +'Topic :: Desktop Environment :: K Desktop Environment (KDE)', +'Topic :: Desktop Environment :: K Desktop Environment (KDE) :: Themes', +'Topic :: Desktop Environment :: PicoGUI', +'Topic :: Desktop Environment :: PicoGUI :: Applications', +'Topic :: Desktop Environment :: PicoGUI :: Themes', +'Topic :: Desktop Environment :: Screen Savers', +'Topic :: Desktop Environment :: Window Managers', +'Topic :: Desktop Environment :: Window Managers :: Afterstep', +'Topic :: Desktop Environment :: Window Managers :: Afterstep :: Themes', +'Topic :: Desktop Environment :: Window Managers :: Applets', +'Topic :: Desktop Environment :: Window Managers :: Blackbox', +'Topic :: Desktop Environment :: Window Managers :: Blackbox :: Themes', +'Topic :: Desktop Environment :: Window Managers :: CTWM', +'Topic :: Desktop Environment :: Window Managers :: CTWM :: Themes', +'Topic :: Desktop Environment :: Window Managers :: Enlightenment', +'Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Epplets', +'Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Themes DR15', +'Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Themes DR16', +'Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Themes DR17', +'Topic :: Desktop Environment :: Window Managers :: Fluxbox', +'Topic :: Desktop Environment :: Window Managers :: Fluxbox :: Themes', +'Topic :: Desktop Environment :: Window Managers :: FVWM', +'Topic :: Desktop Environment :: Window Managers :: FVWM :: Themes', +'Topic :: Desktop Environment :: Window Managers :: IceWM', +'Topic :: Desktop Environment :: Window Managers :: IceWM :: Themes', +'Topic :: Desktop Environment :: Window Managers :: MetaCity', +'Topic :: Desktop Environment :: Window Managers :: MetaCity :: Themes', +'Topic :: Desktop Environment :: Window Managers :: Oroborus', +'Topic :: Desktop Environment :: Window Managers :: Oroborus :: Themes', +'Topic :: Desktop Environment :: Window Managers :: Sawfish', +'Topic :: Desktop Environment :: Window Managers :: Sawfish :: Themes 0.30', +'Topic :: Desktop Environment :: Window Managers :: Sawfish :: Themes pre-0.30', +'Topic :: Desktop Environment :: Window Managers :: Waimea', +'Topic :: Desktop Environment :: Window Managers :: Waimea :: Themes', +'Topic :: Desktop Environment :: Window Managers :: Window Maker', +'Topic :: Desktop Environment :: Window Managers :: Window Maker :: Applets', +'Topic :: Desktop Environment :: Window Managers :: Window Maker :: Themes', +'Topic :: Desktop Environment :: Window Managers :: XFCE', +'Topic :: Desktop Environment :: Window Managers :: XFCE :: Themes', +'Topic :: Documentation', +'Topic :: Education', +'Topic :: Education :: Computer Aided Instruction (CAI)', +'Topic :: Education :: Testing', +'Topic :: Games/Entertainment', +'Topic :: Games/Entertainment :: Arcade', +'Topic :: Games/Entertainment :: Board Games', +'Topic :: Games/Entertainment :: First Person Shooters', +'Topic :: Games/Entertainment :: Fortune Cookies', +'Topic :: Games/Entertainment :: Multi-User Dungeons (MUD)', +'Topic :: Games/Entertainment :: Puzzle Games', +'Topic :: Games/Entertainment :: Real Time Strategy', +'Topic :: Games/Entertainment :: Role-Playing', +'Topic :: Games/Entertainment :: Side-Scrolling/Arcade Games', +'Topic :: Games/Entertainment :: Simulation', +'Topic :: Games/Entertainment :: Turn Based Strategy', +'Topic :: Home Automation', +'Topic :: Internet', +'Topic :: Internet :: File Transfer Protocol (FTP)', +'Topic :: Internet :: Finger', +'Topic :: Internet :: Log Analysis', +'Topic :: Internet :: Name Service (DNS)', +'Topic :: Internet :: Proxy Servers', +'Topic :: Internet :: WAP', +'Topic :: Internet :: WWW/HTTP', +'Topic :: Internet :: WWW/HTTP :: Browsers', +'Topic :: Internet :: WWW/HTTP :: Dynamic Content', +'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: CGI Tools/Libraries', +'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: Message Boards', +'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: News/Diary', +'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: Page Counters', +'Topic :: Internet :: WWW/HTTP :: HTTP Servers', +'Topic :: Internet :: WWW/HTTP :: Indexing/Search', +'Topic :: Internet :: WWW/HTTP :: Session', +'Topic :: Internet :: WWW/HTTP :: Site Management', +'Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking', +'Topic :: Internet :: WWW/HTTP :: WSGI', +'Topic :: Internet :: WWW/HTTP :: WSGI :: Application', +'Topic :: Internet :: WWW/HTTP :: WSGI :: Middleware', +'Topic :: Internet :: WWW/HTTP :: WSGI :: Server', +'Topic :: Internet :: Z39.50', +'Topic :: Multimedia', +'Topic :: Multimedia :: Graphics', +'Topic :: Multimedia :: Graphics :: 3D Modeling', +'Topic :: Multimedia :: Graphics :: 3D Rendering', +'Topic :: Multimedia :: Graphics :: Capture', +'Topic :: Multimedia :: Graphics :: Capture :: Digital Camera', +'Topic :: Multimedia :: Graphics :: Capture :: Scanners', +'Topic :: Multimedia :: Graphics :: Capture :: Screen Capture', +'Topic :: Multimedia :: Graphics :: Editors', +'Topic :: Multimedia :: Graphics :: Editors :: Raster-Based', +'Topic :: Multimedia :: Graphics :: Editors :: Vector-Based', +'Topic :: Multimedia :: Graphics :: Graphics Conversion', +'Topic :: Multimedia :: Graphics :: Presentation', +'Topic :: Multimedia :: Graphics :: Viewers', +'Topic :: Multimedia :: Sound/Audio', +'Topic :: Multimedia :: Sound/Audio :: Analysis', +'Topic :: Multimedia :: Sound/Audio :: Capture/Recording', +'Topic :: Multimedia :: Sound/Audio :: CD Audio', +'Topic :: Multimedia :: Sound/Audio :: CD Audio :: CD Playing', +'Topic :: Multimedia :: Sound/Audio :: CD Audio :: CD Ripping', +'Topic :: Multimedia :: Sound/Audio :: CD Audio :: CD Writing', +'Topic :: Multimedia :: Sound/Audio :: Conversion', +'Topic :: Multimedia :: Sound/Audio :: Editors', +'Topic :: Multimedia :: Sound/Audio :: MIDI', +'Topic :: Multimedia :: Sound/Audio :: Mixers', +'Topic :: Multimedia :: Sound/Audio :: Players', +'Topic :: Multimedia :: Sound/Audio :: Players :: MP3', +'Topic :: Multimedia :: Sound/Audio :: Sound Synthesis', +'Topic :: Multimedia :: Sound/Audio :: Speech', +'Topic :: Multimedia :: Video', +'Topic :: Multimedia :: Video :: Capture', +'Topic :: Multimedia :: Video :: Conversion', +'Topic :: Multimedia :: Video :: Display', +'Topic :: Multimedia :: Video :: Non-Linear Editor', +'Topic :: Office/Business', +'Topic :: Office/Business :: Financial', +'Topic :: Office/Business :: Financial :: Accounting', +'Topic :: Office/Business :: Financial :: Investment', +'Topic :: Office/Business :: Financial :: Point-Of-Sale', +'Topic :: Office/Business :: Financial :: Spreadsheet', +'Topic :: Office/Business :: Groupware', +'Topic :: Office/Business :: News/Diary', +'Topic :: Office/Business :: Office Suites', +'Topic :: Office/Business :: Scheduling', +'Topic :: Other/Nonlisted Topic', +'Topic :: Printing', +'Topic :: Religion', +'Topic :: Scientific/Engineering', +'Topic :: Scientific/Engineering :: Artificial Life', +'Topic :: Scientific/Engineering :: Artificial Intelligence', +'Topic :: Scientific/Engineering :: Astronomy', +'Topic :: Scientific/Engineering :: Atmospheric Science', +'Topic :: Scientific/Engineering :: Bio-Informatics', +'Topic :: Scientific/Engineering :: Chemistry', +'Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)', +'Topic :: Scientific/Engineering :: GIS', +'Topic :: Scientific/Engineering :: Human Machine Interfaces', +'Topic :: Scientific/Engineering :: Image Recognition', +'Topic :: Scientific/Engineering :: Information Analysis', +'Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator', +'Topic :: Scientific/Engineering :: Mathematics', +'Topic :: Scientific/Engineering :: Medical Science Apps.', +'Topic :: Scientific/Engineering :: Physics', +'Topic :: Scientific/Engineering :: Visualization', +'Topic :: Security', +'Topic :: Security :: Cryptography', +'Topic :: Sociology', +'Topic :: Sociology :: Genealogy', +'Topic :: Sociology :: History', +'Topic :: Software Development', +'Topic :: Software Development :: Assemblers', +'Topic :: Software Development :: Bug Tracking', +'Topic :: Software Development :: Build Tools', +'Topic :: Software Development :: Code Generators', +'Topic :: Software Development :: Compilers', +'Topic :: Software Development :: Debuggers', +'Topic :: Software Development :: Disassemblers', +'Topic :: Software Development :: Documentation', +'Topic :: Software Development :: Embedded Systems', +'Topic :: Software Development :: Internationalization', +'Topic :: Software Development :: Interpreters', +'Topic :: Software Development :: Libraries', +'Topic :: Software Development :: Libraries :: Application Frameworks', +'Topic :: Software Development :: Libraries :: Java Libraries', +'Topic :: Software Development :: Libraries :: Perl Modules', +'Topic :: Software Development :: Libraries :: PHP Classes', +'Topic :: Software Development :: Libraries :: Pike Modules', +'Topic :: Software Development :: Libraries :: pygame', +'Topic :: Software Development :: Libraries :: Python Modules', +'Topic :: Software Development :: Libraries :: Ruby Modules', +'Topic :: Software Development :: Libraries :: Tcl Extensions', +'Topic :: Software Development :: Localization', +'Topic :: Software Development :: Object Brokering', +'Topic :: Software Development :: Object Brokering :: CORBA', +'Topic :: Software Development :: Pre-processors', +'Topic :: Software Development :: Quality Assurance', +'Topic :: Software Development :: Testing', +'Topic :: Software Development :: Testing :: Traffic Generation', +'Topic :: Software Development :: User Interfaces', +'Topic :: Software Development :: Version Control', +'Topic :: Software Development :: Version Control :: CVS', +'Topic :: Software Development :: Version Control :: RCS', +'Topic :: Software Development :: Version Control :: SCCS', +'Topic :: Software Development :: Widget Sets', +'Topic :: System', +'Topic :: System :: Archiving', +'Topic :: System :: Archiving :: Backup', +'Topic :: System :: Archiving :: Compression', +'Topic :: System :: Archiving :: Mirroring', +'Topic :: System :: Archiving :: Packaging', +'Topic :: System :: Benchmark', +'Topic :: System :: Boot', +'Topic :: System :: Boot :: Init', +'Topic :: System :: Clustering', +'Topic :: System :: Console Fonts', +'Topic :: System :: Distributed Computing', +'Topic :: System :: Emulators', +'Topic :: System :: Filesystems', +'Topic :: System :: Hardware', +'Topic :: System :: Hardware :: Hardware Drivers', +'Topic :: System :: Hardware :: Mainframes', +'Topic :: System :: Hardware :: Symmetric Multi-processing', +'Topic :: System :: Installation/Setup', +'Topic :: System :: Logging', +'Topic :: System :: Monitoring', +'Topic :: System :: Networking', +'Topic :: System :: Networking :: Firewalls', +'Topic :: System :: Networking :: Monitoring', +'Topic :: System :: Networking :: Monitoring :: Hardware Watchdog', +'Topic :: System :: Networking :: Time Synchronization', +'Topic :: System :: Operating System', +'Topic :: System :: Operating System Kernels', +'Topic :: System :: Operating System Kernels :: BSD', +'Topic :: System :: Operating System Kernels :: GNU Hurd', +'Topic :: System :: Operating System Kernels :: Linux', +'Topic :: System :: Power (UPS)', +'Topic :: System :: Recovery Tools', +'Topic :: System :: Shells', +'Topic :: System :: Software Distribution', +'Topic :: System :: Systems Administration', +'Topic :: System :: Systems Administration :: Authentication/Directory', +'Topic :: System :: Systems Administration :: Authentication/Directory :: LDAP', +'Topic :: System :: Systems Administration :: Authentication/Directory :: NIS', +'Topic :: System :: System Shells', +'Topic :: Terminals', +'Topic :: Terminals :: Serial', +'Topic :: Terminals :: Telnet', +'Topic :: Terminals :: Terminal Emulators/X Terminals', +'Topic :: Text Editors', +'Topic :: Text Editors :: Documentation', +'Topic :: Text Editors :: Emacs', +'Topic :: Text Editors :: Integrated Development Environments (IDE)', +'Topic :: Text Editors :: Text Processing', +'Topic :: Text Editors :: Word Processors', +'Topic :: Text Processing', +'Topic :: Text Processing :: Filters', +'Topic :: Text Processing :: Fonts', +'Topic :: Text Processing :: General', +'Topic :: Text Processing :: Indexing', +'Topic :: Text Processing :: Linguistic', +'Topic :: Text Processing :: Markup', +'Topic :: Text Processing :: Markup :: HTML', +'Topic :: Text Processing :: Markup :: LaTeX', +'Topic :: Text Processing :: Markup :: SGML', +'Topic :: Text Processing :: Markup :: VRML', +'Topic :: Text Processing :: Markup :: XML', +'Topic :: Utilities', +] diff --git a/Lib/packaging/command/__init__.py b/Lib/packaging/command/__init__.py new file mode 100644 index 0000000000..8f8c08c553 --- /dev/null +++ b/Lib/packaging/command/__init__.py @@ -0,0 +1,63 @@ +"""Subpackage containing all standard commands.""" +import os +from packaging.errors import PackagingModuleError +from packaging.util import resolve_name + +__all__ = ['get_command_names', 'set_command', 'get_command_class', + 'STANDARD_COMMANDS'] + +_COMMANDS = { + 'check': 'packaging.command.check.check', + 'test': 'packaging.command.test.test', + 'build': 'packaging.command.build.build', + 'build_py': 'packaging.command.build_py.build_py', + 'build_ext': 'packaging.command.build_ext.build_ext', + 'build_clib': 'packaging.command.build_clib.build_clib', + 'build_scripts': 'packaging.command.build_scripts.build_scripts', + 'clean': 'packaging.command.clean.clean', + 'install_dist': 'packaging.command.install_dist.install_dist', + 'install_lib': 'packaging.command.install_lib.install_lib', + 'install_headers': 'packaging.command.install_headers.install_headers', + 'install_scripts': 'packaging.command.install_scripts.install_scripts', + 'install_data': 'packaging.command.install_data.install_data', + 'install_distinfo': + 'packaging.command.install_distinfo.install_distinfo', + 'sdist': 'packaging.command.sdist.sdist', + 'bdist': 'packaging.command.bdist.bdist', + 'bdist_dumb': 'packaging.command.bdist_dumb.bdist_dumb', + 'bdist_wininst': 'packaging.command.bdist_wininst.bdist_wininst', + 'register': 'packaging.command.register.register', + 'upload': 'packaging.command.upload.upload', + 'upload_docs': 'packaging.command.upload_docs.upload_docs', +} + +# XXX this is crappy +if os.name == 'nt': + _COMMANDS['bdist_msi'] = 'packaging.command.bdist_msi' + +# XXX use OrderedDict to preserve the grouping (build-related, install-related, +# distribution-related) +STANDARD_COMMANDS = set(_COMMANDS) + + +def get_command_names(): + """Return registered commands""" + return sorted(_COMMANDS) + + +def set_command(location): + cls = resolve_name(location) + # XXX we want to do the duck-type checking here + _COMMANDS[cls.get_command_name()] = cls + + +def get_command_class(name): + """Return the registered command""" + try: + cls = _COMMANDS[name] + except KeyError: + raise PackagingModuleError("Invalid command %s" % name) + if isinstance(cls, str): + cls = resolve_name(cls) + _COMMANDS[name] = cls + return cls diff --git a/Lib/packaging/command/bdist.py b/Lib/packaging/command/bdist.py new file mode 100644 index 0000000000..b9d550b86b --- /dev/null +++ b/Lib/packaging/command/bdist.py @@ -0,0 +1,141 @@ +"""Create a built (binary) distribution. + +If a --formats option was given on the command line, this command will +call the corresponding bdist_* commands; if the option was absent, a +bdist_* command depending on the current platform will be called. +""" + +import os + +from packaging import util +from packaging.command.cmd import Command +from packaging.errors import PackagingPlatformError, PackagingOptionError + + +def show_formats(): + """Print list of available formats (arguments to "--format" option). + """ + from packaging.fancy_getopt import FancyGetopt + formats = [] + for format in bdist.format_commands: + formats.append(("formats=" + format, None, + bdist.format_command[format][1])) + pretty_printer = FancyGetopt(formats) + pretty_printer.print_help("List of available distribution formats:") + + +class bdist(Command): + + description = "create a built (binary) distribution" + + user_options = [('bdist-base=', 'b', + "temporary directory for creating built distributions"), + ('plat-name=', 'p', + "platform name to embed in generated filenames " + "(default: %s)" % util.get_platform()), + ('formats=', None, + "formats for distribution (comma-separated list)"), + ('dist-dir=', 'd', + "directory to put final built distributions in " + "[default: dist]"), + ('skip-build', None, + "skip rebuilding everything (for testing/debugging)"), + ('owner=', 'u', + "Owner name used when creating a tar file" + " [default: current user]"), + ('group=', 'g', + "Group name used when creating a tar file" + " [default: current group]"), + ] + + boolean_options = ['skip-build'] + + help_options = [ + ('help-formats', None, + "lists available distribution formats", show_formats), + ] + + # This is of course very simplistic. The various UNIX family operating + # systems have their specific formats, but they are out of scope for us; + # bdist_dumb is, well, dumb; it's more a building block for other + # packaging tools than a real end-user binary format. + default_format = {'posix': 'gztar', + 'nt': 'zip', + 'os2': 'zip'} + + # Establish the preferred order (for the --help-formats option). + format_commands = ['gztar', 'bztar', 'tar', + 'wininst', 'zip', 'msi'] + + # And the real information. + format_command = {'gztar': ('bdist_dumb', "gzip'ed tar file"), + 'bztar': ('bdist_dumb', "bzip2'ed tar file"), + 'tar': ('bdist_dumb', "tar file"), + 'wininst': ('bdist_wininst', + "Windows executable installer"), + 'zip': ('bdist_dumb', "ZIP file"), + 'msi': ('bdist_msi', "Microsoft Installer"), + } + + def initialize_options(self): + self.bdist_base = None + self.plat_name = None + self.formats = None + self.dist_dir = None + self.skip_build = False + self.group = None + self.owner = None + + def finalize_options(self): + # have to finalize 'plat_name' before 'bdist_base' + if self.plat_name is None: + if self.skip_build: + self.plat_name = util.get_platform() + else: + self.plat_name = self.get_finalized_command('build').plat_name + + # 'bdist_base' -- parent of per-built-distribution-format + # temporary directories (eg. we'll probably have + # "build/bdist.<plat>/dumb", etc.) + if self.bdist_base is None: + build_base = self.get_finalized_command('build').build_base + self.bdist_base = os.path.join(build_base, + 'bdist.' + self.plat_name) + + self.ensure_string_list('formats') + if self.formats is None: + try: + self.formats = [self.default_format[os.name]] + except KeyError: + raise PackagingPlatformError( + "don't know how to create built distributions " + "on platform %s" % os.name) + + if self.dist_dir is None: + self.dist_dir = "dist" + + def run(self): + # Figure out which sub-commands we need to run. + commands = [] + for format in self.formats: + try: + commands.append(self.format_command[format][0]) + except KeyError: + raise PackagingOptionError("invalid format '%s'" % format) + + # Reinitialize and run each command. + for i in range(len(self.formats)): + cmd_name = commands[i] + sub_cmd = self.get_reinitialized_command(cmd_name) + sub_cmd.format = self.formats[i] + + # passing the owner and group names for tar archiving + if cmd_name == 'bdist_dumb': + sub_cmd.owner = self.owner + sub_cmd.group = self.group + + # If we're going to need to run this command again, tell it to + # keep its temporary files around so subsequent runs go faster. + if cmd_name in commands[i+1:]: + sub_cmd.keep_temp = True + self.run_command(cmd_name) diff --git a/Lib/packaging/command/bdist_dumb.py b/Lib/packaging/command/bdist_dumb.py new file mode 100644 index 0000000000..d5773f01a7 --- /dev/null +++ b/Lib/packaging/command/bdist_dumb.py @@ -0,0 +1,139 @@ +"""Create a "dumb" built distribution. + +A dumb distribution is just an archive meant to be unpacked under +sys.prefix or sys.exec_prefix. +""" + +import os + +from shutil import rmtree +from sysconfig import get_python_version +from packaging.util import get_platform +from packaging.command.cmd import Command +from packaging.errors import PackagingPlatformError +from packaging import logger + + +class bdist_dumb(Command): + + description = 'create a "dumb" built distribution' + + user_options = [('bdist-dir=', 'd', + "temporary directory for creating the distribution"), + ('plat-name=', 'p', + "platform name to embed in generated filenames " + "(default: %s)" % get_platform()), + ('format=', 'f', + "archive format to create (tar, gztar, zip)"), + ('keep-temp', 'k', + "keep the pseudo-installation tree around after " + + "creating the distribution archive"), + ('dist-dir=', 'd', + "directory to put final built distributions in"), + ('skip-build', None, + "skip rebuilding everything (for testing/debugging)"), + ('relative', None, + "build the archive using relative paths" + "(default: false)"), + ('owner=', 'u', + "Owner name used when creating a tar file" + " [default: current user]"), + ('group=', 'g', + "Group name used when creating a tar file" + " [default: current group]"), + ] + + boolean_options = ['keep-temp', 'skip-build', 'relative'] + + default_format = {'posix': 'gztar', + 'nt': 'zip', + 'os2': 'zip'} + + def initialize_options(self): + self.bdist_dir = None + self.plat_name = None + self.format = None + self.keep_temp = False + self.dist_dir = None + self.skip_build = None + self.relative = False + self.owner = None + self.group = None + + def finalize_options(self): + if self.bdist_dir is None: + bdist_base = self.get_finalized_command('bdist').bdist_base + self.bdist_dir = os.path.join(bdist_base, 'dumb') + + if self.format is None: + try: + self.format = self.default_format[os.name] + except KeyError: + raise PackagingPlatformError( + "don't know how to create dumb built distributions " + "on platform %s" % os.name) + + self.set_undefined_options('bdist', + 'dist_dir', 'plat_name', 'skip_build') + + def run(self): + if not self.skip_build: + self.run_command('build') + + install = self.get_reinitialized_command('install_dist', + reinit_subcommands=True) + install.root = self.bdist_dir + install.skip_build = self.skip_build + install.warn_dir = False + + logger.info("installing to %s", self.bdist_dir) + self.run_command('install_dist') + + # And make an archive relative to the root of the + # pseudo-installation tree. + archive_basename = "%s.%s" % (self.distribution.get_fullname(), + self.plat_name) + + # OS/2 objects to any ":" characters in a filename (such as when + # a timestamp is used in a version) so change them to hyphens. + if os.name == "os2": + archive_basename = archive_basename.replace(":", "-") + + pseudoinstall_root = os.path.join(self.dist_dir, archive_basename) + if not self.relative: + archive_root = self.bdist_dir + else: + if (self.distribution.has_ext_modules() and + (install.install_base != install.install_platbase)): + raise PackagingPlatformError( + "can't make a dumb built distribution where base and " + "platbase are different (%r, %r)" % + (install.install_base, install.install_platbase)) + else: + archive_root = os.path.join( + self.bdist_dir, + self._ensure_relative(install.install_base)) + + # Make the archive + filename = self.make_archive(pseudoinstall_root, + self.format, root_dir=archive_root, + owner=self.owner, group=self.group) + if self.distribution.has_ext_modules(): + pyversion = get_python_version() + else: + pyversion = 'any' + self.distribution.dist_files.append(('bdist_dumb', pyversion, + filename)) + + if not self.keep_temp: + if self.dry_run: + logger.info('removing %s', self.bdist_dir) + else: + rmtree(self.bdist_dir) + + def _ensure_relative(self, path): + # copied from dir_util, deleted + drive, path = os.path.splitdrive(path) + if path[0:1] == os.sep: + path = drive + path[1:] + return path diff --git a/Lib/packaging/command/bdist_msi.py b/Lib/packaging/command/bdist_msi.py new file mode 100644 index 0000000000..eaeb458a8f --- /dev/null +++ b/Lib/packaging/command/bdist_msi.py @@ -0,0 +1,744 @@ +"""Create a Microsoft Installer (.msi) binary distribution.""" + +# Copyright (C) 2005, 2006 Martin von Löwis +# Licensed to PSF under a Contributor Agreement. + +import sys +import os +import msilib + + +from sysconfig import get_python_version +from shutil import rmtree +from packaging.command.cmd import Command +from packaging.version import NormalizedVersion +from packaging.errors import PackagingOptionError +from packaging import logger as log +from packaging.util import get_platform +from msilib import schema, sequence, text +from msilib import Directory, Feature, Dialog, add_data + +class MSIVersion(NormalizedVersion): + """ + MSI ProductVersion must be strictly numeric. + MSIVersion disallows prerelease and postrelease versions. + """ + def __init__(self, *args, **kwargs): + super(MSIVersion, self).__init__(*args, **kwargs) + if not self.is_final: + raise ValueError("ProductVersion must be strictly numeric") + +class PyDialog(Dialog): + """Dialog class with a fixed layout: controls at the top, then a ruler, + then a list of buttons: back, next, cancel. Optionally a bitmap at the + left.""" + def __init__(self, *args, **kw): + """Dialog(database, name, x, y, w, h, attributes, title, first, + default, cancel, bitmap=true)""" + Dialog.__init__(self, *args) + ruler = self.h - 36 + #if kw.get("bitmap", True): + # self.bitmap("Bitmap", 0, 0, bmwidth, ruler, "PythonWin") + self.line("BottomLine", 0, ruler, self.w, 0) + + def title(self, title): + "Set the title text of the dialog at the top." + # name, x, y, w, h, flags=Visible|Enabled|Transparent|NoPrefix, + # text, in VerdanaBold10 + self.text("Title", 15, 10, 320, 60, 0x30003, + r"{\VerdanaBold10}%s" % title) + + def back(self, title, next, name = "Back", active = 1): + """Add a back button with a given title, the tab-next button, + its name in the Control table, possibly initially disabled. + + Return the button, so that events can be associated""" + if active: + flags = 3 # Visible|Enabled + else: + flags = 1 # Visible + return self.pushbutton(name, 180, self.h-27 , 56, 17, flags, title, next) + + def cancel(self, title, next, name = "Cancel", active = 1): + """Add a cancel button with a given title, the tab-next button, + its name in the Control table, possibly initially disabled. + + Return the button, so that events can be associated""" + if active: + flags = 3 # Visible|Enabled + else: + flags = 1 # Visible + return self.pushbutton(name, 304, self.h-27, 56, 17, flags, title, next) + + def next(self, title, next, name = "Next", active = 1): + """Add a Next button with a given title, the tab-next button, + its name in the Control table, possibly initially disabled. + + Return the button, so that events can be associated""" + if active: + flags = 3 # Visible|Enabled + else: + flags = 1 # Visible + return self.pushbutton(name, 236, self.h-27, 56, 17, flags, title, next) + + def xbutton(self, name, title, next, xpos): + """Add a button with a given title, the tab-next button, + its name in the Control table, giving its x position; the + y-position is aligned with the other buttons. + + Return the button, so that events can be associated""" + return self.pushbutton(name, int(self.w*xpos - 28), self.h-27, 56, 17, 3, title, next) + +class bdist_msi(Command): + + description = "create a Microsoft Installer (.msi) binary distribution" + + user_options = [('bdist-dir=', None, + "temporary directory for creating the distribution"), + ('plat-name=', 'p', + "platform name to embed in generated filenames " + "(default: %s)" % get_platform()), + ('keep-temp', 'k', + "keep the pseudo-installation tree around after " + + "creating the distribution archive"), + ('target-version=', None, + "require a specific python version" + + " on the target system"), + ('no-target-compile', 'c', + "do not compile .py to .pyc on the target system"), + ('no-target-optimize', 'o', + "do not compile .py to .pyo (optimized)" + "on the target system"), + ('dist-dir=', 'd', + "directory to put final built distributions in"), + ('skip-build', None, + "skip rebuilding everything (for testing/debugging)"), + ('install-script=', None, + "basename of installation script to be run after" + "installation or before deinstallation"), + ('pre-install-script=', None, + "Fully qualified filename of a script to be run before " + "any files are installed. This script need not be in the " + "distribution"), + ] + + boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', + 'skip-build'] + + all_versions = ['2.0', '2.1', '2.2', '2.3', '2.4', + '2.5', '2.6', '2.7', '2.8', '2.9', + '3.0', '3.1', '3.2', '3.3', '3.4', + '3.5', '3.6', '3.7', '3.8', '3.9'] + other_version = 'X' + + def initialize_options(self): + self.bdist_dir = None + self.plat_name = None + self.keep_temp = False + self.no_target_compile = False + self.no_target_optimize = False + self.target_version = None + self.dist_dir = None + self.skip_build = None + self.install_script = None + self.pre_install_script = None + self.versions = None + + def finalize_options(self): + self.set_undefined_options('bdist', 'skip_build') + + if self.bdist_dir is None: + bdist_base = self.get_finalized_command('bdist').bdist_base + self.bdist_dir = os.path.join(bdist_base, 'msi') + + short_version = get_python_version() + if (not self.target_version) and self.distribution.has_ext_modules(): + self.target_version = short_version + + if self.target_version: + self.versions = [self.target_version] + if not self.skip_build and self.distribution.has_ext_modules()\ + and self.target_version != short_version: + raise PackagingOptionError("target version can only be %s, or the '--skip-build'" \ + " option must be specified" % (short_version,)) + else: + self.versions = list(self.all_versions) + + self.set_undefined_options('bdist', 'dist_dir', 'plat_name') + + if self.pre_install_script: + raise PackagingOptionError("the pre-install-script feature is not yet implemented") + + if self.install_script: + for script in self.distribution.scripts: + if self.install_script == os.path.basename(script): + break + else: + raise PackagingOptionError("install_script '%s' not found in scripts" % \ + self.install_script) + self.install_script_key = None + + + def run(self): + if not self.skip_build: + self.run_command('build') + + install = self.get_reinitialized_command('install_dist', + reinit_subcommands=True) + install.prefix = self.bdist_dir + install.skip_build = self.skip_build + install.warn_dir = False + + install_lib = self.get_reinitialized_command('install_lib') + # we do not want to include pyc or pyo files + install_lib.compile = False + install_lib.optimize = 0 + + if self.distribution.has_ext_modules(): + # If we are building an installer for a Python version other + # than the one we are currently running, then we need to ensure + # our build_lib reflects the other Python version rather than ours. + # Note that for target_version!=sys.version, we must have skipped the + # build step, so there is no issue with enforcing the build of this + # version. + target_version = self.target_version + if not target_version: + assert self.skip_build, "Should have already checked this" + target_version = sys.version[0:3] + plat_specifier = ".%s-%s" % (self.plat_name, target_version) + build = self.get_finalized_command('build') + build.build_lib = os.path.join(build.build_base, + 'lib' + plat_specifier) + + log.info("installing to %s", self.bdist_dir) + install.ensure_finalized() + + # avoid warning of 'install_lib' about installing + # into a directory not in sys.path + sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) + + install.run() + + del sys.path[0] + + self.mkpath(self.dist_dir) + fullname = self.distribution.get_fullname() + installer_name = self.get_installer_filename(fullname) + installer_name = os.path.abspath(installer_name) + if os.path.exists(installer_name): os.unlink(installer_name) + + metadata = self.distribution.metadata + author = metadata.author + if not author: + author = metadata.maintainer + if not author: + author = "UNKNOWN" + version = MSIVersion(metadata.get_version()) + # Prefix ProductName with Python x.y, so that + # it sorts together with the other Python packages + # in Add-Remove-Programs (APR) + fullname = self.distribution.get_fullname() + if self.target_version: + product_name = "Python %s %s" % (self.target_version, fullname) + else: + product_name = "Python %s" % (fullname) + self.db = msilib.init_database(installer_name, schema, + product_name, msilib.gen_uuid(), + str(version), author) + msilib.add_tables(self.db, sequence) + props = [('DistVersion', version)] + email = metadata.author_email or metadata.maintainer_email + if email: + props.append(("ARPCONTACT", email)) + if metadata.url: + props.append(("ARPURLINFOABOUT", metadata.url)) + if props: + add_data(self.db, 'Property', props) + + self.add_find_python() + self.add_files() + self.add_scripts() + self.add_ui() + self.db.Commit() + + if hasattr(self.distribution, 'dist_files'): + tup = 'bdist_msi', self.target_version or 'any', fullname + self.distribution.dist_files.append(tup) + + if not self.keep_temp: + log.info("removing temporary build directory %s", self.bdist_dir) + if not self.dry_run: + rmtree(self.bdist_dir) + + def add_files(self): + db = self.db + cab = msilib.CAB("distfiles") + rootdir = os.path.abspath(self.bdist_dir) + + root = Directory(db, cab, None, rootdir, "TARGETDIR", "SourceDir") + f = Feature(db, "Python", "Python", "Everything", + 0, 1, directory="TARGETDIR") + + items = [(f, root, '')] + for version in self.versions + [self.other_version]: + target = "TARGETDIR" + version + name = default = "Python" + version + desc = "Everything" + if version is self.other_version: + title = "Python from another location" + level = 2 + else: + title = "Python %s from registry" % version + level = 1 + f = Feature(db, name, title, desc, 1, level, directory=target) + dir = Directory(db, cab, root, rootdir, target, default) + items.append((f, dir, version)) + db.Commit() + + seen = {} + for feature, dir, version in items: + todo = [dir] + while todo: + dir = todo.pop() + for file in os.listdir(dir.absolute): + afile = os.path.join(dir.absolute, file) + if os.path.isdir(afile): + short = "%s|%s" % (dir.make_short(file), file) + default = file + version + newdir = Directory(db, cab, dir, file, default, short) + todo.append(newdir) + else: + if not dir.component: + dir.start_component(dir.logical, feature, 0) + if afile not in seen: + key = seen[afile] = dir.add_file(file) + if file==self.install_script: + if self.install_script_key: + raise PackagingOptionError( + "Multiple files with name %s" % file) + self.install_script_key = '[#%s]' % key + else: + key = seen[afile] + add_data(self.db, "DuplicateFile", + [(key + version, dir.component, key, None, dir.logical)]) + db.Commit() + cab.commit(db) + + def add_find_python(self): + """Adds code to the installer to compute the location of Python. + + Properties PYTHON.MACHINE.X.Y and PYTHON.USER.X.Y will be set from the + registry for each version of Python. + + Properties TARGETDIRX.Y will be set from PYTHON.USER.X.Y if defined, + else from PYTHON.MACHINE.X.Y. + + Properties PYTHONX.Y will be set to TARGETDIRX.Y\\python.exe""" + + start = 402 + for ver in self.versions: + install_path = r"SOFTWARE\Python\PythonCore\%s\InstallPath" % ver + machine_reg = "python.machine." + ver + user_reg = "python.user." + ver + machine_prop = "PYTHON.MACHINE." + ver + user_prop = "PYTHON.USER." + ver + machine_action = "PythonFromMachine" + ver + user_action = "PythonFromUser" + ver + exe_action = "PythonExe" + ver + target_dir_prop = "TARGETDIR" + ver + exe_prop = "PYTHON" + ver + if msilib.Win64: + # type: msidbLocatorTypeRawValue + msidbLocatorType64bit + Type = 2+16 + else: + Type = 2 + add_data(self.db, "RegLocator", + [(machine_reg, 2, install_path, None, Type), + (user_reg, 1, install_path, None, Type)]) + add_data(self.db, "AppSearch", + [(machine_prop, machine_reg), + (user_prop, user_reg)]) + add_data(self.db, "CustomAction", + [(machine_action, 51+256, target_dir_prop, "[" + machine_prop + "]"), + (user_action, 51+256, target_dir_prop, "[" + user_prop + "]"), + (exe_action, 51+256, exe_prop, "[" + target_dir_prop + "]\\python.exe"), + ]) + add_data(self.db, "InstallExecuteSequence", + [(machine_action, machine_prop, start), + (user_action, user_prop, start + 1), + (exe_action, None, start + 2), + ]) + add_data(self.db, "InstallUISequence", + [(machine_action, machine_prop, start), + (user_action, user_prop, start + 1), + (exe_action, None, start + 2), + ]) + add_data(self.db, "Condition", + [("Python" + ver, 0, "NOT TARGETDIR" + ver)]) + start += 4 + assert start < 500 + + def add_scripts(self): + if self.install_script: + start = 6800 + for ver in self.versions + [self.other_version]: + install_action = "install_script." + ver + exe_prop = "PYTHON" + ver + add_data(self.db, "CustomAction", + [(install_action, 50, exe_prop, self.install_script_key)]) + add_data(self.db, "InstallExecuteSequence", + [(install_action, "&Python%s=3" % ver, start)]) + start += 1 + # XXX pre-install scripts are currently refused in finalize_options() + # but if this feature is completed, it will also need to add + # entries for each version as the above code does + if self.pre_install_script: + scriptfn = os.path.join(self.bdist_dir, "preinstall.bat") + with open(scriptfn, "w") as f: + # The batch file will be executed with [PYTHON], so that %1 + # is the path to the Python interpreter; %0 will be the path + # of the batch file. + # rem =""" + # %1 %0 + # exit + # """ + # <actual script> + f.write('rem ="""\n%1 %0\nexit\n"""\n') + with open(self.pre_install_script) as fp: + f.write(fp.read()) + add_data(self.db, "Binary", + [("PreInstall", msilib.Binary(scriptfn)), + ]) + add_data(self.db, "CustomAction", + [("PreInstall", 2, "PreInstall", None), + ]) + add_data(self.db, "InstallExecuteSequence", + [("PreInstall", "NOT Installed", 450), + ]) + + def add_ui(self): + db = self.db + x = y = 50 + w = 370 + h = 300 + title = "[ProductName] Setup" + + # see "Dialog Style Bits" + modal = 3 # visible | modal + modeless = 1 # visible + + # UI customization properties + add_data(db, "Property", + # See "DefaultUIFont Property" + [("DefaultUIFont", "DlgFont8"), + # See "ErrorDialog Style Bit" + ("ErrorDialog", "ErrorDlg"), + ("Progress1", "Install"), # modified in maintenance type dlg + ("Progress2", "installs"), + ("MaintenanceForm_Action", "Repair"), + # possible values: ALL, JUSTME + ("WhichUsers", "ALL") + ]) + + # Fonts, see "TextStyle Table" + add_data(db, "TextStyle", + [("DlgFont8", "Tahoma", 9, None, 0), + ("DlgFontBold8", "Tahoma", 8, None, 1), #bold + ("VerdanaBold10", "Verdana", 10, None, 1), + ("VerdanaRed9", "Verdana", 9, 255, 0), + ]) + + # UI Sequences, see "InstallUISequence Table", "Using a Sequence Table" + # Numbers indicate sequence; see sequence.py for how these action integrate + add_data(db, "InstallUISequence", + [("PrepareDlg", "Not Privileged or Windows9x or Installed", 140), + ("WhichUsersDlg", "Privileged and not Windows9x and not Installed", 141), + # In the user interface, assume all-users installation if privileged. + ("SelectFeaturesDlg", "Not Installed", 1230), + # XXX no support for resume installations yet + #("ResumeDlg", "Installed AND (RESUME OR Preselected)", 1240), + ("MaintenanceTypeDlg", "Installed AND NOT RESUME AND NOT Preselected", 1250), + ("ProgressDlg", None, 1280)]) + + add_data(db, 'ActionText', text.ActionText) + add_data(db, 'UIText', text.UIText) + ##################################################################### + # Standard dialogs: FatalError, UserExit, ExitDialog + fatal=PyDialog(db, "FatalError", x, y, w, h, modal, title, + "Finish", "Finish", "Finish") + fatal.title("[ProductName] Installer ended prematurely") + fatal.back("< Back", "Finish", active = 0) + fatal.cancel("Cancel", "Back", active = 0) + fatal.text("Description1", 15, 70, 320, 80, 0x30003, + "[ProductName] setup ended prematurely because of an error. Your system has not been modified. To install this program at a later time, please run the installation again.") + fatal.text("Description2", 15, 155, 320, 20, 0x30003, + "Click the Finish button to exit the Installer.") + c=fatal.next("Finish", "Cancel", name="Finish") + c.event("EndDialog", "Exit") + + user_exit=PyDialog(db, "UserExit", x, y, w, h, modal, title, + "Finish", "Finish", "Finish") + user_exit.title("[ProductName] Installer was interrupted") + user_exit.back("< Back", "Finish", active = 0) + user_exit.cancel("Cancel", "Back", active = 0) + user_exit.text("Description1", 15, 70, 320, 80, 0x30003, + "[ProductName] setup was interrupted. Your system has not been modified. " + "To install this program at a later time, please run the installation again.") + user_exit.text("Description2", 15, 155, 320, 20, 0x30003, + "Click the Finish button to exit the Installer.") + c = user_exit.next("Finish", "Cancel", name="Finish") + c.event("EndDialog", "Exit") + + exit_dialog = PyDialog(db, "ExitDialog", x, y, w, h, modal, title, + "Finish", "Finish", "Finish") + exit_dialog.title("Completing the [ProductName] Installer") + exit_dialog.back("< Back", "Finish", active = 0) + exit_dialog.cancel("Cancel", "Back", active = 0) + exit_dialog.text("Description", 15, 235, 320, 20, 0x30003, + "Click the Finish button to exit the Installer.") + c = exit_dialog.next("Finish", "Cancel", name="Finish") + c.event("EndDialog", "Return") + + ##################################################################### + # Required dialog: FilesInUse, ErrorDlg + inuse = PyDialog(db, "FilesInUse", + x, y, w, h, + 19, # KeepModeless|Modal|Visible + title, + "Retry", "Retry", "Retry", bitmap=False) + inuse.text("Title", 15, 6, 200, 15, 0x30003, + r"{\DlgFontBold8}Files in Use") + inuse.text("Description", 20, 23, 280, 20, 0x30003, + "Some files that need to be updated are currently in use.") + inuse.text("Text", 20, 55, 330, 50, 3, + "The following applications are using files that need to be updated by this setup. Close these applications and then click Retry to continue the installation or Cancel to exit it.") + inuse.control("List", "ListBox", 20, 107, 330, 130, 7, "FileInUseProcess", + None, None, None) + c=inuse.back("Exit", "Ignore", name="Exit") + c.event("EndDialog", "Exit") + c=inuse.next("Ignore", "Retry", name="Ignore") + c.event("EndDialog", "Ignore") + c=inuse.cancel("Retry", "Exit", name="Retry") + c.event("EndDialog","Retry") + + # See "Error Dialog". See "ICE20" for the required names of the controls. + error = Dialog(db, "ErrorDlg", + 50, 10, 330, 101, + 65543, # Error|Minimize|Modal|Visible + title, + "ErrorText", None, None) + error.text("ErrorText", 50,9,280,48,3, "") + #error.control("ErrorIcon", "Icon", 15, 9, 24, 24, 5242881, None, "py.ico", None, None) + error.pushbutton("N",120,72,81,21,3,"No",None).event("EndDialog","ErrorNo") + error.pushbutton("Y",240,72,81,21,3,"Yes",None).event("EndDialog","ErrorYes") + error.pushbutton("A",0,72,81,21,3,"Abort",None).event("EndDialog","ErrorAbort") + error.pushbutton("C",42,72,81,21,3,"Cancel",None).event("EndDialog","ErrorCancel") + error.pushbutton("I",81,72,81,21,3,"Ignore",None).event("EndDialog","ErrorIgnore") + error.pushbutton("O",159,72,81,21,3,"Ok",None).event("EndDialog","ErrorOk") + error.pushbutton("R",198,72,81,21,3,"Retry",None).event("EndDialog","ErrorRetry") + + ##################################################################### + # Global "Query Cancel" dialog + cancel = Dialog(db, "CancelDlg", 50, 10, 260, 85, 3, title, + "No", "No", "No") + cancel.text("Text", 48, 15, 194, 30, 3, + "Are you sure you want to cancel [ProductName] installation?") + #cancel.control("Icon", "Icon", 15, 15, 24, 24, 5242881, None, + # "py.ico", None, None) + c=cancel.pushbutton("Yes", 72, 57, 56, 17, 3, "Yes", "No") + c.event("EndDialog", "Exit") + + c=cancel.pushbutton("No", 132, 57, 56, 17, 3, "No", "Yes") + c.event("EndDialog", "Return") + + ##################################################################### + # Global "Wait for costing" dialog + costing = Dialog(db, "WaitForCostingDlg", 50, 10, 260, 85, modal, title, + "Return", "Return", "Return") + costing.text("Text", 48, 15, 194, 30, 3, + "Please wait while the installer finishes determining your disk space requirements.") + c = costing.pushbutton("Return", 102, 57, 56, 17, 3, "Return", None) + c.event("EndDialog", "Exit") + + ##################################################################### + # Preparation dialog: no user input except cancellation + prep = PyDialog(db, "PrepareDlg", x, y, w, h, modeless, title, + "Cancel", "Cancel", "Cancel") + prep.text("Description", 15, 70, 320, 40, 0x30003, + "Please wait while the Installer prepares to guide you through the installation.") + prep.title("Welcome to the [ProductName] Installer") + c=prep.text("ActionText", 15, 110, 320, 20, 0x30003, "Pondering...") + c.mapping("ActionText", "Text") + c=prep.text("ActionData", 15, 135, 320, 30, 0x30003, None) + c.mapping("ActionData", "Text") + prep.back("Back", None, active=0) + prep.next("Next", None, active=0) + c=prep.cancel("Cancel", None) + c.event("SpawnDialog", "CancelDlg") + + ##################################################################### + # Feature (Python directory) selection + seldlg = PyDialog(db, "SelectFeaturesDlg", x, y, w, h, modal, title, + "Next", "Next", "Cancel") + seldlg.title("Select Python Installations") + + seldlg.text("Hint", 15, 30, 300, 20, 3, + "Select the Python locations where %s should be installed." + % self.distribution.get_fullname()) + + seldlg.back("< Back", None, active=0) + c = seldlg.next("Next >", "Cancel") + order = 1 + c.event("[TARGETDIR]", "[SourceDir]", ordering=order) + for version in self.versions + [self.other_version]: + order += 1 + c.event("[TARGETDIR]", "[TARGETDIR%s]" % version, + "FEATURE_SELECTED AND &Python%s=3" % version, + ordering=order) + c.event("SpawnWaitDialog", "WaitForCostingDlg", ordering=order + 1) + c.event("EndDialog", "Return", ordering=order + 2) + c = seldlg.cancel("Cancel", "Features") + c.event("SpawnDialog", "CancelDlg") + + c = seldlg.control("Features", "SelectionTree", 15, 60, 300, 120, 3, + "FEATURE", None, "PathEdit", None) + c.event("[FEATURE_SELECTED]", "1") + ver = self.other_version + install_other_cond = "FEATURE_SELECTED AND &Python%s=3" % ver + dont_install_other_cond = "FEATURE_SELECTED AND &Python%s<>3" % ver + + c = seldlg.text("Other", 15, 200, 300, 15, 3, + "Provide an alternate Python location") + c.condition("Enable", install_other_cond) + c.condition("Show", install_other_cond) + c.condition("Disable", dont_install_other_cond) + c.condition("Hide", dont_install_other_cond) + + c = seldlg.control("PathEdit", "PathEdit", 15, 215, 300, 16, 1, + "TARGETDIR" + ver, None, "Next", None) + c.condition("Enable", install_other_cond) + c.condition("Show", install_other_cond) + c.condition("Disable", dont_install_other_cond) + c.condition("Hide", dont_install_other_cond) + + ##################################################################### + # Disk cost + cost = PyDialog(db, "DiskCostDlg", x, y, w, h, modal, title, + "OK", "OK", "OK", bitmap=False) + cost.text("Title", 15, 6, 200, 15, 0x30003, + "{\DlgFontBold8}Disk Space Requirements") + cost.text("Description", 20, 20, 280, 20, 0x30003, + "The disk space required for the installation of the selected features.") + cost.text("Text", 20, 53, 330, 60, 3, + "The highlighted volumes (if any) do not have enough disk space " + "available for the currently selected features. You can either " + "remove some files from the highlighted volumes, or choose to " + "install less features onto local drive(s), or select different " + "destination drive(s).") + cost.control("VolumeList", "VolumeCostList", 20, 100, 330, 150, 393223, + None, "{120}{70}{70}{70}{70}", None, None) + cost.xbutton("OK", "Ok", None, 0.5).event("EndDialog", "Return") + + ##################################################################### + # WhichUsers Dialog. Only available on NT, and for privileged users. + # This must be run before FindRelatedProducts, because that will + # take into account whether the previous installation was per-user + # or per-machine. We currently don't support going back to this + # dialog after "Next" was selected; to support this, we would need to + # find how to reset the ALLUSERS property, and how to re-run + # FindRelatedProducts. + # On Windows9x, the ALLUSERS property is ignored on the command line + # and in the Property table, but installer fails according to the documentation + # if a dialog attempts to set ALLUSERS. + whichusers = PyDialog(db, "WhichUsersDlg", x, y, w, h, modal, title, + "AdminInstall", "Next", "Cancel") + whichusers.title("Select whether to install [ProductName] for all users of this computer.") + # A radio group with two options: allusers, justme + g = whichusers.radiogroup("AdminInstall", 15, 60, 260, 50, 3, + "WhichUsers", "", "Next") + g.add("ALL", 0, 5, 150, 20, "Install for all users") + g.add("JUSTME", 0, 25, 150, 20, "Install just for me") + + whichusers.back("Back", None, active=0) + + c = whichusers.next("Next >", "Cancel") + c.event("[ALLUSERS]", "1", 'WhichUsers="ALL"', 1) + c.event("EndDialog", "Return", ordering = 2) + + c = whichusers.cancel("Cancel", "AdminInstall") + c.event("SpawnDialog", "CancelDlg") + + ##################################################################### + # Installation Progress dialog (modeless) + progress = PyDialog(db, "ProgressDlg", x, y, w, h, modeless, title, + "Cancel", "Cancel", "Cancel", bitmap=False) + progress.text("Title", 20, 15, 200, 15, 0x30003, + "{\DlgFontBold8}[Progress1] [ProductName]") + progress.text("Text", 35, 65, 300, 30, 3, + "Please wait while the Installer [Progress2] [ProductName]. " + "This may take several minutes.") + progress.text("StatusLabel", 35, 100, 35, 20, 3, "Status:") + + c=progress.text("ActionText", 70, 100, w-70, 20, 3, "Pondering...") + c.mapping("ActionText", "Text") + + #c=progress.text("ActionData", 35, 140, 300, 20, 3, None) + #c.mapping("ActionData", "Text") + + c=progress.control("ProgressBar", "ProgressBar", 35, 120, 300, 10, 65537, + None, "Progress done", None, None) + c.mapping("SetProgress", "Progress") + + progress.back("< Back", "Next", active=False) + progress.next("Next >", "Cancel", active=False) + progress.cancel("Cancel", "Back").event("SpawnDialog", "CancelDlg") + + ################################################################### + # Maintenance type: repair/uninstall + maint = PyDialog(db, "MaintenanceTypeDlg", x, y, w, h, modal, title, + "Next", "Next", "Cancel") + maint.title("Welcome to the [ProductName] Setup Wizard") + maint.text("BodyText", 15, 63, 330, 42, 3, + "Select whether you want to repair or remove [ProductName].") + g=maint.radiogroup("RepairRadioGroup", 15, 108, 330, 60, 3, + "MaintenanceForm_Action", "", "Next") + #g.add("Change", 0, 0, 200, 17, "&Change [ProductName]") + g.add("Repair", 0, 18, 200, 17, "&Repair [ProductName]") + g.add("Remove", 0, 36, 200, 17, "Re&move [ProductName]") + + maint.back("< Back", None, active=False) + c=maint.next("Finish", "Cancel") + # Change installation: Change progress dialog to "Change", then ask + # for feature selection + #c.event("[Progress1]", "Change", 'MaintenanceForm_Action="Change"', 1) + #c.event("[Progress2]", "changes", 'MaintenanceForm_Action="Change"', 2) + + # Reinstall: Change progress dialog to "Repair", then invoke reinstall + # Also set list of reinstalled features to "ALL" + c.event("[REINSTALL]", "ALL", 'MaintenanceForm_Action="Repair"', 5) + c.event("[Progress1]", "Repairing", 'MaintenanceForm_Action="Repair"', 6) + c.event("[Progress2]", "repairs", 'MaintenanceForm_Action="Repair"', 7) + c.event("Reinstall", "ALL", 'MaintenanceForm_Action="Repair"', 8) + + # Uninstall: Change progress to "Remove", then invoke uninstall + # Also set list of removed features to "ALL" + c.event("[REMOVE]", "ALL", 'MaintenanceForm_Action="Remove"', 11) + c.event("[Progress1]", "Removing", 'MaintenanceForm_Action="Remove"', 12) + c.event("[Progress2]", "removes", 'MaintenanceForm_Action="Remove"', 13) + c.event("Remove", "ALL", 'MaintenanceForm_Action="Remove"', 14) + + # Close dialog when maintenance action scheduled + c.event("EndDialog", "Return", 'MaintenanceForm_Action<>"Change"', 20) + #c.event("NewDialog", "SelectFeaturesDlg", 'MaintenanceForm_Action="Change"', 21) + + maint.cancel("Cancel", "RepairRadioGroup").event("SpawnDialog", "CancelDlg") + + def get_installer_filename(self, fullname): + # Factored out to allow overriding in subclasses + if self.target_version: + base_name = "%s.%s-py%s.msi" % (fullname, self.plat_name, + self.target_version) + else: + base_name = "%s.%s.msi" % (fullname, self.plat_name) + installer_name = os.path.join(self.dist_dir, base_name) + return installer_name diff --git a/Lib/packaging/command/bdist_wininst.py b/Lib/packaging/command/bdist_wininst.py new file mode 100644 index 0000000000..7dbb39b311 --- /dev/null +++ b/Lib/packaging/command/bdist_wininst.py @@ -0,0 +1,346 @@ +"""Create an executable installer for Windows.""" + +# FIXME synchronize bytes/str use with same file in distutils + +import sys +import os + +from shutil import rmtree +from sysconfig import get_python_version +from packaging.command.cmd import Command +from packaging.errors import PackagingOptionError, PackagingPlatformError +from packaging import logger +from packaging.util import get_platform + + +class bdist_wininst(Command): + + description = "create an executable installer for Windows" + + user_options = [('bdist-dir=', None, + "temporary directory for creating the distribution"), + ('plat-name=', 'p', + "platform name to embed in generated filenames " + "(default: %s)" % get_platform()), + ('keep-temp', 'k', + "keep the pseudo-installation tree around after " + + "creating the distribution archive"), + ('target-version=', None, + "require a specific python version" + + " on the target system"), + ('no-target-compile', 'c', + "do not compile .py to .pyc on the target system"), + ('no-target-optimize', 'o', + "do not compile .py to .pyo (optimized)" + "on the target system"), + ('dist-dir=', 'd', + "directory to put final built distributions in"), + ('bitmap=', 'b', + "bitmap to use for the installer instead of python-powered logo"), + ('title=', 't', + "title to display on the installer background instead of default"), + ('skip-build', None, + "skip rebuilding everything (for testing/debugging)"), + ('install-script=', None, + "basename of installation script to be run after" + "installation or before deinstallation"), + ('pre-install-script=', None, + "Fully qualified filename of a script to be run before " + "any files are installed. This script need not be in the " + "distribution"), + ('user-access-control=', None, + "specify Vista's UAC handling - 'none'/default=no " + "handling, 'auto'=use UAC if target Python installed for " + "all users, 'force'=always use UAC"), + ] + + boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', + 'skip-build'] + + def initialize_options(self): + self.bdist_dir = None + self.plat_name = None + self.keep_temp = False + self.no_target_compile = False + self.no_target_optimize = False + self.target_version = None + self.dist_dir = None + self.bitmap = None + self.title = None + self.skip_build = None + self.install_script = None + self.pre_install_script = None + self.user_access_control = None + + + def finalize_options(self): + self.set_undefined_options('bdist', 'skip_build') + + if self.bdist_dir is None: + if self.skip_build and self.plat_name: + # If build is skipped and plat_name is overridden, bdist will + # not see the correct 'plat_name' - so set that up manually. + bdist = self.distribution.get_command_obj('bdist') + bdist.plat_name = self.plat_name + # next the command will be initialized using that name + bdist_base = self.get_finalized_command('bdist').bdist_base + self.bdist_dir = os.path.join(bdist_base, 'wininst') + + if not self.target_version: + self.target_version = "" + + if not self.skip_build and self.distribution.has_ext_modules(): + short_version = get_python_version() + if self.target_version and self.target_version != short_version: + raise PackagingOptionError("target version can only be %s, or the '--skip-build'" \ + " option must be specified" % (short_version,)) + self.target_version = short_version + + self.set_undefined_options('bdist', 'dist_dir', 'plat_name') + + if self.install_script: + for script in self.distribution.scripts: + if self.install_script == os.path.basename(script): + break + else: + raise PackagingOptionError("install_script '%s' not found in scripts" % \ + self.install_script) + + def run(self): + if (sys.platform != "win32" and + (self.distribution.has_ext_modules() or + self.distribution.has_c_libraries())): + raise PackagingPlatformError \ + ("distribution contains extensions and/or C libraries; " + "must be compiled on a Windows 32 platform") + + if not self.skip_build: + self.run_command('build') + + install = self.get_reinitialized_command('install', + reinit_subcommands=True) + install.root = self.bdist_dir + install.skip_build = self.skip_build + install.warn_dir = False + install.plat_name = self.plat_name + + install_lib = self.get_reinitialized_command('install_lib') + # we do not want to include pyc or pyo files + install_lib.compile = False + install_lib.optimize = 0 + + if self.distribution.has_ext_modules(): + # If we are building an installer for a Python version other + # than the one we are currently running, then we need to ensure + # our build_lib reflects the other Python version rather than ours. + # Note that for target_version!=sys.version, we must have skipped the + # build step, so there is no issue with enforcing the build of this + # version. + target_version = self.target_version + if not target_version: + assert self.skip_build, "Should have already checked this" + target_version = sys.version[0:3] + plat_specifier = ".%s-%s" % (self.plat_name, target_version) + build = self.get_finalized_command('build') + build.build_lib = os.path.join(build.build_base, + 'lib' + plat_specifier) + + # Use a custom scheme for the zip-file, because we have to decide + # at installation time which scheme to use. + for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'): + value = key.upper() + if key == 'headers': + value = value + '/Include/$dist_name' + setattr(install, + 'install_' + key, + value) + + logger.info("installing to %s", self.bdist_dir) + install.ensure_finalized() + + # avoid warning of 'install_lib' about installing + # into a directory not in sys.path + sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) + + install.run() + + del sys.path[0] + + # And make an archive relative to the root of the + # pseudo-installation tree. + from tempfile import NamedTemporaryFile + archive_basename = NamedTemporaryFile().name + fullname = self.distribution.get_fullname() + arcname = self.make_archive(archive_basename, "zip", + root_dir=self.bdist_dir) + # create an exe containing the zip-file + self.create_exe(arcname, fullname, self.bitmap) + if self.distribution.has_ext_modules(): + pyversion = get_python_version() + else: + pyversion = 'any' + self.distribution.dist_files.append(('bdist_wininst', pyversion, + self.get_installer_filename(fullname))) + # remove the zip-file again + logger.debug("removing temporary file '%s'", arcname) + os.remove(arcname) + + if not self.keep_temp: + if self.dry_run: + logger.info('removing %s', self.bdist_dir) + else: + rmtree(self.bdist_dir) + + def get_inidata(self): + # Return data describing the installation. + + lines = [] + metadata = self.distribution.metadata + + # Write the [metadata] section. + lines.append("[metadata]") + + # 'info' will be displayed in the installer's dialog box, + # describing the items to be installed. + info = (metadata.long_description or '') + '\n' + + # Escape newline characters + def escape(s): + return s.replace("\n", "\\n") + + for name in ["author", "author_email", "description", "maintainer", + "maintainer_email", "name", "url", "version"]: + data = getattr(metadata, name, "") + if data: + info = info + ("\n %s: %s" % \ + (name.capitalize(), escape(data))) + lines.append("%s=%s" % (name, escape(data))) + + # The [setup] section contains entries controlling + # the installer runtime. + lines.append("\n[Setup]") + if self.install_script: + lines.append("install_script=%s" % self.install_script) + lines.append("info=%s" % escape(info)) + lines.append("target_compile=%d" % (not self.no_target_compile)) + lines.append("target_optimize=%d" % (not self.no_target_optimize)) + if self.target_version: + lines.append("target_version=%s" % self.target_version) + if self.user_access_control: + lines.append("user_access_control=%s" % self.user_access_control) + + title = self.title or self.distribution.get_fullname() + lines.append("title=%s" % escape(title)) + import time + import packaging + build_info = "Built %s with packaging-%s" % \ + (time.ctime(time.time()), packaging.__version__) + lines.append("build_info=%s" % build_info) + return "\n".join(lines) + + def create_exe(self, arcname, fullname, bitmap=None): + import struct + + self.mkpath(self.dist_dir) + + cfgdata = self.get_inidata() + + installer_name = self.get_installer_filename(fullname) + logger.info("creating %s", installer_name) + + if bitmap: + with open(bitmap, "rb") as fp: + bitmapdata = fp.read() + bitmaplen = len(bitmapdata) + else: + bitmaplen = 0 + + with open(installer_name, "wb") as file: + file.write(self.get_exe_bytes()) + if bitmap: + file.write(bitmapdata) + + # Convert cfgdata from unicode to ascii, mbcs encoded + if isinstance(cfgdata, str): + cfgdata = cfgdata.encode("mbcs") + + # Append the pre-install script + cfgdata = cfgdata + "\0" + if self.pre_install_script: + with open(self.pre_install_script) as fp: + script_data = fp.read() + cfgdata = cfgdata + script_data + "\n\0" + else: + # empty pre-install script + cfgdata = cfgdata + "\0" + file.write(cfgdata) + + # The 'magic number' 0x1234567B is used to make sure that the + # binary layout of 'cfgdata' is what the wininst.exe binary + # expects. If the layout changes, increment that number, make + # the corresponding changes to the wininst.exe sources, and + # recompile them. + header = struct.pack("<iii", + 0x1234567B, # tag + len(cfgdata), # length + bitmaplen, # number of bytes in bitmap + ) + file.write(header) + with open(arcname, "rb") as fp: + file.write(fp.read()) + + def get_installer_filename(self, fullname): + # Factored out to allow overriding in subclasses + if self.target_version: + # if we create an installer for a specific python version, + # it's better to include this in the name + installer_name = os.path.join(self.dist_dir, + "%s.%s-py%s.exe" % + (fullname, self.plat_name, self.target_version)) + else: + installer_name = os.path.join(self.dist_dir, + "%s.%s.exe" % (fullname, self.plat_name)) + return installer_name + + def get_exe_bytes(self): + from packaging.compiler.msvccompiler import get_build_version + # If a target-version other than the current version has been + # specified, then using the MSVC version from *this* build is no good. + # Without actually finding and executing the target version and parsing + # its sys.version, we just hard-code our knowledge of old versions. + # NOTE: Possible alternative is to allow "--target-version" to + # specify a Python executable rather than a simple version string. + # We can then execute this program to obtain any info we need, such + # as the real sys.version string for the build. + cur_version = get_python_version() + if self.target_version and self.target_version != cur_version: + # If the target version is *later* than us, then we assume they + # use what we use + # string compares seem wrong, but are what sysconfig.py itself uses + if self.target_version > cur_version: + bv = get_build_version() + else: + if self.target_version < "2.4": + bv = 6.0 + else: + bv = 7.1 + else: + # for current version - use authoritative check. + bv = get_build_version() + + # wininst-x.y.exe is in the same directory as this file + directory = os.path.dirname(__file__) + # we must use a wininst-x.y.exe built with the same C compiler + # used for python. XXX What about mingw, borland, and so on? + + # if plat_name starts with "win" but is not "win32" + # we want to strip "win" and leave the rest (e.g. -amd64) + # for all other cases, we don't want any suffix + if self.plat_name != 'win32' and self.plat_name[:3] == 'win': + sfix = self.plat_name[3:] + else: + sfix = '' + + filename = os.path.join(directory, "wininst-%.1f%s.exe" % (bv, sfix)) + with open(filename, "rb") as fp: + return fp.read() diff --git a/Lib/packaging/command/build.py b/Lib/packaging/command/build.py new file mode 100644 index 0000000000..6580fd1286 --- /dev/null +++ b/Lib/packaging/command/build.py @@ -0,0 +1,151 @@ +"""Main build command, which calls the other build_* commands.""" + +import sys +import os + +from packaging.util import get_platform +from packaging.command.cmd import Command +from packaging.errors import PackagingOptionError +from packaging.compiler import show_compilers + + +class build(Command): + + description = "build everything needed to install" + + user_options = [ + ('build-base=', 'b', + "base directory for build library"), + ('build-purelib=', None, + "build directory for platform-neutral distributions"), + ('build-platlib=', None, + "build directory for platform-specific distributions"), + ('build-lib=', None, + "build directory for all distribution (defaults to either " + + "build-purelib or build-platlib"), + ('build-scripts=', None, + "build directory for scripts"), + ('build-temp=', 't', + "temporary build directory"), + ('plat-name=', 'p', + "platform name to build for, if supported " + "(default: %s)" % get_platform()), + ('compiler=', 'c', + "specify the compiler type"), + ('debug', 'g', + "compile extensions and libraries with debugging information"), + ('force', 'f', + "forcibly build everything (ignore file timestamps)"), + ('executable=', 'e', + "specify final destination interpreter path (build.py)"), + ('use-2to3', None, + "use 2to3 to make source python 3.x compatible"), + ('convert-2to3-doctests', None, + "use 2to3 to convert doctests in seperate text files"), + ('use-2to3-fixers', None, + "list additional fixers opted for during 2to3 conversion"), + ] + + boolean_options = ['debug', 'force'] + + help_options = [ + ('help-compiler', None, + "list available compilers", show_compilers), + ] + + def initialize_options(self): + self.build_base = 'build' + # these are decided only after 'build_base' has its final value + # (unless overridden by the user or client) + self.build_purelib = None + self.build_platlib = None + self.build_lib = None + self.build_temp = None + self.build_scripts = None + self.compiler = None + self.plat_name = None + self.debug = None + self.force = False + self.executable = None + self.use_2to3 = False + self.convert_2to3_doctests = None + self.use_2to3_fixers = None + + def finalize_options(self): + if self.plat_name is None: + self.plat_name = get_platform() + else: + # plat-name only supported for windows (other platforms are + # supported via ./configure flags, if at all). Avoid misleading + # other platforms. + if os.name != 'nt': + raise PackagingOptionError( + "--plat-name only supported on Windows (try " + "using './configure --help' on your platform)") + + plat_specifier = ".%s-%s" % (self.plat_name, sys.version[0:3]) + + # Make it so Python 2.x and Python 2.x with --with-pydebug don't + # share the same build directories. Doing so confuses the build + # process for C modules + if hasattr(sys, 'gettotalrefcount'): + plat_specifier += '-pydebug' + + # 'build_purelib' and 'build_platlib' just default to 'lib' and + # 'lib.<plat>' under the base build directory. We only use one of + # them for a given distribution, though -- + if self.build_purelib is None: + self.build_purelib = os.path.join(self.build_base, 'lib') + if self.build_platlib is None: + self.build_platlib = os.path.join(self.build_base, + 'lib' + plat_specifier) + + # 'build_lib' is the actual directory that we will use for this + # particular module distribution -- if user didn't supply it, pick + # one of 'build_purelib' or 'build_platlib'. + if self.build_lib is None: + if self.distribution.ext_modules: + self.build_lib = self.build_platlib + else: + self.build_lib = self.build_purelib + + # 'build_temp' -- temporary directory for compiler turds, + # "build/temp.<plat>" + if self.build_temp is None: + self.build_temp = os.path.join(self.build_base, + 'temp' + plat_specifier) + if self.build_scripts is None: + self.build_scripts = os.path.join(self.build_base, + 'scripts-' + sys.version[0:3]) + + if self.executable is None: + self.executable = os.path.normpath(sys.executable) + + def run(self): + # Run all relevant sub-commands. This will be some subset of: + # - build_py - pure Python modules + # - build_clib - standalone C libraries + # - build_ext - Python extension modules + # - build_scripts - Python scripts + for cmd_name in self.get_sub_commands(): + self.run_command(cmd_name) + + # -- Predicates for the sub-command list --------------------------- + + def has_pure_modules(self): + return self.distribution.has_pure_modules() + + def has_c_libraries(self): + return self.distribution.has_c_libraries() + + def has_ext_modules(self): + return self.distribution.has_ext_modules() + + def has_scripts(self): + return self.distribution.has_scripts() + + sub_commands = [('build_py', has_pure_modules), + ('build_clib', has_c_libraries), + ('build_ext', has_ext_modules), + ('build_scripts', has_scripts), + ] diff --git a/Lib/packaging/command/build_clib.py b/Lib/packaging/command/build_clib.py new file mode 100644 index 0000000000..4a249964c5 --- /dev/null +++ b/Lib/packaging/command/build_clib.py @@ -0,0 +1,198 @@ +"""Build C/C++ libraries. + +This command is useful to build libraries that are included in the +distribution and needed by extension modules. +""" + +# XXX this module has *lots* of code ripped-off quite transparently from +# build_ext.py -- not surprisingly really, as the work required to build +# a static library from a collection of C source files is not really all +# that different from what's required to build a shared object file from +# a collection of C source files. Nevertheless, I haven't done the +# necessary refactoring to account for the overlap in code between the +# two modules, mainly because a number of subtle details changed in the +# cut 'n paste. Sigh. + +import os +from packaging.command.cmd import Command +from packaging.errors import PackagingSetupError +from packaging.compiler import customize_compiler +from packaging import logger + + +def show_compilers(): + from packaging.compiler import show_compilers + show_compilers() + + +class build_clib(Command): + + description = "build C/C++ libraries used by extension modules" + + user_options = [ + ('build-clib=', 'b', + "directory to build C/C++ libraries to"), + ('build-temp=', 't', + "directory to put temporary build by-products"), + ('debug', 'g', + "compile with debugging information"), + ('force', 'f', + "forcibly build everything (ignore file timestamps)"), + ('compiler=', 'c', + "specify the compiler type"), + ] + + boolean_options = ['debug', 'force'] + + help_options = [ + ('help-compiler', None, + "list available compilers", show_compilers), + ] + + def initialize_options(self): + self.build_clib = None + self.build_temp = None + + # List of libraries to build + self.libraries = None + + # Compilation options for all libraries + self.include_dirs = None + self.define = None + self.undef = None + self.debug = None + self.force = False + self.compiler = None + + + def finalize_options(self): + # This might be confusing: both build-clib and build-temp default + # to build-temp as defined by the "build" command. This is because + # I think that C libraries are really just temporary build + # by-products, at least from the point of view of building Python + # extensions -- but I want to keep my options open. + self.set_undefined_options('build', + ('build_temp', 'build_clib'), + ('build_temp', 'build_temp'), + 'compiler', 'debug', 'force') + + self.libraries = self.distribution.libraries + if self.libraries: + self.check_library_list(self.libraries) + + if self.include_dirs is None: + self.include_dirs = self.distribution.include_dirs or [] + if isinstance(self.include_dirs, str): + self.include_dirs = self.include_dirs.split(os.pathsep) + + # XXX same as for build_ext -- what about 'self.define' and + # 'self.undef' ? + + def run(self): + if not self.libraries: + return + + # Yech -- this is cut 'n pasted from build_ext.py! + from packaging.compiler import new_compiler + self.compiler = new_compiler(compiler=self.compiler, + dry_run=self.dry_run, + force=self.force) + customize_compiler(self.compiler) + + if self.include_dirs is not None: + self.compiler.set_include_dirs(self.include_dirs) + if self.define is not None: + # 'define' option is a list of (name,value) tuples + for name, value in self.define: + self.compiler.define_macro(name, value) + if self.undef is not None: + for macro in self.undef: + self.compiler.undefine_macro(macro) + + self.build_libraries(self.libraries) + + + def check_library_list(self, libraries): + """Ensure that the list of libraries is valid. + + `library` is presumably provided as a command option 'libraries'. + This method checks that it is a list of 2-tuples, where the tuples + are (library_name, build_info_dict). + + Raise PackagingSetupError if the structure is invalid anywhere; + just returns otherwise. + """ + if not isinstance(libraries, list): + raise PackagingSetupError("'libraries' option must be a list of tuples") + + for lib in libraries: + if not isinstance(lib, tuple) and len(lib) != 2: + raise PackagingSetupError("each element of 'libraries' must a 2-tuple") + + name, build_info = lib + + if not isinstance(name, str): + raise PackagingSetupError("first element of each tuple in 'libraries' " + \ + "must be a string (the library name)") + if '/' in name or (os.sep != '/' and os.sep in name): + raise PackagingSetupError(("bad library name '%s': " + + "may not contain directory separators") % \ + lib[0]) + + if not isinstance(build_info, dict): + raise PackagingSetupError("second element of each tuple in 'libraries' " + \ + "must be a dictionary (build info)") + + def get_library_names(self): + # Assume the library list is valid -- 'check_library_list()' is + # called from 'finalize_options()', so it should be! + if not self.libraries: + return None + + lib_names = [] + for lib_name, build_info in self.libraries: + lib_names.append(lib_name) + return lib_names + + + def get_source_files(self): + self.check_library_list(self.libraries) + filenames = [] + for lib_name, build_info in self.libraries: + sources = build_info.get('sources') + if sources is None or not isinstance(sources, (list, tuple)): + raise PackagingSetupError(("in 'libraries' option (library '%s'), " + "'sources' must be present and must be " + "a list of source filenames") % lib_name) + + filenames.extend(sources) + return filenames + + def build_libraries(self, libraries): + for lib_name, build_info in libraries: + sources = build_info.get('sources') + if sources is None or not isinstance(sources, (list, tuple)): + raise PackagingSetupError(("in 'libraries' option (library '%s'), " + + "'sources' must be present and must be " + + "a list of source filenames") % lib_name) + sources = list(sources) + + logger.info("building '%s' library", lib_name) + + # First, compile the source code to object files in the library + # directory. (This should probably change to putting object + # files in a temporary build directory.) + macros = build_info.get('macros') + include_dirs = build_info.get('include_dirs') + objects = self.compiler.compile(sources, + output_dir=self.build_temp, + macros=macros, + include_dirs=include_dirs, + debug=self.debug) + + # Now "link" the object files together into a static library. + # (On Unix at least, this isn't really linking -- it just + # builds an archive. Whatever.) + self.compiler.create_static_lib(objects, lib_name, + output_dir=self.build_clib, + debug=self.debug) diff --git a/Lib/packaging/command/build_ext.py b/Lib/packaging/command/build_ext.py new file mode 100644 index 0000000000..fa8d11f39c --- /dev/null +++ b/Lib/packaging/command/build_ext.py @@ -0,0 +1,657 @@ +"""Build extension modules.""" + +# FIXME Is this module limited to C extensions or do C++ extensions work too? +# The docstring of this module said that C++ was not supported, but other +# comments contradict that. + +import os +import re +import sys +import logging +import sysconfig + +from packaging.util import get_platform +from packaging.command.cmd import Command +from packaging.errors import (CCompilerError, CompileError, PackagingError, + PackagingPlatformError, PackagingSetupError) +from packaging.compiler import customize_compiler, show_compilers +from packaging.util import newer_group +from packaging.compiler.extension import Extension +from packaging import logger + +import site +HAS_USER_SITE = True + +if os.name == 'nt': + from packaging.compiler.msvccompiler import get_build_version + MSVC_VERSION = int(get_build_version()) + +# An extension name is just a dot-separated list of Python NAMEs (ie. +# the same as a fully-qualified module name). +extension_name_re = re.compile \ + (r'^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$') + + +class build_ext(Command): + + description = "build C/C++ extension modules (compile/link to build directory)" + + # XXX thoughts on how to deal with complex command-line options like + # these, i.e. how to make it so fancy_getopt can suck them off the + # command line and turn them into the appropriate + # lists of tuples of what-have-you. + # - each command needs a callback to process its command-line options + # - Command.__init__() needs access to its share of the whole + # command line (must ultimately come from + # Distribution.parse_command_line()) + # - it then calls the current command class' option-parsing + # callback to deal with weird options like -D, which have to + # parse the option text and churn out some custom data + # structure + # - that data structure (in this case, a list of 2-tuples) + # will then be present in the command object by the time + # we get to finalize_options() (i.e. the constructor + # takes care of both command-line and client options + # in between initialize_options() and finalize_options()) + + sep_by = " (separated by '%s')" % os.pathsep + user_options = [ + ('build-lib=', 'b', + "directory for compiled extension modules"), + ('build-temp=', 't', + "directory for temporary files (build by-products)"), + ('plat-name=', 'p', + "platform name to cross-compile for, if supported " + "(default: %s)" % get_platform()), + ('inplace', 'i', + "ignore build-lib and put compiled extensions into the source " + + "directory alongside your pure Python modules"), + ('include-dirs=', 'I', + "list of directories to search for header files" + sep_by), + ('define=', 'D', + "C preprocessor macros to define"), + ('undef=', 'U', + "C preprocessor macros to undefine"), + ('libraries=', 'l', + "external C libraries to link with"), + ('library-dirs=', 'L', + "directories to search for external C libraries" + sep_by), + ('rpath=', 'R', + "directories to search for shared C libraries at runtime"), + ('link-objects=', 'O', + "extra explicit link objects to include in the link"), + ('debug', 'g', + "compile/link with debugging information"), + ('force', 'f', + "forcibly build everything (ignore file timestamps)"), + ('compiler=', 'c', + "specify the compiler type"), + ('swig-opts=', None, + "list of SWIG command-line options"), + ('swig=', None, + "path to the SWIG executable"), + ] + + boolean_options = ['inplace', 'debug', 'force'] + + if HAS_USER_SITE: + user_options.append(('user', None, + "add user include, library and rpath")) + boolean_options.append('user') + + help_options = [ + ('help-compiler', None, + "list available compilers", show_compilers), + ] + + def initialize_options(self): + self.extensions = None + self.build_lib = None + self.plat_name = None + self.build_temp = None + self.inplace = False + self.package = None + + self.include_dirs = None + self.define = None + self.undef = None + self.libraries = None + self.library_dirs = None + self.rpath = None + self.link_objects = None + self.debug = None + self.force = None + self.compiler = None + self.swig = None + self.swig_opts = None + if HAS_USER_SITE: + self.user = None + + def finalize_options(self): + self.set_undefined_options('build', + 'build_lib', 'build_temp', 'compiler', + 'debug', 'force', 'plat_name') + + if self.package is None: + self.package = self.distribution.ext_package + + # Ensure that the list of extensions is valid, i.e. it is a list of + # Extension objects. + self.extensions = self.distribution.ext_modules + if self.extensions: + if not isinstance(self.extensions, (list, tuple)): + type_name = (self.extensions is None and 'None' + or type(self.extensions).__name__) + raise PackagingSetupError( + "'ext_modules' must be a sequence of Extension instances," + " not %s" % (type_name,)) + for i, ext in enumerate(self.extensions): + if isinstance(ext, Extension): + continue # OK! (assume type-checking done + # by Extension constructor) + type_name = (ext is None and 'None' or type(ext).__name__) + raise PackagingSetupError( + "'ext_modules' item %d must be an Extension instance," + " not %s" % (i, type_name)) + + # Make sure Python's include directories (for Python.h, pyconfig.h, + # etc.) are in the include search path. + py_include = sysconfig.get_path('include') + plat_py_include = sysconfig.get_path('platinclude') + if self.include_dirs is None: + self.include_dirs = self.distribution.include_dirs or [] + if isinstance(self.include_dirs, str): + self.include_dirs = self.include_dirs.split(os.pathsep) + + # Put the Python "system" include dir at the end, so that + # any local include dirs take precedence. + self.include_dirs.append(py_include) + if plat_py_include != py_include: + self.include_dirs.append(plat_py_include) + + if isinstance(self.libraries, str): + self.libraries = [self.libraries] + + # Life is easier if we're not forever checking for None, so + # simplify these options to empty lists if unset + if self.libraries is None: + self.libraries = [] + if self.library_dirs is None: + self.library_dirs = [] + elif isinstance(self.library_dirs, str): + self.library_dirs = self.library_dirs.split(os.pathsep) + + if self.rpath is None: + self.rpath = [] + elif isinstance(self.rpath, str): + self.rpath = self.rpath.split(os.pathsep) + + # for extensions under windows use different directories + # for Release and Debug builds. + # also Python's library directory must be appended to library_dirs + if os.name == 'nt': + # the 'libs' directory is for binary installs - we assume that + # must be the *native* platform. But we don't really support + # cross-compiling via a binary install anyway, so we let it go. + self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) + if self.debug: + self.build_temp = os.path.join(self.build_temp, "Debug") + else: + self.build_temp = os.path.join(self.build_temp, "Release") + + # Append the source distribution include and library directories, + # this allows distutils on windows to work in the source tree + self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) + if MSVC_VERSION == 9: + # Use the .lib files for the correct architecture + if self.plat_name == 'win32': + suffix = '' + else: + # win-amd64 or win-ia64 + suffix = self.plat_name[4:] + new_lib = os.path.join(sys.exec_prefix, 'PCbuild') + if suffix: + new_lib = os.path.join(new_lib, suffix) + self.library_dirs.append(new_lib) + + elif MSVC_VERSION == 8: + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VS8.0')) + elif MSVC_VERSION == 7: + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VS7.1')) + else: + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VC6')) + + # OS/2 (EMX) doesn't support Debug vs Release builds, but has the + # import libraries in its "Config" subdirectory + if os.name == 'os2': + self.library_dirs.append(os.path.join(sys.exec_prefix, 'Config')) + + # for extensions under Cygwin and AtheOS Python's library directory must be + # appended to library_dirs + if sys.platform[:6] == 'cygwin' or sys.platform[:6] == 'atheos': + if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + # building third party extensions + self.library_dirs.append(os.path.join(sys.prefix, "lib", + "python" + sysconfig.get_python_version(), + "config")) + else: + # building python standard extensions + self.library_dirs.append(os.curdir) + + # for extensions under Linux or Solaris with a shared Python library, + # Python's library directory must be appended to library_dirs + sysconfig.get_config_var('Py_ENABLE_SHARED') + if (sys.platform.startswith(('linux', 'gnu', 'sunos')) + and sysconfig.get_config_var('Py_ENABLE_SHARED')): + if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + # building third party extensions + self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) + else: + # building python standard extensions + self.library_dirs.append(os.curdir) + + # The argument parsing will result in self.define being a string, but + # it has to be a list of 2-tuples. All the preprocessor symbols + # specified by the 'define' option will be set to '1'. Multiple + # symbols can be separated with commas. + + if self.define: + defines = self.define.split(',') + self.define = [(symbol, '1') for symbol in defines] + + # The option for macros to undefine is also a string from the + # option parsing, but has to be a list. Multiple symbols can also + # be separated with commas here. + if self.undef: + self.undef = self.undef.split(',') + + if self.swig_opts is None: + self.swig_opts = [] + else: + self.swig_opts = self.swig_opts.split(' ') + + # Finally add the user include and library directories if requested + if HAS_USER_SITE and self.user: + user_include = os.path.join(site.USER_BASE, "include") + user_lib = os.path.join(site.USER_BASE, "lib") + if os.path.isdir(user_include): + self.include_dirs.append(user_include) + if os.path.isdir(user_lib): + self.library_dirs.append(user_lib) + self.rpath.append(user_lib) + + def run(self): + from packaging.compiler import new_compiler + + if not self.extensions: + return + + # If we were asked to build any C/C++ libraries, make sure that the + # directory where we put them is in the library search path for + # linking extensions. + if self.distribution.has_c_libraries(): + build_clib = self.get_finalized_command('build_clib') + self.libraries.extend(build_clib.get_library_names() or []) + self.library_dirs.append(build_clib.build_clib) + + # Temporary kludge until we remove the verbose arguments and use + # logging everywhere + verbose = logger.getEffectiveLevel() >= logging.DEBUG + + # Setup the CCompiler object that we'll use to do all the + # compiling and linking + self.compiler_obj = new_compiler(compiler=self.compiler, + verbose=verbose, + dry_run=self.dry_run, + force=self.force) + + customize_compiler(self.compiler_obj) + # If we are cross-compiling, init the compiler now (if we are not + # cross-compiling, init would not hurt, but people may rely on + # late initialization of compiler even if they shouldn't...) + if os.name == 'nt' and self.plat_name != get_platform(): + self.compiler_obj.initialize(self.plat_name) + + # And make sure that any compile/link-related options (which might + # come from the command line or from the setup script) are set in + # that CCompiler object -- that way, they automatically apply to + # all compiling and linking done here. + if self.include_dirs is not None: + self.compiler_obj.set_include_dirs(self.include_dirs) + if self.define is not None: + # 'define' option is a list of (name,value) tuples + for name, value in self.define: + self.compiler_obj.define_macro(name, value) + if self.undef is not None: + for macro in self.undef: + self.compiler_obj.undefine_macro(macro) + if self.libraries is not None: + self.compiler_obj.set_libraries(self.libraries) + if self.library_dirs is not None: + self.compiler_obj.set_library_dirs(self.library_dirs) + if self.rpath is not None: + self.compiler_obj.set_runtime_library_dirs(self.rpath) + if self.link_objects is not None: + self.compiler_obj.set_link_objects(self.link_objects) + + # Now actually compile and link everything. + self.build_extensions() + + def get_source_files(self): + filenames = [] + + # Wouldn't it be neat if we knew the names of header files too... + for ext in self.extensions: + filenames.extend(ext.sources) + + return filenames + + def get_outputs(self): + # And build the list of output (built) filenames. Note that this + # ignores the 'inplace' flag, and assumes everything goes in the + # "build" tree. + outputs = [] + for ext in self.extensions: + outputs.append(self.get_ext_fullpath(ext.name)) + return outputs + + def build_extensions(self): + for ext in self.extensions: + try: + self.build_extension(ext) + except (CCompilerError, PackagingError, CompileError) as e: + if not ext.optional: + raise + logger.warning('%s: building extension %r failed: %s', + self.get_command_name(), ext.name, e) + + def build_extension(self, ext): + sources = ext.sources + if sources is None or not isinstance(sources, (list, tuple)): + raise PackagingSetupError(("in 'ext_modules' option (extension '%s'), " + + "'sources' must be present and must be " + + "a list of source filenames") % ext.name) + sources = list(sources) + + ext_path = self.get_ext_fullpath(ext.name) + depends = sources + ext.depends + if not (self.force or newer_group(depends, ext_path, 'newer')): + logger.debug("skipping '%s' extension (up-to-date)", ext.name) + return + else: + logger.info("building '%s' extension", ext.name) + + # First, scan the sources for SWIG definition files (.i), run + # SWIG on 'em to create .c files, and modify the sources list + # accordingly. + sources = self.swig_sources(sources, ext) + + # Next, compile the source code to object files. + + # XXX not honouring 'define_macros' or 'undef_macros' -- the + # CCompiler API needs to change to accommodate this, and I + # want to do one thing at a time! + + # Two possible sources for extra compiler arguments: + # - 'extra_compile_args' in Extension object + # - CFLAGS environment variable (not particularly + # elegant, but people seem to expect it and I + # guess it's useful) + # The environment variable should take precedence, and + # any sensible compiler will give precedence to later + # command-line args. Hence we combine them in order: + extra_args = ext.extra_compile_args or [] + + macros = ext.define_macros[:] + for undef in ext.undef_macros: + macros.append((undef,)) + + objects = self.compiler_obj.compile(sources, + output_dir=self.build_temp, + macros=macros, + include_dirs=ext.include_dirs, + debug=self.debug, + extra_postargs=extra_args, + depends=ext.depends) + + # XXX -- this is a Vile HACK! + # + # The setup.py script for Python on Unix needs to be able to + # get this list so it can perform all the clean up needed to + # avoid keeping object files around when cleaning out a failed + # build of an extension module. Since Packaging does not + # track dependencies, we have to get rid of intermediates to + # ensure all the intermediates will be properly re-built. + # + self._built_objects = objects[:] + + # Now link the object files together into a "shared object" -- + # of course, first we have to figure out all the other things + # that go into the mix. + if ext.extra_objects: + objects.extend(ext.extra_objects) + extra_args = ext.extra_link_args or [] + + # Detect target language, if not provided + language = ext.language or self.compiler_obj.detect_language(sources) + + self.compiler_obj.link_shared_object( + objects, ext_path, + libraries=self.get_libraries(ext), + library_dirs=ext.library_dirs, + runtime_library_dirs=ext.runtime_library_dirs, + extra_postargs=extra_args, + export_symbols=self.get_export_symbols(ext), + debug=self.debug, + build_temp=self.build_temp, + target_lang=language) + + + def swig_sources(self, sources, extension): + """Walk the list of source files in 'sources', looking for SWIG + interface (.i) files. Run SWIG on all that are found, and + return a modified 'sources' list with SWIG source files replaced + by the generated C (or C++) files. + """ + new_sources = [] + swig_sources = [] + swig_targets = {} + + # XXX this drops generated C/C++ files into the source tree, which + # is fine for developers who want to distribute the generated + # source -- but there should be an option to put SWIG output in + # the temp dir. + + if ('-c++' in self.swig_opts or '-c++' in extension.swig_opts): + target_ext = '.cpp' + else: + target_ext = '.c' + + for source in sources: + base, ext = os.path.splitext(source) + if ext == ".i": # SWIG interface file + new_sources.append(base + '_wrap' + target_ext) + swig_sources.append(source) + swig_targets[source] = new_sources[-1] + else: + new_sources.append(source) + + if not swig_sources: + return new_sources + + swig = self.swig or self.find_swig() + swig_cmd = [swig, "-python"] + swig_cmd.extend(self.swig_opts) + + # Do not override commandline arguments + if not self.swig_opts: + for o in extension.swig_opts: + swig_cmd.append(o) + + for source in swig_sources: + target = swig_targets[source] + logger.info("swigging %s to %s", source, target) + self.spawn(swig_cmd + ["-o", target, source]) + + return new_sources + + def find_swig(self): + """Return the name of the SWIG executable. On Unix, this is + just "swig" -- it should be in the PATH. Tries a bit harder on + Windows. + """ + + if os.name == "posix": + return "swig" + elif os.name == "nt": + + # Look for SWIG in its standard installation directory on + # Windows (or so I presume!). If we find it there, great; + # if not, act like Unix and assume it's in the PATH. + for vers in ("1.3", "1.2", "1.1"): + fn = os.path.join("c:\\swig%s" % vers, "swig.exe") + if os.path.isfile(fn): + return fn + else: + return "swig.exe" + + elif os.name == "os2": + # assume swig available in the PATH. + return "swig.exe" + + else: + raise PackagingPlatformError(("I don't know how to find (much less run) SWIG " + "on platform '%s'") % os.name) + + # -- Name generators ----------------------------------------------- + # (extension names, filenames, whatever) + def get_ext_fullpath(self, ext_name): + """Returns the path of the filename for a given extension. + + The file is located in `build_lib` or directly in the package + (inplace option). + """ + fullname = self.get_ext_fullname(ext_name) + modpath = fullname.split('.') + filename = self.get_ext_filename(modpath[-1]) + + if not self.inplace: + # no further work needed + # returning : + # build_dir/package/path/filename + filename = os.path.join(*modpath[:-1]+[filename]) + return os.path.join(self.build_lib, filename) + + # the inplace option requires to find the package directory + # using the build_py command for that + package = '.'.join(modpath[0:-1]) + build_py = self.get_finalized_command('build_py') + package_dir = os.path.abspath(build_py.get_package_dir(package)) + + # returning + # package_dir/filename + return os.path.join(package_dir, filename) + + def get_ext_fullname(self, ext_name): + """Returns the fullname of a given extension name. + + Adds the `package.` prefix""" + if self.package is None: + return ext_name + else: + return self.package + '.' + ext_name + + def get_ext_filename(self, ext_name): + r"""Convert the name of an extension (eg. "foo.bar") into the name + of the file from which it will be loaded (eg. "foo/bar.so", or + "foo\bar.pyd"). + """ + ext_path = ext_name.split('.') + # OS/2 has an 8 character module (extension) limit :-( + if os.name == "os2": + ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8] + # extensions in debug_mode are named 'module_d.pyd' under windows + so_ext = sysconfig.get_config_var('SO') + if os.name == 'nt' and self.debug: + return os.path.join(*ext_path) + '_d' + so_ext + return os.path.join(*ext_path) + so_ext + + def get_export_symbols(self, ext): + """Return the list of symbols that a shared extension has to + export. This either uses 'ext.export_symbols' or, if it's not + provided, "init" + module_name. Only relevant on Windows, where + the .pyd file (DLL) must export the module "init" function. + """ + initfunc_name = "PyInit_" + ext.name.split('.')[-1] + if initfunc_name not in ext.export_symbols: + ext.export_symbols.append(initfunc_name) + return ext.export_symbols + + def get_libraries(self, ext): + """Return the list of libraries to link against when building a + shared extension. On most platforms, this is just 'ext.libraries'; + on Windows and OS/2, we add the Python library (eg. python20.dll). + """ + # The python library is always needed on Windows. For MSVC, this + # is redundant, since the library is mentioned in a pragma in + # pyconfig.h that MSVC groks. The other Windows compilers all seem + # to need it mentioned explicitly, though, so that's what we do. + # Append '_d' to the python import library on debug builds. + if sys.platform == "win32": + from packaging.compiler.msvccompiler import MSVCCompiler + if not isinstance(self.compiler_obj, MSVCCompiler): + template = "python%d%d" + if self.debug: + template = template + '_d' + pythonlib = template % sys.version_info[:2] + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] + else: + return ext.libraries + elif sys.platform == "os2emx": + # EMX/GCC requires the python library explicitly, and I + # believe VACPP does as well (though not confirmed) - AIM Apr01 + template = "python%d%d" + # debug versions of the main DLL aren't supported, at least + # not at this time - AIM Apr01 + #if self.debug: + # template = template + '_d' + pythonlib = template % sys.version_info[:2] + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] + elif sys.platform[:6] == "cygwin": + template = "python%d.%d" + pythonlib = template % sys.version_info[:2] + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] + elif sys.platform[:6] == "atheos": + template = "python%d.%d" + pythonlib = template % sys.version_info[:2] + # Get SHLIBS from Makefile + extra = [] + for lib in sysconfig.get_config_var('SHLIBS').split(): + if lib.startswith('-l'): + extra.append(lib[2:]) + else: + extra.append(lib) + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib, "m"] + extra + + elif sys.platform == 'darwin': + # Don't use the default code below + return ext.libraries + + else: + if sysconfig.get_config_var('Py_ENABLE_SHARED'): + template = 'python%d.%d' + sys.abiflags + pythonlib = template % sys.version_info[:2] + return ext.libraries + [pythonlib] + else: + return ext.libraries diff --git a/Lib/packaging/command/build_py.py b/Lib/packaging/command/build_py.py new file mode 100644 index 0000000000..7baa6e46a0 --- /dev/null +++ b/Lib/packaging/command/build_py.py @@ -0,0 +1,410 @@ +"""Build pure Python modules (just copy to build directory).""" + +import os +import sys +from glob import glob + +from packaging import logger +from packaging.command.cmd import Command +from packaging.errors import PackagingOptionError, PackagingFileError +from packaging.util import convert_path +from packaging.compat import Mixin2to3 + +# marking public APIs +__all__ = ['build_py'] + +class build_py(Command, Mixin2to3): + + description = "build pure Python modules (copy to build directory)" + + user_options = [ + ('build-lib=', 'd', "directory to build (copy) to"), + ('compile', 'c', "compile .py to .pyc"), + ('no-compile', None, "don't compile .py files [default]"), + ('optimize=', 'O', + "also compile with optimization: -O1 for \"python -O\", " + "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), + ('force', 'f', "forcibly build everything (ignore file timestamps)"), + ('use-2to3', None, + "use 2to3 to make source python 3.x compatible"), + ('convert-2to3-doctests', None, + "use 2to3 to convert doctests in seperate text files"), + ('use-2to3-fixers', None, + "list additional fixers opted for during 2to3 conversion"), + ] + + boolean_options = ['compile', 'force'] + negative_opt = {'no-compile' : 'compile'} + + def initialize_options(self): + self.build_lib = None + self.py_modules = None + self.package = None + self.package_data = None + self.package_dir = None + self.compile = False + self.optimize = 0 + self.force = None + self._updated_files = [] + self._doctests_2to3 = [] + self.use_2to3 = False + self.convert_2to3_doctests = None + self.use_2to3_fixers = None + + def finalize_options(self): + self.set_undefined_options('build', + 'use_2to3', 'use_2to3_fixers', + 'convert_2to3_doctests', 'build_lib', + 'force') + + # Get the distribution options that are aliases for build_py + # options -- list of packages and list of modules. + self.packages = self.distribution.packages + self.py_modules = self.distribution.py_modules + self.package_data = self.distribution.package_data + self.package_dir = None + if self.distribution.package_dir is not None: + self.package_dir = convert_path(self.distribution.package_dir) + self.data_files = self.get_data_files() + + # Ick, copied straight from install_lib.py (fancy_getopt needs a + # type system! Hell, *everything* needs a type system!!!) + if not isinstance(self.optimize, int): + try: + self.optimize = int(self.optimize) + assert 0 <= self.optimize <= 2 + except (ValueError, AssertionError): + raise PackagingOptionError("optimize must be 0, 1, or 2") + + def run(self): + # XXX copy_file by default preserves atime and mtime. IMHO this is + # the right thing to do, but perhaps it should be an option -- in + # particular, a site administrator might want installed files to + # reflect the time of installation rather than the last + # modification time before the installed release. + + # XXX copy_file by default preserves mode, which appears to be the + # wrong thing to do: if a file is read-only in the working + # directory, we want it to be installed read/write so that the next + # installation of the same module distribution can overwrite it + # without problems. (This might be a Unix-specific issue.) Thus + # we turn off 'preserve_mode' when copying to the build directory, + # since the build directory is supposed to be exactly what the + # installation will look like (ie. we preserve mode when + # installing). + + # Two options control which modules will be installed: 'packages' + # and 'py_modules'. The former lets us work with whole packages, not + # specifying individual modules at all; the latter is for + # specifying modules one-at-a-time. + + if self.py_modules: + self.build_modules() + if self.packages: + self.build_packages() + self.build_package_data() + + if self.use_2to3 and self._updated_files: + self.run_2to3(self._updated_files, self._doctests_2to3, + self.use_2to3_fixers) + + self.byte_compile(self.get_outputs(include_bytecode=False)) + + # -- Top-level worker functions ------------------------------------ + + def get_data_files(self): + """Generate list of '(package,src_dir,build_dir,filenames)' tuples. + + Helper function for `finalize_options()`. + """ + data = [] + if not self.packages: + return data + for package in self.packages: + # Locate package source directory + src_dir = self.get_package_dir(package) + + # Compute package build directory + build_dir = os.path.join(*([self.build_lib] + package.split('.'))) + + # Length of path to strip from found files + plen = 0 + if src_dir: + plen = len(src_dir)+1 + + # Strip directory from globbed filenames + filenames = [ + file[plen:] for file in self.find_data_files(package, src_dir) + ] + data.append((package, src_dir, build_dir, filenames)) + return data + + def find_data_files(self, package, src_dir): + """Return filenames for package's data files in 'src_dir'. + + Helper function for `get_data_files()`. + """ + globs = (self.package_data.get('', []) + + self.package_data.get(package, [])) + files = [] + for pattern in globs: + # Each pattern has to be converted to a platform-specific path + filelist = glob(os.path.join(src_dir, convert_path(pattern))) + # Files that match more than one pattern are only added once + files.extend(fn for fn in filelist if fn not in files) + return files + + def build_package_data(self): + """Copy data files into build directory. + + Helper function for `run()`. + """ + # FIXME add tests for this method + for package, src_dir, build_dir, filenames in self.data_files: + for filename in filenames: + target = os.path.join(build_dir, filename) + srcfile = os.path.join(src_dir, filename) + self.mkpath(os.path.dirname(target)) + outf, copied = self.copy_file(srcfile, + target, preserve_mode=False) + if copied and srcfile in self.distribution.convert_2to3.doctests: + self._doctests_2to3.append(outf) + + # XXX - this should be moved to the Distribution class as it is not + # only needed for build_py. It also has no dependencies on this class. + def get_package_dir(self, package): + """Return the directory, relative to the top of the source + distribution, where package 'package' should be found + (at least according to the 'package_dir' option, if any).""" + + path = package.split('.') + if self.package_dir is not None: + path.insert(0, self.package_dir) + + if len(path) > 0: + return os.path.join(*path) + + return '' + + def check_package(self, package, package_dir): + """Helper function for `find_package_modules()` and `find_modules()'. + """ + # Empty dir name means current directory, which we can probably + # assume exists. Also, os.path.exists and isdir don't know about + # my "empty string means current dir" convention, so we have to + # circumvent them. + if package_dir != "": + if not os.path.exists(package_dir): + raise PackagingFileError( + "package directory '%s' does not exist" % package_dir) + if not os.path.isdir(package_dir): + raise PackagingFileError( + "supposed package directory '%s' exists, " + "but is not a directory" % package_dir) + + # Require __init__.py for all but the "root package" + if package: + init_py = os.path.join(package_dir, "__init__.py") + if os.path.isfile(init_py): + return init_py + else: + logger.warning(("package init file '%s' not found " + + "(or not a regular file)"), init_py) + + # Either not in a package at all (__init__.py not expected), or + # __init__.py doesn't exist -- so don't return the filename. + return None + + def check_module(self, module, module_file): + if not os.path.isfile(module_file): + logger.warning("file %s (for module %s) not found", + module_file, module) + return False + else: + return True + + def find_package_modules(self, package, package_dir): + self.check_package(package, package_dir) + module_files = glob(os.path.join(package_dir, "*.py")) + modules = [] + if self.distribution.script_name is not None: + setup_script = os.path.abspath(self.distribution.script_name) + else: + setup_script = None + + for f in module_files: + abs_f = os.path.abspath(f) + if abs_f != setup_script: + module = os.path.splitext(os.path.basename(f))[0] + modules.append((package, module, f)) + else: + logger.debug("excluding %s", setup_script) + return modules + + def find_modules(self): + """Finds individually-specified Python modules, ie. those listed by + module name in 'self.py_modules'. Returns a list of tuples (package, + module_base, filename): 'package' is a tuple of the path through + package-space to the module; 'module_base' is the bare (no + packages, no dots) module name, and 'filename' is the path to the + ".py" file (relative to the distribution root) that implements the + module. + """ + # Map package names to tuples of useful info about the package: + # (package_dir, checked) + # package_dir - the directory where we'll find source files for + # this package + # checked - true if we have checked that the package directory + # is valid (exists, contains __init__.py, ... ?) + packages = {} + + # List of (package, module, filename) tuples to return + modules = [] + + # We treat modules-in-packages almost the same as toplevel modules, + # just the "package" for a toplevel is empty (either an empty + # string or empty list, depending on context). Differences: + # - don't check for __init__.py in directory for empty package + for module in self.py_modules: + path = module.split('.') + package = '.'.join(path[0:-1]) + module_base = path[-1] + + try: + package_dir, checked = packages[package] + except KeyError: + package_dir = self.get_package_dir(package) + checked = False + + if not checked: + init_py = self.check_package(package, package_dir) + packages[package] = (package_dir, 1) + if init_py: + modules.append((package, "__init__", init_py)) + + # XXX perhaps we should also check for just .pyc files + # (so greedy closed-source bastards can distribute Python + # modules too) + module_file = os.path.join(package_dir, module_base + ".py") + if not self.check_module(module, module_file): + continue + + modules.append((package, module_base, module_file)) + + return modules + + def find_all_modules(self): + """Compute the list of all modules that will be built, whether + they are specified one-module-at-a-time ('self.py_modules') or + by whole packages ('self.packages'). Return a list of tuples + (package, module, module_file), just like 'find_modules()' and + 'find_package_modules()' do.""" + modules = [] + if self.py_modules: + modules.extend(self.find_modules()) + if self.packages: + for package in self.packages: + package_dir = self.get_package_dir(package) + m = self.find_package_modules(package, package_dir) + modules.extend(m) + return modules + + def get_source_files(self): + sources = [module[-1] for module in self.find_all_modules()] + sources += [ + os.path.join(src_dir, filename) + for package, src_dir, build_dir, filenames in self.data_files + for filename in filenames] + return sources + + def get_module_outfile(self, build_dir, package, module): + outfile_path = [build_dir] + list(package) + [module + ".py"] + return os.path.join(*outfile_path) + + def get_outputs(self, include_bytecode=True): + modules = self.find_all_modules() + outputs = [] + for package, module, module_file in modules: + package = package.split('.') + filename = self.get_module_outfile(self.build_lib, package, module) + outputs.append(filename) + if include_bytecode: + if self.compile: + outputs.append(filename + "c") + if self.optimize > 0: + outputs.append(filename + "o") + + outputs += [ + os.path.join(build_dir, filename) + for package, src_dir, build_dir, filenames in self.data_files + for filename in filenames] + + return outputs + + def build_module(self, module, module_file, package): + if isinstance(package, str): + package = package.split('.') + elif not isinstance(package, (list, tuple)): + raise TypeError( + "'package' must be a string (dot-separated), list, or tuple") + + # Now put the module source file into the "build" area -- this is + # easy, we just copy it somewhere under self.build_lib (the build + # directory for Python source). + outfile = self.get_module_outfile(self.build_lib, package, module) + dir = os.path.dirname(outfile) + self.mkpath(dir) + return self.copy_file(module_file, outfile, preserve_mode=False) + + def build_modules(self): + modules = self.find_modules() + for package, module, module_file in modules: + + # Now "build" the module -- ie. copy the source file to + # self.build_lib (the build directory for Python source). + # (Actually, it gets copied to the directory for this package + # under self.build_lib.) + self.build_module(module, module_file, package) + + def build_packages(self): + for package in self.packages: + + # Get list of (package, module, module_file) tuples based on + # scanning the package directory. 'package' is only included + # in the tuple so that 'find_modules()' and + # 'find_package_tuples()' have a consistent interface; it's + # ignored here (apart from a sanity check). Also, 'module' is + # the *unqualified* module name (ie. no dots, no package -- we + # already know its package!), and 'module_file' is the path to + # the .py file, relative to the current directory + # (ie. including 'package_dir'). + package_dir = self.get_package_dir(package) + modules = self.find_package_modules(package, package_dir) + + # Now loop over the modules we found, "building" each one (just + # copy it to self.build_lib). + for package_, module, module_file in modules: + assert package == package_ + self.build_module(module, module_file, package) + + def byte_compile(self, files): + if hasattr(sys, 'dont_write_bytecode') and sys.dont_write_bytecode: + logger.warning('%s: byte-compiling is disabled, skipping.', + self.get_command_name()) + return + + from packaging.util import byte_compile # FIXME use compileall + prefix = self.build_lib + if prefix[-1] != os.sep: + prefix = prefix + os.sep + + # XXX this code is essentially the same as the 'byte_compile() + # method of the "install_lib" command, except for the determination + # of the 'prefix' string. Hmmm. + + if self.compile: + byte_compile(files, optimize=0, + force=self.force, prefix=prefix, dry_run=self.dry_run) + if self.optimize > 0: + byte_compile(files, optimize=self.optimize, + force=self.force, prefix=prefix, dry_run=self.dry_run) diff --git a/Lib/packaging/command/build_scripts.py b/Lib/packaging/command/build_scripts.py new file mode 100644 index 0000000000..fe14e0a9cb --- /dev/null +++ b/Lib/packaging/command/build_scripts.py @@ -0,0 +1,154 @@ +"""Build scripts (copy to build dir and fix up shebang line).""" + +import os +import re +import sysconfig +import tokenize + +from packaging.command.cmd import Command +from packaging.util import convert_path, newer +from packaging import logger +from packaging.compat import Mixin2to3 + + +# check if Python is called on the first line with this expression +first_line_re = re.compile(b'^#!.*python[0-9.]*([ \t].*)?$') + +class build_scripts(Command, Mixin2to3): + + description = "build scripts (copy and fix up shebang line)" + + user_options = [ + ('build-dir=', 'd', "directory to build (copy) to"), + ('force', 'f', "forcibly build everything (ignore file timestamps"), + ('executable=', 'e', "specify final destination interpreter path"), + ] + + boolean_options = ['force'] + + + def initialize_options(self): + self.build_dir = None + self.scripts = None + self.force = None + self.executable = None + self.outfiles = None + self.use_2to3 = False + self.convert_2to3_doctests = None + self.use_2to3_fixers = None + + def finalize_options(self): + self.set_undefined_options('build', + ('build_scripts', 'build_dir'), + 'use_2to3', 'use_2to3_fixers', + 'convert_2to3_doctests', 'force', + 'executable') + self.scripts = self.distribution.scripts + + def get_source_files(self): + return self.scripts + + def run(self): + if not self.scripts: + return + copied_files = self.copy_scripts() + if self.use_2to3 and copied_files: + self._run_2to3(copied_files, fixers=self.use_2to3_fixers) + + def copy_scripts(self): + """Copy each script listed in 'self.scripts'; if it's marked as a + Python script in the Unix way (first line matches 'first_line_re', + ie. starts with "\#!" and contains "python"), then adjust the first + line to refer to the current Python interpreter as we copy. + """ + self.mkpath(self.build_dir) + outfiles = [] + for script in self.scripts: + adjust = False + script = convert_path(script) + outfile = os.path.join(self.build_dir, os.path.basename(script)) + outfiles.append(outfile) + + if not self.force and not newer(script, outfile): + logger.debug("not copying %s (up-to-date)", script) + continue + + # Always open the file, but ignore failures in dry-run mode -- + # that way, we'll get accurate feedback if we can read the + # script. + try: + f = open(script, "rb") + except IOError: + if not self.dry_run: + raise + f = None + else: + encoding, lines = tokenize.detect_encoding(f.readline) + f.seek(0) + first_line = f.readline() + if not first_line: + logger.warning('%s: %s is an empty file (skipping)', + self.get_command_name(), script) + continue + + match = first_line_re.match(first_line) + if match: + adjust = True + post_interp = match.group(1) or b'' + + if adjust: + logger.info("copying and adjusting %s -> %s", script, + self.build_dir) + if not self.dry_run: + if not sysconfig.is_python_build(): + executable = self.executable + else: + executable = os.path.join( + sysconfig.get_config_var("BINDIR"), + "python%s%s" % (sysconfig.get_config_var("VERSION"), + sysconfig.get_config_var("EXE"))) + executable = os.fsencode(executable) + shebang = b"#!" + executable + post_interp + b"\n" + # Python parser starts to read a script using UTF-8 until + # it gets a #coding:xxx cookie. The shebang has to be the + # first line of a file, the #coding:xxx cookie cannot be + # written before. So the shebang has to be decodable from + # UTF-8. + try: + shebang.decode('utf-8') + except UnicodeDecodeError: + raise ValueError( + "The shebang ({!r}) is not decodable " + "from utf-8".format(shebang)) + # If the script is encoded to a custom encoding (use a + # #coding:xxx cookie), the shebang has to be decodable from + # the script encoding too. + try: + shebang.decode(encoding) + except UnicodeDecodeError: + raise ValueError( + "The shebang ({!r}) is not decodable " + "from the script encoding ({})" + .format(shebang, encoding)) + with open(outfile, "wb") as outf: + outf.write(shebang) + outf.writelines(f.readlines()) + if f: + f.close() + else: + if f: + f.close() + self.copy_file(script, outfile) + + if os.name == 'posix': + for file in outfiles: + if self.dry_run: + logger.info("changing mode of %s", file) + else: + oldmode = os.stat(file).st_mode & 0o7777 + newmode = (oldmode | 0o555) & 0o7777 + if newmode != oldmode: + logger.info("changing mode of %s from %o to %o", + file, oldmode, newmode) + os.chmod(file, newmode) + return outfiles diff --git a/Lib/packaging/command/check.py b/Lib/packaging/command/check.py new file mode 100644 index 0000000000..6715db90b3 --- /dev/null +++ b/Lib/packaging/command/check.py @@ -0,0 +1,88 @@ +"""Check PEP compliance of metadata.""" + +from packaging import logger +from packaging.command.cmd import Command +from packaging.errors import PackagingSetupError +from packaging.util import resolve_name + +class check(Command): + + description = "check PEP compliance of metadata" + + user_options = [('metadata', 'm', 'Verify metadata'), + ('all', 'a', + ('runs extended set of checks')), + ('strict', 's', + 'Will exit with an error if a check fails')] + + boolean_options = ['metadata', 'all', 'strict'] + + def initialize_options(self): + """Sets default values for options.""" + self.all = False + self.metadata = True + self.strict = False + self._warnings = [] + + def finalize_options(self): + pass + + def warn(self, msg, *args): + """Wrapper around logging that also remembers messages.""" + # XXX we could use a special handler for this, but would need to test + # if it works even if the logger has a too high level + self._warnings.append((msg, args)) + return logger.warning('%s: %s' % (self.get_command_name(), msg), *args) + + def run(self): + """Runs the command.""" + # perform the various tests + if self.metadata: + self.check_metadata() + if self.all: + self.check_restructuredtext() + self.check_hooks_resolvable() + + # let's raise an error in strict mode, if we have at least + # one warning + if self.strict and len(self._warnings) > 0: + msg = '\n'.join(msg % args for msg, args in self._warnings) + raise PackagingSetupError(msg) + + def check_metadata(self): + """Ensures that all required elements of metadata are supplied. + + name, version, URL, author + + Warns if any are missing. + """ + missing, warnings = self.distribution.metadata.check(strict=True) + if missing != []: + self.warn('missing required metadata: %s', ', '.join(missing)) + for warning in warnings: + self.warn(warning) + + def check_restructuredtext(self): + """Checks if the long string fields are reST-compliant.""" + missing, warnings = self.distribution.metadata.check(restructuredtext=True) + if self.distribution.metadata.docutils_support: + for warning in warnings: + line = warning[-1].get('line') + if line is None: + warning = warning[1] + else: + warning = '%s (line %s)' % (warning[1], line) + self.warn(warning) + elif self.strict: + raise PackagingSetupError('The docutils package is needed.') + + def check_hooks_resolvable(self): + for options in self.distribution.command_options.values(): + for hook_kind in ("pre_hook", "post_hook"): + if hook_kind not in options: + break + for hook_name in options[hook_kind][1].values(): + try: + resolve_name(hook_name) + except ImportError: + self.warn('name %r cannot be resolved', hook_name) diff --git a/Lib/packaging/command/clean.py b/Lib/packaging/command/clean.py new file mode 100644 index 0000000000..4f60f4ea93 --- /dev/null +++ b/Lib/packaging/command/clean.py @@ -0,0 +1,76 @@ +"""Clean up temporary files created by the build command.""" + +# Contributed by Bastian Kleineidam <calvin@cs.uni-sb.de> + +import os +from shutil import rmtree +from packaging.command.cmd import Command +from packaging import logger + +class clean(Command): + + description = "clean up temporary files from 'build' command" + user_options = [ + ('build-base=', 'b', + "base build directory (default: 'build.build-base')"), + ('build-lib=', None, + "build directory for all modules (default: 'build.build-lib')"), + ('build-temp=', 't', + "temporary build directory (default: 'build.build-temp')"), + ('build-scripts=', None, + "build directory for scripts (default: 'build.build-scripts')"), + ('bdist-base=', None, + "temporary directory for built distributions"), + ('all', 'a', + "remove all build output, not just temporary by-products") + ] + + boolean_options = ['all'] + + def initialize_options(self): + self.build_base = None + self.build_lib = None + self.build_temp = None + self.build_scripts = None + self.bdist_base = None + self.all = None + + def finalize_options(self): + self.set_undefined_options('build', 'build_base', 'build_lib', + 'build_scripts', 'build_temp') + self.set_undefined_options('bdist', 'bdist_base') + + def run(self): + # remove the build/temp.<plat> directory (unless it's already + # gone) + if os.path.exists(self.build_temp): + if self.dry_run: + logger.info('removing %s', self.build_temp) + else: + rmtree(self.build_temp) + else: + logger.debug("'%s' does not exist -- can't clean it", + self.build_temp) + + if self.all: + # remove build directories + for directory in (self.build_lib, + self.bdist_base, + self.build_scripts): + if os.path.exists(directory): + if self.dry_run: + logger.info('removing %s', directory) + else: + rmtree(directory) + else: + logger.warning("'%s' does not exist -- can't clean it", + directory) + + # just for the heck of it, try to remove the base build directory: + # we might have emptied it right now, but if not we don't care + if not self.dry_run: + try: + os.rmdir(self.build_base) + logger.info("removing '%s'", self.build_base) + except OSError: + pass diff --git a/Lib/packaging/command/cmd.py b/Lib/packaging/command/cmd.py new file mode 100644 index 0000000000..1053ac390a --- /dev/null +++ b/Lib/packaging/command/cmd.py @@ -0,0 +1,441 @@ +"""Base class for commands.""" + +import os +import re +from shutil import copyfile, move, make_archive +from packaging import util +from packaging import logger +from packaging.errors import PackagingOptionError + + +class Command: + """Abstract base class for defining command classes, the "worker bees" + of the Packaging. A useful analogy for command classes is to think of + them as subroutines with local variables called "options". The options + are "declared" in 'initialize_options()' and "defined" (given their + final values, aka "finalized") in 'finalize_options()', both of which + must be defined by every command class. The distinction between the + two is necessary because option values might come from the outside + world (command line, config file, ...), and any options dependent on + other options must be computed *after* these outside influences have + been processed -- hence 'finalize_options()'. The "body" of the + subroutine, where it does all its work based on the values of its + options, is the 'run()' method, which must also be implemented by every + command class. + """ + + # 'sub_commands' formalizes the notion of a "family" of commands, + # eg. "install_dist" as the parent with sub-commands "install_lib", + # "install_headers", etc. The parent of a family of commands + # defines 'sub_commands' as a class attribute; it's a list of + # (command_name : string, predicate : unbound_method | string | None) + # tuples, where 'predicate' is a method of the parent command that + # determines whether the corresponding command is applicable in the + # current situation. (Eg. we "install_headers" is only applicable if + # we have any C header files to install.) If 'predicate' is None, + # that command is always applicable. + # + # 'sub_commands' is usually defined at the *end* of a class, because + # predicates can be unbound methods, so they must already have been + # defined. The canonical example is the "install_dist" command. + sub_commands = [] + + # Pre and post command hooks are run just before or just after the command + # itself. They are simple functions that receive the command instance. They + # are specified as callable objects or dotted strings (for lazy loading). + pre_hook = None + post_hook = None + + # -- Creation/initialization methods ------------------------------- + + def __init__(self, dist): + """Create and initialize a new Command object. Most importantly, + invokes the 'initialize_options()' method, which is the real + initializer and depends on the actual command being instantiated. + """ + # late import because of mutual dependence between these classes + from packaging.dist import Distribution + + if not isinstance(dist, Distribution): + raise TypeError("dist must be an instance of Distribution, not %r" + % type(dist)) + if self.__class__ is Command: + raise RuntimeError("Command is an abstract class") + + self.distribution = dist + self.initialize_options() + + # Per-command versions of the global flags, so that the user can + # customize Packaging' behaviour command-by-command and let some + # commands fall back on the Distribution's behaviour. None means + # "not defined, check self.distribution's copy", while 0 or 1 mean + # false and true (duh). Note that this means figuring out the real + # value of each flag is a touch complicated -- hence "self._dry_run" + # will be handled by a property, below. + # XXX This needs to be fixed. [I changed it to a property--does that + # "fix" it?] + self._dry_run = None + + # Some commands define a 'self.force' option to ignore file + # timestamps, but methods defined *here* assume that + # 'self.force' exists for all commands. So define it here + # just to be safe. + self.force = None + + # The 'help' flag is just used for command line parsing, so + # none of that complicated bureaucracy is needed. + self.help = False + + # 'finalized' records whether or not 'finalize_options()' has been + # called. 'finalize_options()' itself should not pay attention to + # this flag: it is the business of 'ensure_finalized()', which + # always calls 'finalize_options()', to respect/update it. + self.finalized = False + + # XXX A more explicit way to customize dry_run would be better. + @property + def dry_run(self): + if self._dry_run is None: + return getattr(self.distribution, 'dry_run') + else: + return self._dry_run + + def ensure_finalized(self): + if not self.finalized: + self.finalize_options() + self.finalized = True + + # Subclasses must define: + # initialize_options() + # provide default values for all options; may be customized by + # setup script, by options from config file(s), or by command-line + # options + # finalize_options() + # decide on the final values for all options; this is called + # after all possible intervention from the outside world + # (command line, option file, etc.) has been processed + # run() + # run the command: do whatever it is we're here to do, + # controlled by the command's various option values + + def initialize_options(self): + """Set default values for all the options that this command + supports. Note that these defaults may be overridden by other + commands, by the setup script, by config files, or by the + command line. Thus, this is not the place to code dependencies + between options; generally, 'initialize_options()' implementations + are just a bunch of "self.foo = None" assignments. + + This method must be implemented by all command classes. + """ + raise RuntimeError( + "abstract method -- subclass %s must override" % self.__class__) + + def finalize_options(self): + """Set final values for all the options that this command supports. + This is always called as late as possible, ie. after any option + assignments from the command line or from other commands have been + done. Thus, this is the place to code option dependencies: if + 'foo' depends on 'bar', then it is safe to set 'foo' from 'bar' as + long as 'foo' still has the same value it was assigned in + 'initialize_options()'. + + This method must be implemented by all command classes. + """ + raise RuntimeError( + "abstract method -- subclass %s must override" % self.__class__) + + def dump_options(self, header=None, indent=""): + if header is None: + header = "command options for '%s':" % self.get_command_name() + logger.info(indent + header) + indent = indent + " " + negative_opt = getattr(self, 'negative_opt', ()) + for option, _, _ in self.user_options: + if option in negative_opt: + continue + option = option.replace('-', '_') + if option[-1] == "=": + option = option[:-1] + value = getattr(self, option) + logger.info(indent + "%s = %s", option, value) + + def run(self): + """A command's raison d'etre: carry out the action it exists to + perform, controlled by the options initialized in + 'initialize_options()', customized by other commands, the setup + script, the command line and config files, and finalized in + 'finalize_options()'. All terminal output and filesystem + interaction should be done by 'run()'. + + This method must be implemented by all command classes. + """ + raise RuntimeError( + "abstract method -- subclass %s must override" % self.__class__) + + # -- External interface -------------------------------------------- + # (called by outsiders) + + def get_source_files(self): + """Return the list of files that are used as inputs to this command, + i.e. the files used to generate the output files. The result is used + by the `sdist` command in determining the set of default files. + + Command classes should implement this method if they operate on files + from the source tree. + """ + return [] + + def get_outputs(self): + """Return the list of files that would be produced if this command + were actually run. Not affected by the "dry-run" flag or whether + any other commands have been run. + + Command classes should implement this method if they produce any + output files that get consumed by another command. e.g., `build_ext` + returns the list of built extension modules, but not any temporary + files used in the compilation process. + """ + return [] + + # -- Option validation methods ------------------------------------- + # (these are very handy in writing the 'finalize_options()' method) + # + # NB. the general philosophy here is to ensure that a particular option + # value meets certain type and value constraints. If not, we try to + # force it into conformance (eg. if we expect a list but have a string, + # split the string on comma and/or whitespace). If we can't force the + # option into conformance, raise PackagingOptionError. Thus, command + # classes need do nothing more than (eg.) + # self.ensure_string_list('foo') + # and they can be guaranteed that thereafter, self.foo will be + # a list of strings. + + def _ensure_stringlike(self, option, what, default=None): + val = getattr(self, option) + if val is None: + setattr(self, option, default) + return default + elif not isinstance(val, str): + raise PackagingOptionError("'%s' must be a %s (got `%s`)" % + (option, what, val)) + return val + + def ensure_string(self, option, default=None): + """Ensure that 'option' is a string; if not defined, set it to + 'default'. + """ + self._ensure_stringlike(option, "string", default) + + def ensure_string_list(self, option): + r"""Ensure that 'option' is a list of strings. If 'option' is + currently a string, we split it either on /,\s*/ or /\s+/, so + "foo bar baz", "foo,bar,baz", and "foo, bar baz" all become + ["foo", "bar", "baz"]. + """ + val = getattr(self, option) + if val is None: + return + elif isinstance(val, str): + setattr(self, option, re.split(r',\s*|\s+', val)) + else: + if isinstance(val, list): + # checks if all elements are str + ok = True + for element in val: + if not isinstance(element, str): + ok = False + break + else: + ok = False + + if not ok: + raise PackagingOptionError( + "'%s' must be a list of strings (got %r)" % (option, val)) + + def _ensure_tested_string(self, option, tester, + what, error_fmt, default=None): + val = self._ensure_stringlike(option, what, default) + if val is not None and not tester(val): + raise PackagingOptionError( + ("error in '%s' option: " + error_fmt) % (option, val)) + + def ensure_filename(self, option): + """Ensure that 'option' is the name of an existing file.""" + self._ensure_tested_string(option, os.path.isfile, + "filename", + "'%s' does not exist or is not a file") + + def ensure_dirname(self, option): + self._ensure_tested_string(option, os.path.isdir, + "directory name", + "'%s' does not exist or is not a directory") + + # -- Convenience methods for commands ------------------------------ + + @classmethod + def get_command_name(cls): + if hasattr(cls, 'command_name'): + return cls.command_name + else: + return cls.__name__ + + def set_undefined_options(self, src_cmd, *options): + """Set values of undefined options from another command. + + Undefined options are options set to None, which is the convention + used to indicate that an option has not been changed between + 'initialize_options()' and 'finalize_options()'. This method is + usually called from 'finalize_options()' for options that depend on + some other command rather than another option of the same command, + typically subcommands. + + The 'src_cmd' argument is the other command from which option values + will be taken (a command object will be created for it if necessary); + the remaining positional arguments are strings that give the name of + the option to set. If the name is different on the source and target + command, you can pass a tuple with '(name_on_source, name_on_dest)' so + that 'self.name_on_dest' will be set from 'src_cmd.name_on_source'. + """ + src_cmd_obj = self.distribution.get_command_obj(src_cmd) + src_cmd_obj.ensure_finalized() + for obj in options: + if isinstance(obj, tuple): + src_option, dst_option = obj + else: + src_option, dst_option = obj, obj + if getattr(self, dst_option) is None: + setattr(self, dst_option, + getattr(src_cmd_obj, src_option)) + + def get_finalized_command(self, command, create=True): + """Wrapper around Distribution's 'get_command_obj()' method: find + (create if necessary and 'create' is true) the command object for + 'command', call its 'ensure_finalized()' method, and return the + finalized command object. + """ + cmd_obj = self.distribution.get_command_obj(command, create) + cmd_obj.ensure_finalized() + return cmd_obj + + def get_reinitialized_command(self, command, reinit_subcommands=False): + return self.distribution.get_reinitialized_command( + command, reinit_subcommands) + + def run_command(self, command): + """Run some other command: uses the 'run_command()' method of + Distribution, which creates and finalizes the command object if + necessary and then invokes its 'run()' method. + """ + self.distribution.run_command(command) + + def get_sub_commands(self): + """Determine the sub-commands that are relevant in the current + distribution (ie., that need to be run). This is based on the + 'sub_commands' class attribute: each tuple in that list may include + a method that we call to determine if the subcommand needs to be + run for the current distribution. Return a list of command names. + """ + commands = [] + for sub_command in self.sub_commands: + if len(sub_command) == 2: + cmd_name, method = sub_command + if method is None or method(self): + commands.append(cmd_name) + else: + commands.append(sub_command) + return commands + + # -- External world manipulation ----------------------------------- + + def execute(self, func, args, msg=None, level=1): + util.execute(func, args, msg, dry_run=self.dry_run) + + def mkpath(self, name, mode=0o777, dry_run=None, verbose=0): + if dry_run is None: + dry_run = self.dry_run + name = os.path.normpath(name) + if os.path.isdir(name) or name == '': + return + if dry_run: + head = '' + for part in name.split(os.sep): + logger.info("created directory %s%s", head, part) + head += part + os.sep + return + os.makedirs(name, mode) + + def copy_file(self, infile, outfile, + preserve_mode=True, preserve_times=True, link=None, level=1): + """Copy a file respecting verbose, dry-run and force flags. (The + former two default to whatever is in the Distribution object, and + the latter defaults to false for commands that don't define it.)""" + if self.dry_run: + # XXX add a comment + return + if os.path.isdir(outfile): + outfile = os.path.join(outfile, os.path.split(infile)[-1]) + copyfile(infile, outfile) + return outfile, None # XXX + + def copy_tree(self, infile, outfile, preserve_mode=True, + preserve_times=True, preserve_symlinks=False, level=1): + """Copy an entire directory tree respecting verbose, dry-run, + and force flags. + """ + if self.dry_run: + return # see if we want to display something + + + return util.copy_tree(infile, outfile, preserve_mode, preserve_times, + preserve_symlinks, not self.force, dry_run=self.dry_run) + + def move_file(self, src, dst, level=1): + """Move a file respecting the dry-run flag.""" + if self.dry_run: + return # XXX log ? + return move(src, dst) + + def spawn(self, cmd, search_path=True, level=1): + """Spawn an external command respecting dry-run flag.""" + from packaging.util import spawn + spawn(cmd, search_path, dry_run=self.dry_run) + + def make_archive(self, base_name, format, root_dir=None, base_dir=None, + owner=None, group=None): + return make_archive(base_name, format, root_dir, + base_dir, dry_run=self.dry_run, + owner=owner, group=group) + + def make_file(self, infiles, outfile, func, args, + exec_msg=None, skip_msg=None, level=1): + """Special case of 'execute()' for operations that process one or + more input files and generate one output file. Works just like + 'execute()', except the operation is skipped and a different + message printed if 'outfile' already exists and is newer than all + files listed in 'infiles'. If the command defined 'self.force', + and it is true, then the command is unconditionally run -- does no + timestamp checks. + """ + if skip_msg is None: + skip_msg = "skipping %s (inputs unchanged)" % outfile + + # Allow 'infiles' to be a single string + if isinstance(infiles, str): + infiles = (infiles,) + elif not isinstance(infiles, (list, tuple)): + raise TypeError( + "'infiles' must be a string, or a list or tuple of strings") + + if exec_msg is None: + exec_msg = "generating %s from %s" % (outfile, ', '.join(infiles)) + + # If 'outfile' must be regenerated (either because it doesn't + # exist, is out-of-date, or the 'force' flag is true) then + # perform the action that presumably regenerates it + if self.force or util.newer_group(infiles, outfile): + self.execute(func, args, exec_msg, level) + + # Otherwise, print the "skip" message + else: + logger.debug(skip_msg) diff --git a/Lib/packaging/command/command_template b/Lib/packaging/command/command_template new file mode 100644 index 0000000000..a12d32bfb3 --- /dev/null +++ b/Lib/packaging/command/command_template @@ -0,0 +1,35 @@ +"""Do X and Y.""" + +from packaging import logger +from packaging.command.cmd import Command + + +class x(Command): + + # Brief (40-50 characters) description of the command + description = "" + + # List of option tuples: long name, short name (None if no short + # name), and help string. + user_options = [ + ('', '', # long option, short option (one letter) or None + ""), # help text + ] + + def initialize_options(self): + self. = None + self. = None + self. = None + + def finalize_options(self): + if self.x is None: + self.x = ... + + def run(self): + ... + logger.info(...) + + if not self.dry_run: + ... + + self.execute(..., dry_run=self.dry_run) diff --git a/Lib/packaging/command/config.py b/Lib/packaging/command/config.py new file mode 100644 index 0000000000..264c1396b2 --- /dev/null +++ b/Lib/packaging/command/config.py @@ -0,0 +1,349 @@ +"""Prepare the build. + +This module provides config, a (mostly) empty command class +that exists mainly to be sub-classed by specific module distributions and +applications. The idea is that while every "config" command is different, +at least they're all named the same, and users always see "config" in the +list of standard commands. Also, this is a good place to put common +configure-like tasks: "try to compile this C code", or "figure out where +this header file lives". +""" + +import os +import re + +from packaging.command.cmd import Command +from packaging.errors import PackagingExecError +from packaging.compiler import customize_compiler +from packaging import logger + +LANG_EXT = {'c': '.c', 'c++': '.cxx'} + +class config(Command): + + description = "prepare the build" + + user_options = [ + ('compiler=', None, + "specify the compiler type"), + ('cc=', None, + "specify the compiler executable"), + ('include-dirs=', 'I', + "list of directories to search for header files"), + ('define=', 'D', + "C preprocessor macros to define"), + ('undef=', 'U', + "C preprocessor macros to undefine"), + ('libraries=', 'l', + "external C libraries to link with"), + ('library-dirs=', 'L', + "directories to search for external C libraries"), + + ('noisy', None, + "show every action (compile, link, run, ...) taken"), + ('dump-source', None, + "dump generated source files before attempting to compile them"), + ] + + + # The three standard command methods: since the "config" command + # does nothing by default, these are empty. + + def initialize_options(self): + self.compiler = None + self.cc = None + self.include_dirs = None + self.libraries = None + self.library_dirs = None + + # maximal output for now + self.noisy = True + self.dump_source = True + + # list of temporary files generated along-the-way that we have + # to clean at some point + self.temp_files = [] + + def finalize_options(self): + if self.include_dirs is None: + self.include_dirs = self.distribution.include_dirs or [] + elif isinstance(self.include_dirs, str): + self.include_dirs = self.include_dirs.split(os.pathsep) + + if self.libraries is None: + self.libraries = [] + elif isinstance(self.libraries, str): + self.libraries = [self.libraries] + + if self.library_dirs is None: + self.library_dirs = [] + elif isinstance(self.library_dirs, str): + self.library_dirs = self.library_dirs.split(os.pathsep) + + def run(self): + pass + + + # Utility methods for actual "config" commands. The interfaces are + # loosely based on Autoconf macros of similar names. Sub-classes + # may use these freely. + + def _check_compiler(self): + """Check that 'self.compiler' really is a CCompiler object; + if not, make it one. + """ + # We do this late, and only on-demand, because this is an expensive + # import. + from packaging.compiler.ccompiler import CCompiler + from packaging.compiler import new_compiler + if not isinstance(self.compiler, CCompiler): + self.compiler = new_compiler(compiler=self.compiler, + dry_run=self.dry_run, force=True) + customize_compiler(self.compiler) + if self.include_dirs: + self.compiler.set_include_dirs(self.include_dirs) + if self.libraries: + self.compiler.set_libraries(self.libraries) + if self.library_dirs: + self.compiler.set_library_dirs(self.library_dirs) + + + def _gen_temp_sourcefile(self, body, headers, lang): + filename = "_configtest" + LANG_EXT[lang] + with open(filename, "w") as file: + if headers: + for header in headers: + file.write("#include <%s>\n" % header) + file.write("\n") + file.write(body) + if body[-1] != "\n": + file.write("\n") + return filename + + def _preprocess(self, body, headers, include_dirs, lang): + src = self._gen_temp_sourcefile(body, headers, lang) + out = "_configtest.i" + self.temp_files.extend((src, out)) + self.compiler.preprocess(src, out, include_dirs=include_dirs) + return src, out + + def _compile(self, body, headers, include_dirs, lang): + src = self._gen_temp_sourcefile(body, headers, lang) + if self.dump_source: + dump_file(src, "compiling '%s':" % src) + obj = self.compiler.object_filenames([src])[0] + self.temp_files.extend((src, obj)) + self.compiler.compile([src], include_dirs=include_dirs) + return src, obj + + def _link(self, body, headers, include_dirs, libraries, library_dirs, + lang): + src, obj = self._compile(body, headers, include_dirs, lang) + prog = os.path.splitext(os.path.basename(src))[0] + self.compiler.link_executable([obj], prog, + libraries=libraries, + library_dirs=library_dirs, + target_lang=lang) + + if self.compiler.exe_extension is not None: + prog = prog + self.compiler.exe_extension + self.temp_files.append(prog) + + return src, obj, prog + + def _clean(self, *filenames): + if not filenames: + filenames = self.temp_files + self.temp_files = [] + logger.info("removing: %s", ' '.join(filenames)) + for filename in filenames: + try: + os.remove(filename) + except OSError: + pass + + + # XXX these ignore the dry-run flag: what to do, what to do? even if + # you want a dry-run build, you still need some sort of configuration + # info. My inclination is to make it up to the real config command to + # consult 'dry_run', and assume a default (minimal) configuration if + # true. The problem with trying to do it here is that you'd have to + # return either true or false from all the 'try' methods, neither of + # which is correct. + + # XXX need access to the header search path and maybe default macros. + + def try_cpp(self, body=None, headers=None, include_dirs=None, lang="c"): + """Construct a source file from 'body' (a string containing lines + of C/C++ code) and 'headers' (a list of header files to include) + and run it through the preprocessor. Return true if the + preprocessor succeeded, false if there were any errors. + ('body' probably isn't of much use, but what the heck.) + """ + from packaging.compiler.ccompiler import CompileError + self._check_compiler() + ok = True + try: + self._preprocess(body, headers, include_dirs, lang) + except CompileError: + ok = False + + self._clean() + return ok + + def search_cpp(self, pattern, body=None, headers=None, include_dirs=None, + lang="c"): + """Construct a source file (just like 'try_cpp()'), run it through + the preprocessor, and return true if any line of the output matches + 'pattern'. 'pattern' should either be a compiled regex object or a + string containing a regex. If both 'body' and 'headers' are None, + preprocesses an empty file -- which can be useful to determine the + symbols the preprocessor and compiler set by default. + """ + self._check_compiler() + src, out = self._preprocess(body, headers, include_dirs, lang) + + if isinstance(pattern, str): + pattern = re.compile(pattern) + + with open(out) as file: + match = False + while True: + line = file.readline() + if line == '': + break + if pattern.search(line): + match = True + break + + self._clean() + return match + + def try_compile(self, body, headers=None, include_dirs=None, lang="c"): + """Try to compile a source file built from 'body' and 'headers'. + Return true on success, false otherwise. + """ + from packaging.compiler.ccompiler import CompileError + self._check_compiler() + try: + self._compile(body, headers, include_dirs, lang) + ok = True + except CompileError: + ok = False + + logger.info(ok and "success!" or "failure.") + self._clean() + return ok + + def try_link(self, body, headers=None, include_dirs=None, libraries=None, + library_dirs=None, lang="c"): + """Try to compile and link a source file, built from 'body' and + 'headers', to executable form. Return true on success, false + otherwise. + """ + from packaging.compiler.ccompiler import CompileError, LinkError + self._check_compiler() + try: + self._link(body, headers, include_dirs, + libraries, library_dirs, lang) + ok = True + except (CompileError, LinkError): + ok = False + + logger.info(ok and "success!" or "failure.") + self._clean() + return ok + + def try_run(self, body, headers=None, include_dirs=None, libraries=None, + library_dirs=None, lang="c"): + """Try to compile, link to an executable, and run a program + built from 'body' and 'headers'. Return true on success, false + otherwise. + """ + from packaging.compiler.ccompiler import CompileError, LinkError + self._check_compiler() + try: + src, obj, exe = self._link(body, headers, include_dirs, + libraries, library_dirs, lang) + self.spawn([exe]) + ok = True + except (CompileError, LinkError, PackagingExecError): + ok = False + + logger.info(ok and "success!" or "failure.") + self._clean() + return ok + + + # -- High-level methods -------------------------------------------- + # (these are the ones that are actually likely to be useful + # when implementing a real-world config command!) + + def check_func(self, func, headers=None, include_dirs=None, + libraries=None, library_dirs=None, decl=False, call=False): + + """Determine if function 'func' is available by constructing a + source file that refers to 'func', and compiles and links it. + If everything succeeds, returns true; otherwise returns false. + + The constructed source file starts out by including the header + files listed in 'headers'. If 'decl' is true, it then declares + 'func' (as "int func()"); you probably shouldn't supply 'headers' + and set 'decl' true in the same call, or you might get errors about + a conflicting declarations for 'func'. Finally, the constructed + 'main()' function either references 'func' or (if 'call' is true) + calls it. 'libraries' and 'library_dirs' are used when + linking. + """ + + self._check_compiler() + body = [] + if decl: + body.append("int %s ();" % func) + body.append("int main () {") + if call: + body.append(" %s();" % func) + else: + body.append(" %s;" % func) + body.append("}") + body = "\n".join(body) + "\n" + + return self.try_link(body, headers, include_dirs, + libraries, library_dirs) + + def check_lib(self, library, library_dirs=None, headers=None, + include_dirs=None, other_libraries=[]): + """Determine if 'library' is available to be linked against, + without actually checking that any particular symbols are provided + by it. 'headers' will be used in constructing the source file to + be compiled, but the only effect of this is to check if all the + header files listed are available. Any libraries listed in + 'other_libraries' will be included in the link, in case 'library' + has symbols that depend on other libraries. + """ + self._check_compiler() + return self.try_link("int main (void) { }", + headers, include_dirs, + [library]+other_libraries, library_dirs) + + def check_header(self, header, include_dirs=None, library_dirs=None, + lang="c"): + """Determine if the system header file named by 'header_file' + exists and can be found by the preprocessor; return true if so, + false otherwise. + """ + return self.try_cpp(body="/* No body */", headers=[header], + include_dirs=include_dirs) + + +def dump_file(filename, head=None): + """Dumps a file content into log.info. + + If head is not None, will be dumped before the file content. + """ + if head is None: + logger.info(filename) + else: + logger.info(head) + with open(filename) as file: + logger.info(file.read()) diff --git a/Lib/packaging/command/install_data.py b/Lib/packaging/command/install_data.py new file mode 100644 index 0000000000..9ca6279533 --- /dev/null +++ b/Lib/packaging/command/install_data.py @@ -0,0 +1,79 @@ +"""Install platform-independent data files.""" + +# Contributed by Bastian Kleineidam + +import os +from shutil import Error +from sysconfig import get_paths, format_value +from packaging import logger +from packaging.util import convert_path +from packaging.command.cmd import Command + + +class install_data(Command): + + description = "install platform-independent data files" + + user_options = [ + ('install-dir=', 'd', + "base directory for installing data files " + "(default: installation base dir)"), + ('root=', None, + "install everything relative to this alternate root directory"), + ('force', 'f', "force installation (overwrite existing files)"), + ] + + boolean_options = ['force'] + + def initialize_options(self): + self.install_dir = None + self.outfiles = [] + self.data_files_out = [] + self.root = None + self.force = False + self.data_files = self.distribution.data_files + self.warn_dir = True + + def finalize_options(self): + self.set_undefined_options('install_dist', + ('install_data', 'install_dir'), + 'root', 'force') + + def run(self): + self.mkpath(self.install_dir) + for _file in self.data_files.items(): + destination = convert_path(self.expand_categories(_file[1])) + dir_dest = os.path.abspath(os.path.dirname(destination)) + + self.mkpath(dir_dest) + try: + out = self.copy_file(_file[0], dir_dest)[0] + except Error as e: + logger.warning('%s: %s', self.get_command_name(), e) + out = destination + + self.outfiles.append(out) + self.data_files_out.append((_file[0], destination)) + + def expand_categories(self, path_with_categories): + local_vars = get_paths() + local_vars['distribution.name'] = self.distribution.metadata['Name'] + expanded_path = format_value(path_with_categories, local_vars) + expanded_path = format_value(expanded_path, local_vars) + if '{' in expanded_path and '}' in expanded_path: + logger.warning( + '%s: unable to expand %s, some categories may be missing', + self.get_command_name(), path_with_categories) + return expanded_path + + def get_source_files(self): + return list(self.data_files) + + def get_inputs(self): + return list(self.data_files) + + def get_outputs(self): + return self.outfiles + + def get_resources_out(self): + return self.data_files_out diff --git a/Lib/packaging/command/install_dist.py b/Lib/packaging/command/install_dist.py new file mode 100644 index 0000000000..dfe6df2d5a --- /dev/null +++ b/Lib/packaging/command/install_dist.py @@ -0,0 +1,625 @@ +"""Main install command, which calls the other install_* commands.""" + +import sys +import os + +import sysconfig +from sysconfig import get_config_vars, get_paths, get_path, get_config_var + +from packaging import logger +from packaging.command.cmd import Command +from packaging.errors import PackagingPlatformError +from packaging.util import write_file +from packaging.util import convert_path, change_root, get_platform +from packaging.errors import PackagingOptionError + + +HAS_USER_SITE = True + + +class install_dist(Command): + + description = "install everything from build directory" + + user_options = [ + # Select installation scheme and set base director(y|ies) + ('prefix=', None, + "installation prefix"), + ('exec-prefix=', None, + "(Unix only) prefix for platform-specific files"), + ('home=', None, + "(Unix only) home directory to install under"), + + # Or just set the base director(y|ies) + ('install-base=', None, + "base installation directory (instead of --prefix or --home)"), + ('install-platbase=', None, + "base installation directory for platform-specific files " + + "(instead of --exec-prefix or --home)"), + ('root=', None, + "install everything relative to this alternate root directory"), + + # Or explicitly set the installation scheme + ('install-purelib=', None, + "installation directory for pure Python module distributions"), + ('install-platlib=', None, + "installation directory for non-pure module distributions"), + ('install-lib=', None, + "installation directory for all module distributions " + + "(overrides --install-purelib and --install-platlib)"), + + ('install-headers=', None, + "installation directory for C/C++ headers"), + ('install-scripts=', None, + "installation directory for Python scripts"), + ('install-data=', None, + "installation directory for data files"), + + # Byte-compilation options -- see install_lib.py for details, as + # these are duplicated from there (but only install_lib does + # anything with them). + ('compile', 'c', "compile .py to .pyc [default]"), + ('no-compile', None, "don't compile .py files"), + ('optimize=', 'O', + 'also compile with optimization: -O1 for "python -O", ' + '-O2 for "python -OO", and -O0 to disable [default: -O0]'), + + # Miscellaneous control options + ('force', 'f', + "force installation (overwrite any existing files)"), + ('skip-build', None, + "skip rebuilding everything (for testing/debugging)"), + + # Where to install documentation (eventually!) + #('doc-format=', None, "format of documentation to generate"), + #('install-man=', None, "directory for Unix man pages"), + #('install-html=', None, "directory for HTML documentation"), + #('install-info=', None, "directory for GNU info files"), + + # XXX use a name that makes clear this is the old format + ('record=', None, + "filename in which to record a list of installed files " + "(not PEP 376-compliant)"), + ('resources=', None, + "data files mapping"), + + # .dist-info related arguments, read by install_dist_info + ('no-distinfo', None, + "do not create a .dist-info directory"), + ('installer=', None, + "the name of the installer"), + ('requested', None, + "generate a REQUESTED file (i.e."), + ('no-requested', None, + "do not generate a REQUESTED file"), + ('no-record', None, + "do not generate a RECORD file"), + ] + + boolean_options = ['compile', 'force', 'skip-build', 'no-distinfo', + 'requested', 'no-record'] + + if HAS_USER_SITE: + user_options.append( + ('user', None, + "install in user site-packages directory [%s]" % + get_path('purelib', '%s_user' % os.name))) + + boolean_options.append('user') + + negative_opt = {'no-compile': 'compile', 'no-requested': 'requested'} + + def initialize_options(self): + # High-level options: these select both an installation base + # and scheme. + self.prefix = None + self.exec_prefix = None + self.home = None + if HAS_USER_SITE: + self.user = False + + # These select only the installation base; it's up to the user to + # specify the installation scheme (currently, that means supplying + # the --install-{platlib,purelib,scripts,data} options). + self.install_base = None + self.install_platbase = None + self.root = None + + # These options are the actual installation directories; if not + # supplied by the user, they are filled in using the installation + # scheme implied by prefix/exec-prefix/home and the contents of + # that installation scheme. + self.install_purelib = None # for pure module distributions + self.install_platlib = None # non-pure (dists w/ extensions) + self.install_headers = None # for C/C++ headers + self.install_lib = None # set to either purelib or platlib + self.install_scripts = None + self.install_data = None + if HAS_USER_SITE: + self.install_userbase = get_config_var('userbase') + self.install_usersite = get_path('purelib', '%s_user' % os.name) + + self.compile = None + self.optimize = None + + # These two are for putting non-packagized distributions into their + # own directory and creating a .pth file if it makes sense. + # 'extra_path' comes from the setup file; 'install_path_file' can + # be turned off if it makes no sense to install a .pth file. (But + # better to install it uselessly than to guess wrong and not + # install it when it's necessary and would be used!) Currently, + # 'install_path_file' is always true unless some outsider meddles + # with it. + self.extra_path = None + self.install_path_file = True + + # 'force' forces installation, even if target files are not + # out-of-date. 'skip_build' skips running the "build" command, + # handy if you know it's not necessary. 'warn_dir' (which is *not* + # a user option, it's just there so the bdist_* commands can turn + # it off) determines whether we warn about installing to a + # directory not in sys.path. + self.force = False + self.skip_build = False + self.warn_dir = True + + # These are only here as a conduit from the 'build' command to the + # 'install_*' commands that do the real work. ('build_base' isn't + # actually used anywhere, but it might be useful in future.) They + # are not user options, because if the user told the install + # command where the build directory is, that wouldn't affect the + # build command. + self.build_base = None + self.build_lib = None + + # Not defined yet because we don't know anything about + # documentation yet. + #self.install_man = None + #self.install_html = None + #self.install_info = None + + self.record = None + self.resources = None + + # .dist-info related options + self.no_distinfo = None + self.installer = None + self.requested = None + self.no_record = None + self.no_resources = None + + # -- Option finalizing methods ------------------------------------- + # (This is rather more involved than for most commands, + # because this is where the policy for installing third- + # party Python modules on various platforms given a wide + # array of user input is decided. Yes, it's quite complex!) + + def finalize_options(self): + # This method (and its pliant slaves, like 'finalize_unix()', + # 'finalize_other()', and 'select_scheme()') is where the default + # installation directories for modules, extension modules, and + # anything else we care to install from a Python module + # distribution. Thus, this code makes a pretty important policy + # statement about how third-party stuff is added to a Python + # installation! Note that the actual work of installation is done + # by the relatively simple 'install_*' commands; they just take + # their orders from the installation directory options determined + # here. + + # Check for errors/inconsistencies in the options; first, stuff + # that's wrong on any platform. + + if ((self.prefix or self.exec_prefix or self.home) and + (self.install_base or self.install_platbase)): + raise PackagingOptionError( + "must supply either prefix/exec-prefix/home or " + "install-base/install-platbase -- not both") + + if self.home and (self.prefix or self.exec_prefix): + raise PackagingOptionError( + "must supply either home or prefix/exec-prefix -- not both") + + if HAS_USER_SITE and self.user and ( + self.prefix or self.exec_prefix or self.home or + self.install_base or self.install_platbase): + raise PackagingOptionError( + "can't combine user with prefix/exec_prefix/home or " + "install_base/install_platbase") + + # Next, stuff that's wrong (or dubious) only on certain platforms. + if os.name != "posix": + if self.exec_prefix: + logger.warning( + '%s: exec-prefix option ignored on this platform', + self.get_command_name()) + self.exec_prefix = None + + # Now the interesting logic -- so interesting that we farm it out + # to other methods. The goal of these methods is to set the final + # values for the install_{lib,scripts,data,...} options, using as + # input a heady brew of prefix, exec_prefix, home, install_base, + # install_platbase, user-supplied versions of + # install_{purelib,platlib,lib,scripts,data,...}, and the + # INSTALL_SCHEME dictionary above. Phew! + + self.dump_dirs("pre-finalize_{unix,other}") + + if os.name == 'posix': + self.finalize_unix() + else: + self.finalize_other() + + self.dump_dirs("post-finalize_{unix,other}()") + + # Expand configuration variables, tilde, etc. in self.install_base + # and self.install_platbase -- that way, we can use $base or + # $platbase in the other installation directories and not worry + # about needing recursive variable expansion (shudder). + + py_version = sys.version.split()[0] + prefix, exec_prefix, srcdir, projectbase = get_config_vars( + 'prefix', 'exec_prefix', 'srcdir', 'projectbase') + + metadata = self.distribution.metadata + self.config_vars = { + 'dist_name': metadata['Name'], + 'dist_version': metadata['Version'], + 'dist_fullname': metadata.get_fullname(), + 'py_version': py_version, + 'py_version_short': py_version[:3], + 'py_version_nodot': py_version[:3:2], + 'sys_prefix': prefix, + 'prefix': prefix, + 'sys_exec_prefix': exec_prefix, + 'exec_prefix': exec_prefix, + 'srcdir': srcdir, + 'projectbase': projectbase, + } + + if HAS_USER_SITE: + self.config_vars['userbase'] = self.install_userbase + self.config_vars['usersite'] = self.install_usersite + + self.expand_basedirs() + + self.dump_dirs("post-expand_basedirs()") + + # Now define config vars for the base directories so we can expand + # everything else. + self.config_vars['base'] = self.install_base + self.config_vars['platbase'] = self.install_platbase + + # Expand "~" and configuration variables in the installation + # directories. + self.expand_dirs() + + self.dump_dirs("post-expand_dirs()") + + # Create directories in the home dir: + if HAS_USER_SITE and self.user: + self.create_home_path() + + # Pick the actual directory to install all modules to: either + # install_purelib or install_platlib, depending on whether this + # module distribution is pure or not. Of course, if the user + # already specified install_lib, use their selection. + if self.install_lib is None: + if self.distribution.ext_modules: # has extensions: non-pure + self.install_lib = self.install_platlib + else: + self.install_lib = self.install_purelib + + # Convert directories from Unix /-separated syntax to the local + # convention. + self.convert_paths('lib', 'purelib', 'platlib', + 'scripts', 'data', 'headers') + if HAS_USER_SITE: + self.convert_paths('userbase', 'usersite') + + # Well, we're not actually fully completely finalized yet: we still + # have to deal with 'extra_path', which is the hack for allowing + # non-packagized module distributions (hello, Numerical Python!) to + # get their own directories. + self.handle_extra_path() + self.install_libbase = self.install_lib # needed for .pth file + self.install_lib = os.path.join(self.install_lib, self.extra_dirs) + + # If a new root directory was supplied, make all the installation + # dirs relative to it. + if self.root is not None: + self.change_roots('libbase', 'lib', 'purelib', 'platlib', + 'scripts', 'data', 'headers') + + self.dump_dirs("after prepending root") + + # Find out the build directories, ie. where to install from. + self.set_undefined_options('build', 'build_base', 'build_lib') + + # Punt on doc directories for now -- after all, we're punting on + # documentation completely! + + if self.no_distinfo is None: + self.no_distinfo = False + + def finalize_unix(self): + """Finalize options for posix platforms.""" + if self.install_base is not None or self.install_platbase is not None: + if ((self.install_lib is None and + self.install_purelib is None and + self.install_platlib is None) or + self.install_headers is None or + self.install_scripts is None or + self.install_data is None): + raise PackagingOptionError( + "install-base or install-platbase supplied, but " + "installation scheme is incomplete") + return + + if HAS_USER_SITE and self.user: + if self.install_userbase is None: + raise PackagingPlatformError( + "user base directory is not specified") + self.install_base = self.install_platbase = self.install_userbase + self.select_scheme("posix_user") + elif self.home is not None: + self.install_base = self.install_platbase = self.home + self.select_scheme("posix_home") + else: + if self.prefix is None: + if self.exec_prefix is not None: + raise PackagingOptionError( + "must not supply exec-prefix without prefix") + + self.prefix = os.path.normpath(sys.prefix) + self.exec_prefix = os.path.normpath(sys.exec_prefix) + + else: + if self.exec_prefix is None: + self.exec_prefix = self.prefix + + self.install_base = self.prefix + self.install_platbase = self.exec_prefix + self.select_scheme("posix_prefix") + + def finalize_other(self): + """Finalize options for non-posix platforms""" + if HAS_USER_SITE and self.user: + if self.install_userbase is None: + raise PackagingPlatformError( + "user base directory is not specified") + self.install_base = self.install_platbase = self.install_userbase + self.select_scheme(os.name + "_user") + elif self.home is not None: + self.install_base = self.install_platbase = self.home + self.select_scheme("posix_home") + else: + if self.prefix is None: + self.prefix = os.path.normpath(sys.prefix) + + self.install_base = self.install_platbase = self.prefix + try: + self.select_scheme(os.name) + except KeyError: + raise PackagingPlatformError( + "no support for installation on '%s'" % os.name) + + def dump_dirs(self, msg): + """Dump the list of user options.""" + logger.debug(msg + ":") + for opt in self.user_options: + opt_name = opt[0] + if opt_name[-1] == "=": + opt_name = opt_name[0:-1] + if opt_name in self.negative_opt: + opt_name = self.negative_opt[opt_name] + opt_name = opt_name.replace('-', '_') + val = not getattr(self, opt_name) + else: + opt_name = opt_name.replace('-', '_') + val = getattr(self, opt_name) + logger.debug(" %s: %s", opt_name, val) + + def select_scheme(self, name): + """Set the install directories by applying the install schemes.""" + # it's the caller's problem if they supply a bad name! + scheme = get_paths(name, expand=False) + for key, value in scheme.items(): + if key == 'platinclude': + key = 'headers' + value = os.path.join(value, self.distribution.metadata['Name']) + attrname = 'install_' + key + if hasattr(self, attrname): + if getattr(self, attrname) is None: + setattr(self, attrname, value) + + def _expand_attrs(self, attrs): + for attr in attrs: + val = getattr(self, attr) + if val is not None: + if os.name == 'posix' or os.name == 'nt': + val = os.path.expanduser(val) + # see if we want to push this work in sysconfig XXX + val = sysconfig._subst_vars(val, self.config_vars) + setattr(self, attr, val) + + def expand_basedirs(self): + """Call `os.path.expanduser` on install_{base,platbase} and root.""" + self._expand_attrs(['install_base', 'install_platbase', 'root']) + + def expand_dirs(self): + """Call `os.path.expanduser` on install dirs.""" + self._expand_attrs(['install_purelib', 'install_platlib', + 'install_lib', 'install_headers', + 'install_scripts', 'install_data']) + + def convert_paths(self, *names): + """Call `convert_path` over `names`.""" + for name in names: + attr = "install_" + name + setattr(self, attr, convert_path(getattr(self, attr))) + + def handle_extra_path(self): + """Set `path_file` and `extra_dirs` using `extra_path`.""" + if self.extra_path is None: + self.extra_path = self.distribution.extra_path + + if self.extra_path is not None: + if isinstance(self.extra_path, str): + self.extra_path = self.extra_path.split(',') + + if len(self.extra_path) == 1: + path_file = extra_dirs = self.extra_path[0] + elif len(self.extra_path) == 2: + path_file, extra_dirs = self.extra_path + else: + raise PackagingOptionError( + "'extra_path' option must be a list, tuple, or " + "comma-separated string with 1 or 2 elements") + + # convert to local form in case Unix notation used (as it + # should be in setup scripts) + extra_dirs = convert_path(extra_dirs) + else: + path_file = None + extra_dirs = '' + + # XXX should we warn if path_file and not extra_dirs? (in which + # case the path file would be harmless but pointless) + self.path_file = path_file + self.extra_dirs = extra_dirs + + def change_roots(self, *names): + """Change the install direcories pointed by name using root.""" + for name in names: + attr = "install_" + name + setattr(self, attr, change_root(self.root, getattr(self, attr))) + + def create_home_path(self): + """Create directories under ~.""" + if HAS_USER_SITE and not self.user: + return + home = convert_path(os.path.expanduser("~")) + for name, path in self.config_vars.items(): + if path.startswith(home) and not os.path.isdir(path): + os.makedirs(path, 0o700) + + # -- Command execution methods ------------------------------------- + + def run(self): + """Runs the command.""" + # Obviously have to build before we can install + if not self.skip_build: + self.run_command('build') + # If we built for any other platform, we can't install. + build_plat = self.distribution.get_command_obj('build').plat_name + # check warn_dir - it is a clue that the 'install_dist' is happening + # internally, and not to sys.path, so we don't check the platform + # matches what we are running. + if self.warn_dir and build_plat != get_platform(): + raise PackagingPlatformError("Can't install when " + "cross-compiling") + + # Run all sub-commands (at least those that need to be run) + for cmd_name in self.get_sub_commands(): + self.run_command(cmd_name) + + if self.path_file: + self.create_path_file() + + # write list of installed files, if requested. + if self.record: + outputs = self.get_outputs() + if self.root: # strip any package prefix + root_len = len(self.root) + for counter in range(len(outputs)): + outputs[counter] = outputs[counter][root_len:] + self.execute(write_file, + (self.record, outputs), + "writing list of installed files to '%s'" % + self.record) + + normpath, normcase = os.path.normpath, os.path.normcase + sys_path = [normcase(normpath(p)) for p in sys.path] + install_lib = normcase(normpath(self.install_lib)) + if (self.warn_dir and + not (self.path_file and self.install_path_file) and + install_lib not in sys_path): + logger.debug(("modules installed to '%s', which is not in " + "Python's module search path (sys.path) -- " + "you'll have to change the search path yourself"), + self.install_lib) + + def create_path_file(self): + """Creates the .pth file""" + filename = os.path.join(self.install_libbase, + self.path_file + ".pth") + if self.install_path_file: + self.execute(write_file, + (filename, [self.extra_dirs]), + "creating %s" % filename) + else: + logger.warning('%s: path file %r not created', + self.get_command_name(), filename) + + # -- Reporting methods --------------------------------------------- + + def get_outputs(self): + """Assembles the outputs of all the sub-commands.""" + outputs = [] + for cmd_name in self.get_sub_commands(): + cmd = self.get_finalized_command(cmd_name) + # Add the contents of cmd.get_outputs(), ensuring + # that outputs doesn't contain duplicate entries + for filename in cmd.get_outputs(): + if filename not in outputs: + outputs.append(filename) + + if self.path_file and self.install_path_file: + outputs.append(os.path.join(self.install_libbase, + self.path_file + ".pth")) + + return outputs + + def get_inputs(self): + """Returns the inputs of all the sub-commands""" + # XXX gee, this looks familiar ;-( + inputs = [] + for cmd_name in self.get_sub_commands(): + cmd = self.get_finalized_command(cmd_name) + inputs.extend(cmd.get_inputs()) + + return inputs + + # -- Predicates for sub-command list ------------------------------- + + def has_lib(self): + """Returns true if the current distribution has any Python + modules to install.""" + return (self.distribution.has_pure_modules() or + self.distribution.has_ext_modules()) + + def has_headers(self): + """Returns true if the current distribution has any headers to + install.""" + return self.distribution.has_headers() + + def has_scripts(self): + """Returns true if the current distribution has any scripts to. + install.""" + return self.distribution.has_scripts() + + def has_data(self): + """Returns true if the current distribution has any data to. + install.""" + return self.distribution.has_data_files() + + # 'sub_commands': a list of commands this command might have to run to + # get its work done. See cmd.py for more info. + sub_commands = [('install_lib', has_lib), + ('install_headers', has_headers), + ('install_scripts', has_scripts), + ('install_data', has_data), + # keep install_distinfo last, as it needs the record + # with files to be completely generated + ('install_distinfo', lambda self: not self.no_distinfo), + ] diff --git a/Lib/packaging/command/install_distinfo.py b/Lib/packaging/command/install_distinfo.py new file mode 100644 index 0000000000..0e1577e7dc --- /dev/null +++ b/Lib/packaging/command/install_distinfo.py @@ -0,0 +1,175 @@ +"""Create the PEP 376-compliant .dist-info directory.""" + +# Forked from the former install_egg_info command by Josip Djolonga + +import csv +import os +import re +import hashlib + +from packaging.command.cmd import Command +from packaging import logger +from shutil import rmtree + + +class install_distinfo(Command): + + description = 'create a .dist-info directory for the distribution' + + user_options = [ + ('distinfo-dir=', None, + "directory where the the .dist-info directory will be installed"), + ('installer=', None, + "the name of the installer"), + ('requested', None, + "generate a REQUESTED file"), + ('no-requested', None, + "do not generate a REQUESTED file"), + ('no-record', None, + "do not generate a RECORD file"), + ('no-resources', None, + "do not generate a RESSOURCES list installed file") + ] + + boolean_options = ['requested', 'no-record', 'no-resources'] + + negative_opt = {'no-requested': 'requested'} + + def initialize_options(self): + self.distinfo_dir = None + self.installer = None + self.requested = None + self.no_record = None + self.no_resources = None + self.outfiles = [] + + def finalize_options(self): + self.set_undefined_options('install_dist', + 'installer', 'requested', 'no_record') + + self.set_undefined_options('install_lib', + ('install_dir', 'distinfo_dir')) + + if self.installer is None: + # FIXME distutils or packaging? + # + document default in the option help text above and in install + self.installer = 'distutils' + if self.requested is None: + self.requested = True + if self.no_record is None: + self.no_record = False + if self.no_resources is None: + self.no_resources = False + + metadata = self.distribution.metadata + + basename = "%s-%s.dist-info" % ( + to_filename(safe_name(metadata['Name'])), + to_filename(safe_version(metadata['Version']))) + + self.distinfo_dir = os.path.join(self.distinfo_dir, basename) + + def run(self): + # FIXME dry-run should be used at a finer level, so that people get + # useful logging output and can have an idea of what the command would + # have done + if not self.dry_run: + target = self.distinfo_dir + + if os.path.isdir(target) and not os.path.islink(target): + rmtree(target) + elif os.path.exists(target): + self.execute(os.unlink, (self.distinfo_dir,), + "removing " + target) + + self.execute(os.makedirs, (target,), "creating " + target) + + metadata_path = os.path.join(self.distinfo_dir, 'METADATA') + logger.info('creating %s', metadata_path) + self.distribution.metadata.write(metadata_path) + self.outfiles.append(metadata_path) + + installer_path = os.path.join(self.distinfo_dir, 'INSTALLER') + logger.info('creating %s', installer_path) + with open(installer_path, 'w') as f: + f.write(self.installer) + self.outfiles.append(installer_path) + + if self.requested: + requested_path = os.path.join(self.distinfo_dir, 'REQUESTED') + logger.info('creating %s', requested_path) + open(requested_path, 'wb').close() + self.outfiles.append(requested_path) + + + if not self.no_resources: + install_data = self.get_finalized_command('install_data') + if install_data.get_resources_out() != []: + resources_path = os.path.join(self.distinfo_dir, + 'RESOURCES') + logger.info('creating %s', resources_path) + with open(resources_path, 'wb') as f: + writer = csv.writer(f, delimiter=',', + lineterminator='\n', + quotechar='"') + for tuple in install_data.get_resources_out(): + writer.writerow(tuple) + + self.outfiles.append(resources_path) + + if not self.no_record: + record_path = os.path.join(self.distinfo_dir, 'RECORD') + logger.info('creating %s', record_path) + with open(record_path, 'w', encoding='utf-8') as f: + writer = csv.writer(f, delimiter=',', + lineterminator='\n', + quotechar='"') + + install = self.get_finalized_command('install_dist') + + for fpath in install.get_outputs(): + if fpath.endswith('.pyc') or fpath.endswith('.pyo'): + # do not put size and md5 hash, as in PEP-376 + writer.writerow((fpath, '', '')) + else: + size = os.path.getsize(fpath) + with open(fpath, 'rb') as fp: + hash = hashlib.md5() + hash.update(fp.read()) + md5sum = hash.hexdigest() + writer.writerow((fpath, md5sum, size)) + + # add the RECORD file itself + writer.writerow((record_path, '', '')) + self.outfiles.append(record_path) + + def get_outputs(self): + return self.outfiles + + +# The following functions are taken from setuptools' pkg_resources module. + +def safe_name(name): + """Convert an arbitrary string to a standard distribution name + + Any runs of non-alphanumeric/. characters are replaced with a single '-'. + """ + return re.sub('[^A-Za-z0-9.]+', '-', name) + + +def safe_version(version): + """Convert an arbitrary string to a standard version string + + Spaces become dots, and all other non-alphanumeric characters become + dashes, with runs of multiple dashes condensed to a single dash. + """ + version = version.replace(' ', '.') + return re.sub('[^A-Za-z0-9.]+', '-', version) + + +def to_filename(name): + """Convert a project or version name to its filename-escaped form + + Any '-' characters are currently replaced with '_'. + """ + return name.replace('-', '_') diff --git a/Lib/packaging/command/install_headers.py b/Lib/packaging/command/install_headers.py new file mode 100644 index 0000000000..e043d6b8ed --- /dev/null +++ b/Lib/packaging/command/install_headers.py @@ -0,0 +1,43 @@ +"""Install C/C++ header files to the Python include directory.""" + +from packaging.command.cmd import Command + + +# XXX force is never used +class install_headers(Command): + + description = "install C/C++ header files" + + user_options = [('install-dir=', 'd', + "directory to install header files to"), + ('force', 'f', + "force installation (overwrite existing files)"), + ] + + boolean_options = ['force'] + + def initialize_options(self): + self.install_dir = None + self.force = False + self.outfiles = [] + + def finalize_options(self): + self.set_undefined_options('install_dist', + ('install_headers', 'install_dir'), + 'force') + + def run(self): + headers = self.distribution.headers + if not headers: + return + + self.mkpath(self.install_dir) + for header in headers: + out = self.copy_file(header, self.install_dir)[0] + self.outfiles.append(out) + + def get_inputs(self): + return self.distribution.headers or [] + + def get_outputs(self): + return self.outfiles diff --git a/Lib/packaging/command/install_lib.py b/Lib/packaging/command/install_lib.py new file mode 100644 index 0000000000..978f0ef8a8 --- /dev/null +++ b/Lib/packaging/command/install_lib.py @@ -0,0 +1,222 @@ +"""Install all modules (extensions and pure Python).""" + +import os +import sys +import logging + +from packaging import logger +from packaging.command.cmd import Command +from packaging.errors import PackagingOptionError + + +# Extension for Python source files. +if hasattr(os, 'extsep'): + PYTHON_SOURCE_EXTENSION = os.extsep + "py" +else: + PYTHON_SOURCE_EXTENSION = ".py" + +class install_lib(Command): + + description = "install all modules (extensions and pure Python)" + + # The byte-compilation options are a tad confusing. Here are the + # possible scenarios: + # 1) no compilation at all (--no-compile --no-optimize) + # 2) compile .pyc only (--compile --no-optimize; default) + # 3) compile .pyc and "level 1" .pyo (--compile --optimize) + # 4) compile "level 1" .pyo only (--no-compile --optimize) + # 5) compile .pyc and "level 2" .pyo (--compile --optimize-more) + # 6) compile "level 2" .pyo only (--no-compile --optimize-more) + # + # The UI for this is two option, 'compile' and 'optimize'. + # 'compile' is strictly boolean, and only decides whether to + # generate .pyc files. 'optimize' is three-way (0, 1, or 2), and + # decides both whether to generate .pyo files and what level of + # optimization to use. + + user_options = [ + ('install-dir=', 'd', "directory to install to"), + ('build-dir=','b', "build directory (where to install from)"), + ('force', 'f', "force installation (overwrite existing files)"), + ('compile', 'c', "compile .py to .pyc [default]"), + ('no-compile', None, "don't compile .py files"), + ('optimize=', 'O', + "also compile with optimization: -O1 for \"python -O\", " + "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), + ('skip-build', None, "skip the build steps"), + ] + + boolean_options = ['force', 'compile', 'skip-build'] + negative_opt = {'no-compile' : 'compile'} + + def initialize_options(self): + # let the 'install_dist' command dictate our installation directory + self.install_dir = None + self.build_dir = None + self.force = False + self.compile = None + self.optimize = None + self.skip_build = None + + def finalize_options(self): + # Get all the information we need to install pure Python modules + # from the umbrella 'install_dist' command -- build (source) directory, + # install (target) directory, and whether to compile .py files. + self.set_undefined_options('install_dist', + ('build_lib', 'build_dir'), + ('install_lib', 'install_dir'), + 'force', 'compile', 'optimize', 'skip_build') + + if self.compile is None: + self.compile = True + if self.optimize is None: + self.optimize = 0 + + if not isinstance(self.optimize, int): + try: + self.optimize = int(self.optimize) + if self.optimize not in (0, 1, 2): + raise AssertionError + except (ValueError, AssertionError): + raise PackagingOptionError("optimize must be 0, 1, or 2") + + def run(self): + # Make sure we have built everything we need first + self.build() + + # Install everything: simply dump the entire contents of the build + # directory to the installation directory (that's the beauty of + # having a build directory!) + outfiles = self.install() + + # (Optionally) compile .py to .pyc + if outfiles is not None and self.distribution.has_pure_modules(): + self.byte_compile(outfiles) + + # -- Top-level worker functions ------------------------------------ + # (called from 'run()') + + def build(self): + if not self.skip_build: + if self.distribution.has_pure_modules(): + self.run_command('build_py') + if self.distribution.has_ext_modules(): + self.run_command('build_ext') + + def install(self): + if os.path.isdir(self.build_dir): + outfiles = self.copy_tree(self.build_dir, self.install_dir) + else: + logger.warning( + '%s: %r does not exist -- no Python modules to install', + self.get_command_name(), self.build_dir) + return + return outfiles + + def byte_compile(self, files): + if getattr(sys, 'dont_write_bytecode'): + # XXX do we want this? because a Python runs without bytecode + # doesn't mean that the *dists should not contain bytecode + #--or does it? + logger.warning('%s: byte-compiling is disabled, skipping.', + self.get_command_name()) + return + + from packaging.util import byte_compile # FIXME use compileall + + # Get the "--root" directory supplied to the "install_dist" command, + # and use it as a prefix to strip off the purported filename + # encoded in bytecode files. This is far from complete, but it + # should at least generate usable bytecode in RPM distributions. + install_root = self.get_finalized_command('install_dist').root + + # Temporary kludge until we remove the verbose arguments and use + # logging everywhere + verbose = logger.getEffectiveLevel() >= logging.DEBUG + + if self.compile: + byte_compile(files, optimize=0, + force=self.force, prefix=install_root, + dry_run=self.dry_run) + if self.optimize > 0: + byte_compile(files, optimize=self.optimize, + force=self.force, prefix=install_root, + verbose=verbose, + dry_run=self.dry_run) + + + # -- Utility methods ----------------------------------------------- + + def _mutate_outputs(self, has_any, build_cmd, cmd_option, output_dir): + if not has_any: + return [] + + build_cmd = self.get_finalized_command(build_cmd) + build_files = build_cmd.get_outputs() + build_dir = getattr(build_cmd, cmd_option) + + prefix_len = len(build_dir) + len(os.sep) + outputs = [] + for file in build_files: + outputs.append(os.path.join(output_dir, file[prefix_len:])) + + return outputs + + def _bytecode_filenames(self, py_filenames): + bytecode_files = [] + for py_file in py_filenames: + # Since build_py handles package data installation, the + # list of outputs can contain more than just .py files. + # Make sure we only report bytecode for the .py files. + ext = os.path.splitext(os.path.normcase(py_file))[1] + if ext != PYTHON_SOURCE_EXTENSION: + continue + if self.compile: + bytecode_files.append(py_file + "c") + if self.optimize > 0: + bytecode_files.append(py_file + "o") + + return bytecode_files + + + # -- External interface -------------------------------------------- + # (called by outsiders) + + def get_outputs(self): + """Return the list of files that would be installed if this command + were actually run. Not affected by the "dry-run" flag or whether + modules have actually been built yet. + """ + pure_outputs = \ + self._mutate_outputs(self.distribution.has_pure_modules(), + 'build_py', 'build_lib', + self.install_dir) + if self.compile: + bytecode_outputs = self._bytecode_filenames(pure_outputs) + else: + bytecode_outputs = [] + + ext_outputs = \ + self._mutate_outputs(self.distribution.has_ext_modules(), + 'build_ext', 'build_lib', + self.install_dir) + + return pure_outputs + bytecode_outputs + ext_outputs + + def get_inputs(self): + """Get the list of files that are input to this command, ie. the + files that get installed as they are named in the build tree. + The files in this list correspond one-to-one to the output + filenames returned by 'get_outputs()'. + """ + inputs = [] + + if self.distribution.has_pure_modules(): + build_py = self.get_finalized_command('build_py') + inputs.extend(build_py.get_outputs()) + + if self.distribution.has_ext_modules(): + build_ext = self.get_finalized_command('build_ext') + inputs.extend(build_ext.get_outputs()) + + return inputs diff --git a/Lib/packaging/command/install_scripts.py b/Lib/packaging/command/install_scripts.py new file mode 100644 index 0000000000..cfacbe25fb --- /dev/null +++ b/Lib/packaging/command/install_scripts.py @@ -0,0 +1,59 @@ +"""Install scripts.""" + +# Contributed by Bastian Kleineidam + +import os +from packaging.command.cmd import Command +from packaging import logger + +class install_scripts(Command): + + description = "install scripts (Python or otherwise)" + + user_options = [ + ('install-dir=', 'd', "directory to install scripts to"), + ('build-dir=','b', "build directory (where to install from)"), + ('force', 'f', "force installation (overwrite existing files)"), + ('skip-build', None, "skip the build steps"), + ] + + boolean_options = ['force', 'skip-build'] + + + def initialize_options(self): + self.install_dir = None + self.force = False + self.build_dir = None + self.skip_build = None + + def finalize_options(self): + self.set_undefined_options('build', ('build_scripts', 'build_dir')) + self.set_undefined_options('install_dist', + ('install_scripts', 'install_dir'), + 'force', 'skip_build') + + def run(self): + if not self.skip_build: + self.run_command('build_scripts') + + if not os.path.exists(self.build_dir): + self.outfiles = [] + return + + self.outfiles = self.copy_tree(self.build_dir, self.install_dir) + if os.name == 'posix': + # Set the executable bits (owner, group, and world) on + # all the scripts we just installed. + for file in self.get_outputs(): + if self.dry_run: + logger.info("changing mode of %s", file) + else: + mode = (os.stat(file).st_mode | 0o555) & 0o7777 + logger.info("changing mode of %s to %o", file, mode) + os.chmod(file, mode) + + def get_inputs(self): + return self.distribution.scripts or [] + + def get_outputs(self): + return self.outfiles or [] diff --git a/Lib/packaging/command/register.py b/Lib/packaging/command/register.py new file mode 100644 index 0000000000..67cda80c8b --- /dev/null +++ b/Lib/packaging/command/register.py @@ -0,0 +1,263 @@ +"""Register a release with a project index.""" + +# Contributed by Richard Jones + +import getpass +import urllib.error +import urllib.parse +import urllib.request + +from packaging import logger +from packaging.util import (read_pypirc, generate_pypirc, DEFAULT_REPOSITORY, + DEFAULT_REALM, get_pypirc_path, encode_multipart) +from packaging.command.cmd import Command + +class register(Command): + + description = "register a release with PyPI" + user_options = [ + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), + ('show-response', None, + "display full response text from server"), + ('list-classifiers', None, + "list valid Trove classifiers"), + ('strict', None , + "stop the registration if the metadata is not fully compliant") + ] + + boolean_options = ['show-response', 'list-classifiers', 'strict'] + + def initialize_options(self): + self.repository = None + self.realm = None + self.show_response = False + self.list_classifiers = False + self.strict = False + + def finalize_options(self): + if self.repository is None: + self.repository = DEFAULT_REPOSITORY + if self.realm is None: + self.realm = DEFAULT_REALM + + def run(self): + self._set_config() + + # Check the package metadata + check = self.distribution.get_command_obj('check') + if check.strict != self.strict and not check.all: + # If check was already run but with different options, + # re-run it + check.strict = self.strict + check.all = True + self.distribution.have_run.pop('check', None) + self.run_command('check') + + if self.dry_run: + self.verify_metadata() + elif self.list_classifiers: + self.classifiers() + else: + self.send_metadata() + + def _set_config(self): + ''' Reads the configuration file and set attributes. + ''' + config = read_pypirc(self.repository, self.realm) + if config != {}: + self.username = config['username'] + self.password = config['password'] + self.repository = config['repository'] + self.realm = config['realm'] + self.has_config = True + else: + if self.repository not in ('pypi', DEFAULT_REPOSITORY): + raise ValueError('%s not found in .pypirc' % self.repository) + if self.repository == 'pypi': + self.repository = DEFAULT_REPOSITORY + self.has_config = False + + def classifiers(self): + ''' Fetch the list of classifiers from the server. + ''' + response = urllib.request.urlopen(self.repository+'?:action=list_classifiers') + logger.info(response.read()) + + def verify_metadata(self): + ''' Send the metadata to the package index server to be checked. + ''' + # send the info to the server and report the result + code, result = self.post_to_server(self.build_post_data('verify')) + logger.info('server response (%s): %s', code, result) + + + def send_metadata(self): + ''' Send the metadata to the package index server. + + Well, do the following: + 1. figure who the user is, and then + 2. send the data as a Basic auth'ed POST. + + First we try to read the username/password from $HOME/.pypirc, + which is a ConfigParser-formatted file with a section + [distutils] containing username and password entries (both + in clear text). Eg: + + [distutils] + index-servers = + pypi + + [pypi] + username: fred + password: sekrit + + Otherwise, to figure who the user is, we offer the user three + choices: + + 1. use existing login, + 2. register as a new user, or + 3. set the password to a random string and email the user. + + ''' + # TODO factor registration out into another method + # TODO use print to print, not logging + + # see if we can short-cut and get the username/password from the + # config + if self.has_config: + choice = '1' + username = self.username + password = self.password + else: + choice = 'x' + username = password = '' + + # get the user's login info + choices = '1 2 3 4'.split() + while choice not in choices: + logger.info('''\ +We need to know who you are, so please choose either: + 1. use your existing login, + 2. register as a new user, + 3. have the server generate a new password for you (and email it to you), or + 4. quit +Your selection [default 1]: ''') + + choice = input() + if not choice: + choice = '1' + elif choice not in choices: + print('Please choose one of the four options!') + + if choice == '1': + # get the username and password + while not username: + username = input('Username: ') + while not password: + password = getpass.getpass('Password: ') + + # set up the authentication + auth = urllib.request.HTTPPasswordMgr() + host = urllib.parse.urlparse(self.repository)[1] + auth.add_password(self.realm, host, username, password) + # send the info to the server and report the result + code, result = self.post_to_server(self.build_post_data('submit'), + auth) + logger.info('Server response (%s): %s', code, result) + + # possibly save the login + if code == 200: + if self.has_config: + # sharing the password in the distribution instance + # so the upload command can reuse it + self.distribution.password = password + else: + logger.info( + 'I can store your PyPI login so future submissions ' + 'will be faster.\n(the login will be stored in %s)', + get_pypirc_path()) + choice = 'X' + while choice.lower() not in 'yn': + choice = input('Save your login (y/N)?') + if not choice: + choice = 'n' + if choice.lower() == 'y': + generate_pypirc(username, password) + + elif choice == '2': + data = {':action': 'user'} + data['name'] = data['password'] = data['email'] = '' + data['confirm'] = None + while not data['name']: + data['name'] = input('Username: ') + while data['password'] != data['confirm']: + while not data['password']: + data['password'] = getpass.getpass('Password: ') + while not data['confirm']: + data['confirm'] = getpass.getpass(' Confirm: ') + if data['password'] != data['confirm']: + data['password'] = '' + data['confirm'] = None + print("Password and confirm don't match!") + while not data['email']: + data['email'] = input(' EMail: ') + code, result = self.post_to_server(data) + if code != 200: + logger.info('server response (%s): %s', code, result) + else: + logger.info('you will receive an email shortly; follow the ' + 'instructions in it to complete registration.') + elif choice == '3': + data = {':action': 'password_reset'} + data['email'] = '' + while not data['email']: + data['email'] = input('Your email address: ') + code, result = self.post_to_server(data) + logger.info('server response (%s): %s', code, result) + + def build_post_data(self, action): + # figure the data to send - the metadata plus some additional + # information used by the package server + data = self.distribution.metadata.todict() + data[':action'] = action + return data + + # XXX to be refactored with upload.upload_file + def post_to_server(self, data, auth=None): + ''' Post a query to the server, and return a string response. + ''' + if 'name' in data: + logger.info('Registering %s to %s', data['name'], self.repository) + # Build up the MIME payload for the urllib2 POST data + content_type, body = encode_multipart(data.items(), []) + + # build the Request + headers = { + 'Content-type': content_type, + 'Content-length': str(len(body)) + } + req = urllib.request.Request(self.repository, body, headers) + + # handle HTTP and include the Basic Auth handler + opener = urllib.request.build_opener( + urllib.request.HTTPBasicAuthHandler(password_mgr=auth) + ) + data = '' + try: + result = opener.open(req) + except urllib.error.HTTPError as e: + if self.show_response: + data = e.fp.read() + result = e.code, e.msg + except urllib.error.URLError as e: + result = 500, str(e) + else: + if self.show_response: + data = result.read() + result = 200, 'OK' + if self.show_response: + dashes = '-' * 75 + logger.info('%s%s%s', dashes, data, dashes) + + return result diff --git a/Lib/packaging/command/sdist.py b/Lib/packaging/command/sdist.py new file mode 100644 index 0000000000..09b26db2d8 --- /dev/null +++ b/Lib/packaging/command/sdist.py @@ -0,0 +1,348 @@ +"""Create a source distribution.""" + +import os +import re +import sys +from io import StringIO +from shutil import get_archive_formats, rmtree + +from packaging import logger +from packaging.util import resolve_name +from packaging.errors import (PackagingPlatformError, PackagingOptionError, + PackagingModuleError, PackagingFileError) +from packaging.command import get_command_names +from packaging.command.cmd import Command +from packaging.manifest import Manifest + + +def show_formats(): + """Print all possible values for the 'formats' option (used by + the "--help-formats" command-line option). + """ + from packaging.fancy_getopt import FancyGetopt + formats = sorted(('formats=' + name, None, desc) + for name, desc in get_archive_formats()) + FancyGetopt(formats).print_help( + "List of available source distribution formats:") + +# a \ followed by some spaces + EOL +_COLLAPSE_PATTERN = re.compile('\\\w\n', re.M) +_COMMENTED_LINE = re.compile('^#.*\n$|^\w*\n$', re.M) + + +class sdist(Command): + + description = "create a source distribution (tarball, zip file, etc.)" + + user_options = [ + ('manifest=', 'm', + "name of manifest file [default: MANIFEST]"), + ('use-defaults', None, + "include the default file set in the manifest " + "[default; disable with --no-defaults]"), + ('no-defaults', None, + "don't include the default file set"), + ('prune', None, + "specifically exclude files/directories that should not be " + "distributed (build tree, RCS/CVS dirs, etc.) " + "[default; disable with --no-prune]"), + ('no-prune', None, + "don't automatically exclude anything"), + ('manifest-only', 'o', + "just regenerate the manifest and then stop "), + ('formats=', None, + "formats for source distribution (comma-separated list)"), + ('keep-temp', 'k', + "keep the distribution tree around after creating " + + "archive file(s)"), + ('dist-dir=', 'd', + "directory to put the source distribution archive(s) in " + "[default: dist]"), + ('check-metadata', None, + "Ensure that all required elements of metadata " + "are supplied. Warn if any missing. [default]"), + ('owner=', 'u', + "Owner name used when creating a tar file [default: current user]"), + ('group=', 'g', + "Group name used when creating a tar file [default: current group]"), + ('manifest-builders=', None, + "manifest builders (comma-separated list)"), + ] + + boolean_options = ['use-defaults', 'prune', + 'manifest-only', 'keep-temp', 'check-metadata'] + + help_options = [ + ('help-formats', None, + "list available distribution formats", show_formats), + ] + + negative_opt = {'no-defaults': 'use-defaults', + 'no-prune': 'prune'} + + default_format = {'posix': 'gztar', + 'nt': 'zip'} + + def initialize_options(self): + self.manifest = None + # 'use_defaults': if true, we will include the default file set + # in the manifest + self.use_defaults = True + self.prune = True + self.manifest_only = False + self.formats = None + self.keep_temp = False + self.dist_dir = None + + self.archive_files = None + self.metadata_check = True + self.owner = None + self.group = None + self.filelist = None + self.manifest_builders = None + + def _check_archive_formats(self, formats): + supported_formats = [name for name, desc in get_archive_formats()] + for format in formats: + if format not in supported_formats: + return format + return None + + def finalize_options(self): + if self.manifest is None: + self.manifest = "MANIFEST" + + self.ensure_string_list('formats') + if self.formats is None: + try: + self.formats = [self.default_format[os.name]] + except KeyError: + raise PackagingPlatformError("don't know how to create source " + "distributions on platform %s" % os.name) + + bad_format = self._check_archive_formats(self.formats) + if bad_format: + raise PackagingOptionError("unknown archive format '%s'" \ + % bad_format) + + if self.dist_dir is None: + self.dist_dir = "dist" + + if self.filelist is None: + self.filelist = Manifest() + + if self.manifest_builders is None: + self.manifest_builders = [] + else: + if isinstance(self.manifest_builders, str): + self.manifest_builders = self.manifest_builders.split(',') + builders = [] + for builder in self.manifest_builders: + builder = builder.strip() + if builder == '': + continue + try: + builder = resolve_name(builder) + except ImportError as e: + raise PackagingModuleError(e) + + builders.append(builder) + + self.manifest_builders = builders + + def run(self): + # 'filelist' contains the list of files that will make up the + # manifest + self.filelist.clear() + + # Check the package metadata + if self.metadata_check: + self.run_command('check') + + # Do whatever it takes to get the list of files to process + # (process the manifest template, read an existing manifest, + # whatever). File list is accumulated in 'self.filelist'. + self.get_file_list() + + # If user just wanted us to regenerate the manifest, stop now. + if self.manifest_only: + return + + # Otherwise, go ahead and create the source distribution tarball, + # or zipfile, or whatever. + self.make_distribution() + + def get_file_list(self): + """Figure out the list of files to include in the source + distribution, and put it in 'self.filelist'. This might involve + reading the manifest template (and writing the manifest), or just + reading the manifest, or just using the default file set -- it all + depends on the user's options. + """ + template_exists = len(self.distribution.extra_files) > 0 + if not template_exists: + logger.warning('%s: using default file list', + self.get_command_name()) + self.filelist.findall() + + if self.use_defaults: + self.add_defaults() + if template_exists: + template = '\n'.join(self.distribution.extra_files) + self.filelist.read_template(StringIO(template)) + + # call manifest builders, if any. + for builder in self.manifest_builders: + builder(self.distribution, self.filelist) + + if self.prune: + self.prune_file_list() + + self.filelist.write(self.manifest) + + def add_defaults(self): + """Add all default files to self.filelist. + + In addition to the setup.cfg file, this will include all files returned + by the get_source_files of every registered command. This will find + Python modules and packages, data files listed in package_data_, + data_files and extra_files, scripts, C sources of extension modules or + C libraries (headers are missing). + """ + if os.path.exists('setup.cfg'): + self.filelist.append('setup.cfg') + else: + logger.warning("%s: standard 'setup.cfg' file not found", + self.get_command_name()) + + for cmd_name in get_command_names(): + try: + cmd_obj = self.get_finalized_command(cmd_name) + except PackagingOptionError: + pass + else: + self.filelist.extend(cmd_obj.get_source_files()) + + def prune_file_list(self): + """Prune off branches that might slip into the file list as created + by 'read_template()', but really don't belong there: + * the build tree (typically "build") + * the release tree itself (only an issue if we ran "sdist" + previously with --keep-temp, or it aborted) + * any RCS, CVS, .svn, .hg, .git, .bzr, _darcs directories + """ + build = self.get_finalized_command('build') + base_dir = self.distribution.get_fullname() + + self.filelist.exclude_pattern(None, prefix=build.build_base) + self.filelist.exclude_pattern(None, prefix=base_dir) + + # pruning out vcs directories + # both separators are used under win32 + if sys.platform == 'win32': + seps = r'/|\\' + else: + seps = '/' + + vcs_dirs = ['RCS', 'CVS', r'\.svn', r'\.hg', r'\.git', r'\.bzr', + '_darcs'] + vcs_ptrn = r'(^|%s)(%s)(%s).*' % (seps, '|'.join(vcs_dirs), seps) + self.filelist.exclude_pattern(vcs_ptrn, is_regex=True) + + def make_release_tree(self, base_dir, files): + """Create the directory tree that will become the source + distribution archive. All directories implied by the filenames in + 'files' are created under 'base_dir', and then we hard link or copy + (if hard linking is unavailable) those files into place. + Essentially, this duplicates the developer's source tree, but in a + directory named after the distribution, containing only the files + to be distributed. + """ + # Create all the directories under 'base_dir' necessary to + # put 'files' there; the 'mkpath()' is just so we don't die + # if the manifest happens to be empty. + self.mkpath(base_dir) + self.create_tree(base_dir, files, dry_run=self.dry_run) + + # And walk over the list of files, either making a hard link (if + # os.link exists) to each one that doesn't already exist in its + # corresponding location under 'base_dir', or copying each file + # that's out-of-date in 'base_dir'. (Usually, all files will be + # out-of-date, because by default we blow away 'base_dir' when + # we're done making the distribution archives.) + + if hasattr(os, 'link'): # can make hard links on this system + link = 'hard' + msg = "making hard links in %s..." % base_dir + else: # nope, have to copy + link = None + msg = "copying files to %s..." % base_dir + + if not files: + logger.warning("no files to distribute -- empty manifest?") + else: + logger.info(msg) + + for file in self.distribution.metadata.requires_files: + if file not in files: + msg = "'%s' must be included explicitly in 'extra_files'" \ + % file + raise PackagingFileError(msg) + + for file in files: + if not os.path.isfile(file): + logger.warning("'%s' not a regular file -- skipping", file) + else: + dest = os.path.join(base_dir, file) + self.copy_file(file, dest, link=link) + + self.distribution.metadata.write(os.path.join(base_dir, 'PKG-INFO')) + + def make_distribution(self): + """Create the source distribution(s). First, we create the release + tree with 'make_release_tree()'; then, we create all required + archive files (according to 'self.formats') from the release tree. + Finally, we clean up by blowing away the release tree (unless + 'self.keep_temp' is true). The list of archive files created is + stored so it can be retrieved later by 'get_archive_files()'. + """ + # Don't warn about missing metadata here -- should be (and is!) + # done elsewhere. + base_dir = self.distribution.get_fullname() + base_name = os.path.join(self.dist_dir, base_dir) + + self.make_release_tree(base_dir, self.filelist.files) + archive_files = [] # remember names of files we create + # tar archive must be created last to avoid overwrite and remove + if 'tar' in self.formats: + self.formats.append(self.formats.pop(self.formats.index('tar'))) + + for fmt in self.formats: + file = self.make_archive(base_name, fmt, base_dir=base_dir, + owner=self.owner, group=self.group) + archive_files.append(file) + self.distribution.dist_files.append(('sdist', '', file)) + + self.archive_files = archive_files + + if not self.keep_temp: + if self.dry_run: + logger.info('removing %s', base_dir) + else: + rmtree(base_dir) + + def get_archive_files(self): + """Return the list of archive files created when the command + was run, or None if the command hasn't run yet. + """ + return self.archive_files + + def create_tree(self, base_dir, files, mode=0o777, verbose=1, + dry_run=False): + need_dir = set() + for file in files: + need_dir.add(os.path.join(base_dir, os.path.dirname(file))) + + # Now create them + for dir in sorted(need_dir): + self.mkpath(dir, mode, verbose=verbose, dry_run=dry_run) diff --git a/Lib/packaging/command/test.py b/Lib/packaging/command/test.py new file mode 100644 index 0000000000..7f9015bc8e --- /dev/null +++ b/Lib/packaging/command/test.py @@ -0,0 +1,81 @@ +"""Run the project's test suite.""" + +import os +import sys +import logging +import unittest + +from packaging import logger +from packaging.command.cmd import Command +from packaging.database import get_distribution +from packaging.errors import PackagingOptionError +from packaging.util import resolve_name + + +class test(Command): + + description = "run the project's test suite" + + user_options = [ + ('suite=', 's', + "test suite to run (for example: 'some_module.test_suite')"), + ('runner=', None, + "test runner to be called."), + ('tests-require=', None, + "list of distributions required to run the test suite."), + ] + + def initialize_options(self): + self.suite = None + self.runner = None + self.tests_require = [] + + def finalize_options(self): + self.build_lib = self.get_finalized_command("build").build_lib + for requirement in self.tests_require: + if get_distribution(requirement) is None: + logger.warning("test dependency %s is not installed, " + "tests may fail", requirement) + if (not self.suite and not self.runner and + self.get_ut_with_discovery() is None): + raise PackagingOptionError( + "no test discovery available, please give a 'suite' or " + "'runner' option or install unittest2") + + def get_ut_with_discovery(self): + if hasattr(unittest.TestLoader, "discover"): + return unittest + else: + try: + import unittest2 + return unittest2 + except ImportError: + return None + + def run(self): + prev_syspath = sys.path[:] + try: + # build release + build = self.get_reinitialized_command('build') + self.run_command('build') + sys.path.insert(0, build.build_lib) + + # Temporary kludge until we remove the verbose arguments and use + # logging everywhere + logger = logging.getLogger('packaging') + verbose = logger.getEffectiveLevel() >= logging.DEBUG + verbosity = verbose + 1 + + # run the tests + if self.runner: + resolve_name(self.runner)() + elif self.suite: + runner = unittest.TextTestRunner(verbosity=verbosity) + runner.run(resolve_name(self.suite)()) + elif self.get_ut_with_discovery(): + ut = self.get_ut_with_discovery() + test_suite = ut.TestLoader().discover(os.curdir) + runner = ut.TextTestRunner(verbosity=verbosity) + runner.run(test_suite) + finally: + sys.path[:] = prev_syspath diff --git a/Lib/packaging/command/upload.py b/Lib/packaging/command/upload.py new file mode 100644 index 0000000000..f56d2c69fe --- /dev/null +++ b/Lib/packaging/command/upload.py @@ -0,0 +1,168 @@ +"""Upload a distribution to a project index.""" + +import os +import socket +import logging +import platform +import urllib.parse +from base64 import standard_b64encode +from hashlib import md5 +from urllib.error import HTTPError +from urllib.request import urlopen, Request + +from packaging import logger +from packaging.errors import PackagingOptionError +from packaging.util import (spawn, read_pypirc, DEFAULT_REPOSITORY, + DEFAULT_REALM, encode_multipart) +from packaging.command.cmd import Command + + +class upload(Command): + + description = "upload distribution to PyPI" + + user_options = [ + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), + ('show-response', None, + "display full response text from server"), + ('sign', 's', + "sign files to upload using gpg"), + ('identity=', 'i', + "GPG identity used to sign files"), + ('upload-docs', None, + "upload documentation too"), + ] + + boolean_options = ['show-response', 'sign'] + + def initialize_options(self): + self.repository = None + self.realm = None + self.show_response = False + self.username = '' + self.password = '' + self.show_response = False + self.sign = False + self.identity = None + self.upload_docs = False + + def finalize_options(self): + if self.repository is None: + self.repository = DEFAULT_REPOSITORY + if self.realm is None: + self.realm = DEFAULT_REALM + if self.identity and not self.sign: + raise PackagingOptionError( + "Must use --sign for --identity to have meaning") + config = read_pypirc(self.repository, self.realm) + if config != {}: + self.username = config['username'] + self.password = config['password'] + self.repository = config['repository'] + self.realm = config['realm'] + + # getting the password from the distribution + # if previously set by the register command + if not self.password and self.distribution.password: + self.password = self.distribution.password + + def run(self): + if not self.distribution.dist_files: + raise PackagingOptionError( + "No dist file created in earlier command") + for command, pyversion, filename in self.distribution.dist_files: + self.upload_file(command, pyversion, filename) + if self.upload_docs: + upload_docs = self.get_finalized_command("upload_docs") + upload_docs.repository = self.repository + upload_docs.username = self.username + upload_docs.password = self.password + upload_docs.run() + + # XXX to be refactored with register.post_to_server + def upload_file(self, command, pyversion, filename): + # Makes sure the repository URL is compliant + scheme, netloc, url, params, query, fragments = \ + urllib.parse.urlparse(self.repository) + if params or query or fragments: + raise AssertionError("Incompatible url %s" % self.repository) + + if scheme not in ('http', 'https'): + raise AssertionError("unsupported scheme " + scheme) + + # Sign if requested + if self.sign: + gpg_args = ["gpg", "--detach-sign", "-a", filename] + if self.identity: + gpg_args[2:2] = ["--local-user", self.identity] + spawn(gpg_args, + dry_run=self.dry_run) + + # Fill in the data - send all the metadata in case we need to + # register a new release + with open(filename, 'rb') as f: + content = f.read() + + data = self.distribution.metadata.todict() + + # extra upload infos + data[':action'] = 'file_upload' + data['protcol_version'] = '1' + data['content'] = (os.path.basename(filename), content) + data['filetype'] = command + data['pyversion'] = pyversion + data['md5_digest'] = md5(content).hexdigest() + + if command == 'bdist_dumb': + data['comment'] = 'built for %s' % platform.platform(terse=True) + + if self.sign: + with open(filename + '.asc') as fp: + sig = fp.read() + data['gpg_signature'] = [ + (os.path.basename(filename) + ".asc", sig)] + + # set up the authentication + # The exact encoding of the authentication string is debated. + # Anyway PyPI only accepts ascii for both username or password. + user_pass = (self.username + ":" + self.password).encode('ascii') + auth = b"Basic " + standard_b64encode(user_pass) + + # Build up the MIME payload for the POST data + files = [] + for key in ('content', 'gpg_signature'): + if key in data: + filename_, value = data.pop(key) + files.append((key, filename_, value)) + + content_type, body = encode_multipart(data.items(), files) + + logger.info("Submitting %s to %s", filename, self.repository) + + # build the Request + headers = {'Content-type': content_type, + 'Content-length': str(len(body)), + 'Authorization': auth} + + request = Request(self.repository, body, headers) + # send the data + try: + result = urlopen(request) + status = result.code + reason = result.msg + except socket.error as e: + logger.error(e) + return + except HTTPError as e: + status = e.code + reason = e.msg + + if status == 200: + logger.info('Server response (%s): %s', status, reason) + else: + logger.error('Upload failed (%s): %s', status, reason) + + if self.show_response and logger.isEnabledFor(logging.INFO): + sep = '-' * 75 + logger.info('%s\n%s\n%s', sep, result.read().decode(), sep) diff --git a/Lib/packaging/command/upload_docs.py b/Lib/packaging/command/upload_docs.py new file mode 100644 index 0000000000..03dd3ec374 --- /dev/null +++ b/Lib/packaging/command/upload_docs.py @@ -0,0 +1,130 @@ +"""Upload HTML documentation to a project index.""" + +import os +import base64 +import socket +import zipfile +import logging +import http.client +import urllib.parse +from io import BytesIO + +from packaging import logger +from packaging.util import (read_pypirc, DEFAULT_REPOSITORY, DEFAULT_REALM, + encode_multipart) +from packaging.errors import PackagingFileError +from packaging.command.cmd import Command + + +def zip_dir(directory): + """Compresses recursively contents of directory into a BytesIO object""" + destination = BytesIO() + with zipfile.ZipFile(destination, "w") as zip_file: + for root, dirs, files in os.walk(directory): + for name in files: + full = os.path.join(root, name) + relative = root[len(directory):].lstrip(os.path.sep) + dest = os.path.join(relative, name) + zip_file.write(full, dest) + return destination + + +class upload_docs(Command): + + description = "upload HTML documentation to PyPI" + + user_options = [ + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), + ('show-response', None, + "display full response text from server"), + ('upload-dir=', None, + "directory to upload"), + ] + + def initialize_options(self): + self.repository = None + self.realm = None + self.show_response = False + self.upload_dir = None + self.username = '' + self.password = '' + + def finalize_options(self): + if self.repository is None: + self.repository = DEFAULT_REPOSITORY + if self.realm is None: + self.realm = DEFAULT_REALM + if self.upload_dir is None: + build = self.get_finalized_command('build') + self.upload_dir = os.path.join(build.build_base, "docs") + if not os.path.isdir(self.upload_dir): + self.upload_dir = os.path.join(build.build_base, "doc") + logger.info('Using upload directory %s', self.upload_dir) + self.verify_upload_dir(self.upload_dir) + config = read_pypirc(self.repository, self.realm) + if config != {}: + self.username = config['username'] + self.password = config['password'] + self.repository = config['repository'] + self.realm = config['realm'] + + def verify_upload_dir(self, upload_dir): + self.ensure_dirname('upload_dir') + index_location = os.path.join(upload_dir, "index.html") + if not os.path.exists(index_location): + mesg = "No 'index.html found in docs directory (%s)" + raise PackagingFileError(mesg % upload_dir) + + def run(self): + name = self.distribution.metadata['Name'] + version = self.distribution.metadata['Version'] + zip_file = zip_dir(self.upload_dir) + + fields = [(':action', 'doc_upload'), + ('name', name), ('version', version)] + files = [('content', name, zip_file.getvalue())] + content_type, body = encode_multipart(fields, files) + + credentials = self.username + ':' + self.password + auth = b"Basic " + base64.encodebytes(credentials.encode()).strip() + + logger.info("Submitting documentation to %s", self.repository) + + scheme, netloc, url, params, query, fragments = urllib.parse.urlparse( + self.repository) + if scheme == "http": + conn = http.client.HTTPConnection(netloc) + elif scheme == "https": + conn = http.client.HTTPSConnection(netloc) + else: + raise AssertionError("unsupported scheme %r" % scheme) + + try: + conn.connect() + conn.putrequest("POST", url) + conn.putheader('Content-type', content_type) + conn.putheader('Content-length', str(len(body))) + conn.putheader('Authorization', auth) + conn.endheaders() + conn.send(body) + + except socket.error as e: + logger.error(e) + return + + r = conn.getresponse() + + if r.status == 200: + logger.info('Server response (%s): %s', r.status, r.reason) + elif r.status == 301: + location = r.getheader('Location') + if location is None: + location = 'http://packages.python.org/%s/' % name + logger.info('Upload successful. Visit %s', location) + else: + logger.error('Upload failed (%s): %s', r.status, r.reason) + + if self.show_response and logger.isEnabledFor(logging.INFO): + sep = '-' * 75 + logger.info('%s\n%s\n%s', sep, r.read().decode('utf-8'), sep) diff --git a/Lib/packaging/command/wininst-10.0-amd64.exe b/Lib/packaging/command/wininst-10.0-amd64.exe Binary files differnew file mode 100644 index 0000000000..11f98cd2ad --- /dev/null +++ b/Lib/packaging/command/wininst-10.0-amd64.exe diff --git a/Lib/packaging/command/wininst-10.0.exe b/Lib/packaging/command/wininst-10.0.exe Binary files differnew file mode 100644 index 0000000000..8ac6e19b8e --- /dev/null +++ b/Lib/packaging/command/wininst-10.0.exe diff --git a/Lib/packaging/command/wininst-6.0.exe b/Lib/packaging/command/wininst-6.0.exe Binary files differnew file mode 100644 index 0000000000..f57c855a61 --- /dev/null +++ b/Lib/packaging/command/wininst-6.0.exe diff --git a/Lib/packaging/command/wininst-7.1.exe b/Lib/packaging/command/wininst-7.1.exe Binary files differnew file mode 100644 index 0000000000..1433bc1ad3 --- /dev/null +++ b/Lib/packaging/command/wininst-7.1.exe diff --git a/Lib/packaging/command/wininst-8.0.exe b/Lib/packaging/command/wininst-8.0.exe Binary files differnew file mode 100644 index 0000000000..7403bfabf5 --- /dev/null +++ b/Lib/packaging/command/wininst-8.0.exe diff --git a/Lib/packaging/command/wininst-9.0-amd64.exe b/Lib/packaging/command/wininst-9.0-amd64.exe Binary files differnew file mode 100644 index 0000000000..11d8011c71 --- /dev/null +++ b/Lib/packaging/command/wininst-9.0-amd64.exe diff --git a/Lib/packaging/command/wininst-9.0.exe b/Lib/packaging/command/wininst-9.0.exe Binary files differnew file mode 100644 index 0000000000..dadb31d893 --- /dev/null +++ b/Lib/packaging/command/wininst-9.0.exe diff --git a/Lib/packaging/compat.py b/Lib/packaging/compat.py new file mode 100644 index 0000000000..a82efd3691 --- /dev/null +++ b/Lib/packaging/compat.py @@ -0,0 +1,57 @@ +"""Compatibility helpers. + +This module provides classes, variables and imports which are used to +support packaging across Python 2.x and 3.x. +""" + +from packaging import logger + + +# XXX Having two classes with the same name is not a good thing. +# XXX 2to3-related code should move from util to this module + +# TODO Move common code here: PY3 (bool indicating if we're on 3.x), any, etc. + +try: + from packaging.util import Mixin2to3 as _Mixin2to3 + _CONVERT = True + _KLASS = _Mixin2to3 +except ImportError: + _CONVERT = False + _KLASS = object + +__all__ = ['Mixin2to3'] + + +class Mixin2to3(_KLASS): + """ The base class which can be used for refactoring. When run under + Python 3.0, the run_2to3 method provided by Mixin2to3 is overridden. + When run on Python 2.x, it merely creates a class which overrides run_2to3, + yet does nothing in particular with it. + """ + if _CONVERT: + + def _run_2to3(self, files, doctests=[], fixers=[]): + """ Takes a list of files and doctests, and performs conversion + on those. + - First, the files which contain the code(`files`) are converted. + - Second, the doctests in `files` are converted. + - Thirdly, the doctests in `doctests` are converted. + """ + if fixers: + self.fixer_names = fixers + + logger.info('converting Python code') + _KLASS.run_2to3(self, files) + + logger.info('converting doctests in Python files') + _KLASS.run_2to3(self, files, doctests_only=True) + + if doctests != []: + logger.info('converting doctest in text files') + _KLASS.run_2to3(self, doctests, doctests_only=True) + else: + # If run on Python 2.x, there is nothing to do. + + def _run_2to3(self, files, doctests=[], fixers=[]): + pass diff --git a/Lib/packaging/compiler/__init__.py b/Lib/packaging/compiler/__init__.py new file mode 100644 index 0000000000..e267e9ffde --- /dev/null +++ b/Lib/packaging/compiler/__init__.py @@ -0,0 +1,279 @@ +"""Compiler abstraction model used by packaging. + +An abstract base class is defined in the ccompiler submodule, and +concrete implementations suitable for various platforms are defined in +the other submodules. The extension module is also placed in this +package. + +In general, code should not instantiate compiler classes directly but +use the new_compiler and customize_compiler functions provided in this +module. + +The compiler system has a registration API: get_default_compiler, +set_compiler, show_compilers. +""" + +import os +import sys +import re +import sysconfig + +from packaging.util import resolve_name +from packaging.errors import PackagingPlatformError +from packaging import logger + +def customize_compiler(compiler): + """Do any platform-specific customization of a CCompiler instance. + + Mainly needed on Unix, so we can plug in the information that + varies across Unices and is stored in Python's Makefile. + """ + if compiler.name == "unix": + cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags = ( + sysconfig.get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS', + 'CCSHARED', 'LDSHARED', 'SO', 'AR', + 'ARFLAGS')) + + if 'CC' in os.environ: + cc = os.environ['CC'] + if 'CXX' in os.environ: + cxx = os.environ['CXX'] + if 'LDSHARED' in os.environ: + ldshared = os.environ['LDSHARED'] + if 'CPP' in os.environ: + cpp = os.environ['CPP'] + else: + cpp = cc + " -E" # not always + if 'LDFLAGS' in os.environ: + ldshared = ldshared + ' ' + os.environ['LDFLAGS'] + if 'CFLAGS' in os.environ: + cflags = opt + ' ' + os.environ['CFLAGS'] + ldshared = ldshared + ' ' + os.environ['CFLAGS'] + if 'CPPFLAGS' in os.environ: + cpp = cpp + ' ' + os.environ['CPPFLAGS'] + cflags = cflags + ' ' + os.environ['CPPFLAGS'] + ldshared = ldshared + ' ' + os.environ['CPPFLAGS'] + if 'AR' in os.environ: + ar = os.environ['AR'] + if 'ARFLAGS' in os.environ: + archiver = ar + ' ' + os.environ['ARFLAGS'] + else: + if ar_flags is not None: + archiver = ar + ' ' + ar_flags + else: + # see if its the proper default value + # mmm I don't want to backport the makefile + archiver = ar + ' rc' + + cc_cmd = cc + ' ' + cflags + compiler.set_executables( + preprocessor=cpp, + compiler=cc_cmd, + compiler_so=cc_cmd + ' ' + ccshared, + compiler_cxx=cxx, + linker_so=ldshared, + linker_exe=cc, + archiver=archiver) + + compiler.shared_lib_extension = so_ext + + +# Map a sys.platform/os.name ('posix', 'nt') to the default compiler +# type for that platform. Keys are interpreted as re match +# patterns. Order is important; platform mappings are preferred over +# OS names. +_default_compilers = ( + # Platform string mappings + + # on a cygwin built python we can use gcc like an ordinary UNIXish + # compiler + ('cygwin.*', 'unix'), + + # OS name mappings + ('posix', 'unix'), + ('nt', 'msvc'), +) + +def get_default_compiler(osname=None, platform=None): + """ Determine the default compiler to use for the given platform. + + osname should be one of the standard Python OS names (i.e. the + ones returned by os.name) and platform the common value + returned by sys.platform for the platform in question. + + The default values are os.name and sys.platform in case the + parameters are not given. + + """ + if osname is None: + osname = os.name + if platform is None: + platform = sys.platform + for pattern, compiler in _default_compilers: + if re.match(pattern, platform) is not None or \ + re.match(pattern, osname) is not None: + return compiler + # Defaults to Unix compiler + return 'unix' + + +# compiler mapping +# XXX useful to expose them? (i.e. get_compiler_names) +_COMPILERS = { + 'unix': 'packaging.compiler.unixccompiler.UnixCCompiler', + 'msvc': 'packaging.compiler.msvccompiler.MSVCCompiler', + 'cygwin': 'packaging.compiler.cygwinccompiler.CygwinCCompiler', + 'mingw32': 'packaging.compiler.cygwinccompiler.Mingw32CCompiler', + 'bcpp': 'packaging.compiler.bcppcompiler.BCPPCompiler', +} + +def set_compiler(location): + """Add or change a compiler""" + cls = resolve_name(location) + # XXX we want to check the class here + _COMPILERS[cls.name] = cls + + +def show_compilers(): + """Print list of available compilers (used by the "--help-compiler" + options to "build", "build_ext", "build_clib"). + """ + from packaging.fancy_getopt import FancyGetopt + compilers = [] + + for name, cls in _COMPILERS.items(): + if isinstance(cls, str): + cls = resolve_name(cls) + _COMPILERS[name] = cls + + compilers.append(("compiler=" + name, None, cls.description)) + + compilers.sort() + pretty_printer = FancyGetopt(compilers) + pretty_printer.print_help("List of available compilers:") + + +def new_compiler(plat=None, compiler=None, verbose=0, dry_run=False, + force=False): + """Generate an instance of some CCompiler subclass for the supplied + platform/compiler combination. 'plat' defaults to 'os.name' + (eg. 'posix', 'nt'), and 'compiler' defaults to the default compiler + for that platform. Currently only 'posix' and 'nt' are supported, and + the default compilers are "traditional Unix interface" (UnixCCompiler + class) and Visual C++ (MSVCCompiler class). Note that it's perfectly + possible to ask for a Unix compiler object under Windows, and a + Microsoft compiler object under Unix -- if you supply a value for + 'compiler', 'plat' is ignored. + """ + if plat is None: + plat = os.name + + try: + if compiler is None: + compiler = get_default_compiler(plat) + + cls = _COMPILERS[compiler] + except KeyError: + msg = "don't know how to compile C/C++ code on platform '%s'" % plat + if compiler is not None: + msg = msg + " with '%s' compiler" % compiler + raise PackagingPlatformError(msg) + + if isinstance(cls, str): + cls = resolve_name(cls) + _COMPILERS[compiler] = cls + + + # XXX The None is necessary to preserve backwards compatibility + # with classes that expect verbose to be the first positional + # argument. + return cls(None, dry_run, force) + + +def gen_preprocess_options(macros, include_dirs): + """Generate C pre-processor options (-D, -U, -I) as used by at least + two types of compilers: the typical Unix compiler and Visual C++. + 'macros' is the usual thing, a list of 1- or 2-tuples, where (name,) + means undefine (-U) macro 'name', and (name,value) means define (-D) + macro 'name' to 'value'. 'include_dirs' is just a list of directory + names to be added to the header file search path (-I). Returns a list + of command-line options suitable for either Unix compilers or Visual + C++. + """ + # XXX it would be nice (mainly aesthetic, and so we don't generate + # stupid-looking command lines) to go over 'macros' and eliminate + # redundant definitions/undefinitions (ie. ensure that only the + # latest mention of a particular macro winds up on the command + # line). I don't think it's essential, though, since most (all?) + # Unix C compilers only pay attention to the latest -D or -U + # mention of a macro on their command line. Similar situation for + # 'include_dirs'. I'm punting on both for now. Anyways, weeding out + # redundancies like this should probably be the province of + # CCompiler, since the data structures used are inherited from it + # and therefore common to all CCompiler classes. + + pp_opts = [] + for macro in macros: + + if not isinstance(macro, tuple) and 1 <= len(macro) <= 2: + raise TypeError( + "bad macro definition '%s': each element of 'macros'" + "list must be a 1- or 2-tuple" % macro) + + if len(macro) == 1: # undefine this macro + pp_opts.append("-U%s" % macro[0]) + elif len(macro) == 2: + if macro[1] is None: # define with no explicit value + pp_opts.append("-D%s" % macro[0]) + else: + # XXX *don't* need to be clever about quoting the + # macro value here, because we're going to avoid the + # shell at all costs when we spawn the command! + pp_opts.append("-D%s=%s" % macro) + + for dir in include_dirs: + pp_opts.append("-I%s" % dir) + + return pp_opts + + +def gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries): + """Generate linker options for searching library directories and + linking with specific libraries. + + 'libraries' and 'library_dirs' are, respectively, lists of library names + (not filenames!) and search directories. Returns a list of command-line + options suitable for use with some compiler (depending on the two format + strings passed in). + """ + lib_opts = [] + + for dir in library_dirs: + lib_opts.append(compiler.library_dir_option(dir)) + + for dir in runtime_library_dirs: + opt = compiler.runtime_library_dir_option(dir) + if isinstance(opt, list): + lib_opts.extend(opt) + else: + lib_opts.append(opt) + + # XXX it's important that we *not* remove redundant library mentions! + # sometimes you really do have to say "-lfoo -lbar -lfoo" in order to + # resolve all symbols. I just hope we never have to say "-lfoo obj.o + # -lbar" to get things to work -- that's certainly a possibility, but a + # pretty nasty way to arrange your C code. + + for lib in libraries: + lib_dir, lib_name = os.path.split(lib) + if lib_dir != '': + lib_file = compiler.find_library_file([lib_dir], lib_name) + if lib_file is not None: + lib_opts.append(lib_file) + else: + logger.warning("no library file corresponding to " + "'%s' found (skipping)" % lib) + else: + lib_opts.append(compiler.library_option(lib)) + + return lib_opts diff --git a/Lib/packaging/compiler/bcppcompiler.py b/Lib/packaging/compiler/bcppcompiler.py new file mode 100644 index 0000000000..63b6d8b192 --- /dev/null +++ b/Lib/packaging/compiler/bcppcompiler.py @@ -0,0 +1,356 @@ +"""CCompiler implementation for the Borland C++ compiler.""" + +# This implementation by Lyle Johnson, based on the original msvccompiler.py +# module and using the directions originally published by Gordon Williams. + +# XXX looks like there's a LOT of overlap between these two classes: +# someone should sit down and factor out the common code as +# WindowsCCompiler! --GPW + +import os + +from packaging.errors import (PackagingExecError, CompileError, LibError, + LinkError, UnknownFileError) +from packaging.compiler.ccompiler import CCompiler +from packaging.compiler import gen_preprocess_options +from packaging.file_util import write_file +from packaging.dep_util import newer +from packaging import logger + + +class BCPPCompiler(CCompiler) : + """Concrete class that implements an interface to the Borland C/C++ + compiler, as defined by the CCompiler abstract class. + """ + + name = 'bcpp' + description = 'Borland C++ Compiler' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = _c_extensions + _cpp_extensions + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + + def __init__(self, verbose=0, dry_run=False, force=False): + CCompiler.__init__(self, verbose, dry_run, force) + + # These executables are assumed to all be in the path. + # Borland doesn't seem to use any special registry settings to + # indicate their installation locations. + + self.cc = "bcc32.exe" + self.linker = "ilink32.exe" + self.lib = "tlib.exe" + + self.preprocess_options = None + self.compile_options = ['/tWM', '/O2', '/q', '/g0'] + self.compile_options_debug = ['/tWM', '/Od', '/q', '/g0'] + + self.ldflags_shared = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_shared_debug = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_static = [] + self.ldflags_exe = ['/Gn', '/q', '/x'] + self.ldflags_exe_debug = ['/Gn', '/q', '/x','/r'] + + + # -- Worker methods ------------------------------------------------ + + def compile(self, sources, + output_dir=None, macros=None, include_dirs=None, debug=False, + extra_preargs=None, extra_postargs=None, depends=None): + + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + compile_opts = extra_preargs or [] + compile_opts.append('-c') + if debug: + compile_opts.extend(self.compile_options_debug) + else: + compile_opts.extend(self.compile_options) + + for obj in objects: + try: + src, ext = build[obj] + except KeyError: + continue + # XXX why do the normpath here? + src = os.path.normpath(src) + obj = os.path.normpath(obj) + # XXX _setup_compile() did a mkpath() too but before the normpath. + # Is it possible to skip the normpath? + self.mkpath(os.path.dirname(obj)) + + if ext == '.res': + # This is already a binary file -- skip it. + continue # the 'for' loop + if ext == '.rc': + # This needs to be compiled to a .res file -- do it now. + try: + self.spawn(["brcc32", "-fo", obj, src]) + except PackagingExecError as msg: + raise CompileError(msg) + continue # the 'for' loop + + # The next two are both for the real compiler. + if ext in self._c_extensions: + input_opt = "" + elif ext in self._cpp_extensions: + input_opt = "-P" + else: + # Unknown file type -- no extra options. The compiler + # will probably fail, but let it just in case this is a + # file the compiler recognizes even if we don't. + input_opt = "" + + output_opt = "-o" + obj + + # Compiler command line syntax is: "bcc32 [options] file(s)". + # Note that the source file names must appear at the end of + # the command line. + try: + self.spawn([self.cc] + compile_opts + pp_opts + + [input_opt, output_opt] + + extra_postargs + [src]) + except PackagingExecError as msg: + raise CompileError(msg) + + return objects + + + def create_static_lib(self, objects, output_libname, output_dir=None, + debug=False, target_lang=None): + objects, output_dir = self._fix_object_args(objects, output_dir) + output_filename = \ + self.library_filename(output_libname, output_dir=output_dir) + + if self._need_link(objects, output_filename): + lib_args = [output_filename, '/u'] + objects + if debug: + pass # XXX what goes here? + try: + self.spawn([self.lib] + lib_args) + except PackagingExecError as msg: + raise LibError(msg) + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + + def link(self, target_desc, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + + # XXX this ignores 'build_temp'! should follow the lead of + # msvccompiler.py + + objects, output_dir = self._fix_object_args(objects, output_dir) + libraries, library_dirs, runtime_library_dirs = \ + self._fix_lib_args(libraries, library_dirs, runtime_library_dirs) + + if runtime_library_dirs: + logger.warning("don't know what to do with " + "'runtime_library_dirs': %r", runtime_library_dirs) + + if output_dir is not None: + output_filename = os.path.join(output_dir, output_filename) + + if self._need_link(objects, output_filename): + + # Figure out linker args based on type of target. + if target_desc == CCompiler.EXECUTABLE: + startup_obj = 'c0w32' + if debug: + ld_args = self.ldflags_exe_debug[:] + else: + ld_args = self.ldflags_exe[:] + else: + startup_obj = 'c0d32' + if debug: + ld_args = self.ldflags_shared_debug[:] + else: + ld_args = self.ldflags_shared[:] + + + # Create a temporary exports file for use by the linker + if export_symbols is None: + def_file = '' + else: + head, tail = os.path.split(output_filename) + modname, ext = os.path.splitext(tail) + temp_dir = os.path.dirname(objects[0]) # preserve tree structure + def_file = os.path.join(temp_dir, '%s.def' % modname) + contents = ['EXPORTS'] + for sym in (export_symbols or []): + contents.append(' %s=_%s' % (sym, sym)) + self.execute(write_file, (def_file, contents), + "writing %s" % def_file) + + # Borland C++ has problems with '/' in paths + objects2 = [os.path.normpath(o) for o in objects] + # split objects in .obj and .res files + # Borland C++ needs them at different positions in the command line + objects = [startup_obj] + resources = [] + for file in objects2: + base, ext = os.path.splitext(os.path.normcase(file)) + if ext == '.res': + resources.append(file) + else: + objects.append(file) + + + for l in library_dirs: + ld_args.append("/L%s" % os.path.normpath(l)) + ld_args.append("/L.") # we sometimes use relative paths + + # list of object files + ld_args.extend(objects) + + # XXX the command line syntax for Borland C++ is a bit wonky; + # certain filenames are jammed together in one big string, but + # comma-delimited. This doesn't mesh too well with the + # Unix-centric attitude (with a DOS/Windows quoting hack) of + # 'spawn()', so constructing the argument list is a bit + # awkward. Note that doing the obvious thing and jamming all + # the filenames and commas into one argument would be wrong, + # because 'spawn()' would quote any filenames with spaces in + # them. Arghghh!. Apparently it works fine as coded... + + # name of dll/exe file + ld_args.extend((',',output_filename)) + # no map file and start libraries + ld_args.append(',,') + + for lib in libraries: + # see if we find it and if there is a bcpp specific lib + # (xxx_bcpp.lib) + libfile = self.find_library_file(library_dirs, lib, debug) + if libfile is None: + ld_args.append(lib) + # probably a BCPP internal library -- don't warn + else: + # full name which prefers bcpp_xxx.lib over xxx.lib + ld_args.append(libfile) + + # some default libraries + ld_args.append('import32') + ld_args.append('cw32mt') + + # def file for export symbols + ld_args.extend((',',def_file)) + # add resource files + ld_args.append(',') + ld_args.extend(resources) + + + if extra_preargs: + ld_args[:0] = extra_preargs + if extra_postargs: + ld_args.extend(extra_postargs) + + self.mkpath(os.path.dirname(output_filename)) + try: + self.spawn([self.linker] + ld_args) + except PackagingExecError as msg: + raise LinkError(msg) + + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + # -- Miscellaneous methods ----------------------------------------- + + + def find_library_file(self, dirs, lib, debug=False): + # List of effective library names to try, in order of preference: + # xxx_bcpp.lib is better than xxx.lib + # and xxx_d.lib is better than xxx.lib if debug is set + # + # The "_bcpp" suffix is to handle a Python installation for people + # with multiple compilers (primarily Packaging hackers, I suspect + # ;-). The idea is they'd have one static library for each + # compiler they care about, since (almost?) every Windows compiler + # seems to have a different format for static libraries. + if debug: + dlib = (lib + "_d") + try_names = (dlib + "_bcpp", lib + "_bcpp", dlib, lib) + else: + try_names = (lib + "_bcpp", lib) + + for dir in dirs: + for name in try_names: + libfile = os.path.join(dir, self.library_filename(name)) + if os.path.exists(libfile): + return libfile + else: + # Oops, didn't find it in *any* of 'dirs' + return None + + # overwrite the one from CCompiler to support rc and res-files + def object_filenames(self, source_filenames, strip_dir=False, + output_dir=''): + if output_dir is None: + output_dir = '' + obj_names = [] + for src_name in source_filenames: + # use normcase to make sure '.rc' is really '.rc' and not '.RC' + base, ext = os.path.splitext(os.path.normcase(src_name)) + if ext not in (self.src_extensions + ['.rc','.res']): + raise UnknownFileError("unknown file type '%s' (from '%s')" % \ + (ext, src_name)) + if strip_dir: + base = os.path.basename(base) + if ext == '.res': + # these can go unchanged + obj_names.append(os.path.join(output_dir, base + ext)) + elif ext == '.rc': + # these need to be compiled to .res-files + obj_names.append(os.path.join(output_dir, base + '.res')) + else: + obj_names.append(os.path.join(output_dir, + base + self.obj_extension)) + return obj_names + + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, + extra_postargs=None): + _, macros, include_dirs = \ + self._fix_compile_args(None, macros, include_dirs) + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = ['cpp32.exe'] + pp_opts + if output_file is not None: + pp_args.append('-o' + output_file) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or the + # source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except PackagingExecError as msg: + print(msg) + raise CompileError(msg) diff --git a/Lib/packaging/compiler/ccompiler.py b/Lib/packaging/compiler/ccompiler.py new file mode 100644 index 0000000000..d2743274fd --- /dev/null +++ b/Lib/packaging/compiler/ccompiler.py @@ -0,0 +1,865 @@ +"""Abstract base class for compilers. + +This modules contains CCompiler, an abstract base class that defines the +interface for the compiler abstraction model used by packaging. +""" + +import os +import sys +from shutil import move +from packaging import logger +from packaging.util import split_quoted, execute, newer_group, spawn +from packaging.errors import (CompileError, LinkError, UnknownFileError) +from packaging.compiler import gen_preprocess_options + + +class CCompiler: + """Abstract base class to define the interface that must be implemented + by real compiler classes. Also has some utility methods used by + several compiler classes. + + The basic idea behind a compiler abstraction class is that each + instance can be used for all the compile/link steps in building a + single project. Thus, attributes common to all of those compile and + link steps -- include directories, macros to define, libraries to link + against, etc. -- are attributes of the compiler instance. To allow for + variability in how individual files are treated, most of those + attributes may be varied on a per-compilation or per-link basis. + """ + + # 'name' is a class attribute that identifies this class. It + # keeps code that wants to know what kind of compiler it's dealing with + # from having to import all possible compiler classes just to do an + # 'isinstance'. + name = None + description = None + + # XXX things not handled by this compiler abstraction model: + # * client can't provide additional options for a compiler, + # e.g. warning, optimization, debugging flags. Perhaps this + # should be the domain of concrete compiler abstraction classes + # (UnixCCompiler, MSVCCompiler, etc.) -- or perhaps the base + # class should have methods for the common ones. + # * can't completely override the include or library searchg + # path, ie. no "cc -I -Idir1 -Idir2" or "cc -L -Ldir1 -Ldir2". + # I'm not sure how widely supported this is even by Unix + # compilers, much less on other platforms. And I'm even less + # sure how useful it is; maybe for cross-compiling, but + # support for that is a ways off. (And anyways, cross + # compilers probably have a dedicated binary with the + # right paths compiled in. I hope.) + # * can't do really freaky things with the library list/library + # dirs, e.g. "-Ldir1 -lfoo -Ldir2 -lfoo" to link against + # different versions of libfoo.a in different locations. I + # think this is useless without the ability to null out the + # library search path anyways. + + + # Subclasses that rely on the standard filename generation methods + # implemented below should override these; see the comment near + # those methods ('object_filenames()' et. al.) for details: + src_extensions = None # list of strings + obj_extension = None # string + static_lib_extension = None + shared_lib_extension = None # string + static_lib_format = None # format string + shared_lib_format = None # prob. same as static_lib_format + exe_extension = None # string + + # Default language settings. language_map is used to detect a source + # file or Extension target language, checking source filenames. + # language_order is used to detect the language precedence, when deciding + # what language to use when mixing source types. For example, if some + # extension has two files with ".c" extension, and one with ".cpp", it + # is still linked as c++. + language_map = {".c": "c", + ".cc": "c++", + ".cpp": "c++", + ".cxx": "c++", + ".m": "objc", + } + language_order = ["c++", "objc", "c"] + + def __init__(self, verbose=0, dry_run=False, force=False): + self.dry_run = dry_run + self.force = force + self.verbose = verbose + + # 'output_dir': a common output directory for object, library, + # shared object, and shared library files + self.output_dir = None + + # 'macros': a list of macro definitions (or undefinitions). A + # macro definition is a 2-tuple (name, value), where the value is + # either a string or None (no explicit value). A macro + # undefinition is a 1-tuple (name,). + self.macros = [] + + # 'include_dirs': a list of directories to search for include files + self.include_dirs = [] + + # 'libraries': a list of libraries to include in any link + # (library names, not filenames: eg. "foo" not "libfoo.a") + self.libraries = [] + + # 'library_dirs': a list of directories to search for libraries + self.library_dirs = [] + + # 'runtime_library_dirs': a list of directories to search for + # shared libraries/objects at runtime + self.runtime_library_dirs = [] + + # 'objects': a list of object files (or similar, such as explicitly + # named library files) to include on any link + self.objects = [] + + for key, value in self.executables.items(): + self.set_executable(key, value) + + def set_executables(self, **args): + """Define the executables (and options for them) that will be run + to perform the various stages of compilation. The exact set of + executables that may be specified here depends on the compiler + class (via the 'executables' class attribute), but most will have: + compiler the C/C++ compiler + linker_so linker used to create shared objects and libraries + linker_exe linker used to create binary executables + archiver static library creator + + On platforms with a command line (Unix, DOS/Windows), each of these + is a string that will be split into executable name and (optional) + list of arguments. (Splitting the string is done similarly to how + Unix shells operate: words are delimited by spaces, but quotes and + backslashes can override this. See + 'distutils.util.split_quoted()'.) + """ + + # Note that some CCompiler implementation classes will define class + # attributes 'cpp', 'cc', etc. with hard-coded executable names; + # this is appropriate when a compiler class is for exactly one + # compiler/OS combination (eg. MSVCCompiler). Other compiler + # classes (UnixCCompiler, in particular) are driven by information + # discovered at run-time, since there are many different ways to do + # basically the same things with Unix C compilers. + + for key, value in args.items(): + if key not in self.executables: + raise ValueError("unknown executable '%s' for class %s" % \ + (key, self.__class__.__name__)) + self.set_executable(key, value) + + def set_executable(self, key, value): + if isinstance(value, str): + setattr(self, key, split_quoted(value)) + else: + setattr(self, key, value) + + def _find_macro(self, name): + i = 0 + for defn in self.macros: + if defn[0] == name: + return i + i = i + 1 + return None + + def _check_macro_definitions(self, definitions): + """Ensures that every element of 'definitions' is a valid macro + definition, ie. either (name,value) 2-tuple or a (name,) tuple. Do + nothing if all definitions are OK, raise TypeError otherwise. + """ + for defn in definitions: + if not (isinstance(defn, tuple) and + (len(defn) == 1 or + (len(defn) == 2 and + (isinstance(defn[1], str) or defn[1] is None))) and + isinstance(defn[0], str)): + raise TypeError(("invalid macro definition '%s': " % defn) + \ + "must be tuple (string,), (string, string), or " + \ + "(string, None)") + + + # -- Bookkeeping methods ------------------------------------------- + + def define_macro(self, name, value=None): + """Define a preprocessor macro for all compilations driven by this + compiler object. The optional parameter 'value' should be a + string; if it is not supplied, then the macro will be defined + without an explicit value and the exact outcome depends on the + compiler used (XXX true? does ANSI say anything about this?) + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro(name) + if i is not None: + del self.macros[i] + + defn = (name, value) + self.macros.append(defn) + + def undefine_macro(self, name): + """Undefine a preprocessor macro for all compilations driven by + this compiler object. If the same macro is defined by + 'define_macro()' and undefined by 'undefine_macro()' the last call + takes precedence (including multiple redefinitions or + undefinitions). If the macro is redefined/undefined on a + per-compilation basis (ie. in the call to 'compile()'), then that + takes precedence. + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro(name) + if i is not None: + del self.macros[i] + + undefn = (name,) + self.macros.append(undefn) + + def add_include_dir(self, dir): + """Add 'dir' to the list of directories that will be searched for + header files. The compiler is instructed to search directories in + the order in which they are supplied by successive calls to + 'add_include_dir()'. + """ + self.include_dirs.append(dir) + + def set_include_dirs(self, dirs): + """Set the list of directories that will be searched to 'dirs' (a + list of strings). Overrides any preceding calls to + 'add_include_dir()'; subsequence calls to 'add_include_dir()' add + to the list passed to 'set_include_dirs()'. This does not affect + any list of standard include directories that the compiler may + search by default. + """ + self.include_dirs = dirs[:] + + def add_library(self, libname): + """Add 'libname' to the list of libraries that will be included in + all links driven by this compiler object. Note that 'libname' + should *not* be the name of a file containing a library, but the + name of the library itself: the actual filename will be inferred by + the linker, the compiler, or the compiler class (depending on the + platform). + + The linker will be instructed to link against libraries in the + order they were supplied to 'add_library()' and/or + 'set_libraries()'. It is perfectly valid to duplicate library + names; the linker will be instructed to link against libraries as + many times as they are mentioned. + """ + self.libraries.append(libname) + + def set_libraries(self, libnames): + """Set the list of libraries to be included in all links driven by + this compiler object to 'libnames' (a list of strings). This does + not affect any standard system libraries that the linker may + include by default. + """ + self.libraries = libnames[:] + + + def add_library_dir(self, dir): + """Add 'dir' to the list of directories that will be searched for + libraries specified to 'add_library()' and 'set_libraries()'. The + linker will be instructed to search for libraries in the order they + are supplied to 'add_library_dir()' and/or 'set_library_dirs()'. + """ + self.library_dirs.append(dir) + + def set_library_dirs(self, dirs): + """Set the list of library search directories to 'dirs' (a list of + strings). This does not affect any standard library search path + that the linker may search by default. + """ + self.library_dirs = dirs[:] + + def add_runtime_library_dir(self, dir): + """Add 'dir' to the list of directories that will be searched for + shared libraries at runtime. + """ + self.runtime_library_dirs.append(dir) + + def set_runtime_library_dirs(self, dirs): + """Set the list of directories to search for shared libraries at + runtime to 'dirs' (a list of strings). This does not affect any + standard search path that the runtime linker may search by + default. + """ + self.runtime_library_dirs = dirs[:] + + def add_link_object(self, object): + """Add 'object' to the list of object files (or analogues, such as + explicitly named library files or the output of "resource + compilers") to be included in every link driven by this compiler + object. + """ + self.objects.append(object) + + def set_link_objects(self, objects): + """Set the list of object files (or analogues) to be included in + every link to 'objects'. This does not affect any standard object + files that the linker may include by default (such as system + libraries). + """ + self.objects = objects[:] + + + # -- Private utility methods -------------------------------------- + # (here for the convenience of subclasses) + + # Helper method to prep compiler in subclass compile() methods + def _setup_compile(self, outdir, macros, incdirs, sources, depends, + extra): + """Process arguments and decide which source files to compile.""" + if outdir is None: + outdir = self.output_dir + elif not isinstance(outdir, str): + raise TypeError("'output_dir' must be a string or None") + + if macros is None: + macros = self.macros + elif isinstance(macros, list): + macros = macros + (self.macros or []) + else: + raise TypeError("'macros' (if supplied) must be a list of tuples") + + if incdirs is None: + incdirs = self.include_dirs + elif isinstance(incdirs, (list, tuple)): + incdirs = list(incdirs) + (self.include_dirs or []) + else: + raise TypeError( + "'include_dirs' (if supplied) must be a list of strings") + + if extra is None: + extra = [] + + # Get the list of expected output (object) files + objects = self.object_filenames(sources, + strip_dir=False, + output_dir=outdir) + assert len(objects) == len(sources) + + pp_opts = gen_preprocess_options(macros, incdirs) + + build = {} + for i in range(len(sources)): + src = sources[i] + obj = objects[i] + ext = os.path.splitext(src)[1] + self.mkpath(os.path.dirname(obj)) + build[obj] = (src, ext) + + return macros, objects, extra, pp_opts, build + + def _get_cc_args(self, pp_opts, debug, before): + # works for unixccompiler and cygwinccompiler + cc_args = pp_opts + ['-c'] + if debug: + cc_args[:0] = ['-g'] + if before: + cc_args[:0] = before + return cc_args + + def _fix_compile_args(self, output_dir, macros, include_dirs): + """Typecheck and fix-up some of the arguments to the 'compile()' + method, and return fixed-up values. Specifically: if 'output_dir' + is None, replaces it with 'self.output_dir'; ensures that 'macros' + is a list, and augments it with 'self.macros'; ensures that + 'include_dirs' is a list, and augments it with 'self.include_dirs'. + Guarantees that the returned values are of the correct type, + i.e. for 'output_dir' either string or None, and for 'macros' and + 'include_dirs' either list or None. + """ + if output_dir is None: + output_dir = self.output_dir + elif not isinstance(output_dir, str): + raise TypeError("'output_dir' must be a string or None") + + if macros is None: + macros = self.macros + elif isinstance(macros, list): + macros = macros + (self.macros or []) + else: + raise TypeError("'macros' (if supplied) must be a list of tuples") + + if include_dirs is None: + include_dirs = self.include_dirs + elif isinstance(include_dirs, (list, tuple)): + include_dirs = list(include_dirs) + (self.include_dirs or []) + else: + raise TypeError( + "'include_dirs' (if supplied) must be a list of strings") + + return output_dir, macros, include_dirs + + def _fix_object_args(self, objects, output_dir): + """Typecheck and fix up some arguments supplied to various methods. + Specifically: ensure that 'objects' is a list; if output_dir is + None, replace with self.output_dir. Return fixed versions of + 'objects' and 'output_dir'. + """ + if not isinstance(objects, (list, tuple)): + raise TypeError("'objects' must be a list or tuple of strings") + objects = list(objects) + + if output_dir is None: + output_dir = self.output_dir + elif not isinstance(output_dir, str): + raise TypeError("'output_dir' must be a string or None") + + return objects, output_dir + + def _fix_lib_args(self, libraries, library_dirs, runtime_library_dirs): + """Typecheck and fix up some of the arguments supplied to the + 'link_*' methods. Specifically: ensure that all arguments are + lists, and augment them with their permanent versions + (eg. 'self.libraries' augments 'libraries'). Return a tuple with + fixed versions of all arguments. + """ + if libraries is None: + libraries = self.libraries + elif isinstance(libraries, (list, tuple)): + libraries = list(libraries) + (self.libraries or []) + else: + raise TypeError( + "'libraries' (if supplied) must be a list of strings") + + if library_dirs is None: + library_dirs = self.library_dirs + elif isinstance(library_dirs, (list, tuple)): + library_dirs = list(library_dirs) + (self.library_dirs or []) + else: + raise TypeError( + "'library_dirs' (if supplied) must be a list of strings") + + if runtime_library_dirs is None: + runtime_library_dirs = self.runtime_library_dirs + elif isinstance(runtime_library_dirs, (list, tuple)): + runtime_library_dirs = (list(runtime_library_dirs) + + (self.runtime_library_dirs or [])) + else: + raise TypeError("'runtime_library_dirs' (if supplied) " + "must be a list of strings") + + return libraries, library_dirs, runtime_library_dirs + + def _need_link(self, objects, output_file): + """Return true if we need to relink the files listed in 'objects' + to recreate 'output_file'. + """ + if self.force: + return True + else: + if self.dry_run: + newer = newer_group(objects, output_file, missing='newer') + else: + newer = newer_group(objects, output_file) + return newer + + def detect_language(self, sources): + """Detect the language of a given file, or list of files. Uses + language_map, and language_order to do the job. + """ + if not isinstance(sources, list): + sources = [sources] + lang = None + index = len(self.language_order) + for source in sources: + base, ext = os.path.splitext(source) + extlang = self.language_map.get(ext) + try: + extindex = self.language_order.index(extlang) + if extindex < index: + lang = extlang + index = extindex + except ValueError: + pass + return lang + + # -- Worker methods ------------------------------------------------ + # (must be implemented by subclasses) + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + """Preprocess a single C/C++ source file, named in 'source'. + Output will be written to file named 'output_file', or stdout if + 'output_file' not supplied. 'macros' is a list of macro + definitions as for 'compile()', which will augment the macros set + with 'define_macro()' and 'undefine_macro()'. 'include_dirs' is a + list of directory names that will be added to the default list. + + Raises PreprocessError on failure. + """ + pass + + def compile(self, sources, output_dir=None, macros=None, + include_dirs=None, debug=False, extra_preargs=None, + extra_postargs=None, depends=None): + """Compile one or more source files. + + 'sources' must be a list of filenames, most likely C/C++ + files, but in reality anything that can be handled by a + particular compiler and compiler class (eg. MSVCCompiler can + handle resource files in 'sources'). Return a list of object + filenames, one per source filename in 'sources'. Depending on + the implementation, not all source files will necessarily be + compiled, but all corresponding object filenames will be + returned. + + If 'output_dir' is given, object files will be put under it, while + retaining their original path component. That is, "foo/bar.c" + normally compiles to "foo/bar.o" (for a Unix implementation); if + 'output_dir' is "build", then it would compile to + "build/foo/bar.o". + + 'macros', if given, must be a list of macro definitions. A macro + definition is either a (name, value) 2-tuple or a (name,) 1-tuple. + The former defines a macro; if the value is None, the macro is + defined without an explicit value. The 1-tuple case undefines a + macro. Later definitions/redefinitions/ undefinitions take + precedence. + + 'include_dirs', if given, must be a list of strings, the + directories to add to the default include file search path for this + compilation only. + + 'debug' is a boolean; if true, the compiler will be instructed to + output debug symbols in (or alongside) the object file(s). + + 'extra_preargs' and 'extra_postargs' are implementation- dependent. + On platforms that have the notion of a command line (e.g. Unix, + DOS/Windows), they are most likely lists of strings: extra + command-line arguments to prepand/append to the compiler command + line. On other platforms, consult the implementation class + documentation. In any event, they are intended as an escape hatch + for those occasions when the abstract compiler framework doesn't + cut the mustard. + + 'depends', if given, is a list of filenames that all targets + depend on. If a source file is older than any file in + depends, then the source file will be recompiled. This + supports dependency tracking, but only at a coarse + granularity. + + Raises CompileError on failure. + """ + # A concrete compiler class can either override this method + # entirely or implement _compile(). + + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + for obj in objects: + try: + src, ext = build[obj] + except KeyError: + continue + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # Return *all* object filenames, not just the ones we just built. + return objects + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + """Compile 'src' to product 'obj'.""" + + # A concrete compiler class that does not override compile() + # should implement _compile(). + pass + + def create_static_lib(self, objects, output_libname, output_dir=None, + debug=False, target_lang=None): + """Link a bunch of stuff together to create a static library file. + The "bunch of stuff" consists of the list of object files supplied + as 'objects', the extra object files supplied to + 'add_link_object()' and/or 'set_link_objects()', the libraries + supplied to 'add_library()' and/or 'set_libraries()', and the + libraries supplied as 'libraries' (if any). + + 'output_libname' should be a library name, not a filename; the + filename will be inferred from the library name. 'output_dir' is + the directory where the library file will be put. + + 'debug' is a boolean; if true, debugging information will be + included in the library (note that on most platforms, it is the + compile step where this matters: the 'debug' flag is included here + just for consistency). + + 'target_lang' is the target language for which the given objects + are being compiled. This allows specific linkage time treatment of + certain languages. + + Raises LibError on failure. + """ + pass + + # values for target_desc parameter in link() + SHARED_OBJECT = "shared_object" + SHARED_LIBRARY = "shared_library" + EXECUTABLE = "executable" + + def link(self, target_desc, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + """Link a bunch of stuff together to create an executable or + shared library file. + + The "bunch of stuff" consists of the list of object files supplied + as 'objects'. 'output_filename' should be a filename. If + 'output_dir' is supplied, 'output_filename' is relative to it + (i.e. 'output_filename' can provide directory components if + needed). + + 'libraries' is a list of libraries to link against. These are + library names, not filenames, since they're translated into + filenames in a platform-specific way (eg. "foo" becomes "libfoo.a" + on Unix and "foo.lib" on DOS/Windows). However, they can include a + directory component, which means the linker will look in that + specific directory rather than searching all the normal locations. + + 'library_dirs', if supplied, should be a list of directories to + search for libraries that were specified as bare library names + (ie. no directory component). These are on top of the system + default and those supplied to 'add_library_dir()' and/or + 'set_library_dirs()'. 'runtime_library_dirs' is a list of + directories that will be embedded into the shared library and used + to search for other shared libraries that *it* depends on at + run-time. (This may only be relevant on Unix.) + + 'export_symbols' is a list of symbols that the shared library will + export. (This appears to be relevant only on Windows.) + + 'debug' is as for 'compile()' and 'create_static_lib()', with the + slight distinction that it actually matters on most platforms (as + opposed to 'create_static_lib()', which includes a 'debug' flag + mostly for form's sake). + + 'extra_preargs' and 'extra_postargs' are as for 'compile()' (except + of course that they supply command-line arguments for the + particular linker being used). + + 'target_lang' is the target language for which the given objects + are being compiled. This allows specific linkage time treatment of + certain languages. + + Raises LinkError on failure. + """ + raise NotImplementedError + + + # Old 'link_*()' methods, rewritten to use the new 'link()' method. + + def link_shared_lib(self, objects, output_libname, output_dir=None, + libraries=None, library_dirs=None, + runtime_library_dirs=None, export_symbols=None, + debug=False, extra_preargs=None, extra_postargs=None, + build_temp=None, target_lang=None): + self.link(CCompiler.SHARED_LIBRARY, objects, + self.library_filename(output_libname, lib_type='shared'), + output_dir, + libraries, library_dirs, runtime_library_dirs, + export_symbols, debug, + extra_preargs, extra_postargs, build_temp, target_lang) + + def link_shared_object(self, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, + runtime_library_dirs=None, export_symbols=None, + debug=False, extra_preargs=None, extra_postargs=None, + build_temp=None, target_lang=None): + self.link(CCompiler.SHARED_OBJECT, objects, + output_filename, output_dir, + libraries, library_dirs, runtime_library_dirs, + export_symbols, debug, + extra_preargs, extra_postargs, build_temp, target_lang) + + def link_executable(self, objects, output_progname, output_dir=None, + libraries=None, library_dirs=None, + runtime_library_dirs=None, debug=False, + extra_preargs=None, extra_postargs=None, + target_lang=None): + self.link(CCompiler.EXECUTABLE, objects, + self.executable_filename(output_progname), output_dir, + libraries, library_dirs, runtime_library_dirs, None, + debug, extra_preargs, extra_postargs, None, target_lang) + + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function; there is + # no appropriate default implementation so subclasses should + # implement all of these. + + def library_dir_option(self, dir): + """Return the compiler option to add 'dir' to the list of + directories searched for libraries. + """ + raise NotImplementedError + + def runtime_library_dir_option(self, dir): + """Return the compiler option to add 'dir' to the list of + directories searched for runtime libraries. + """ + raise NotImplementedError + + def library_option(self, lib): + """Return the compiler option to add 'dir' to the list of libraries + linked into the shared library or executable. + """ + raise NotImplementedError + + def has_function(self, funcname, includes=None, include_dirs=None, + libraries=None, library_dirs=None): + """Return a boolean indicating whether funcname is supported on + the current platform. The optional arguments can be used to + augment the compilation environment. + """ + + # this can't be included at module scope because it tries to + # import math which might not be available at that point - maybe + # the necessary logic should just be inlined? + import tempfile + if includes is None: + includes = [] + if include_dirs is None: + include_dirs = [] + if libraries is None: + libraries = [] + if library_dirs is None: + library_dirs = [] + fd, fname = tempfile.mkstemp(".c", funcname, text=True) + with os.fdopen(fd, "w") as f: + for incl in includes: + f.write("""#include "%s"\n""" % incl) + f.write("""\ +main (int argc, char **argv) { + %s(); +} +""" % funcname) + try: + objects = self.compile([fname], include_dirs=include_dirs) + except CompileError: + return False + + try: + self.link_executable(objects, "a.out", + libraries=libraries, + library_dirs=library_dirs) + except (LinkError, TypeError): + return False + return True + + def find_library_file(self, dirs, lib, debug=False): + """Search the specified list of directories for a static or shared + library file 'lib' and return the full path to that file. If + 'debug' is true, look for a debugging version (if that makes sense on + the current platform). Return None if 'lib' wasn't found in any of + the specified directories. + """ + raise NotImplementedError + + # -- Filename generation methods ----------------------------------- + + # The default implementation of the filename generating methods are + # prejudiced towards the Unix/DOS/Windows view of the world: + # * object files are named by replacing the source file extension + # (eg. .c/.cpp -> .o/.obj) + # * library files (shared or static) are named by plugging the + # library name and extension into a format string, eg. + # "lib%s.%s" % (lib_name, ".a") for Unix static libraries + # * executables are named by appending an extension (possibly + # empty) to the program name: eg. progname + ".exe" for + # Windows + # + # To reduce redundant code, these methods expect to find + # several attributes in the current object (presumably defined + # as class attributes): + # * src_extensions - + # list of C/C++ source file extensions, eg. ['.c', '.cpp'] + # * obj_extension - + # object file extension, eg. '.o' or '.obj' + # * static_lib_extension - + # extension for static library files, eg. '.a' or '.lib' + # * shared_lib_extension - + # extension for shared library/object files, eg. '.so', '.dll' + # * static_lib_format - + # format string for generating static library filenames, + # eg. 'lib%s.%s' or '%s.%s' + # * shared_lib_format + # format string for generating shared library filenames + # (probably same as static_lib_format, since the extension + # is one of the intended parameters to the format string) + # * exe_extension - + # extension for executable files, eg. '' or '.exe' + + def object_filenames(self, source_filenames, strip_dir=False, output_dir=''): + if output_dir is None: + output_dir = '' + obj_names = [] + for src_name in source_filenames: + base, ext = os.path.splitext(src_name) + base = os.path.splitdrive(base)[1] # Chop off the drive + base = base[os.path.isabs(base):] # If abs, chop off leading / + if ext not in self.src_extensions: + raise UnknownFileError("unknown file type '%s' (from '%s')" % + (ext, src_name)) + if strip_dir: + base = os.path.basename(base) + obj_names.append(os.path.join(output_dir, + base + self.obj_extension)) + return obj_names + + def shared_object_filename(self, basename, strip_dir=False, output_dir=''): + assert output_dir is not None + if strip_dir: + basename = os.path.basename(basename) + return os.path.join(output_dir, basename + self.shared_lib_extension) + + def executable_filename(self, basename, strip_dir=False, output_dir=''): + assert output_dir is not None + if strip_dir: + basename = os.path.basename(basename) + return os.path.join(output_dir, basename + (self.exe_extension or '')) + + def library_filename(self, libname, lib_type='static', # or 'shared' + strip_dir=False, output_dir=''): + assert output_dir is not None + if lib_type not in ("static", "shared", "dylib"): + raise ValueError( + "'lib_type' must be 'static', 'shared' or 'dylib'") + fmt = getattr(self, lib_type + "_lib_format") + ext = getattr(self, lib_type + "_lib_extension") + + dir, base = os.path.split(libname) + filename = fmt % (base, ext) + if strip_dir: + dir = '' + + return os.path.join(output_dir, dir, filename) + + + # -- Utility methods ----------------------------------------------- + + def execute(self, func, args, msg=None, level=1): + execute(func, args, msg, self.dry_run) + + def spawn(self, cmd): + spawn(cmd, dry_run=self.dry_run) + + def move_file(self, src, dst): + logger.info("moving %r to %r", src, dst) + if self.dry_run: + return + return move(src, dst) + + def mkpath(self, name, mode=0o777): + name = os.path.normpath(name) + if os.path.isdir(name) or name == '': + return + if self.dry_run: + head = '' + for part in name.split(os.sep): + logger.info("created directory %s%s", head, part) + head += part + os.sep + return + os.makedirs(name, mode) diff --git a/Lib/packaging/compiler/cygwinccompiler.py b/Lib/packaging/compiler/cygwinccompiler.py new file mode 100644 index 0000000000..348dbe782f --- /dev/null +++ b/Lib/packaging/compiler/cygwinccompiler.py @@ -0,0 +1,354 @@ +"""CCompiler implementations for Cygwin and mingw32 versions of GCC. + +This module contains the CygwinCCompiler class, a subclass of +UnixCCompiler that handles the Cygwin port of the GNU C compiler to +Windows, and the Mingw32CCompiler class which handles the mingw32 port +of GCC (same as cygwin in no-cygwin mode). +""" + +# problems: +# +# * if you use a msvc compiled python version (1.5.2) +# 1. you have to insert a __GNUC__ section in its config.h +# 2. you have to generate a import library for its dll +# - create a def-file for python??.dll +# - create a import library using +# dlltool --dllname python15.dll --def python15.def \ +# --output-lib libpython15.a +# +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# +# * We put export_symbols in a def-file, and don't use +# --export-all-symbols because it doesn't worked reliable in some +# tested configurations. And because other windows compilers also +# need their symbols specified this no serious problem. +# +# tested configurations: +# +# * cygwin gcc 2.91.57/ld 2.9.4/dllwrap 0.2.4 works +# (after patching python's config.h and for C++ some other include files) +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# * mingw32 gcc 2.95.2/ld 2.9.4/dllwrap 0.2.4 works +# (ld doesn't support -shared, so we use dllwrap) +# * cygwin gcc 2.95.2/ld 2.10.90/dllwrap 2.10.90 works now +# - its dllwrap doesn't work, there is a bug in binutils 2.10.90 +# see also http://sources.redhat.com/ml/cygwin/2000-06/msg01274.html +# - using gcc -mdll instead dllwrap doesn't work without -static because +# it tries to link against dlls instead their import libraries. (If +# it finds the dll first.) +# By specifying -static we force ld to link against the import libraries, +# this is windows standard and there are normally not the necessary symbols +# in the dlls. +# *** only the version of June 2000 shows these problems +# * cygwin gcc 3.2/ld 2.13.90 works +# (ld supports -shared) +# * mingw gcc 3.2/ld 2.13 works +# (ld supports -shared) + + +import os +import sys + +from packaging import logger +from packaging.compiler.unixccompiler import UnixCCompiler +from packaging.util import write_file +from packaging.errors import PackagingExecError, CompileError, UnknownFileError +from packaging.util import get_compiler_versions +import sysconfig + + +def get_msvcr(): + """Include the appropriate MSVC runtime library if Python was built + with MSVC 7.0 or later. + """ + msc_pos = sys.version.find('MSC v.') + if msc_pos != -1: + msc_ver = sys.version[msc_pos+6:msc_pos+10] + if msc_ver == '1300': + # MSVC 7.0 + return ['msvcr70'] + elif msc_ver == '1310': + # MSVC 7.1 + return ['msvcr71'] + elif msc_ver == '1400': + # VS2005 / MSVC 8.0 + return ['msvcr80'] + elif msc_ver == '1500': + # VS2008 / MSVC 9.0 + return ['msvcr90'] + else: + raise ValueError("Unknown MS Compiler version %s " % msc_ver) + + +class CygwinCCompiler(UnixCCompiler): + """ Handles the Cygwin port of the GNU C compiler to Windows. + """ + name = 'cygwin' + description = 'Cygwin port of GNU C Compiler for Win32' + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".dll" + static_lib_format = "lib%s%s" + shared_lib_format = "%s%s" + exe_extension = ".exe" + + def __init__(self, verbose=0, dry_run=False, force=False): + + UnixCCompiler.__init__(self, verbose, dry_run, force) + + status, details = check_config_h() + logger.debug("Python's GCC status: %s (details: %s)", status, details) + if status is not CONFIG_H_OK: + self.warn( + "Python's pyconfig.h doesn't seem to support your compiler. " + "Reason: %s. " + "Compiling may fail because of undefined preprocessor macros." + % details) + + self.gcc_version, self.ld_version, self.dllwrap_version = \ + get_compiler_versions() + logger.debug(self.name + ": gcc %s, ld %s, dllwrap %s\n", + self.gcc_version, + self.ld_version, + self.dllwrap_version) + + # ld_version >= "2.10.90" and < "2.13" should also be able to use + # gcc -mdll instead of dllwrap + # Older dllwraps had own version numbers, newer ones use the + # same as the rest of binutils ( also ld ) + # dllwrap 2.10.90 is buggy + if self.ld_version >= "2.10.90": + self.linker_dll = "gcc" + else: + self.linker_dll = "dllwrap" + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # Hard-code GCC because that's what this is all about. + # XXX optimization, warnings etc. should be customizable. + self.set_executables(compiler='gcc -mcygwin -O -Wall', + compiler_so='gcc -mcygwin -mdll -O -Wall', + compiler_cxx='g++ -mcygwin -O -Wall', + linker_exe='gcc -mcygwin', + linker_so=('%s -mcygwin %s' % + (self.linker_dll, shared_option))) + + # cygwin and mingw32 need different sets of libraries + if self.gcc_version == "2.91.57": + # cygwin shouldn't need msvcrt, but without the dlls will crash + # (gcc version 2.91.57) -- perhaps something about initialization + self.dll_libraries=["msvcrt"] + self.warn( + "Consider upgrading to a newer version of gcc") + else: + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + """Compile the source by spawning GCC and windres if needed.""" + if ext == '.rc' or ext == '.res': + # gcc needs '.res' and '.rc' compiled to object files !!! + try: + self.spawn(["windres", "-i", src, "-o", obj]) + except PackagingExecError as msg: + raise CompileError(msg) + else: # for other files use the C-compiler + try: + self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + + extra_postargs) + except PackagingExecError as msg: + raise CompileError(msg) + + def link(self, target_desc, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + """Link the objects.""" + # use separate copies, so we can modify the lists + extra_preargs = list(extra_preargs or []) + libraries = list(libraries or []) + objects = list(objects or []) + + # Additional libraries + libraries.extend(self.dll_libraries) + + # handle export symbols by creating a def-file + # with executables this only works with gcc/ld as linker + if ((export_symbols is not None) and + (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): + # (The linker doesn't do anything if output is up-to-date. + # So it would probably better to check if we really need this, + # but for this we had to insert some unchanged parts of + # UnixCCompiler, and this is not what we want.) + + # we want to put some files in the same directory as the + # object files are, build_temp doesn't help much + # where are the object files + temp_dir = os.path.dirname(objects[0]) + # name of dll to give the helper files the same base name + dll_name, dll_extension = os.path.splitext( + os.path.basename(output_filename)) + + # generate the filenames for these files + def_file = os.path.join(temp_dir, dll_name + ".def") + lib_file = os.path.join(temp_dir, 'lib' + dll_name + ".a") + + # Generate .def file + contents = [ + "LIBRARY %s" % os.path.basename(output_filename), + "EXPORTS"] + for sym in export_symbols: + contents.append(sym) + self.execute(write_file, (def_file, contents), + "writing %s" % def_file) + + # next add options for def-file and to creating import libraries + + # dllwrap uses different options than gcc/ld + if self.linker_dll == "dllwrap": + extra_preargs.extend(("--output-lib", lib_file)) + # for dllwrap we have to use a special option + extra_preargs.extend(("--def", def_file)) + # we use gcc/ld here and can be sure ld is >= 2.9.10 + else: + # doesn't work: bfd_close build\...\libfoo.a: Invalid operation + #extra_preargs.extend(("-Wl,--out-implib,%s" % lib_file)) + # for gcc/ld the def-file is specified as any object files + objects.append(def_file) + + #end: if ((export_symbols is not None) and + # (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): + + # who wants symbols and a many times larger output file + # should explicitly switch the debug mode on + # otherwise we let dllwrap/ld strip the output file + # (On my machine: 10KB < stripped_file < ??100KB + # unstripped_file = stripped_file + XXX KB + # ( XXX=254 for a typical python extension)) + if not debug: + extra_preargs.append("-s") + + UnixCCompiler.link(self, target_desc, objects, output_filename, + output_dir, libraries, library_dirs, + runtime_library_dirs, + None, # export_symbols, we do this in our def-file + debug, extra_preargs, extra_postargs, build_temp, + target_lang) + + # -- Miscellaneous methods ----------------------------------------- + + def object_filenames(self, source_filenames, strip_dir=False, + output_dir=''): + """Adds supports for rc and res files.""" + if output_dir is None: + output_dir = '' + obj_names = [] + for src_name in source_filenames: + # use normcase to make sure '.rc' is really '.rc' and not '.RC' + base, ext = os.path.splitext(os.path.normcase(src_name)) + if ext not in (self.src_extensions + ['.rc','.res']): + raise UnknownFileError("unknown file type '%s' (from '%s')" % (ext, src_name)) + if strip_dir: + base = os.path.basename (base) + if ext in ('.res', '.rc'): + # these need to be compiled to object files + obj_names.append (os.path.join(output_dir, + base + ext + self.obj_extension)) + else: + obj_names.append (os.path.join(output_dir, + base + self.obj_extension)) + return obj_names + +# the same as cygwin plus some additional parameters +class Mingw32CCompiler(CygwinCCompiler): + """ Handles the Mingw32 port of the GNU C compiler to Windows. + """ + name = 'mingw32' + description = 'MinGW32 compiler' + + def __init__(self, verbose=0, dry_run=False, force=False): + + CygwinCCompiler.__init__ (self, verbose, dry_run, force) + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # A real mingw32 doesn't need to specify a different entry point, + # but cygwin 2.91.57 in no-cygwin-mode needs it. + if self.gcc_version <= "2.91.57": + entry_point = '--entry _DllMain@12' + else: + entry_point = '' + + self.set_executables(compiler='gcc -mno-cygwin -O -Wall', + compiler_so='gcc -mno-cygwin -mdll -O -Wall', + compiler_cxx='g++ -mno-cygwin -O -Wall', + linker_exe='gcc -mno-cygwin', + linker_so='%s -mno-cygwin %s %s' + % (self.linker_dll, shared_option, + entry_point)) + # Maybe we should also append -mthreads, but then the finished + # dlls need another dll (mingwm10.dll see Mingw32 docs) + # (-mthreads: Support thread-safe exception handling on `Mingw32') + + # no additional libraries needed + self.dll_libraries=[] + + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + +# Because these compilers aren't configured in Python's pyconfig.h file by +# default, we should at least warn the user if he is using a unmodified +# version. + +CONFIG_H_OK = "ok" +CONFIG_H_NOTOK = "not ok" +CONFIG_H_UNCERTAIN = "uncertain" + +def check_config_h(): + """Check if the current Python installation appears amenable to building + extensions with GCC. + + Returns a tuple (status, details), where 'status' is one of the following + constants: + + - CONFIG_H_OK: all is well, go ahead and compile + - CONFIG_H_NOTOK: doesn't look good + - CONFIG_H_UNCERTAIN: not sure -- unable to read pyconfig.h + + 'details' is a human-readable string explaining the situation. + + Note there are two ways to conclude "OK": either 'sys.version' contains + the string "GCC" (implying that this Python was built with GCC), or the + installed "pyconfig.h" contains the string "__GNUC__". + """ + + # XXX since this function also checks sys.version, it's not strictly a + # "pyconfig.h" check -- should probably be renamed... + # if sys.version contains GCC then python was compiled with GCC, and the + # pyconfig.h file should be OK + if "GCC" in sys.version: + return CONFIG_H_OK, "sys.version mentions 'GCC'" + + # let's see if __GNUC__ is mentioned in python.h + fn = sysconfig.get_config_h_filename() + try: + with open(fn) as config_h: + if "__GNUC__" in config_h.read(): + return CONFIG_H_OK, "'%s' mentions '__GNUC__'" % fn + else: + return CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn + except IOError as exc: + return (CONFIG_H_UNCERTAIN, + "couldn't read '%s': %s" % (fn, exc.strerror)) diff --git a/Lib/packaging/compiler/extension.py b/Lib/packaging/compiler/extension.py new file mode 100644 index 0000000000..66f6e9a6bb --- /dev/null +++ b/Lib/packaging/compiler/extension.py @@ -0,0 +1,121 @@ +"""Class representing C/C++ extension modules.""" + +from packaging import logger + +# This class is really only used by the "build_ext" command, so it might +# make sense to put it in distutils.command.build_ext. However, that +# module is already big enough, and I want to make this class a bit more +# complex to simplify some common cases ("foo" module in "foo.c") and do +# better error-checking ("foo.c" actually exists). +# +# Also, putting this in build_ext.py means every setup script would have to +# import that large-ish module (indirectly, through distutils.core) in +# order to do anything. + + +class Extension: + """Just a collection of attributes that describes an extension + module and everything needed to build it (hopefully in a portable + way, but there are hooks that let you be as unportable as you need). + + Instance attributes: + name : string + the full name of the extension, including any packages -- ie. + *not* a filename or pathname, but Python dotted name + sources : [string] + list of source filenames, relative to the distribution root + (where the setup script lives), in Unix form (slash-separated) + for portability. Source files may be C, C++, SWIG (.i), + platform-specific resource files, or whatever else is recognized + by the "build_ext" command as source for a Python extension. + include_dirs : [string] + list of directories to search for C/C++ header files (in Unix + form for portability) + define_macros : [(name : string, value : string|None)] + list of macros to define; each macro is defined using a 2-tuple, + where 'value' is either the string to define it to or None to + define it without a particular value (equivalent of "#define + FOO" in source or -DFOO on Unix C compiler command line) + undef_macros : [string] + list of macros to undefine explicitly + library_dirs : [string] + list of directories to search for C/C++ libraries at link time + libraries : [string] + list of library names (not filenames or paths) to link against + runtime_library_dirs : [string] + list of directories to search for C/C++ libraries at run time + (for shared extensions, this is when the extension is loaded) + extra_objects : [string] + list of extra files to link with (eg. object files not implied + by 'sources', static library that must be explicitly specified, + binary resource files, etc.) + extra_compile_args : [string] + any extra platform- and compiler-specific information to use + when compiling the source files in 'sources'. For platforms and + compilers where "command line" makes sense, this is typically a + list of command-line arguments, but for other platforms it could + be anything. + extra_link_args : [string] + any extra platform- and compiler-specific information to use + when linking object files together to create the extension (or + to create a new static Python interpreter). Similar + interpretation as for 'extra_compile_args'. + export_symbols : [string] + list of symbols to be exported from a shared extension. Not + used on all platforms, and not generally necessary for Python + extensions, which typically export exactly one symbol: "init" + + extension_name. + swig_opts : [string] + any extra options to pass to SWIG if a source file has the .i + extension. + depends : [string] + list of files that the extension depends on + language : string + extension language (i.e. "c", "c++", "objc"). Will be detected + from the source extensions if not provided. + optional : boolean + specifies that a build failure in the extension should not abort the + build process, but simply not install the failing extension. + """ + + # **kwargs are allowed so that a warning is emitted instead of an + # exception + def __init__(self, name, sources, include_dirs=None, define_macros=None, + undef_macros=None, library_dirs=None, libraries=None, + runtime_library_dirs=None, extra_objects=None, + extra_compile_args=None, extra_link_args=None, + export_symbols=None, swig_opts=None, depends=None, + language=None, optional=None, **kw): + if not isinstance(name, str): + raise AssertionError("'name' must be a string") + + if not isinstance(sources, list): + raise AssertionError("'sources' must be a list of strings") + + for v in sources: + if not isinstance(v, str): + raise AssertionError("'sources' must be a list of strings") + + self.name = name + self.sources = sources + self.include_dirs = include_dirs or [] + self.define_macros = define_macros or [] + self.undef_macros = undef_macros or [] + self.library_dirs = library_dirs or [] + self.libraries = libraries or [] + self.runtime_library_dirs = runtime_library_dirs or [] + self.extra_objects = extra_objects or [] + self.extra_compile_args = extra_compile_args or [] + self.extra_link_args = extra_link_args or [] + self.export_symbols = export_symbols or [] + self.swig_opts = swig_opts or [] + self.depends = depends or [] + self.language = language + self.optional = optional + + # If there are unknown keyword options, warn about them + if len(kw) > 0: + options = [repr(option) for option in kw] + options = ', '.join(sorted(options)) + logger.warning( + 'unknown arguments given to Extension: %s', options) diff --git a/Lib/packaging/compiler/msvc9compiler.py b/Lib/packaging/compiler/msvc9compiler.py new file mode 100644 index 0000000000..6ca49c21b2 --- /dev/null +++ b/Lib/packaging/compiler/msvc9compiler.py @@ -0,0 +1,720 @@ +"""CCompiler implementation for the Microsoft Visual Studio 2008 compiler. + +The MSVCCompiler class is compatible with VS 2005 and VS 2008. Legacy +support for older versions of VS are in the msvccompiler module. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) +# ported to VS2005 and VS 2008 by Christian Heimes +import os +import subprocess +import sys +import re + +from packaging.errors import (PackagingExecError, PackagingPlatformError, + CompileError, LibError, LinkError) +from packaging.compiler.ccompiler import CCompiler +from packaging.compiler import gen_lib_options +from packaging import logger +from packaging.util import get_platform + +import winreg + +RegOpenKeyEx = winreg.OpenKeyEx +RegEnumKey = winreg.EnumKey +RegEnumValue = winreg.EnumValue +RegError = winreg.error + +HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + +VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" +WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" +NET_BASE = r"Software\Microsoft\.NETFramework" + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is +# the param to cross-compile on x86 targetting amd64.) +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'amd64', + 'win-ia64' : 'ia64', +} + + +class Reg: + """Helper class to read values from the registry + """ + + def get_value(cls, path, key): + for base in HKEYS: + d = cls.read_values(base, path) + if d and key in d: + return d[key] + raise KeyError(key) + get_value = classmethod(get_value) + + def read_keys(cls, base, key): + """Return list of registry keys.""" + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + read_keys = classmethod(read_keys) + + def read_values(cls, base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) + i += 1 + return d + read_values = classmethod(read_values) + + def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + convert_mbcs = staticmethod(convert_mbcs) + +class MacroExpander: + + def __init__(self, version): + self.macros = {} + self.vsbase = VS_BASE % version + self.load_macros(version) + + def set_macro(self, macro, path, key): + self.macros["$(%s)" % macro] = Reg.get_value(path, key) + + def load_macros(self, version): + self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") + self.set_macro("FrameworkDir", NET_BASE, "installroot") + try: + if version >= 8.0: + self.set_macro("FrameworkSDKDir", NET_BASE, + "sdkinstallrootv2.0") + else: + raise KeyError("sdkinstallrootv2.0") + except KeyError: + raise PackagingPlatformError( +"""Python was built with Visual Studio 2008; extensions must be built with a +compiler than can generate compatible binaries. Visual Studio 2008 was not +found on this system. If you have Cygwin installed, you can try compiling +with MingW32, by passing "-c mingw32" to pysetup.""") + + if version >= 9.0: + self.set_macro("FrameworkVersion", self.vsbase, "clr version") + self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") + else: + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = Reg.get_value(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + +def removeDuplicates(variable): + """Remove duplicate values of an environment variable. + """ + oldList = variable.split(os.pathsep) + newList = [] + for i in oldList: + if i not in newList: + newList.append(i) + newVariable = os.pathsep.join(newList) + return newVariable + +def find_vcvarsall(version): + """Find the vcvarsall.bat file + + At first it tries to find the productdir of VS 2008 in the registry. If + that fails it falls back to the VS90COMNTOOLS env var. + """ + vsbase = VS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + logger.debug("Unable to find productdir in registry") + productdir = None + + if not productdir or not os.path.isdir(productdir): + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + + if toolsdir and os.path.isdir(toolsdir): + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) + if not os.path.isdir(productdir): + logger.debug("%s is not a valid directory", productdir) + return None + else: + logger.debug("env var %s is not set or invalid", toolskey) + if not productdir: + logger.debug("no productdir found") + return None + vcvarsall = os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + logger.debug("unable to find vcvarsall.bat") + return None + +def query_vcvarsall(version, arch="x86"): + """Launch vcvarsall.bat and read the settings from its environment + """ + vcvarsall = find_vcvarsall(version) + interesting = set(("include", "lib", "libpath", "path")) + result = {} + + if vcvarsall is None: + raise PackagingPlatformError("Unable to find vcvarsall.bat") + logger.debug("calling 'vcvarsall.bat %s' (version=%s)", arch, version) + popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise PackagingPlatformError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.split("\n"): + line = Reg.convert_mbcs(line) + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key in interesting: + if value.endswith(os.pathsep): + value = value[:-1] + result[key] = removeDuplicates(value) + + if len(result) != len(interesting): + raise ValueError(str(list(result))) + + return result + +# More globals +VERSION = get_build_version() +if VERSION < 8.0: + raise PackagingPlatformError("VC %0.1f is not supported by this module" % VERSION) +# MACROS = MacroExpander(VERSION) + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + name = 'msvc' + description = 'Microsoft Visual C++' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, verbose=0, dry_run=False, force=False): + CCompiler.__init__(self, verbose, dry_run, force) + self.__version = VERSION + self.__root = r"Software\Microsoft\VisualStudio" + # self.__macros = MACROS + self.__paths = [] + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.__arch = None # deprecated name + self.initialized = False + + def initialize(self, plat_name=None): + # multi-init means we would need to check platform same each time... + assert not self.initialized, "don't init multiple times" + if plat_name is None: + plat_name = get_platform() + # sanity check for platforms to prevent obscure errors later. + ok_plats = 'win32', 'win-amd64', 'win-ia64' + if plat_name not in ok_plats: + raise PackagingPlatformError("--plat-name must be one of %s" % + (ok_plats,)) + + if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe("cl.exe"): + # Assume that the SDK set up everything alright; don't try to be + # smarter + self.cc = "cl.exe" + self.linker = "link.exe" + self.lib = "lib.exe" + self.rc = "rc.exe" + self.mc = "mc.exe" + else: + # On x86, 'vcvars32.bat amd64' creates an env that doesn't work; + # to cross compile, you use 'x86_amd64'. + # On AMD64, 'vcvars32.bat amd64' is a native build env; to cross + # compile use 'x86' (ie, it runs the x86 compiler directly) + # No idea how itanium handles this, if at all. + if plat_name == get_platform() or plat_name == 'win32': + # native build or cross-compile to win32 + plat_spec = PLAT_TO_VCVARS[plat_name] + else: + # cross compile from win32 -> some 64bit + plat_spec = PLAT_TO_VCVARS[get_platform()] + '_' + \ + PLAT_TO_VCVARS[plat_name] + + vc_env = query_vcvarsall(VERSION, plat_spec) + + # take care to only use strings in the environment. + self.__paths = vc_env['path'].split(os.pathsep) + os.environ['lib'] = vc_env['lib'] + os.environ['include'] = vc_env['include'] + + if len(self.__paths) == 0: + raise PackagingPlatformError("Python was built with %s, " + "and extensions need to be built with the same " + "version of the compiler, but it isn't installed." + % self.__product) + + self.cc = self.find_exe("cl.exe") + self.linker = self.find_exe("link.exe") + self.lib = self.find_exe("lib.exe") + self.rc = self.find_exe("rc.exe") # resource compiler + self.mc = self.find_exe("mc.exe") # message compiler + #self.set_path_env_var('lib') + #self.set_path_env_var('include') + + # extend the MSVC path with the current path + try: + for p in os.environ['path'].split(';'): + self.__paths.append(p) + except KeyError: + pass + self.__paths = normalize_and_reduce_paths(self.__paths) + os.environ['path'] = ";".join(self.__paths) + + self.preprocess_options = None + if self.__arch == "x86": + self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', + '/DNDEBUG'] + self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', + '/Z7', '/D_DEBUG'] + else: + # Win64 + self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GS-' , + '/DNDEBUG'] + self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', + '/Z7', '/D_DEBUG'] + + self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO'] + if self.__version >= 7: + self.ldflags_shared_debug = [ + '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG', '/pdb:None' + ] + self.ldflags_static = [ '/nologo'] + + self.initialized = True + + # -- Worker methods ------------------------------------------------ + + def object_filenames(self, + source_filenames, + strip_dir=False, + output_dir=''): + # Copied from ccompiler.py, extended to return .res as 'object'-file + # for .rc input file + if output_dir is None: output_dir = '' + obj_names = [] + for src_name in source_filenames: + base, ext = os.path.splitext(src_name) + base = os.path.splitdrive(base)[1] # Chop off the drive + base = base[os.path.isabs(base):] # If abs, chop off leading / + if ext not in self.src_extensions: + # Better to raise an exception instead of silently continuing + # and later complain about sources and targets having + # different lengths + raise CompileError("Don't know how to compile %s" % src_name) + if strip_dir: + base = os.path.basename(base) + if ext in self._rc_extensions: + obj_names.append(os.path.join(output_dir, + base + self.res_extension)) + elif ext in self._mc_extensions: + obj_names.append(os.path.join(output_dir, + base + self.res_extension)) + else: + obj_names.append(os.path.join(output_dir, + base + self.obj_extension)) + return obj_names + + + def compile(self, sources, + output_dir=None, macros=None, include_dirs=None, debug=False, + extra_preargs=None, extra_postargs=None, depends=None): + + if not self.initialized: + self.initialize() + compile_info = self._setup_compile(output_dir, macros, include_dirs, + sources, depends, extra_postargs) + macros, objects, extra_postargs, pp_opts, build = compile_info + + compile_opts = extra_preargs or [] + compile_opts.append('/c') + if debug: + compile_opts.extend(self.compile_options_debug) + else: + compile_opts.extend(self.compile_options) + + for obj in objects: + try: + src, ext = build[obj] + except KeyError: + continue + if debug: + # pass the full pathname to MSVC in debug mode, + # this allows the debugger to find the source file + # without asking the user to browse for it + src = os.path.abspath(src) + + if ext in self._c_extensions: + input_opt = "/Tc" + src + elif ext in self._cpp_extensions: + input_opt = "/Tp" + src + elif ext in self._rc_extensions: + # compile .RC to .RES file + input_opt = src + output_opt = "/fo" + obj + try: + self.spawn([self.rc] + pp_opts + + [output_opt] + [input_opt]) + except PackagingExecError as msg: + raise CompileError(msg) + continue + elif ext in self._mc_extensions: + # Compile .MC to .RC file to .RES file. + # * '-h dir' specifies the directory for the + # generated include file + # * '-r dir' specifies the target directory of the + # generated RC file and the binary message resource + # it includes + # + # For now (since there are no options to change this), + # we use the source-directory for the include file and + # the build directory for the RC file and message + # resources. This works at least for win32all. + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) + try: + # first compile .MC to .RC and .H file + self.spawn([self.mc] + + ['-h', h_dir, '-r', rc_dir] + [src]) + base, _ = os.path.splitext(os.path.basename(src)) + rc_file = os.path.join(rc_dir, base + '.rc') + # then compile .RC to .RES file + self.spawn([self.rc] + + ["/fo" + obj] + [rc_file]) + + except PackagingExecError as msg: + raise CompileError(msg) + continue + else: + # how to handle this file? + raise CompileError("Don't know how to compile %s to %s" + % (src, obj)) + + output_opt = "/Fo" + obj + try: + self.spawn([self.cc] + compile_opts + pp_opts + + [input_opt, output_opt] + + extra_postargs) + except PackagingExecError as msg: + raise CompileError(msg) + + return objects + + + def create_static_lib(self, + objects, + output_libname, + output_dir=None, + debug=False, + target_lang=None): + + if not self.initialized: + self.initialize() + objects, output_dir = self._fix_object_args(objects, output_dir) + output_filename = self.library_filename(output_libname, + output_dir=output_dir) + + if self._need_link(objects, output_filename): + lib_args = objects + ['/OUT:' + output_filename] + if debug: + pass # XXX what goes here? + try: + self.spawn([self.lib] + lib_args) + except PackagingExecError as msg: + raise LibError(msg) + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + + def link(self, target_desc, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + if not self.initialized: + self.initialize() + objects, output_dir = self._fix_object_args(objects, output_dir) + fixed_args = self._fix_lib_args(libraries, library_dirs, + runtime_library_dirs) + libraries, library_dirs, runtime_library_dirs = fixed_args + + if runtime_library_dirs: + self.warn("don't know what to do with 'runtime_library_dirs': " + + str(runtime_library_dirs)) + + lib_opts = gen_lib_options(self, + library_dirs, runtime_library_dirs, + libraries) + if output_dir is not None: + output_filename = os.path.join(output_dir, output_filename) + + if self._need_link(objects, output_filename): + if target_desc == CCompiler.EXECUTABLE: + if debug: + ldflags = self.ldflags_shared_debug[1:] + else: + ldflags = self.ldflags_shared[1:] + else: + if debug: + ldflags = self.ldflags_shared_debug + else: + ldflags = self.ldflags_shared + + export_opts = [] + for sym in (export_symbols or []): + export_opts.append("/EXPORT:" + sym) + + ld_args = (ldflags + lib_opts + export_opts + + objects + ['/OUT:' + output_filename]) + + # The MSVC linker generates .lib and .exp files, which cannot be + # suppressed by any linker switches. The .lib files may even be + # needed! Make sure they are generated in the temporary build + # directory. Since they have different names for debug and release + # builds, they can go into the same directory. + build_temp = os.path.dirname(objects[0]) + if export_symbols is not None: + dll_name, dll_ext = os.path.splitext( + os.path.basename(output_filename)) + implib_file = os.path.join( + build_temp, + self.library_filename(dll_name)) + ld_args.append('/IMPLIB:' + implib_file) + + # Embedded manifests are recommended - see MSDN article titled + # "How to: Embed a Manifest Inside a C/C++ Application" + # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) + # Ask the linker to generate the manifest in the temp dir, so + # we can embed it later. + temp_manifest = os.path.join( + build_temp, + os.path.basename(output_filename) + ".manifest") + ld_args.append('/MANIFESTFILE:' + temp_manifest) + + if extra_preargs: + ld_args[:0] = extra_preargs + if extra_postargs: + ld_args.extend(extra_postargs) + + self.mkpath(os.path.dirname(output_filename)) + try: + self.spawn([self.linker] + ld_args) + except PackagingExecError as msg: + raise LinkError(msg) + + # embed the manifest + # XXX - this is somewhat fragile - if mt.exe fails, distutils + # will still consider the DLL up-to-date, but it will not have a + # manifest. Maybe we should link to a temp file? OTOH, that + # implies a build environment error that shouldn't go undetected. + if target_desc == CCompiler.EXECUTABLE: + mfid = 1 + else: + mfid = 2 + self._remove_visual_c_ref(temp_manifest) + out_arg = '-outputresource:%s;%s' % (output_filename, mfid) + try: + self.spawn(['mt.exe', '-nologo', '-manifest', + temp_manifest, out_arg]) + except PackagingExecError as msg: + raise LinkError(msg) + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + def _remove_visual_c_ref(self, manifest_file): + try: + # Remove references to the Visual C runtime, so they will + # fall through to the Visual C dependency of Python.exe. + # This way, when installed for a restricted user (e.g. + # runtimes are not in WinSxS folder, but in Python's own + # folder), the runtimes do not need to be in every folder + # with .pyd's. + with open(manifest_file) as manifest_f: + manifest_buf = manifest_f.read() + pattern = re.compile( + r"""<assemblyIdentity.*?name=("|')Microsoft\."""\ + r"""VC\d{2}\.CRT("|').*?(/>|</assemblyIdentity>)""", + re.DOTALL) + manifest_buf = re.sub(pattern, "", manifest_buf) + pattern = "<dependentAssembly>\s*</dependentAssembly>" + manifest_buf = re.sub(pattern, "", manifest_buf) + with open(manifest_file, 'w') as manifest_f: + manifest_f.write(manifest_buf) + except IOError: + pass + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function, in + # ccompiler.py. + + def library_dir_option(self, dir): + return "/LIBPATH:" + dir + + def runtime_library_dir_option(self, dir): + raise PackagingPlatformError( + "don't know how to set runtime library search path for MSVC++") + + def library_option(self, lib): + return self.library_filename(lib) + + + def find_library_file(self, dirs, lib, debug=False): + # Prefer a debugging library if found (and requested), but deal + # with it if we don't have one. + if debug: + try_names = [lib + "_d", lib] + else: + try_names = [lib] + for dir in dirs: + for name in try_names: + libfile = os.path.join(dir, self.library_filename(name)) + if os.path.exists(libfile): + return libfile + else: + # Oops, didn't find it in *any* of 'dirs' + return None + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p),exe) + if os.path.isfile(fn): + return fn + + return exe diff --git a/Lib/packaging/compiler/msvccompiler.py b/Lib/packaging/compiler/msvccompiler.py new file mode 100644 index 0000000000..3d4ac3c60b --- /dev/null +++ b/Lib/packaging/compiler/msvccompiler.py @@ -0,0 +1,635 @@ +"""CCompiler implementation for old Microsoft Visual Studio compilers. + +For a compiler compatible with VS 2005 and 2008, use msvc9compiler. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) + + +import sys +import os + +from packaging.errors import (PackagingExecError, PackagingPlatformError, + CompileError, LibError, LinkError) +from packaging.compiler.ccompiler import CCompiler +from packaging.compiler import gen_lib_options +from packaging import logger + +_can_read_reg = False +try: + import winreg + + _can_read_reg = True + hkey_mod = winreg + + RegOpenKeyEx = winreg.OpenKeyEx + RegEnumKey = winreg.EnumKey + RegEnumValue = winreg.EnumValue + RegError = winreg.error + +except ImportError: + try: + import win32api + import win32con + _can_read_reg = True + hkey_mod = win32con + + RegOpenKeyEx = win32api.RegOpenKeyEx + RegEnumKey = win32api.RegEnumKey + RegEnumValue = win32api.RegEnumValue + RegError = win32api.error + + except ImportError: + logger.warning( + "can't read registry to find the necessary compiler setting;\n" + "make sure that Python modules _winreg, win32api or win32con " + "are installed.") + +if _can_read_reg: + HKEYS = (hkey_mod.HKEY_USERS, + hkey_mod.HKEY_CURRENT_USER, + hkey_mod.HKEY_LOCAL_MACHINE, + hkey_mod.HKEY_CLASSES_ROOT) + + +def read_keys(base, key): + """Return list of registry keys.""" + + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i = i + 1 + return L + + +def read_values(base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[convert_mbcs(name)] = convert_mbcs(value) + i = i + 1 + return d + + +def convert_mbcs(s): + enc = getattr(s, "encode", None) + if enc is not None: + try: + s = enc("mbcs") + except UnicodeError: + pass + return s + + +class MacroExpander: + + def __init__(self, version): + self.macros = {} + self.load_macros(version) + + def set_macro(self, macro, path, key): + for base in HKEYS: + d = read_values(base, path) + if d: + self.macros["$(%s)" % macro] = d[key] + break + + def load_macros(self, version): + vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version + self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") + net = r"Software\Microsoft\.NETFramework" + self.set_macro("FrameworkDir", net, "installroot") + try: + if version > 7.0: + self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") + else: + self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") + except KeyError: + raise PackagingPlatformError( +"""Python was built with Visual Studio 2003; extensions must be built with +a compiler than can generate compatible binaries. Visual Studio 2003 was +not found on this system. If you have Cygwin installed, you can try +compiling with MingW32, by passing "-c mingw32" to pysetup.""") + + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = read_values(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + + +def get_build_architecture(): + """Return the processor architecture. + + Possible results are "Intel", "Itanium", or "AMD64". + """ + + prefix = " bit (" + i = sys.version.find(prefix) + if i == -1: + return "Intel" + j = sys.version.find(")", i) + return sys.version[i+len(prefix):j] + + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + + +class MSVCCompiler(CCompiler): + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + name = 'msvc' + description = "Microsoft Visual C++" + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, verbose=0, dry_run=False, force=False): + CCompiler.__init__(self, verbose, dry_run, force) + self.__version = get_build_version() + self.__arch = get_build_architecture() + if self.__arch == "Intel": + # x86 + if self.__version >= 7: + self.__root = r"Software\Microsoft\VisualStudio" + self.__macros = MacroExpander(self.__version) + else: + self.__root = r"Software\Microsoft\Devstudio" + self.__product = "Visual Studio version %s" % self.__version + else: + # Win64. Assume this was built with the platform SDK + self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) + + self.initialized = False + + def initialize(self): + self.__paths = [] + if ("DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and + self.find_exe("cl.exe")): + # Assume that the SDK set up everything alright; don't try to be + # smarter + self.cc = "cl.exe" + self.linker = "link.exe" + self.lib = "lib.exe" + self.rc = "rc.exe" + self.mc = "mc.exe" + else: + self.__paths = self.get_msvc_paths("path") + + if len(self.__paths) == 0: + raise PackagingPlatformError("Python was built with %s " + "and extensions need to be built with the same " + "version of the compiler, but it isn't installed." % + self.__product) + + self.cc = self.find_exe("cl.exe") + self.linker = self.find_exe("link.exe") + self.lib = self.find_exe("lib.exe") + self.rc = self.find_exe("rc.exe") # resource compiler + self.mc = self.find_exe("mc.exe") # message compiler + self.set_path_env_var('lib') + self.set_path_env_var('include') + + # extend the MSVC path with the current path + try: + for p in os.environ['path'].split(';'): + self.__paths.append(p) + except KeyError: + pass + self.__paths = normalize_and_reduce_paths(self.__paths) + os.environ['path'] = ';'.join(self.__paths) + + self.preprocess_options = None + if self.__arch == "Intel": + self.compile_options = ['/nologo', '/Ox', '/MD', '/W3', '/GX', + '/DNDEBUG'] + self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GX', + '/Z7', '/D_DEBUG'] + else: + # Win64 + self.compile_options = ['/nologo', '/Ox', '/MD', '/W3', '/GS-', + '/DNDEBUG'] + self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', + '/Z7', '/D_DEBUG'] + + self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO'] + if self.__version >= 7: + self.ldflags_shared_debug = [ + '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' + ] + else: + self.ldflags_shared_debug = [ + '/DLL', '/nologo', '/INCREMENTAL:no', '/pdb:None', '/DEBUG' + ] + self.ldflags_static = [ '/nologo'] + + self.initialized = True + + # -- Worker methods ------------------------------------------------ + + def object_filenames(self, source_filenames, strip_dir=False, output_dir=''): + # Copied from ccompiler.py, extended to return .res as 'object'-file + # for .rc input file + if output_dir is None: + output_dir = '' + obj_names = [] + for src_name in source_filenames: + base, ext = os.path.splitext(src_name) + base = os.path.splitdrive(base)[1] # Chop off the drive + base = base[os.path.isabs(base):] # If abs, chop off leading / + if ext not in self.src_extensions: + # Better to raise an exception instead of silently continuing + # and later complain about sources and targets having + # different lengths + raise CompileError("Don't know how to compile %s" % src_name) + if strip_dir: + base = os.path.basename(base) + if ext in self._rc_extensions: + obj_names.append(os.path.join(output_dir, + base + self.res_extension)) + elif ext in self._mc_extensions: + obj_names.append(os.path.join(output_dir, + base + self.res_extension)) + else: + obj_names.append(os.path.join(output_dir, + base + self.obj_extension)) + return obj_names + + def compile(self, sources, + output_dir=None, macros=None, include_dirs=None, debug=False, + extra_preargs=None, extra_postargs=None, depends=None): + + if not self.initialized: + self.initialize() + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + + compile_opts = extra_preargs or [] + compile_opts.append('/c') + if debug: + compile_opts.extend(self.compile_options_debug) + else: + compile_opts.extend(self.compile_options) + + for obj in objects: + try: + src, ext = build[obj] + except KeyError: + continue + if debug: + # pass the full pathname to MSVC in debug mode, + # this allows the debugger to find the source file + # without asking the user to browse for it + src = os.path.abspath(src) + + if ext in self._c_extensions: + input_opt = "/Tc" + src + elif ext in self._cpp_extensions: + input_opt = "/Tp" + src + elif ext in self._rc_extensions: + # compile .RC to .RES file + input_opt = src + output_opt = "/fo" + obj + try: + self.spawn([self.rc] + pp_opts + + [output_opt] + [input_opt]) + except PackagingExecError as msg: + raise CompileError(msg) + continue + elif ext in self._mc_extensions: + + # Compile .MC to .RC file to .RES file. + # * '-h dir' specifies the directory for the + # generated include file + # * '-r dir' specifies the target directory of the + # generated RC file and the binary message resource + # it includes + # + # For now (since there are no options to change this), + # we use the source-directory for the include file and + # the build directory for the RC file and message + # resources. This works at least for win32all. + + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) + try: + # first compile .MC to .RC and .H file + self.spawn([self.mc] + + ['-h', h_dir, '-r', rc_dir] + [src]) + base, _ = os.path.splitext(os.path.basename(src)) + rc_file = os.path.join(rc_dir, base + '.rc') + # then compile .RC to .RES file + self.spawn([self.rc] + + ["/fo" + obj] + [rc_file]) + + except PackagingExecError as msg: + raise CompileError(msg) + continue + else: + # how to handle this file? + raise CompileError( + "Don't know how to compile %s to %s" % + (src, obj)) + + output_opt = "/Fo" + obj + try: + self.spawn([self.cc] + compile_opts + pp_opts + + [input_opt, output_opt] + + extra_postargs) + except PackagingExecError as msg: + raise CompileError(msg) + + return objects + + def create_static_lib(self, objects, output_libname, output_dir=None, + debug=False, target_lang=None): + if not self.initialized: + self.initialize() + objects, output_dir = self._fix_object_args(objects, output_dir) + output_filename = \ + self.library_filename(output_libname, output_dir=output_dir) + + if self._need_link(objects, output_filename): + lib_args = objects + ['/OUT:' + output_filename] + if debug: + pass # XXX what goes here? + try: + self.spawn([self.lib] + lib_args) + except PackagingExecError as msg: + raise LibError(msg) + + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + def link(self, target_desc, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + + if not self.initialized: + self.initialize() + objects, output_dir = self._fix_object_args(objects, output_dir) + libraries, library_dirs, runtime_library_dirs = \ + self._fix_lib_args(libraries, library_dirs, runtime_library_dirs) + + if runtime_library_dirs: + self.warn("don't know what to do with 'runtime_library_dirs': %s" + % (runtime_library_dirs,)) + + lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, + libraries) + if output_dir is not None: + output_filename = os.path.join(output_dir, output_filename) + + if self._need_link(objects, output_filename): + + if target_desc == CCompiler.EXECUTABLE: + if debug: + ldflags = self.ldflags_shared_debug[1:] + else: + ldflags = self.ldflags_shared[1:] + else: + if debug: + ldflags = self.ldflags_shared_debug + else: + ldflags = self.ldflags_shared + + export_opts = [] + for sym in (export_symbols or []): + export_opts.append("/EXPORT:" + sym) + + ld_args = (ldflags + lib_opts + export_opts + + objects + ['/OUT:' + output_filename]) + + # The MSVC linker generates .lib and .exp files, which cannot be + # suppressed by any linker switches. The .lib files may even be + # needed! Make sure they are generated in the temporary build + # directory. Since they have different names for debug and release + # builds, they can go into the same directory. + if export_symbols is not None: + dll_name, dll_ext = os.path.splitext( + os.path.basename(output_filename)) + implib_file = os.path.join( + os.path.dirname(objects[0]), + self.library_filename(dll_name)) + ld_args.append('/IMPLIB:' + implib_file) + + if extra_preargs: + ld_args[:0] = extra_preargs + if extra_postargs: + ld_args.extend(extra_postargs) + + self.mkpath(os.path.dirname(output_filename)) + try: + self.spawn([self.linker] + ld_args) + except PackagingExecError as msg: + raise LinkError(msg) + + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function, in + # ccompiler.py. + + def library_dir_option(self, dir): + return "/LIBPATH:" + dir + + def runtime_library_dir_option(self, dir): + raise PackagingPlatformError("don't know how to set runtime library search path for MSVC++") + + def library_option(self, lib): + return self.library_filename(lib) + + def find_library_file(self, dirs, lib, debug=False): + # Prefer a debugging library if found (and requested), but deal + # with it if we don't have one. + if debug: + try_names = [lib + "_d", lib] + else: + try_names = [lib] + for dir in dirs: + for name in try_names: + libfile = os.path.join(dir, self.library_filename(name)) + if os.path.exists(libfile): + return libfile + else: + # Oops, didn't find it in *any* of 'dirs' + return None + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + return exe + + def get_msvc_paths(self, path, platform='x86'): + """Get a list of devstudio directories (include, lib or path). + + Return a list of strings. The list will be empty if unable to + access the registry or appropriate registry keys not found. + """ + + if not _can_read_reg: + return [] + + path = path + " dirs" + if self.__version >= 7: + key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" + % (self.__root, self.__version)) + else: + key = (r"%s\6.0\Build System\Components\Platforms" + r"\Win32 (%s)\Directories" % (self.__root, platform)) + + for base in HKEYS: + d = read_values(base, key) + if d: + if self.__version >= 7: + return self.__macros.sub(d[path]).split(";") + else: + return d[path].split(";") + # MSVC 6 seems to create the registry entries we need only when + # the GUI is run. + if self.__version == 6: + for base in HKEYS: + if read_values(base, r"%s\6.0" % self.__root) is not None: + self.warn("It seems you have Visual Studio 6 installed, " + "but the expected registry settings are not present.\n" + "You must at least run the Visual Studio GUI once " + "so that these entries are created.") + break + return [] + + def set_path_env_var(self, name): + """Set environment variable 'name' to an MSVC path type value. + + This is equivalent to a SET command prior to execution of spawned + commands. + """ + + if name == "lib": + p = self.get_msvc_paths("library") + else: + p = self.get_msvc_paths(name) + if p: + os.environ[name] = ';'.join(p) + + +if get_build_version() >= 8.0: + logger.debug("importing new compiler from distutils.msvc9compiler") + OldMSVCCompiler = MSVCCompiler + from packaging.compiler.msvc9compiler import MSVCCompiler + # get_build_architecture not really relevant now we support cross-compile + from packaging.compiler.msvc9compiler import MacroExpander diff --git a/Lib/packaging/compiler/unixccompiler.py b/Lib/packaging/compiler/unixccompiler.py new file mode 100644 index 0000000000..c857d03e09 --- /dev/null +++ b/Lib/packaging/compiler/unixccompiler.py @@ -0,0 +1,339 @@ +"""CCompiler implementation for Unix compilers. + +This module contains the UnixCCompiler class, a subclass of CCompiler +that handles the "typical" Unix-style command-line C compiler: + * macros defined with -Dname[=value] + * macros undefined with -Uname + * include search directories specified with -Idir + * libraries specified with -lllib + * library search directories specified with -Ldir + * compile handled by 'cc' (or similar) executable with -c option: + compiles .c to .o + * link static library handled by 'ar' command (possibly with 'ranlib') + * link shared library handled by 'cc -shared' +""" + +import os, sys + +from packaging.util import newer +from packaging.compiler.ccompiler import CCompiler +from packaging.compiler import gen_preprocess_options, gen_lib_options +from packaging.errors import (PackagingExecError, CompileError, + LibError, LinkError) +from packaging import logger +import sysconfig + + +# XXX Things not currently handled: +# * optimization/debug/warning flags; we just use whatever's in Python's +# Makefile and live with it. Is this adequate? If not, we might +# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, +# SunCCompiler, and I suspect down that road lies madness. +# * even if we don't know a warning flag from an optimization flag, +# we need some way for outsiders to feed preprocessor/compiler/linker +# flags in to us -- eg. a sysadmin might want to mandate certain flags +# via a site config file, or a user might want to set something for +# compiling this module distribution only via the pysetup command +# line, whatever. As long as these options come from something on the +# current system, they can be as system-dependent as they like, and we +# should just happily stuff them into the preprocessor/compiler/linker +# options and carry on. + +def _darwin_compiler_fixup(compiler_so, cc_args): + """ + This function will strip '-isysroot PATH' and '-arch ARCH' from the + compile flags if the user has specified one them in extra_compile_flags. + + This is needed because '-arch ARCH' adds another architecture to the + build, without a way to remove an architecture. Furthermore GCC will + barf if multiple '-isysroot' arguments are present. + """ + stripArch = stripSysroot = False + + compiler_so = list(compiler_so) + kernel_version = os.uname()[2] # 8.4.3 + major_version = int(kernel_version.split('.')[0]) + + if major_version < 8: + # OSX before 10.4.0, these don't support -arch and -isysroot at + # all. + stripArch = stripSysroot = True + else: + stripArch = '-arch' in cc_args + stripSysroot = '-isysroot' in cc_args + + if stripArch or 'ARCHFLAGS' in os.environ: + while True: + try: + index = compiler_so.index('-arch') + # Strip this argument and the next one: + del compiler_so[index:index+2] + except ValueError: + break + + if 'ARCHFLAGS' in os.environ and not stripArch: + # User specified different -arch flags in the environ, + # see also the sysconfig + compiler_so = compiler_so + os.environ['ARCHFLAGS'].split() + + if stripSysroot: + try: + index = compiler_so.index('-isysroot') + # Strip this argument and the next one: + del compiler_so[index:index+2] + except ValueError: + pass + + # Check if the SDK that is used during compilation actually exists, + # the universal build requires the usage of a universal SDK and not all + # users have that installed by default. + sysroot = None + if '-isysroot' in cc_args: + idx = cc_args.index('-isysroot') + sysroot = cc_args[idx+1] + elif '-isysroot' in compiler_so: + idx = compiler_so.index('-isysroot') + sysroot = compiler_so[idx+1] + + if sysroot and not os.path.isdir(sysroot): + logger.warning( + "compiling with an SDK that doesn't seem to exist: %r;\n" + "please check your Xcode installation", sysroot) + + return compiler_so + +class UnixCCompiler(CCompiler): + + name = 'unix' + description = 'Standard UNIX-style compiler' + + # These are used by CCompiler in two places: the constructor sets + # instance attributes 'preprocessor', 'compiler', etc. from them, and + # 'set_executable()' allows any of these to be set. The defaults here + # are pretty generic; they will probably have to be set by an outsider + # (eg. using information discovered by the sysconfig about building + # Python extensions). + executables = {'preprocessor' : None, + 'compiler' : ["cc"], + 'compiler_so' : ["cc"], + 'compiler_cxx' : ["cc"], + 'linker_so' : ["cc", "-shared"], + 'linker_exe' : ["cc"], + 'archiver' : ["ar", "-cr"], + 'ranlib' : None, + } + + if sys.platform[:6] == "darwin": + executables['ranlib'] = ["ranlib"] + + # Needed for the filename generation methods provided by the base + # class, CCompiler. NB. whoever instantiates/uses a particular + # UnixCCompiler instance should set 'shared_lib_ext' -- we set a + # reasonable common default here, but it's not necessarily used on all + # Unices! + + src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".so" + dylib_lib_extension = ".dylib" + static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" + if sys.platform == "cygwin": + exe_extension = ".exe" + + def preprocess(self, source, + output_file=None, macros=None, include_dirs=None, + extra_preargs=None, extra_postargs=None): + ignore, macros, include_dirs = \ + self._fix_compile_args(None, macros, include_dirs) + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = self.preprocessor + pp_opts + if output_file: + pp_args.extend(('-o', output_file)) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or we're + # generating output to stdout, or there's a target output file and + # the source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except PackagingExecError as msg: + raise CompileError(msg) + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + compiler_so = self.compiler_so + if sys.platform == 'darwin': + compiler_so = _darwin_compiler_fixup(compiler_so, cc_args + extra_postargs) + try: + self.spawn(compiler_so + cc_args + [src, '-o', obj] + + extra_postargs) + except PackagingExecError as msg: + raise CompileError(msg) + + def create_static_lib(self, objects, output_libname, + output_dir=None, debug=False, target_lang=None): + objects, output_dir = self._fix_object_args(objects, output_dir) + + output_filename = \ + self.library_filename(output_libname, output_dir=output_dir) + + if self._need_link(objects, output_filename): + self.mkpath(os.path.dirname(output_filename)) + self.spawn(self.archiver + + [output_filename] + + objects + self.objects) + + # Not many Unices required ranlib anymore -- SunOS 4.x is, I + # think the only major Unix that does. Maybe we need some + # platform intelligence here to skip ranlib if it's not + # needed -- or maybe Python's configure script took care of + # it for us, hence the check for leading colon. + if self.ranlib: + try: + self.spawn(self.ranlib + [output_filename]) + except PackagingExecError as msg: + raise LibError(msg) + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + def link(self, target_desc, objects, + output_filename, output_dir=None, libraries=None, + library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + objects, output_dir = self._fix_object_args(objects, output_dir) + libraries, library_dirs, runtime_library_dirs = \ + self._fix_lib_args(libraries, library_dirs, runtime_library_dirs) + + lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, + libraries) + if type(output_dir) not in (str, type(None)): + raise TypeError("'output_dir' must be a string or None") + if output_dir is not None: + output_filename = os.path.join(output_dir, output_filename) + + if self._need_link(objects, output_filename): + ld_args = (objects + self.objects + + lib_opts + ['-o', output_filename]) + if debug: + ld_args[:0] = ['-g'] + if extra_preargs: + ld_args[:0] = extra_preargs + if extra_postargs: + ld_args.extend(extra_postargs) + self.mkpath(os.path.dirname(output_filename)) + try: + if target_desc == CCompiler.EXECUTABLE: + linker = self.linker_exe[:] + else: + linker = self.linker_so[:] + if target_lang == "c++" and self.compiler_cxx: + # skip over environment variable settings if /usr/bin/env + # is used to set up the linker's environment. + # This is needed on OSX. Note: this assumes that the + # normal and C++ compiler have the same environment + # settings. + i = 0 + if os.path.basename(linker[0]) == "env": + i = 1 + while '=' in linker[i]: + i = i + 1 + + linker[i] = self.compiler_cxx[i] + + if sys.platform == 'darwin': + linker = _darwin_compiler_fixup(linker, ld_args) + + self.spawn(linker + ld_args) + except PackagingExecError as msg: + raise LinkError(msg) + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function, in + # ccompiler.py. + + def library_dir_option(self, dir): + return "-L" + dir + + def _is_gcc(self, compiler_name): + return "gcc" in compiler_name or "g++" in compiler_name + + def runtime_library_dir_option(self, dir): + # XXX Hackish, at the very least. See Python bug #445902: + # http://sourceforge.net/tracker/index.php + # ?func=detail&aid=445902&group_id=5470&atid=105470 + # Linkers on different platforms need different options to + # specify that directories need to be added to the list of + # directories searched for dependencies when a dynamic library + # is sought. GCC on GNU systems (Linux, FreeBSD, ...) has to + # be told to pass the -R option through to the linker, whereas + # other compilers and gcc on other systems just know this. + # Other compilers may need something slightly different. At + # this time, there's no way to determine this information from + # the configuration data stored in the Python installation, so + # we use this hack. + + compiler = os.path.basename(sysconfig.get_config_var("CC")) + if sys.platform[:6] == "darwin": + # MacOSX's linker doesn't understand the -R flag at all + return "-L" + dir + elif sys.platform[:5] == "hp-ux": + if self._is_gcc(compiler): + return ["-Wl,+s", "-L" + dir] + return ["+s", "-L" + dir] + elif sys.platform[:7] == "irix646" or sys.platform[:6] == "osf1V5": + return ["-rpath", dir] + elif self._is_gcc(compiler): + # gcc on non-GNU systems does not need -Wl, but can + # use it anyway. Since distutils has always passed in + # -Wl whenever gcc was used in the past it is probably + # safest to keep doing so. + if sysconfig.get_config_var("GNULD") == "yes": + # GNU ld needs an extra option to get a RUNPATH + # instead of just an RPATH. + return "-Wl,--enable-new-dtags,-R" + dir + else: + return "-Wl,-R" + dir + elif sys.platform[:3] == "aix": + return "-blibpath:" + dir + else: + # No idea how --enable-new-dtags would be passed on to + # ld if this system was using GNU ld. Don't know if a + # system like this even exists. + return "-R" + dir + + def library_option(self, lib): + return "-l" + lib + + def find_library_file(self, dirs, lib, debug=False): + shared_f = self.library_filename(lib, lib_type='shared') + dylib_f = self.library_filename(lib, lib_type='dylib') + static_f = self.library_filename(lib, lib_type='static') + + for dir in dirs: + shared = os.path.join(dir, shared_f) + dylib = os.path.join(dir, dylib_f) + static = os.path.join(dir, static_f) + # We're second-guessing the linker here, with not much hard + # data to go on: GCC seems to prefer the shared library, so I'm + # assuming that *all* Unix C compilers do. And of course I'm + # ignoring even GCC's "-static" option. So sue me. + if os.path.exists(dylib): + return dylib + elif os.path.exists(shared): + return shared + elif os.path.exists(static): + return static + + # Oops, didn't find it in *any* of 'dirs' + return None diff --git a/Lib/packaging/config.py b/Lib/packaging/config.py new file mode 100644 index 0000000000..e02800e960 --- /dev/null +++ b/Lib/packaging/config.py @@ -0,0 +1,379 @@ +"""Utilities to find and read config files used by packaging.""" + +import os +import sys +import logging + +from shlex import split +from configparser import RawConfigParser +from packaging import logger +from packaging.errors import PackagingOptionError +from packaging.compiler.extension import Extension +from packaging.util import (check_environ, iglob, resolve_name, strtobool, + split_multiline) +from packaging.compiler import set_compiler +from packaging.command import set_command +from packaging.markers import interpret + + +def _check_name(name, packages): + if '.' not in name: + return + parts = name.split('.') + modname = parts[-1] + parent = '.'.join(parts[:-1]) + if parent not in packages: + # we could log a warning instead of raising, but what's the use + # of letting people build modules they can't import? + raise PackagingOptionError( + 'parent package for extension %r not found' % name) + + +def _pop_values(values_dct, key): + """Remove values from the dictionary and convert them as a list""" + vals_str = values_dct.pop(key, '') + if not vals_str: + return + fields = [] + # the line separator is \n for setup.cfg files + for field in vals_str.split('\n'): + tmp_vals = field.split('--') + if len(tmp_vals) == 2 and not interpret(tmp_vals[1]): + continue + fields.append(tmp_vals[0]) + # Get bash options like `gcc -print-file-name=libgcc.a` XXX bash options? + vals = split(' '.join(fields)) + if vals: + return vals + + +def _rel_path(base, path): + # normalizes and returns a lstripped-/-separated path + base = base.replace(os.path.sep, '/') + path = path.replace(os.path.sep, '/') + assert path.startswith(base) + return path[len(base):].lstrip('/') + + +def get_resources_dests(resources_root, rules): + """Find destinations for resources files""" + destinations = {} + for base, suffix, dest in rules: + prefix = os.path.join(resources_root, base) + for abs_base in iglob(prefix): + abs_glob = os.path.join(abs_base, suffix) + for abs_path in iglob(abs_glob): + resource_file = _rel_path(resources_root, abs_path) + if dest is None: # remove the entry if it was here + destinations.pop(resource_file, None) + else: + rel_path = _rel_path(abs_base, abs_path) + rel_dest = dest.replace(os.path.sep, '/').rstrip('/') + destinations[resource_file] = rel_dest + '/' + rel_path + return destinations + + +class Config: + """Class used to work with configuration files""" + def __init__(self, dist): + self.dist = dist + self.setup_hooks = [] + + def run_hooks(self, config): + """Run setup hooks in the order defined in the spec.""" + for hook in self.setup_hooks: + hook(config) + + def find_config_files(self): + """Find as many configuration files as should be processed for this + platform, and return a list of filenames in the order in which they + should be parsed. The filenames returned are guaranteed to exist + (modulo nasty race conditions). + + There are three possible config files: packaging.cfg in the + Packaging installation directory (ie. where the top-level + Packaging __inst__.py file lives), a file in the user's home + directory named .pydistutils.cfg on Unix and pydistutils.cfg + on Windows/Mac; and setup.cfg in the current directory. + + The file in the user's home directory can be disabled with the + --no-user-cfg option. + """ + files = [] + check_environ() + + # Where to look for the system-wide Packaging config file + sys_dir = os.path.dirname(sys.modules['packaging'].__file__) + + # Look for the system config file + sys_file = os.path.join(sys_dir, "packaging.cfg") + if os.path.isfile(sys_file): + files.append(sys_file) + + # What to call the per-user config file + if os.name == 'posix': + user_filename = ".pydistutils.cfg" + else: + user_filename = "pydistutils.cfg" + + # And look for the user config file + if self.dist.want_user_cfg: + user_file = os.path.join(os.path.expanduser('~'), user_filename) + if os.path.isfile(user_file): + files.append(user_file) + + # All platforms support local setup.cfg + local_file = "setup.cfg" + if os.path.isfile(local_file): + files.append(local_file) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("using config files: %s", ', '.join(files)) + return files + + def _convert_metadata(self, name, value): + # converts a value found in setup.cfg into a valid metadata + # XXX + return value + + def _read_setup_cfg(self, parser, cfg_filename): + cfg_directory = os.path.dirname(os.path.abspath(cfg_filename)) + content = {} + for section in parser.sections(): + content[section] = dict(parser.items(section)) + + # global setup hooks are called first + if 'global' in content: + if 'setup_hooks' in content['global']: + setup_hooks = split_multiline(content['global']['setup_hooks']) + + # add project directory to sys.path, to allow hooks to be + # distributed with the project + sys.path.insert(0, cfg_directory) + try: + for line in setup_hooks: + try: + hook = resolve_name(line) + except ImportError as e: + logger.warning('cannot find setup hook: %s', + e.args[0]) + else: + self.setup_hooks.append(hook) + self.run_hooks(content) + finally: + sys.path.pop(0) + + metadata = self.dist.metadata + + # setting the metadata values + if 'metadata' in content: + for key, value in content['metadata'].items(): + key = key.replace('_', '-') + if metadata.is_multi_field(key): + value = split_multiline(value) + + if key == 'project-url': + value = [(label.strip(), url.strip()) + for label, url in + [v.split(',') for v in value]] + + if key == 'description-file': + if 'description' in content['metadata']: + msg = ("description and description-file' are " + "mutually exclusive") + raise PackagingOptionError(msg) + + filenames = value.split() + + # concatenate all files + value = [] + for filename in filenames: + # will raise if file not found + with open(filename) as description_file: + value.append(description_file.read().strip()) + # add filename as a required file + if filename not in metadata.requires_files: + metadata.requires_files.append(filename) + value = '\n'.join(value).strip() + key = 'description' + + if metadata.is_metadata_field(key): + metadata[key] = self._convert_metadata(key, value) + + if 'files' in content: + files = content['files'] + self.dist.package_dir = files.pop('packages_root', None) + + files = dict((key, split_multiline(value)) for key, value in + files.items()) + + self.dist.packages = [] + + packages = files.get('packages', []) + if isinstance(packages, str): + packages = [packages] + + for package in packages: + if ':' in package: + dir_, package = package.split(':') + self.dist.package_dir[package] = dir_ + self.dist.packages.append(package) + + self.dist.py_modules = files.get('modules', []) + if isinstance(self.dist.py_modules, str): + self.dist.py_modules = [self.dist.py_modules] + self.dist.scripts = files.get('scripts', []) + if isinstance(self.dist.scripts, str): + self.dist.scripts = [self.dist.scripts] + + self.dist.package_data = {} + for data in files.get('package_data', []): + data = data.split('=') + if len(data) != 2: + continue # FIXME errors should never pass silently + key, value = data + self.dist.package_data[key.strip()] = value.strip() + + self.dist.data_files = [] + for data in files.get('data_files', []): + data = data.split('=') + if len(data) != 2: + continue + key, value = data + values = [v.strip() for v in value.split(',')] + self.dist.data_files.append((key, values)) + + # manifest template + self.dist.extra_files = files.get('extra_files', []) + + resources = [] + for rule in files.get('resources', []): + glob, destination = rule.split('=', 1) + rich_glob = glob.strip().split(' ', 1) + if len(rich_glob) == 2: + prefix, suffix = rich_glob + else: + assert len(rich_glob) == 1 + prefix = '' + suffix = glob + if destination == '<exclude>': + destination = None + resources.append( + (prefix.strip(), suffix.strip(), destination.strip())) + self.dist.data_files = get_resources_dests( + cfg_directory, resources) + + ext_modules = self.dist.ext_modules + for section_key in content: + # no str.partition in 2.4 :( + labels = section_key.split(':') + if len(labels) == 2 and labels[0] == 'extension': + values_dct = content[section_key] + if 'name' in values_dct: + raise PackagingOptionError( + 'extension name should be given as [extension: name], ' + 'not as key') + name = labels[1].strip() + _check_name(name, self.dist.packages) + ext_modules.append(Extension( + name, + _pop_values(values_dct, 'sources'), + _pop_values(values_dct, 'include_dirs'), + _pop_values(values_dct, 'define_macros'), + _pop_values(values_dct, 'undef_macros'), + _pop_values(values_dct, 'library_dirs'), + _pop_values(values_dct, 'libraries'), + _pop_values(values_dct, 'runtime_library_dirs'), + _pop_values(values_dct, 'extra_objects'), + _pop_values(values_dct, 'extra_compile_args'), + _pop_values(values_dct, 'extra_link_args'), + _pop_values(values_dct, 'export_symbols'), + _pop_values(values_dct, 'swig_opts'), + _pop_values(values_dct, 'depends'), + values_dct.pop('language', None), + values_dct.pop('optional', None), + **values_dct)) + + def parse_config_files(self, filenames=None): + if filenames is None: + filenames = self.find_config_files() + + logger.debug("Distribution.parse_config_files():") + + parser = RawConfigParser() + + for filename in filenames: + logger.debug(" reading %s", filename) + parser.read(filename, encoding='utf-8') + + if os.path.split(filename)[-1] == 'setup.cfg': + self._read_setup_cfg(parser, filename) + + for section in parser.sections(): + if section == 'global': + if parser.has_option('global', 'compilers'): + self._load_compilers(parser.get('global', 'compilers')) + + if parser.has_option('global', 'commands'): + self._load_commands(parser.get('global', 'commands')) + + options = parser.options(section) + opt_dict = self.dist.get_option_dict(section) + + for opt in options: + if opt == '__name__': + continue + val = parser.get(section, opt) + opt = opt.replace('-', '_') + + if opt == 'sub_commands': + val = split_multiline(val) + if isinstance(val, str): + val = [val] + + # Hooks use a suffix system to prevent being overriden + # by a config file processed later (i.e. a hook set in + # the user config file cannot be replaced by a hook + # set in a project config file, unless they have the + # same suffix). + if (opt.startswith("pre_hook.") or + opt.startswith("post_hook.")): + hook_type, alias = opt.split(".") + hook_dict = opt_dict.setdefault( + hook_type, (filename, {}))[1] + hook_dict[alias] = val + else: + opt_dict[opt] = filename, val + + # Make the RawConfigParser forget everything (so we retain + # the original filenames that options come from) + parser.__init__() + + # If there was a "global" section in the config file, use it + # to set Distribution options. + if 'global' in self.dist.command_options: + for opt, (src, val) in self.dist.command_options['global'].items(): + alias = self.dist.negative_opt.get(opt) + try: + if alias: + setattr(self.dist, alias, not strtobool(val)) + elif opt == 'dry_run': # FIXME ugh! + setattr(self.dist, opt, strtobool(val)) + else: + setattr(self.dist, opt, val) + except ValueError as msg: + raise PackagingOptionError(msg) + + def _load_compilers(self, compilers): + compilers = split_multiline(compilers) + if isinstance(compilers, str): + compilers = [compilers] + for compiler in compilers: + set_compiler(compiler.strip()) + + def _load_commands(self, commands): + commands = split_multiline(commands) + if isinstance(commands, str): + commands = [commands] + for command in commands: + set_command(command.strip()) diff --git a/Lib/packaging/create.py b/Lib/packaging/create.py new file mode 100644 index 0000000000..ecabca041a --- /dev/null +++ b/Lib/packaging/create.py @@ -0,0 +1,693 @@ +"""Interactive helper used to create a setup.cfg file. + +This script will generate a packaging configuration file by looking at +the current directory and asking the user questions. It is intended to +be called as *pysetup create*. +""" + +# Original code by Sean Reifschneider <jafo@tummy.com> + +# Original TODO list: +# Look for a license file and automatically add the category. +# When a .c file is found during the walk, can we add it as an extension? +# Ask if there is a maintainer different that the author +# Ask for the platform (can we detect this via "import win32" or something?) +# Ask for the dependencies. +# Ask for the Requires-Dist +# Ask for the Provides-Dist +# Ask for a description +# Detect scripts (not sure how. #! outside of package?) + +import os +import re +import imp +import sys +import glob +import shutil +import sysconfig +import tokenize +from hashlib import md5 +from textwrap import dedent +from functools import cmp_to_key +from configparser import RawConfigParser +# importing this with an underscore as it should be replaced by the +# dict form or another structures for all purposes +from packaging._trove import all_classifiers as _CLASSIFIERS_LIST +from packaging.version import is_valid_version + +_FILENAME = 'setup.cfg' +_DEFAULT_CFG = '.pypkgcreate' # FIXME use a section in user .pydistutils.cfg + +_helptext = { + 'name': ''' +The name of the project to be packaged, usually a single word composed +of lower-case characters such as "zope.interface", "sqlalchemy" or +"CherryPy". +''', + 'version': ''' +Version number of the software, typically 2 or 3 numbers separated by +dots such as "1.0", "0.6b3", or "3.2.1". "0.1.0" is recommended for +initial development. +''', + 'summary': ''' +A one-line summary of what this project is or does, typically a sentence +80 characters or less in length. +''', + 'author': ''' +The full name of the author (typically you). +''', + 'author_email': ''' +Email address of the project author. +''', + 'do_classifier': ''' +Trove classifiers are optional identifiers that allow you to specify the +intended audience by saying things like "Beta software with a text UI +for Linux under the PSF license". However, this can be a somewhat +involved process. +''', + 'packages': ''' +Python packages included in the project. +''', + 'modules': ''' +Pure Python modules included in the project. +''', + 'extra_files': ''' +You can provide extra files/dirs contained in your project. +It has to follow the template syntax. XXX add help here. +''', + + 'home_page': ''' +The home page for the project, typically a public Web page. +''', + 'trove_license': ''' +Optionally you can specify a license. Type a string that identifies a +common license, and then you can select a list of license specifiers. +''', + 'trove_generic': ''' +Optionally, you can set other trove identifiers for things such as the +human language, programming language, user interface, etc. +''', + 'setup.py found': ''' +The setup.py script will be executed to retrieve the metadata. +An interactive helper will be run if you answer "n", +''', +} + +PROJECT_MATURITY = ['Development Status :: 1 - Planning', + 'Development Status :: 2 - Pre-Alpha', + 'Development Status :: 3 - Alpha', + 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', + 'Development Status :: 6 - Mature', + 'Development Status :: 7 - Inactive'] + +# XXX everything needs docstrings and tests (both low-level tests of various +# methods and functional tests of running the script) + + +def load_setup(): + """run the setup script (i.e the setup.py file) + + This function load the setup file in all cases (even if it have already + been loaded before, because we are monkey patching its setup function with + a particular one""" + with open("setup.py", "rb") as f: + encoding, lines = tokenize.detect_encoding(f.readline) + with open("setup.py", encoding=encoding) as f: + imp.load_module("setup", f, "setup.py", (".py", "r", imp.PY_SOURCE)) + + +def ask_yn(question, default=None, helptext=None): + question += ' (y/n)' + while True: + answer = ask(question, default, helptext, required=True) + if answer and answer[0].lower() in 'yn': + return answer[0].lower() + + print('\nERROR: You must select "Y" or "N".\n') + + +# XXX use util.ask +# FIXME: if prompt ends with '?', don't add ':' + + +def ask(question, default=None, helptext=None, required=True, + lengthy=False, multiline=False): + prompt = '%s: ' % (question,) + if default: + prompt = '%s [%s]: ' % (question, default) + if default and len(question) + len(default) > 70: + prompt = '%s\n [%s]: ' % (question, default) + if lengthy or multiline: + prompt += '\n > ' + + if not helptext: + helptext = 'No additional help available.' + + helptext = helptext.strip("\n") + + while True: + sys.stdout.write(prompt) + sys.stdout.flush() + + line = sys.stdin.readline().strip() + if line == '?': + print('=' * 70) + print(helptext) + print('=' * 70) + continue + if default and not line: + return default + if not line and required: + print('*' * 70) + print('This value cannot be empty.') + print('===========================') + if helptext: + print(helptext) + print('*' * 70) + continue + return line + + +def convert_yn_to_bool(yn, yes=True, no=False): + """Convert a y/yes or n/no to a boolean value.""" + if yn.lower().startswith('y'): + return yes + else: + return no + + +def _build_classifiers_dict(classifiers): + d = {} + for key in classifiers: + subdict = d + for subkey in key.split(' :: '): + if subkey not in subdict: + subdict[subkey] = {} + subdict = subdict[subkey] + return d + +CLASSIFIERS = _build_classifiers_dict(_CLASSIFIERS_LIST) + + +def _build_licences(classifiers): + res = [] + for index, item in enumerate(classifiers): + if not item.startswith('License :: '): + continue + res.append((index, item.split(' :: ')[-1].lower())) + return res + +LICENCES = _build_licences(_CLASSIFIERS_LIST) + + +class MainProgram: + """Make a project setup configuration file (setup.cfg).""" + + def __init__(self): + self.configparser = None + self.classifiers = set() + self.data = {'name': '', + 'version': '1.0.0', + 'classifier': self.classifiers, + 'packages': [], + 'modules': [], + 'platform': [], + 'resources': [], + 'extra_files': [], + 'scripts': [], + } + self._load_defaults() + + def __call__(self): + setupcfg_defined = False + if self.has_setup_py() and self._prompt_user_for_conversion(): + setupcfg_defined = self.convert_py_to_cfg() + if not setupcfg_defined: + self.define_cfg_values() + self._write_cfg() + + def has_setup_py(self): + """Test for the existence of a setup.py file.""" + return os.path.exists('setup.py') + + def define_cfg_values(self): + self.inspect() + self.query_user() + + def _lookup_option(self, key): + if not self.configparser.has_option('DEFAULT', key): + return None + return self.configparser.get('DEFAULT', key) + + def _load_defaults(self): + # Load default values from a user configuration file + self.configparser = RawConfigParser() + # TODO replace with section in distutils config file + default_cfg = os.path.expanduser(os.path.join('~', _DEFAULT_CFG)) + self.configparser.read(default_cfg) + self.data['author'] = self._lookup_option('author') + self.data['author_email'] = self._lookup_option('author_email') + + def _prompt_user_for_conversion(self): + # Prompt the user about whether they would like to use the setup.py + # conversion utility to generate a setup.cfg or generate the setup.cfg + # from scratch + answer = ask_yn(('A legacy setup.py has been found.\n' + 'Would you like to convert it to a setup.cfg?'), + default="y", + helptext=_helptext['setup.py found']) + return convert_yn_to_bool(answer) + + def _dotted_packages(self, data): + packages = sorted(data) + modified_pkgs = [] + for pkg in packages: + pkg = pkg.lstrip('./') + pkg = pkg.replace('/', '.') + modified_pkgs.append(pkg) + return modified_pkgs + + def _write_cfg(self): + if os.path.exists(_FILENAME): + if os.path.exists('%s.old' % _FILENAME): + print("ERROR: %(name)s.old backup exists, please check that " + "current %(name)s is correct and remove %(name)s.old" % + {'name': _FILENAME}) + return + shutil.move(_FILENAME, '%s.old' % _FILENAME) + + with open(_FILENAME, 'w', encoding='utf-8') as fp: + fp.write('[metadata]\n') + # TODO use metadata module instead of hard-coding field-specific + # behavior here + + # simple string entries + for name in ('name', 'version', 'summary', 'download_url'): + fp.write('%s = %s\n' % (name, self.data.get(name, 'UNKNOWN'))) + + # optional string entries + if 'keywords' in self.data and self.data['keywords']: + fp.write('keywords = %s\n' % ' '.join(self.data['keywords'])) + for name in ('home_page', 'author', 'author_email', + 'maintainer', 'maintainer_email', 'description-file'): + if name in self.data and self.data[name]: + fp.write('%s = %s\n' % (name, self.data[name])) + if 'description' in self.data: + fp.write( + 'description = %s\n' + % '\n |'.join(self.data['description'].split('\n'))) + + # multiple use string entries + for name in ('platform', 'supported-platform', 'classifier', + 'requires-dist', 'provides-dist', 'obsoletes-dist', + 'requires-external'): + if not(name in self.data and self.data[name]): + continue + fp.write('%s = ' % name) + fp.write(''.join(' %s\n' % val + for val in self.data[name]).lstrip()) + fp.write('\n[files]\n') + for name in ('packages', 'modules', 'scripts', + 'package_data', 'extra_files'): + if not(name in self.data and self.data[name]): + continue + fp.write('%s = %s\n' + % (name, '\n '.join(self.data[name]).strip())) + fp.write('\nresources =\n') + for src, dest in self.data['resources']: + fp.write(' %s = %s\n' % (src, dest)) + fp.write('\n') + + os.chmod(_FILENAME, 0o644) + print('Wrote "%s".' % _FILENAME) + + def convert_py_to_cfg(self): + """Generate a setup.cfg from an existing setup.py. + + It only exports the distutils metadata (setuptools specific metadata + is not currently supported). + """ + data = self.data + + def setup_mock(**attrs): + """Mock the setup(**attrs) in order to retrieve metadata.""" + + # TODO use config and metadata instead of Distribution + from distutils.dist import Distribution + dist = Distribution(attrs) + dist.parse_config_files() + + # 1. retrieve metadata fields that are quite similar in + # PEP 314 and PEP 345 + labels = (('name',) * 2, + ('version',) * 2, + ('author',) * 2, + ('author_email',) * 2, + ('maintainer',) * 2, + ('maintainer_email',) * 2, + ('description', 'summary'), + ('long_description', 'description'), + ('url', 'home_page'), + ('platforms', 'platform'), + # backport only for 2.5+ + ('provides', 'provides-dist'), + ('obsoletes', 'obsoletes-dist'), + ('requires', 'requires-dist')) + + get = lambda lab: getattr(dist.metadata, lab.replace('-', '_')) + data.update((new, get(old)) for old, new in labels if get(old)) + + # 2. retrieve data that requires special processing + data['classifier'].update(dist.get_classifiers() or []) + data['scripts'].extend(dist.scripts or []) + data['packages'].extend(dist.packages or []) + data['modules'].extend(dist.py_modules or []) + # 2.1 data_files -> resources + if dist.data_files: + if (len(dist.data_files) < 2 or + isinstance(dist.data_files[1], str)): + dist.data_files = [('', dist.data_files)] + # add tokens in the destination paths + vars = {'distribution.name': data['name']} + path_tokens = list(sysconfig.get_paths(vars=vars).items()) + + # TODO replace this with a key function + def length_comparison(x, y): + len_x = len(x[1]) + len_y = len(y[1]) + if len_x == len_y: + return 0 + elif len_x < len_y: + return -1 + else: + return 1 + + # sort tokens to use the longest one first + path_tokens.sort(key=cmp_to_key(length_comparison)) + for dest, srcs in (dist.data_files or []): + dest = os.path.join(sys.prefix, dest) + dest = dest.replace(os.path.sep, '/') + for tok, path in path_tokens: + path = path.replace(os.path.sep, '/') + if not dest.startswith(path): + continue + + dest = ('{%s}' % tok) + dest[len(path):] + files = [('/ '.join(src.rsplit('/', 1)), dest) + for src in srcs] + data['resources'].extend(files) + + # 2.2 package_data -> extra_files + package_dirs = dist.package_dir or {} + for package, extras in dist.package_data.items() or []: + package_dir = package_dirs.get(package, package) + for file_ in extras: + if package_dir: + file_ = package_dir + '/' + file_ + data['extra_files'].append(file_) + + # Use README file if its content is the desciption + if "description" in data: + ref = md5(re.sub('\s', '', + self.data['description']).lower().encode()) + ref = ref.digest() + for readme in glob.glob('README*'): + with open(readme, encoding='utf-8') as fp: + contents = fp.read() + contents = re.sub('\s', '', contents.lower()).encode() + val = md5(contents).digest() + if val == ref: + del data['description'] + data['description-file'] = readme + break + + # apply monkey patch to distutils (v1) and setuptools (if needed) + # (abort the feature if distutils v1 has been killed) + try: + from distutils import core + core.setup # make sure it's not d2 maskerading as d1 + except (ImportError, AttributeError): + return + saved_setups = [(core, core.setup)] + core.setup = setup_mock + try: + import setuptools + except ImportError: + pass + else: + saved_setups.append((setuptools, setuptools.setup)) + setuptools.setup = setup_mock + # get metadata by executing the setup.py with the patched setup(...) + success = False # for python < 2.4 + try: + load_setup() + success = True + finally: # revert monkey patches + for patched_module, original_setup in saved_setups: + patched_module.setup = original_setup + if not self.data: + raise ValueError('Unable to load metadata from setup.py') + return success + + def inspect(self): + """Inspect the current working diretory for a name and version. + + This information is harvested in where the directory is named + like [name]-[version]. + """ + dir_name = os.path.basename(os.getcwd()) + self.data['name'] = dir_name + match = re.match(r'(.*)-(\d.+)', dir_name) + if match: + self.data['name'] = match.group(1) + self.data['version'] = match.group(2) + # TODO needs testing! + if not is_valid_version(self.data['version']): + msg = "Invalid version discovered: %s" % self.data['version'] + raise ValueError(msg) + + def query_user(self): + self.data['name'] = ask('Project name', self.data['name'], + _helptext['name']) + + self.data['version'] = ask('Current version number', + self.data.get('version'), _helptext['version']) + self.data['summary'] = ask('Project description summary', + self.data.get('summary'), _helptext['summary'], + lengthy=True) + self.data['author'] = ask('Author name', + self.data.get('author'), _helptext['author']) + self.data['author_email'] = ask('Author email address', + self.data.get('author_email'), _helptext['author_email']) + self.data['home_page'] = ask('Project home page', + self.data.get('home_page'), _helptext['home_page'], + required=False) + + if ask_yn('Do you want me to automatically build the file list ' + 'with everything I can find in the current directory? ' + 'If you say no, you will have to define them manually.') == 'y': + self._find_files() + else: + while ask_yn('Do you want to add a single module?' + ' (you will be able to add full packages next)', + helptext=_helptext['modules']) == 'y': + self._set_multi('Module name', 'modules') + + while ask_yn('Do you want to add a package?', + helptext=_helptext['packages']) == 'y': + self._set_multi('Package name', 'packages') + + while ask_yn('Do you want to add an extra file?', + helptext=_helptext['extra_files']) == 'y': + self._set_multi('Extra file/dir name', 'extra_files') + + if ask_yn('Do you want to set Trove classifiers?', + helptext=_helptext['do_classifier']) == 'y': + self.set_classifier() + + def _find_files(self): + # we are looking for python modules and packages, + # other stuff are added as regular files + pkgs = self.data['packages'] + modules = self.data['modules'] + extra_files = self.data['extra_files'] + + def is_package(path): + return os.path.exists(os.path.join(path, '__init__.py')) + + curdir = os.getcwd() + scanned = [] + _pref = ['lib', 'include', 'dist', 'build', '.', '~'] + _suf = ['.pyc'] + + def to_skip(path): + path = relative(path) + + for pref in _pref: + if path.startswith(pref): + return True + + for suf in _suf: + if path.endswith(suf): + return True + + return False + + def relative(path): + return path[len(curdir) + 1:] + + def dotted(path): + res = relative(path).replace(os.path.sep, '.') + if res.endswith('.py'): + res = res[:-len('.py')] + return res + + # first pass: packages + for root, dirs, files in os.walk(curdir): + if to_skip(root): + continue + for dir_ in sorted(dirs): + if to_skip(dir_): + continue + fullpath = os.path.join(root, dir_) + dotted_name = dotted(fullpath) + if is_package(fullpath) and dotted_name not in pkgs: + pkgs.append(dotted_name) + scanned.append(fullpath) + + # modules and extra files + for root, dirs, files in os.walk(curdir): + if to_skip(root): + continue + + if any(root.startswith(path) for path in scanned): + continue + + for file in sorted(files): + fullpath = os.path.join(root, file) + if to_skip(fullpath): + continue + # single module? + if os.path.splitext(file)[-1] == '.py': + modules.append(dotted(fullpath)) + else: + extra_files.append(relative(fullpath)) + + def _set_multi(self, question, name): + existing_values = self.data[name] + value = ask(question, helptext=_helptext[name]).strip() + if value not in existing_values: + existing_values.append(value) + + def set_classifier(self): + self.set_maturity_status(self.classifiers) + self.set_license(self.classifiers) + self.set_other_classifier(self.classifiers) + + def set_other_classifier(self, classifiers): + if ask_yn('Do you want to set other trove identifiers?', 'n', + _helptext['trove_generic']) != 'y': + return + self.walk_classifiers(classifiers, [CLASSIFIERS], '') + + def walk_classifiers(self, classifiers, trovepath, desc): + trove = trovepath[-1] + + if not trove: + return + + for key in sorted(trove): + if len(trove[key]) == 0: + if ask_yn('Add "%s"' % desc[4:] + ' :: ' + key, 'n') == 'y': + classifiers.add(desc[4:] + ' :: ' + key) + continue + + if ask_yn('Do you want to set items under\n "%s" (%d sub-items)?' + % (key, len(trove[key])), 'n', + _helptext['trove_generic']) == 'y': + self.walk_classifiers(classifiers, trovepath + [trove[key]], + desc + ' :: ' + key) + + def set_license(self, classifiers): + while True: + license = ask('What license do you use?', + helptext=_helptext['trove_license'], required=False) + if not license: + return + + license_words = license.lower().split(' ') + found_list = [] + + for index, licence in LICENCES: + for word in license_words: + if word in licence: + found_list.append(index) + break + + if len(found_list) == 0: + print('ERROR: Could not find a matching license for "%s"' % + license) + continue + + question = 'Matching licenses:\n\n' + + for index, list_index in enumerate(found_list): + question += ' %s) %s\n' % (index + 1, + _CLASSIFIERS_LIST[list_index]) + + question += ('\nType the number of the license you wish to use or ' + '? to try again:') + choice = ask(question, required=False) + + if choice == '?': + continue + if choice == '': + return + + try: + index = found_list[int(choice) - 1] + except ValueError: + print("ERROR: Invalid selection, type a number from the list " + "above.") + + classifiers.add(_CLASSIFIERS_LIST[index]) + + def set_maturity_status(self, classifiers): + maturity_name = lambda mat: mat.split('- ')[-1] + maturity_question = '''\ + Please select the project status: + + %s + + Status''' % '\n'.join('%s - %s' % (i, maturity_name(n)) + for i, n in enumerate(PROJECT_MATURITY)) + while True: + choice = ask(dedent(maturity_question), required=False) + + if choice: + try: + choice = int(choice) - 1 + key = PROJECT_MATURITY[choice] + classifiers.add(key) + return + except (IndexError, ValueError): + print("ERROR: Invalid selection, type a single digit " + "number.") + + +def main(): + """Main entry point.""" + program = MainProgram() + # # uncomment when implemented + # if not program.load_existing_setup_script(): + # program.inspect_directory() + # program.query_user() + # program.update_config_file() + # program.write_setup_script() + # packaging.util.cfg_to_args() + program() + + +if __name__ == '__main__': + main() diff --git a/Lib/packaging/database.py b/Lib/packaging/database.py new file mode 100644 index 0000000000..c733b7af67 --- /dev/null +++ b/Lib/packaging/database.py @@ -0,0 +1,647 @@ +"""PEP 376 implementation.""" + +import io +import os +import re +import csv +import sys +import zipimport +from hashlib import md5 +from packaging import logger +from packaging.errors import PackagingError +from packaging.version import suggest_normalized_version, VersionPredicate +from packaging.metadata import Metadata + + +__all__ = [ + 'Distribution', 'EggInfoDistribution', 'distinfo_dirname', + 'get_distributions', 'get_distribution', 'get_file_users', + 'provides_distribution', 'obsoletes_distribution', + 'enable_cache', 'disable_cache', 'clear_cache', + 'get_file_path', 'get_file'] + + +# TODO update docs + +DIST_FILES = ('INSTALLER', 'METADATA', 'RECORD', 'REQUESTED', 'RESOURCES') + +# Cache +_cache_name = {} # maps names to Distribution instances +_cache_name_egg = {} # maps names to EggInfoDistribution instances +_cache_path = {} # maps paths to Distribution instances +_cache_path_egg = {} # maps paths to EggInfoDistribution instances +_cache_generated = False # indicates if .dist-info distributions are cached +_cache_generated_egg = False # indicates if .dist-info and .egg are cached +_cache_enabled = True + + +def enable_cache(): + """ + Enables the internal cache. + + Note that this function will not clear the cache in any case, for that + functionality see :func:`clear_cache`. + """ + global _cache_enabled + + _cache_enabled = True + + +def disable_cache(): + """ + Disables the internal cache. + + Note that this function will not clear the cache in any case, for that + functionality see :func:`clear_cache`. + """ + global _cache_enabled + + _cache_enabled = False + + +def clear_cache(): + """ Clears the internal cache. """ + global _cache_generated, _cache_generated_egg + + _cache_name.clear() + _cache_name_egg.clear() + _cache_path.clear() + _cache_path_egg.clear() + _cache_generated = False + _cache_generated_egg = False + + +def _yield_distributions(include_dist, include_egg, paths): + """ + Yield .dist-info and .egg(-info) distributions, based on the arguments + + :parameter include_dist: yield .dist-info distributions + :parameter include_egg: yield .egg(-info) distributions + """ + for path in paths: + realpath = os.path.realpath(path) + if not os.path.isdir(realpath): + continue + for dir in os.listdir(realpath): + dist_path = os.path.join(realpath, dir) + if include_dist and dir.endswith('.dist-info'): + yield Distribution(dist_path) + elif include_egg and (dir.endswith('.egg-info') or + dir.endswith('.egg')): + yield EggInfoDistribution(dist_path) + + +def _generate_cache(use_egg_info, paths): + global _cache_generated, _cache_generated_egg + + if _cache_generated_egg or (_cache_generated and not use_egg_info): + return + else: + gen_dist = not _cache_generated + gen_egg = use_egg_info + + for dist in _yield_distributions(gen_dist, gen_egg, paths): + if isinstance(dist, Distribution): + _cache_path[dist.path] = dist + if dist.name not in _cache_name: + _cache_name[dist.name] = [] + _cache_name[dist.name].append(dist) + else: + _cache_path_egg[dist.path] = dist + if dist.name not in _cache_name_egg: + _cache_name_egg[dist.name] = [] + _cache_name_egg[dist.name].append(dist) + + if gen_dist: + _cache_generated = True + if gen_egg: + _cache_generated_egg = True + + +class Distribution: + """Created with the *path* of the ``.dist-info`` directory provided to the + constructor. It reads the metadata contained in ``METADATA`` when it is + instantiated.""" + + name = '' + """The name of the distribution.""" + + version = '' + """The version of the distribution.""" + + metadata = None + """A :class:`packaging.metadata.Metadata` instance loaded with + the distribution's ``METADATA`` file.""" + + requested = False + """A boolean that indicates whether the ``REQUESTED`` metadata file is + present (in other words, whether the package was installed by user + request or it was installed as a dependency).""" + + def __init__(self, path): + if _cache_enabled and path in _cache_path: + self.metadata = _cache_path[path].metadata + else: + metadata_path = os.path.join(path, 'METADATA') + self.metadata = Metadata(path=metadata_path) + + self.name = self.metadata['Name'] + self.version = self.metadata['Version'] + self.path = path + + if _cache_enabled and path not in _cache_path: + _cache_path[path] = self + + def __repr__(self): + return '<Distribution %r %s at %r>' % ( + self.name, self.version, self.path) + + def _get_records(self, local=False): + results = [] + with self.get_distinfo_file('RECORD') as record: + record_reader = csv.reader(record, delimiter=',', + lineterminator='\n') + for row in record_reader: + missing = [None for i in range(len(row), 3)] + path, checksum, size = row + missing + if local: + path = path.replace('/', os.sep) + path = os.path.join(sys.prefix, path) + results.append((path, checksum, size)) + return results + + def get_resource_path(self, relative_path): + with self.get_distinfo_file('RESOURCES') as resources_file: + resources_reader = csv.reader(resources_file, delimiter=',', + lineterminator='\n') + for relative, destination in resources_reader: + if relative == relative_path: + return destination + raise KeyError( + 'no resource file with relative path %r is installed' % + relative_path) + + def list_installed_files(self, local=False): + """ + Iterates over the ``RECORD`` entries and returns a tuple + ``(path, md5, size)`` for each line. If *local* is ``True``, + the returned path is transformed into a local absolute path. + Otherwise the raw value from RECORD is returned. + + A local absolute path is an absolute path in which occurrences of + ``'/'`` have been replaced by the system separator given by ``os.sep``. + + :parameter local: flag to say if the path should be returned as a local + absolute path + + :type local: boolean + :returns: iterator of (path, md5, size) + """ + for result in self._get_records(local): + yield result + + def uses(self, path): + """ + Returns ``True`` if path is listed in ``RECORD``. *path* can be a local + absolute path or a relative ``'/'``-separated path. + + :rtype: boolean + """ + for p, checksum, size in self._get_records(): + local_absolute = os.path.join(sys.prefix, p) + if path == p or path == local_absolute: + return True + return False + + def get_distinfo_file(self, path, binary=False): + """ + Returns a file located under the ``.dist-info`` directory. Returns a + ``file`` instance for the file pointed by *path*. + + :parameter path: a ``'/'``-separated path relative to the + ``.dist-info`` directory or an absolute path; + If *path* is an absolute path and doesn't start + with the ``.dist-info`` directory path, + a :class:`PackagingError` is raised + :type path: string + :parameter binary: If *binary* is ``True``, opens the file in read-only + binary mode (``rb``), otherwise opens it in + read-only mode (``r``). + :rtype: file object + """ + open_flags = 'r' + if binary: + open_flags += 'b' + + # Check if it is an absolute path # XXX use relpath, add tests + if path.find(os.sep) >= 0: + # it's an absolute path? + distinfo_dirname, path = path.split(os.sep)[-2:] + if distinfo_dirname != self.path.split(os.sep)[-1]: + raise PackagingError( + 'dist-info file %r does not belong to the %r %s ' + 'distribution' % (path, self.name, self.version)) + + # The file must be relative + if path not in DIST_FILES: + raise PackagingError('invalid path for a dist-info file: %r' % + path) + + path = os.path.join(self.path, path) + return open(path, open_flags) + + def list_distinfo_files(self, local=False): + """ + Iterates over the ``RECORD`` entries and returns paths for each line if + the path is pointing to a file located in the ``.dist-info`` directory + or one of its subdirectories. + + :parameter local: If *local* is ``True``, each returned path is + transformed into a local absolute path. Otherwise the + raw value from ``RECORD`` is returned. + :type local: boolean + :returns: iterator of paths + """ + for path, checksum, size in self._get_records(local): + yield path + + def __eq__(self, other): + return isinstance(other, Distribution) and self.path == other.path + + # See http://docs.python.org/reference/datamodel#object.__hash__ + __hash__ = object.__hash__ + + +class EggInfoDistribution: + """Created with the *path* of the ``.egg-info`` directory or file provided + to the constructor. It reads the metadata contained in the file itself, or + if the given path happens to be a directory, the metadata is read from the + file ``PKG-INFO`` under that directory.""" + + name = '' + """The name of the distribution.""" + + version = '' + """The version of the distribution.""" + + metadata = None + """A :class:`packaging.metadata.Metadata` instance loaded with + the distribution's ``METADATA`` file.""" + + _REQUIREMENT = re.compile( + r'(?P<name>[-A-Za-z0-9_.]+)\s*' + r'(?P<first>(?:<|<=|!=|==|>=|>)[-A-Za-z0-9_.]+)?\s*' + r'(?P<rest>(?:\s*,\s*(?:<|<=|!=|==|>=|>)[-A-Za-z0-9_.]+)*)\s*' + r'(?P<extras>\[.*\])?') + + def __init__(self, path): + self.path = path + if _cache_enabled and path in _cache_path_egg: + self.metadata = _cache_path_egg[path].metadata + self.name = self.metadata['Name'] + self.version = self.metadata['Version'] + return + + # reused from Distribute's pkg_resources + def yield_lines(strs): + """Yield non-empty/non-comment lines of a ``basestring`` + or sequence""" + if isinstance(strs, str): + for s in strs.splitlines(): + s = s.strip() + # skip blank lines/comments + if s and not s.startswith('#'): + yield s + else: + for ss in strs: + for s in yield_lines(ss): + yield s + + requires = None + + if path.endswith('.egg'): + if os.path.isdir(path): + meta_path = os.path.join(path, 'EGG-INFO', 'PKG-INFO') + self.metadata = Metadata(path=meta_path) + try: + req_path = os.path.join(path, 'EGG-INFO', 'requires.txt') + with open(req_path, 'r') as fp: + requires = fp.read() + except IOError: + requires = None + else: + # FIXME handle the case where zipfile is not available + zipf = zipimport.zipimporter(path) + fileobj = io.StringIO( + zipf.get_data('EGG-INFO/PKG-INFO').decode('utf8')) + self.metadata = Metadata(fileobj=fileobj) + try: + requires = zipf.get_data('EGG-INFO/requires.txt') + except IOError: + requires = None + self.name = self.metadata['Name'] + self.version = self.metadata['Version'] + + elif path.endswith('.egg-info'): + if os.path.isdir(path): + path = os.path.join(path, 'PKG-INFO') + try: + with open(os.path.join(path, 'requires.txt'), 'r') as fp: + requires = fp.read() + except IOError: + requires = None + self.metadata = Metadata(path=path) + self.name = self.metadata['Name'] + self.version = self.metadata['Version'] + + else: + raise ValueError('path must end with .egg-info or .egg, got %r' % + path) + + if requires is not None: + if self.metadata['Metadata-Version'] == '1.1': + # we can't have 1.1 metadata *and* Setuptools requires + for field in ('Obsoletes', 'Requires', 'Provides'): + del self.metadata[field] + + reqs = [] + + if requires is not None: + for line in yield_lines(requires): + if line.startswith('['): + logger.warning( + 'extensions in requires.txt are not supported ' + '(used by %r %s)', self.name, self.version) + break + else: + match = self._REQUIREMENT.match(line.strip()) + if not match: + # this happens when we encounter extras; since they + # are written at the end of the file we just exit + break + else: + if match.group('extras'): + msg = ('extra requirements are not supported ' + '(used by %r %s)', self.name, self.version) + logger.warning(msg, self.name) + name = match.group('name') + version = None + if match.group('first'): + version = match.group('first') + if match.group('rest'): + version += match.group('rest') + version = version.replace(' ', '') # trim spaces + if version is None: + reqs.append(name) + else: + reqs.append('%s (%s)' % (name, version)) + + if len(reqs) > 0: + self.metadata['Requires-Dist'] += reqs + + if _cache_enabled: + _cache_path_egg[self.path] = self + + def __repr__(self): + return '<EggInfoDistribution %r %s at %r>' % ( + self.name, self.version, self.path) + + def list_installed_files(self, local=False): + + def _md5(path): + with open(path, 'rb') as f: + content = f.read() + return md5(content).hexdigest() + + def _size(path): + return os.stat(path).st_size + + path = self.path + if local: + path = path.replace('/', os.sep) + + # XXX What about scripts and data files ? + if os.path.isfile(path): + return [(path, _md5(path), _size(path))] + else: + files = [] + for root, dir, files_ in os.walk(path): + for item in files_: + item = os.path.join(root, item) + files.append((item, _md5(item), _size(item))) + return files + + return [] + + def uses(self, path): + return False + + def __eq__(self, other): + return (isinstance(other, EggInfoDistribution) and + self.path == other.path) + + # See http://docs.python.org/reference/datamodel#object.__hash__ + __hash__ = object.__hash__ + + +def distinfo_dirname(name, version): + """ + The *name* and *version* parameters are converted into their + filename-escaped form, i.e. any ``'-'`` characters are replaced + with ``'_'`` other than the one in ``'dist-info'`` and the one + separating the name from the version number. + + :parameter name: is converted to a standard distribution name by replacing + any runs of non- alphanumeric characters with a single + ``'-'``. + :type name: string + :parameter version: is converted to a standard version string. Spaces + become dots, and all other non-alphanumeric characters + (except dots) become dashes, with runs of multiple + dashes condensed to a single dash. + :type version: string + :returns: directory name + :rtype: string""" + file_extension = '.dist-info' + name = name.replace('-', '_') + normalized_version = suggest_normalized_version(version) + # Because this is a lookup procedure, something will be returned even if + # it is a version that cannot be normalized + if normalized_version is None: + # Unable to achieve normality? + normalized_version = version + return '-'.join([name, normalized_version]) + file_extension + + +def get_distributions(use_egg_info=False, paths=None): + """ + Provides an iterator that looks for ``.dist-info`` directories in + ``sys.path`` and returns :class:`Distribution` instances for each one of + them. If the parameters *use_egg_info* is ``True``, then the ``.egg-info`` + files and directores are iterated as well. + + :rtype: iterator of :class:`Distribution` and :class:`EggInfoDistribution` + instances + """ + if paths is None: + paths = sys.path + + if not _cache_enabled: + for dist in _yield_distributions(True, use_egg_info, paths): + yield dist + else: + _generate_cache(use_egg_info, paths) + + for dist in _cache_path.values(): + yield dist + + if use_egg_info: + for dist in _cache_path_egg.values(): + yield dist + + +def get_distribution(name, use_egg_info=False, paths=None): + """ + Scans all elements in ``sys.path`` and looks for all directories + ending with ``.dist-info``. Returns a :class:`Distribution` + corresponding to the ``.dist-info`` directory that contains the + ``METADATA`` that matches *name* for the *name* metadata field. + If no distribution exists with the given *name* and the parameter + *use_egg_info* is set to ``True``, then all files and directories ending + with ``.egg-info`` are scanned. A :class:`EggInfoDistribution` instance is + returned if one is found that has metadata that matches *name* for the + *name* metadata field. + + This function only returns the first result found, as no more than one + value is expected. If the directory is not found, ``None`` is returned. + + :rtype: :class:`Distribution` or :class:`EggInfoDistribution` or None + """ + if paths is None: + paths = sys.path + + if not _cache_enabled: + for dist in _yield_distributions(True, use_egg_info, paths): + if dist.name == name: + return dist + else: + _generate_cache(use_egg_info, paths) + + if name in _cache_name: + return _cache_name[name][0] + elif use_egg_info and name in _cache_name_egg: + return _cache_name_egg[name][0] + else: + return None + + +def obsoletes_distribution(name, version=None, use_egg_info=False): + """ + Iterates over all distributions to find which distributions obsolete + *name*. + + If a *version* is provided, it will be used to filter the results. + If the argument *use_egg_info* is set to ``True``, then ``.egg-info`` + distributions will be considered as well. + + :type name: string + :type version: string + :parameter name: + """ + for dist in get_distributions(use_egg_info): + obsoleted = (dist.metadata['Obsoletes-Dist'] + + dist.metadata['Obsoletes']) + for obs in obsoleted: + o_components = obs.split(' ', 1) + if len(o_components) == 1 or version is None: + if name == o_components[0]: + yield dist + break + else: + try: + predicate = VersionPredicate(obs) + except ValueError: + raise PackagingError( + 'distribution %r has ill-formed obsoletes field: ' + '%r' % (dist.name, obs)) + if name == o_components[0] and predicate.match(version): + yield dist + break + + +def provides_distribution(name, version=None, use_egg_info=False): + """ + Iterates over all distributions to find which distributions provide *name*. + If a *version* is provided, it will be used to filter the results. Scans + all elements in ``sys.path`` and looks for all directories ending with + ``.dist-info``. Returns a :class:`Distribution` corresponding to the + ``.dist-info`` directory that contains a ``METADATA`` that matches *name* + for the name metadata. If the argument *use_egg_info* is set to ``True``, + then all files and directories ending with ``.egg-info`` are considered + as well and returns an :class:`EggInfoDistribution` instance. + + This function only returns the first result found, since no more than + one values are expected. If the directory is not found, returns ``None``. + + :parameter version: a version specifier that indicates the version + required, conforming to the format in ``PEP-345`` + + :type name: string + :type version: string + """ + predicate = None + if not version is None: + try: + predicate = VersionPredicate(name + ' (' + version + ')') + except ValueError: + raise PackagingError('invalid name or version: %r, %r' % + (name, version)) + + for dist in get_distributions(use_egg_info): + provided = dist.metadata['Provides-Dist'] + dist.metadata['Provides'] + + for p in provided: + p_components = p.rsplit(' ', 1) + if len(p_components) == 1 or predicate is None: + if name == p_components[0]: + yield dist + break + else: + p_name, p_ver = p_components + if len(p_ver) < 2 or p_ver[0] != '(' or p_ver[-1] != ')': + raise PackagingError( + 'distribution %r has invalid Provides field: %r' % + (dist.name, p)) + p_ver = p_ver[1:-1] # trim off the parenthesis + if p_name == name and predicate.match(p_ver): + yield dist + break + + +def get_file_users(path): + """ + Iterates over all distributions to find out which distributions use + *path*. + + :parameter path: can be a local absolute path or a relative + ``'/'``-separated path. + :type path: string + :rtype: iterator of :class:`Distribution` instances + """ + for dist in get_distributions(): + if dist.uses(path): + yield dist + + +def get_file_path(distribution_name, relative_path): + """Return the path to a resource file.""" + dist = get_distribution(distribution_name) + if dist is not None: + return dist.get_resource_path(relative_path) + raise LookupError('no distribution named %r found' % distribution_name) + + +def get_file(distribution_name, relative_path, *args, **kwargs): + """Open and return a resource file.""" + return open(get_file_path(distribution_name, relative_path), + *args, **kwargs) diff --git a/Lib/packaging/depgraph.py b/Lib/packaging/depgraph.py new file mode 100644 index 0000000000..843aab47b0 --- /dev/null +++ b/Lib/packaging/depgraph.py @@ -0,0 +1,273 @@ +"""Class and functions dealing with dependencies between distributions. + +This module provides a DependencyGraph class to represent the +dependencies between distributions. Auxiliary functions can generate a +graph, find reverse dependencies, and print a graph in DOT format. +""" + +import sys + +from io import StringIO +from packaging.errors import PackagingError +from packaging.version import VersionPredicate, IrrationalVersionError + +__all__ = ['DependencyGraph', 'generate_graph', 'dependent_dists', + 'graph_to_dot'] + + +class DependencyGraph: + """ + Represents a dependency graph between distributions. + + The dependency relationships are stored in an ``adjacency_list`` that maps + distributions to a list of ``(other, label)`` tuples where ``other`` + is a distribution and the edge is labeled with ``label`` (i.e. the version + specifier, if such was provided). Also, for more efficient traversal, for + every distribution ``x``, a list of predecessors is kept in + ``reverse_list[x]``. An edge from distribution ``a`` to + distribution ``b`` means that ``a`` depends on ``b``. If any missing + dependencies are found, they are stored in ``missing``, which is a + dictionary that maps distributions to a list of requirements that were not + provided by any other distributions. + """ + + def __init__(self): + self.adjacency_list = {} + self.reverse_list = {} + self.missing = {} + + def add_distribution(self, distribution): + """Add the *distribution* to the graph. + + :type distribution: :class:`packaging.database.Distribution` or + :class:`packaging.database.EggInfoDistribution` + """ + self.adjacency_list[distribution] = [] + self.reverse_list[distribution] = [] + self.missing[distribution] = [] + + def add_edge(self, x, y, label=None): + """Add an edge from distribution *x* to distribution *y* with the given + *label*. + + :type x: :class:`packaging.database.Distribution` or + :class:`packaging.database.EggInfoDistribution` + :type y: :class:`packaging.database.Distribution` or + :class:`packaging.database.EggInfoDistribution` + :type label: ``str`` or ``None`` + """ + self.adjacency_list[x].append((y, label)) + # multiple edges are allowed, so be careful + if x not in self.reverse_list[y]: + self.reverse_list[y].append(x) + + def add_missing(self, distribution, requirement): + """ + Add a missing *requirement* for the given *distribution*. + + :type distribution: :class:`packaging.database.Distribution` or + :class:`packaging.database.EggInfoDistribution` + :type requirement: ``str`` + """ + self.missing[distribution].append(requirement) + + def _repr_dist(self, dist): + return '%r %s' % (dist.name, dist.version) + + def repr_node(self, dist, level=1): + """Prints only a subgraph""" + output = [] + output.append(self._repr_dist(dist)) + for other, label in self.adjacency_list[dist]: + dist = self._repr_dist(other) + if label is not None: + dist = '%s [%s]' % (dist, label) + output.append(' ' * level + str(dist)) + suboutput = self.repr_node(other, level + 1) + subs = suboutput.split('\n') + output.extend(subs[1:]) + return '\n'.join(output) + + def __repr__(self): + """Representation of the graph""" + output = [] + for dist, adjs in self.adjacency_list.items(): + output.append(self.repr_node(dist)) + return '\n'.join(output) + + +def graph_to_dot(graph, f, skip_disconnected=True): + """Writes a DOT output for the graph to the provided file *f*. + + If *skip_disconnected* is set to ``True``, then all distributions + that are not dependent on any other distribution are skipped. + + :type f: has to support ``file``-like operations + :type skip_disconnected: ``bool`` + """ + disconnected = [] + + f.write("digraph dependencies {\n") + for dist, adjs in graph.adjacency_list.items(): + if len(adjs) == 0 and not skip_disconnected: + disconnected.append(dist) + for other, label in adjs: + if not label is None: + f.write('"%s" -> "%s" [label="%s"]\n' % + (dist.name, other.name, label)) + else: + f.write('"%s" -> "%s"\n' % (dist.name, other.name)) + if not skip_disconnected and len(disconnected) > 0: + f.write('subgraph disconnected {\n') + f.write('label = "Disconnected"\n') + f.write('bgcolor = red\n') + + for dist in disconnected: + f.write('"%s"' % dist.name) + f.write('\n') + f.write('}\n') + f.write('}\n') + + +def generate_graph(dists): + """Generates a dependency graph from the given distributions. + + :parameter dists: a list of distributions + :type dists: list of :class:`packaging.database.Distribution` and + :class:`packaging.database.EggInfoDistribution` instances + :rtype: a :class:`DependencyGraph` instance + """ + graph = DependencyGraph() + provided = {} # maps names to lists of (version, dist) tuples + + # first, build the graph and find out the provides + for dist in dists: + graph.add_distribution(dist) + provides = (dist.metadata['Provides-Dist'] + + dist.metadata['Provides'] + + ['%s (%s)' % (dist.name, dist.version)]) + + for p in provides: + comps = p.strip().rsplit(" ", 1) + name = comps[0] + version = None + if len(comps) == 2: + version = comps[1] + if len(version) < 3 or version[0] != '(' or version[-1] != ')': + raise PackagingError('distribution %r has ill-formed' + 'provides field: %r' % (dist.name, p)) + version = version[1:-1] # trim off parenthesis + if name not in provided: + provided[name] = [] + provided[name].append((version, dist)) + + # now make the edges + for dist in dists: + requires = dist.metadata['Requires-Dist'] + dist.metadata['Requires'] + for req in requires: + try: + predicate = VersionPredicate(req) + except IrrationalVersionError: + # XXX compat-mode if cannot read the version + name = req.split()[0] + predicate = VersionPredicate(name) + + name = predicate.name + + if name not in provided: + graph.add_missing(dist, req) + else: + matched = False + for version, provider in provided[name]: + try: + match = predicate.match(version) + except IrrationalVersionError: + # XXX small compat-mode + if version.split(' ') == 1: + match = True + else: + match = False + + if match: + graph.add_edge(dist, provider, req) + matched = True + break + if not matched: + graph.add_missing(dist, req) + return graph + + +def dependent_dists(dists, dist): + """Recursively generate a list of distributions from *dists* that are + dependent on *dist*. + + :param dists: a list of distributions + :param dist: a distribution, member of *dists* for which we are interested + """ + if dist not in dists: + raise ValueError('given distribution %r is not a member of the list' % + dist.name) + graph = generate_graph(dists) + + dep = [dist] # dependent distributions + fringe = graph.reverse_list[dist] # list of nodes we should inspect + + while not len(fringe) == 0: + node = fringe.pop() + dep.append(node) + for prev in graph.reverse_list[node]: + if prev not in dep: + fringe.append(prev) + + dep.pop(0) # remove dist from dep, was there to prevent infinite loops + return dep + + +def main(): + from packaging.database import get_distributions + tempout = StringIO() + try: + old = sys.stderr + sys.stderr = tempout + try: + dists = list(get_distributions(use_egg_info=True)) + graph = generate_graph(dists) + finally: + sys.stderr = old + except Exception as e: + tempout.seek(0) + tempout = tempout.read() + print('Could not generate the graph') + print(tempout) + print(e) + sys.exit(1) + + for dist, reqs in graph.missing.items(): + if len(reqs) > 0: + print("Warning: Missing dependencies for %r:" % dist.name, + ", ".join(reqs)) + # XXX replace with argparse + if len(sys.argv) == 1: + print('Dependency graph:') + print(' ', repr(graph).replace('\n', '\n ')) + sys.exit(0) + elif len(sys.argv) > 1 and sys.argv[1] in ('-d', '--dot'): + if len(sys.argv) > 2: + filename = sys.argv[2] + else: + filename = 'depgraph.dot' + + with open(filename, 'w') as f: + graph_to_dot(graph, f, True) + tempout.seek(0) + tempout = tempout.read() + print(tempout) + print('Dot file written at %r' % filename) + sys.exit(0) + else: + print('Supported option: -d [filename]') + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/Lib/packaging/dist.py b/Lib/packaging/dist.py new file mode 100644 index 0000000000..f1441d14cb --- /dev/null +++ b/Lib/packaging/dist.py @@ -0,0 +1,768 @@ +"""Class representing the project being built/installed/etc.""" + +import os +import re + +from packaging import logger +from packaging.util import strtobool, resolve_name +from packaging.config import Config +from packaging.errors import (PackagingOptionError, PackagingArgError, + PackagingModuleError, PackagingClassError) +from packaging.command import get_command_class, STANDARD_COMMANDS +from packaging.command.cmd import Command +from packaging.metadata import Metadata +from packaging.fancy_getopt import FancyGetopt + +# Regex to define acceptable Packaging command names. This is not *quite* +# the same as a Python name -- leading underscores are not allowed. The fact +# that they're very similar is no coincidence: the default naming scheme is +# to look for a Python module named after the command. +command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$') + +USAGE = """\ +usage: %(script)s [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...] + or: %(script)s --help [cmd1 cmd2 ...] + or: %(script)s --help-commands + or: %(script)s cmd --help +""" + + +def gen_usage(script_name): + script = os.path.basename(script_name) + return USAGE % {'script': script} + + +class Distribution: + """Class used to represent a project and work with it. + + Most of the work hiding behind 'pysetup run' is really done within a + Distribution instance, which farms the work out to the commands + specified on the command line. + """ + + # 'global_options' describes the command-line options that may be + # supplied to the setup script prior to any actual commands. + # Eg. "pysetup run -n" or "pysetup run --dry-run" both take advantage of + # these global options. This list should be kept to a bare minimum, + # since every global option is also valid as a command option -- and we + # don't want to pollute the commands with too many options that they + # have minimal control over. + global_options = [ + ('dry-run', 'n', "don't actually do anything"), + ('help', 'h', "show detailed help message"), + ('no-user-cfg', None, 'ignore pydistutils.cfg in your home directory'), + ] + + # 'common_usage' is a short (2-3 line) string describing the common + # usage of the setup script. + common_usage = """\ +Common commands: (see '--help-commands' for more) + + pysetup run build will build the project underneath 'build/' + pysetup run install will install the project +""" + + # options that are not propagated to the commands + display_options = [ + ('help-commands', None, + "list all available commands"), + ('use-2to3', None, + "use 2to3 to make source python 3.x compatible"), + ('convert-2to3-doctests', None, + "use 2to3 to convert doctests in seperate text files"), + ] + display_option_names = [x[0].replace('-', '_') for x in display_options] + + # negative options are options that exclude other options + negative_opt = {} + + # -- Creation/initialization methods ------------------------------- + def __init__(self, attrs=None): + """Construct a new Distribution instance: initialize all the + attributes of a Distribution, and then use 'attrs' (a dictionary + mapping attribute names to values) to assign some of those + attributes their "real" values. (Any attributes not mentioned in + 'attrs' will be assigned to some null value: 0, None, an empty list + or dictionary, etc.) Most importantly, initialize the + 'command_obj' attribute to the empty dictionary; this will be + filled in with real command objects by 'parse_command_line()'. + """ + + # Default values for our command-line options + self.dry_run = False + self.help = False + for attr in self.display_option_names: + setattr(self, attr, False) + + # Store the configuration + self.config = Config(self) + + # Store the distribution metadata (name, version, author, and so + # forth) in a separate object -- we're getting to have enough + # information here (and enough command-line options) that it's + # worth it. + self.metadata = Metadata() + + # 'cmdclass' maps command names to class objects, so we + # can 1) quickly figure out which class to instantiate when + # we need to create a new command object, and 2) have a way + # for the setup script to override command classes + self.cmdclass = {} + + # 'script_name' and 'script_args' are usually set to sys.argv[0] + # and sys.argv[1:], but they can be overridden when the caller is + # not necessarily a setup script run from the command line. + self.script_name = None + self.script_args = None + + # 'command_options' is where we store command options between + # parsing them (from config files, the command line, etc.) and when + # they are actually needed -- ie. when the command in question is + # instantiated. It is a dictionary of dictionaries of 2-tuples: + # command_options = { command_name : { option : (source, value) } } + self.command_options = {} + + # 'dist_files' is the list of (command, pyversion, file) that + # have been created by any dist commands run so far. This is + # filled regardless of whether the run is dry or not. pyversion + # gives sysconfig.get_python_version() if the dist file is + # specific to a Python version, 'any' if it is good for all + # Python versions on the target platform, and '' for a source + # file. pyversion should not be used to specify minimum or + # maximum required Python versions; use the metainfo for that + # instead. + self.dist_files = [] + + # These options are really the business of various commands, rather + # than of the Distribution itself. We provide aliases for them in + # Distribution as a convenience to the developer. + self.packages = [] + self.package_data = {} + self.package_dir = None + self.py_modules = [] + self.libraries = [] + self.headers = [] + self.ext_modules = [] + self.ext_package = None + self.include_dirs = [] + self.extra_path = None + self.scripts = [] + self.data_files = {} + self.password = '' + self.use_2to3 = False + self.convert_2to3_doctests = [] + self.extra_files = [] + + # And now initialize bookkeeping stuff that can't be supplied by + # the caller at all. 'command_obj' maps command names to + # Command instances -- that's how we enforce that every command + # class is a singleton. + self.command_obj = {} + + # 'have_run' maps command names to boolean values; it keeps track + # of whether we have actually run a particular command, to make it + # cheap to "run" a command whenever we think we might need to -- if + # it's already been done, no need for expensive filesystem + # operations, we just check the 'have_run' dictionary and carry on. + # It's only safe to query 'have_run' for a command class that has + # been instantiated -- a false value will be inserted when the + # command object is created, and replaced with a true value when + # the command is successfully run. Thus it's probably best to use + # '.get()' rather than a straight lookup. + self.have_run = {} + + # Now we'll use the attrs dictionary (ultimately, keyword args from + # the setup script) to possibly override any or all of these + # distribution options. + + if attrs is not None: + # Pull out the set of command options and work on them + # specifically. Note that this order guarantees that aliased + # command options will override any supplied redundantly + # through the general options dictionary. + options = attrs.get('options') + if options is not None: + del attrs['options'] + for command, cmd_options in options.items(): + opt_dict = self.get_option_dict(command) + for opt, val in cmd_options.items(): + opt_dict[opt] = ("setup script", val) + + # Now work on the rest of the attributes. Any attribute that's + # not already defined is invalid! + for key, val in attrs.items(): + if self.metadata.is_metadata_field(key): + self.metadata[key] = val + elif hasattr(self, key): + setattr(self, key, val) + else: + logger.warning( + 'unknown argument given to Distribution: %r', key) + + # no-user-cfg is handled before other command line args + # because other args override the config files, and this + # one is needed before we can load the config files. + # If attrs['script_args'] wasn't passed, assume false. + # + # This also make sure we just look at the global options + self.want_user_cfg = True + + if self.script_args is not None: + for arg in self.script_args: + if not arg.startswith('-'): + break + if arg == '--no-user-cfg': + self.want_user_cfg = False + break + + self.finalize_options() + + def get_option_dict(self, command): + """Get the option dictionary for a given command. If that + command's option dictionary hasn't been created yet, then create it + and return the new dictionary; otherwise, return the existing + option dictionary. + """ + d = self.command_options.get(command) + if d is None: + d = self.command_options[command] = {} + return d + + def get_fullname(self): + return self.metadata.get_fullname() + + def dump_option_dicts(self, header=None, commands=None, indent=""): + from pprint import pformat + + if commands is None: # dump all command option dicts + commands = sorted(self.command_options) + + if header is not None: + logger.info(indent + header) + indent = indent + " " + + if not commands: + logger.info(indent + "no commands known yet") + return + + for cmd_name in commands: + opt_dict = self.command_options.get(cmd_name) + if opt_dict is None: + logger.info(indent + "no option dict for %r command", + cmd_name) + else: + logger.info(indent + "option dict for %r command:", cmd_name) + out = pformat(opt_dict) + for line in out.split('\n'): + logger.info(indent + " " + line) + + # -- Config file finding/parsing methods --------------------------- + # XXX to be removed + def parse_config_files(self, filenames=None): + return self.config.parse_config_files(filenames) + + def find_config_files(self): + return self.config.find_config_files() + + # -- Command-line parsing methods ---------------------------------- + + def parse_command_line(self): + """Parse the setup script's command line, taken from the + 'script_args' instance attribute (which defaults to 'sys.argv[1:]' + -- see 'setup()' in run.py). This list is first processed for + "global options" -- options that set attributes of the Distribution + instance. Then, it is alternately scanned for Packaging commands + and options for that command. Each new command terminates the + options for the previous command. The allowed options for a + command are determined by the 'user_options' attribute of the + command class -- thus, we have to be able to load command classes + in order to parse the command line. Any error in that 'options' + attribute raises PackagingGetoptError; any error on the + command line raises PackagingArgError. If no Packaging commands + were found on the command line, raises PackagingArgError. Return + true if command line was successfully parsed and we should carry + on with executing commands; false if no errors but we shouldn't + execute commands (currently, this only happens if user asks for + help). + """ + # + # We now have enough information to show the Macintosh dialog + # that allows the user to interactively specify the "command line". + # + toplevel_options = self._get_toplevel_options() + + # We have to parse the command line a bit at a time -- global + # options, then the first command, then its options, and so on -- + # because each command will be handled by a different class, and + # the options that are valid for a particular class aren't known + # until we have loaded the command class, which doesn't happen + # until we know what the command is. + + self.commands = [] + parser = FancyGetopt(toplevel_options + self.display_options) + parser.set_negative_aliases(self.negative_opt) + args = parser.getopt(args=self.script_args, object=self) + option_order = parser.get_option_order() + + # for display options we return immediately + if self.handle_display_options(option_order): + return + + while args: + args = self._parse_command_opts(parser, args) + if args is None: # user asked for help (and got it) + return + + # Handle the cases of --help as a "global" option, ie. + # "pysetup run --help" and "pysetup run --help command ...". For the + # former, we show global options (--dry-run, etc.) + # and display-only options (--name, --version, etc.); for the + # latter, we omit the display-only options and show help for + # each command listed on the command line. + if self.help: + self._show_help(parser, + display_options=len(self.commands) == 0, + commands=self.commands) + return + + return True + + def _get_toplevel_options(self): + """Return the non-display options recognized at the top level. + + This includes options that are recognized *only* at the top + level as well as options recognized for commands. + """ + return self.global_options + + def _parse_command_opts(self, parser, args): + """Parse the command-line options for a single command. + 'parser' must be a FancyGetopt instance; 'args' must be the list + of arguments, starting with the current command (whose options + we are about to parse). Returns a new version of 'args' with + the next command at the front of the list; will be the empty + list if there are no more commands on the command line. Returns + None if the user asked for help on this command. + """ + # Pull the current command from the head of the command line + command = args[0] + if not command_re.match(command): + raise SystemExit("invalid command name %r" % command) + self.commands.append(command) + + # Dig up the command class that implements this command, so we + # 1) know that it's a valid command, and 2) know which options + # it takes. + try: + cmd_class = get_command_class(command) + except PackagingModuleError as msg: + raise PackagingArgError(msg) + + # XXX We want to push this in packaging.command + # + # Require that the command class be derived from Command -- want + # to be sure that the basic "command" interface is implemented. + for meth in ('initialize_options', 'finalize_options', 'run'): + if hasattr(cmd_class, meth): + continue + raise PackagingClassError( + 'command %r must implement %r' % (cmd_class, meth)) + + # Also make sure that the command object provides a list of its + # known options. + if not (hasattr(cmd_class, 'user_options') and + isinstance(cmd_class.user_options, list)): + raise PackagingClassError( + "command class %s must provide " + "'user_options' attribute (a list of tuples)" % cmd_class) + + # If the command class has a list of negative alias options, + # merge it in with the global negative aliases. + negative_opt = self.negative_opt + if hasattr(cmd_class, 'negative_opt'): + negative_opt = negative_opt.copy() + negative_opt.update(cmd_class.negative_opt) + + # Check for help_options in command class. They have a different + # format (tuple of four) so we need to preprocess them here. + if (hasattr(cmd_class, 'help_options') and + isinstance(cmd_class.help_options, list)): + help_options = cmd_class.help_options[:] + else: + help_options = [] + + # All commands support the global options too, just by adding + # in 'global_options'. + parser.set_option_table(self.global_options + + cmd_class.user_options + + help_options) + parser.set_negative_aliases(negative_opt) + args, opts = parser.getopt(args[1:]) + if hasattr(opts, 'help') and opts.help: + self._show_help(parser, display_options=False, + commands=[cmd_class]) + return + + if (hasattr(cmd_class, 'help_options') and + isinstance(cmd_class.help_options, list)): + help_option_found = False + for help_option, short, desc, func in cmd_class.help_options: + if hasattr(opts, help_option.replace('-', '_')): + help_option_found = True + if hasattr(func, '__call__'): + func() + else: + raise PackagingClassError( + "invalid help function %r for help option %r: " + "must be a callable object (function, etc.)" + % (func, help_option)) + + if help_option_found: + return + + # Put the options from the command line into their official + # holding pen, the 'command_options' dictionary. + opt_dict = self.get_option_dict(command) + for name, value in vars(opts).items(): + opt_dict[name] = ("command line", value) + + return args + + def finalize_options(self): + """Set final values for all the options on the Distribution + instance, analogous to the .finalize_options() method of Command + objects. + """ + if getattr(self, 'convert_2to3_doctests', None): + self.convert_2to3_doctests = [os.path.join(p) + for p in self.convert_2to3_doctests] + else: + self.convert_2to3_doctests = [] + + def _show_help(self, parser, global_options=True, display_options=True, + commands=[]): + """Show help for the setup script command line in the form of + several lists of command-line options. 'parser' should be a + FancyGetopt instance; do not expect it to be returned in the + same state, as its option table will be reset to make it + generate the correct help text. + + If 'global_options' is true, lists the global options: + --dry-run, etc. If 'display_options' is true, lists + the "display-only" options: --help-commands. Finally, + lists per-command help for every command name or command class + in 'commands'. + """ + if global_options: + if display_options: + options = self._get_toplevel_options() + else: + options = self.global_options + parser.set_option_table(options) + parser.print_help(self.common_usage + "\nGlobal options:") + print() + + if display_options: + parser.set_option_table(self.display_options) + parser.print_help( + "Information display options (just display " + + "information, ignore any commands)") + print() + + for command in self.commands: + if isinstance(command, type) and issubclass(command, Command): + cls = command + else: + cls = get_command_class(command) + if (hasattr(cls, 'help_options') and + isinstance(cls.help_options, list)): + parser.set_option_table(cls.user_options + cls.help_options) + else: + parser.set_option_table(cls.user_options) + parser.print_help("Options for %r command:" % cls.__name__) + print() + + print(gen_usage(self.script_name)) + + def handle_display_options(self, option_order): + """If there were any non-global "display-only" options + (--help-commands) on the command line, display the requested info and + return true; else return false. + """ + # User just wants a list of commands -- we'll print it out and stop + # processing now (ie. if they ran "setup --help-commands foo bar", + # we ignore "foo bar"). + if self.help_commands: + self.print_commands() + print() + print(gen_usage(self.script_name)) + return True + + # If user supplied any of the "display metadata" options, then + # display that metadata in the order in which the user supplied the + # metadata options. + any_display_options = False + is_display_option = set() + for option in self.display_options: + is_display_option.add(option[0]) + + for opt, val in option_order: + if val and opt in is_display_option: + opt = opt.replace('-', '_') + value = self.metadata[opt] + if opt in ('keywords', 'platform'): + print(','.join(value)) + elif opt in ('classifier', 'provides', 'requires', + 'obsoletes'): + print('\n'.join(value)) + else: + print(value) + any_display_options = True + + return any_display_options + + def print_command_list(self, commands, header, max_length): + """Print a subset of the list of all commands -- used by + 'print_commands()'. + """ + print(header + ":") + + for cmd in commands: + cls = self.cmdclass.get(cmd) or get_command_class(cmd) + description = getattr(cls, 'description', + '(no description available)') + + print(" %-*s %s" % (max_length, cmd, description)) + + def _get_command_groups(self): + """Helper function to retrieve all the command class names divided + into standard commands (listed in + packaging2.command.STANDARD_COMMANDS) and extra commands (given in + self.cmdclass and not standard commands). + """ + extra_commands = [cmd for cmd in self.cmdclass + if cmd not in STANDARD_COMMANDS] + return STANDARD_COMMANDS, extra_commands + + def print_commands(self): + """Print out a help message listing all available commands with a + description of each. The list is divided into standard commands + (listed in packaging2.command.STANDARD_COMMANDS) and extra commands + (given in self.cmdclass and not standard commands). The + descriptions come from the command class attribute + 'description'. + """ + std_commands, extra_commands = self._get_command_groups() + max_length = 0 + for cmd in (std_commands + extra_commands): + if len(cmd) > max_length: + max_length = len(cmd) + + self.print_command_list(std_commands, + "Standard commands", + max_length) + if extra_commands: + print() + self.print_command_list(extra_commands, + "Extra commands", + max_length) + + # -- Command class/object methods ---------------------------------- + + def get_command_obj(self, command, create=True): + """Return the command object for 'command'. Normally this object + is cached on a previous call to 'get_command_obj()'; if no command + object for 'command' is in the cache, then we either create and + return it (if 'create' is true) or return None. + """ + cmd_obj = self.command_obj.get(command) + if not cmd_obj and create: + logger.debug("Distribution.get_command_obj(): " + "creating %r command object", command) + + cls = get_command_class(command) + cmd_obj = self.command_obj[command] = cls(self) + self.have_run[command] = 0 + + # Set any options that were supplied in config files or on the + # command line. (XXX support for error reporting is suboptimal + # here: errors aren't reported until finalize_options is called, + # which means we won't report the source of the error.) + options = self.command_options.get(command) + if options: + self._set_command_options(cmd_obj, options) + + return cmd_obj + + def _set_command_options(self, command_obj, option_dict=None): + """Set the options for 'command_obj' from 'option_dict'. Basically + this means copying elements of a dictionary ('option_dict') to + attributes of an instance ('command'). + + 'command_obj' must be a Command instance. If 'option_dict' is not + supplied, uses the standard option dictionary for this command + (from 'self.command_options'). + """ + command_name = command_obj.get_command_name() + if option_dict is None: + option_dict = self.get_option_dict(command_name) + + logger.debug(" setting options for %r command:", command_name) + + for option, (source, value) in option_dict.items(): + logger.debug(" %s = %s (from %s)", option, value, source) + try: + bool_opts = [x.replace('-', '_') + for x in command_obj.boolean_options] + except AttributeError: + bool_opts = [] + try: + neg_opt = command_obj.negative_opt + except AttributeError: + neg_opt = {} + + try: + is_string = isinstance(value, str) + if option in neg_opt and is_string: + setattr(command_obj, neg_opt[option], not strtobool(value)) + elif option in bool_opts and is_string: + setattr(command_obj, option, strtobool(value)) + elif hasattr(command_obj, option): + setattr(command_obj, option, value) + else: + raise PackagingOptionError( + "error in %s: command %r has no such option %r" % + (source, command_name, option)) + except ValueError as msg: + raise PackagingOptionError(msg) + + def get_reinitialized_command(self, command, reinit_subcommands=False): + """Reinitializes a command to the state it was in when first + returned by 'get_command_obj()': ie., initialized but not yet + finalized. This provides the opportunity to sneak option + values in programmatically, overriding or supplementing + user-supplied values from the config files and command line. + You'll have to re-finalize the command object (by calling + 'finalize_options()' or 'ensure_finalized()') before using it for + real. + + 'command' should be a command name (string) or command object. If + 'reinit_subcommands' is true, also reinitializes the command's + sub-commands, as declared by the 'sub_commands' class attribute (if + it has one). See the "install_dist" command for an example. Only + reinitializes the sub-commands that actually matter, ie. those + whose test predicates return true. + + Returns the reinitialized command object. + """ + if not isinstance(command, Command): + command_name = command + command = self.get_command_obj(command_name) + else: + command_name = command.get_command_name() + + if not command.finalized: + return command + + command.initialize_options() + self.have_run[command_name] = 0 + command.finalized = False + self._set_command_options(command) + + if reinit_subcommands: + for sub in command.get_sub_commands(): + self.get_reinitialized_command(sub, reinit_subcommands) + + return command + + # -- Methods that operate on the Distribution ---------------------- + + def run_commands(self): + """Run each command that was seen on the setup script command line. + Uses the list of commands found and cache of command objects + created by 'get_command_obj()'. + """ + for cmd in self.commands: + self.run_command(cmd) + + # -- Methods that operate on its Commands -------------------------- + + def run_command(self, command, options=None): + """Do whatever it takes to run a command (including nothing at all, + if the command has already been run). Specifically: if we have + already created and run the command named by 'command', return + silently without doing anything. If the command named by 'command' + doesn't even have a command object yet, create one. Then invoke + 'run()' on that command object (or an existing one). + """ + # Already been here, done that? then return silently. + if self.have_run.get(command): + return + + if options is not None: + self.command_options[command] = options + + cmd_obj = self.get_command_obj(command) + cmd_obj.ensure_finalized() + self.run_command_hooks(cmd_obj, 'pre_hook') + logger.info("running %s", command) + cmd_obj.run() + self.run_command_hooks(cmd_obj, 'post_hook') + self.have_run[command] = 1 + + def run_command_hooks(self, cmd_obj, hook_kind): + """Run hooks registered for that command and phase. + + *cmd_obj* is a finalized command object; *hook_kind* is either + 'pre_hook' or 'post_hook'. + """ + if hook_kind not in ('pre_hook', 'post_hook'): + raise ValueError('invalid hook kind: %r' % hook_kind) + + hooks = getattr(cmd_obj, hook_kind, None) + + if hooks is None: + return + + for hook in hooks.values(): + if isinstance(hook, str): + try: + hook_obj = resolve_name(hook) + except ImportError as e: + raise PackagingModuleError(e) + else: + hook_obj = hook + + if not hasattr(hook_obj, '__call__'): + raise PackagingOptionError('hook %r is not callable' % hook) + + logger.info('running %s %s for command %s', + hook_kind, hook, cmd_obj.get_command_name()) + hook_obj(cmd_obj) + + # -- Distribution query methods ------------------------------------ + def has_pure_modules(self): + return len(self.packages or self.py_modules or []) > 0 + + def has_ext_modules(self): + return self.ext_modules and len(self.ext_modules) > 0 + + def has_c_libraries(self): + return self.libraries and len(self.libraries) > 0 + + def has_modules(self): + return self.has_pure_modules() or self.has_ext_modules() + + def has_headers(self): + return self.headers and len(self.headers) > 0 + + def has_scripts(self): + return self.scripts and len(self.scripts) > 0 + + def has_data_files(self): + return self.data_files and len(self.data_files) > 0 + + def is_pure(self): + return (self.has_pure_modules() and + not self.has_ext_modules() and + not self.has_c_libraries()) diff --git a/Lib/packaging/errors.py b/Lib/packaging/errors.py new file mode 100644 index 0000000000..8924a2dcfc --- /dev/null +++ b/Lib/packaging/errors.py @@ -0,0 +1,142 @@ +"""Exceptions used throughout the package. + +Submodules of packaging may raise exceptions defined in this module as +well as standard exceptions; in particular, SystemExit is usually raised +for errors that are obviously the end-user's fault (e.g. bad +command-line arguments). +""" + + +class PackagingError(Exception): + """The root of all Packaging evil.""" + + +class PackagingModuleError(PackagingError): + """Unable to load an expected module, or to find an expected class + within some module (in particular, command modules and classes).""" + + +class PackagingClassError(PackagingError): + """Some command class (or possibly distribution class, if anyone + feels a need to subclass Distribution) is found not to be holding + up its end of the bargain, ie. implementing some part of the + "command "interface.""" + + +class PackagingGetoptError(PackagingError): + """The option table provided to 'fancy_getopt()' is bogus.""" + + +class PackagingArgError(PackagingError): + """Raised by fancy_getopt in response to getopt.error -- ie. an + error in the command line usage.""" + + +class PackagingFileError(PackagingError): + """Any problems in the filesystem: expected file not found, etc. + Typically this is for problems that we detect before IOError or + OSError could be raised.""" + + +class PackagingOptionError(PackagingError): + """Syntactic/semantic errors in command options, such as use of + mutually conflicting options, or inconsistent options, + badly-spelled values, etc. No distinction is made between option + values originating in the setup script, the command line, config + files, or what-have-you -- but if we *know* something originated in + the setup script, we'll raise PackagingSetupError instead.""" + + +class PackagingSetupError(PackagingError): + """For errors that can be definitely blamed on the setup script, + such as invalid keyword arguments to 'setup()'.""" + + +class PackagingPlatformError(PackagingError): + """We don't know how to do something on the current platform (but + we do know how to do it on some platform) -- eg. trying to compile + C files on a platform not supported by a CCompiler subclass.""" + + +class PackagingExecError(PackagingError): + """Any problems executing an external program (such as the C + compiler, when compiling C files).""" + + +class PackagingInternalError(PackagingError): + """Internal inconsistencies or impossibilities (obviously, this + should never be seen if the code is working!).""" + + +class PackagingTemplateError(PackagingError): + """Syntax error in a file list template.""" + + +class PackagingByteCompileError(PackagingError): + """Byte compile error.""" + + +class PackagingPyPIError(PackagingError): + """Any problem occuring during using the indexes.""" + + +# Exception classes used by the CCompiler implementation classes +class CCompilerError(Exception): + """Some compile/link operation failed.""" + + +class PreprocessError(CCompilerError): + """Failure to preprocess one or more C/C++ files.""" + + +class CompileError(CCompilerError): + """Failure to compile one or more C/C++ source files.""" + + +class LibError(CCompilerError): + """Failure to create a static library from one or more C/C++ object + files.""" + + +class LinkError(CCompilerError): + """Failure to link one or more C/C++ object files into an executable + or shared library file.""" + + +class UnknownFileError(CCompilerError): + """Attempt to process an unknown file type.""" + + +class MetadataMissingError(PackagingError): + """A required metadata is missing""" + + +class MetadataConflictError(PackagingError): + """Attempt to read or write metadata fields that are conflictual.""" + + +class MetadataUnrecognizedVersionError(PackagingError): + """Unknown metadata version number.""" + + +class IrrationalVersionError(Exception): + """This is an irrational version.""" + pass + + +class HugeMajorVersionNumError(IrrationalVersionError): + """An irrational version because the major version number is huge + (often because a year or date was used). + + See `error_on_huge_major_num` option in `NormalizedVersion` for details. + This guard can be disabled by setting that option False. + """ + pass + + +class InstallationException(Exception): + """Base exception for installation scripts""" + + +class InstallationConflict(InstallationException): + """Raised when a conflict is detected""" diff --git a/Lib/packaging/fancy_getopt.py b/Lib/packaging/fancy_getopt.py new file mode 100644 index 0000000000..61dd5fc58e --- /dev/null +++ b/Lib/packaging/fancy_getopt.py @@ -0,0 +1,388 @@ +"""Command line parsing machinery. + +The FancyGetopt class is a Wrapper around the getopt module that +provides the following additional features: + * short and long options are tied together + * options have help strings, so fancy_getopt could potentially + create a complete usage summary + * options set attributes of a passed-in object. + +It is used under the hood by the command classes. Do not use directly. +""" + +import getopt +import re +import sys +import textwrap + +from packaging.errors import PackagingGetoptError, PackagingArgError + +# Much like command_re in packaging.core, this is close to but not quite +# the same as a Python NAME -- except, in the spirit of most GNU +# utilities, we use '-' in place of '_'. (The spirit of LISP lives on!) +# The similarities to NAME are again not a coincidence... +longopt_pat = r'[a-zA-Z](?:[a-zA-Z0-9-]*)' +longopt_re = re.compile(r'^%s$' % longopt_pat) + +# For recognizing "negative alias" options, eg. "quiet=!verbose" +neg_alias_re = re.compile("^(%s)=!(%s)$" % (longopt_pat, longopt_pat)) + + +class FancyGetopt: + """Wrapper around the standard 'getopt()' module that provides some + handy extra functionality: + * short and long options are tied together + * options have help strings, and help text can be assembled + from them + * options set attributes of a passed-in object + * boolean options can have "negative aliases" -- eg. if + --quiet is the "negative alias" of --verbose, then "--quiet" + on the command line sets 'verbose' to false + """ + + def __init__(self, option_table=None): + + # The option table is (currently) a list of tuples. The + # tuples may have 3 or four values: + # (long_option, short_option, help_string [, repeatable]) + # if an option takes an argument, its long_option should have '=' + # appended; short_option should just be a single character, no ':' + # in any case. If a long_option doesn't have a corresponding + # short_option, short_option should be None. All option tuples + # must have long options. + self.option_table = option_table + + # 'option_index' maps long option names to entries in the option + # table (ie. those 3-tuples). + self.option_index = {} + if self.option_table: + self._build_index() + + # 'alias' records (duh) alias options; {'foo': 'bar'} means + # --foo is an alias for --bar + self.alias = {} + + # 'negative_alias' keeps track of options that are the boolean + # opposite of some other option + self.negative_alias = {} + + # These keep track of the information in the option table. We + # don't actually populate these structures until we're ready to + # parse the command line, since the 'option_table' passed in here + # isn't necessarily the final word. + self.short_opts = [] + self.long_opts = [] + self.short2long = {} + self.attr_name = {} + self.takes_arg = {} + + # And 'option_order' is filled up in 'getopt()'; it records the + # original order of options (and their values) on the command line, + # but expands short options, converts aliases, etc. + self.option_order = [] + + def _build_index(self): + self.option_index.clear() + for option in self.option_table: + self.option_index[option[0]] = option + + def set_option_table(self, option_table): + self.option_table = option_table + self._build_index() + + def add_option(self, long_option, short_option=None, help_string=None): + if long_option in self.option_index: + raise PackagingGetoptError( + "option conflict: already an option '%s'" % long_option) + else: + option = (long_option, short_option, help_string) + self.option_table.append(option) + self.option_index[long_option] = option + + def has_option(self, long_option): + """Return true if the option table for this parser has an + option with long name 'long_option'.""" + return long_option in self.option_index + + def _check_alias_dict(self, aliases, what): + assert isinstance(aliases, dict) + for alias, opt in aliases.items(): + if alias not in self.option_index: + raise PackagingGetoptError( + ("invalid %s '%s': " + "option '%s' not defined") % (what, alias, alias)) + if opt not in self.option_index: + raise PackagingGetoptError( + ("invalid %s '%s': " + "aliased option '%s' not defined") % (what, alias, opt)) + + def set_aliases(self, alias): + """Set the aliases for this option parser.""" + self._check_alias_dict(alias, "alias") + self.alias = alias + + def set_negative_aliases(self, negative_alias): + """Set the negative aliases for this option parser. + 'negative_alias' should be a dictionary mapping option names to + option names, both the key and value must already be defined + in the option table.""" + self._check_alias_dict(negative_alias, "negative alias") + self.negative_alias = negative_alias + + def _grok_option_table(self): + """Populate the various data structures that keep tabs on the + option table. Called by 'getopt()' before it can do anything + worthwhile. + """ + self.long_opts = [] + self.short_opts = [] + self.short2long.clear() + self.repeat = {} + + for option in self.option_table: + if len(option) == 3: + longopt, short, help = option + repeat = 0 + elif len(option) == 4: + longopt, short, help, repeat = option + else: + # the option table is part of the code, so simply + # assert that it is correct + raise ValueError("invalid option tuple: %r" % option) + + # Type- and value-check the option names + if not isinstance(longopt, str) or len(longopt) < 2: + raise PackagingGetoptError( + ("invalid long option '%s': " + "must be a string of length >= 2") % longopt) + + if (not ((short is None) or + (isinstance(short, str) and len(short) == 1))): + raise PackagingGetoptError( + ("invalid short option '%s': " + "must be a single character or None") % short) + + self.repeat[longopt] = repeat + self.long_opts.append(longopt) + + if longopt[-1] == '=': # option takes an argument? + if short: + short = short + ':' + longopt = longopt[0:-1] + self.takes_arg[longopt] = 1 + else: + + # Is option is a "negative alias" for some other option (eg. + # "quiet" == "!verbose")? + alias_to = self.negative_alias.get(longopt) + if alias_to is not None: + if self.takes_arg[alias_to]: + raise PackagingGetoptError( + ("invalid negative alias '%s': " + "aliased option '%s' takes a value") % \ + (longopt, alias_to)) + + self.long_opts[-1] = longopt # XXX redundant?! + self.takes_arg[longopt] = 0 + + else: + self.takes_arg[longopt] = 0 + + # If this is an alias option, make sure its "takes arg" flag is + # the same as the option it's aliased to. + alias_to = self.alias.get(longopt) + if alias_to is not None: + if self.takes_arg[longopt] != self.takes_arg[alias_to]: + raise PackagingGetoptError( + ("invalid alias '%s': inconsistent with " + "aliased option '%s' (one of them takes a value, " + "the other doesn't") % (longopt, alias_to)) + + # Now enforce some bondage on the long option name, so we can + # later translate it to an attribute name on some object. Have + # to do this a bit late to make sure we've removed any trailing + # '='. + if not longopt_re.match(longopt): + raise PackagingGetoptError( + ("invalid long option name '%s' " + + "(must be letters, numbers, hyphens only") % longopt) + + self.attr_name[longopt] = longopt.replace('-', '_') + if short: + self.short_opts.append(short) + self.short2long[short[0]] = longopt + + def getopt(self, args=None, object=None): + """Parse command-line options in args. Store as attributes on object. + + If 'args' is None or not supplied, uses 'sys.argv[1:]'. If + 'object' is None or not supplied, creates a new OptionDummy + object, stores option values there, and returns a tuple (args, + object). If 'object' is supplied, it is modified in place and + 'getopt()' just returns 'args'; in both cases, the returned + 'args' is a modified copy of the passed-in 'args' list, which + is left untouched. + """ + if args is None: + args = sys.argv[1:] + if object is None: + object = OptionDummy() + created_object = 1 + else: + created_object = 0 + + self._grok_option_table() + + short_opts = ' '.join(self.short_opts) + + try: + opts, args = getopt.getopt(args, short_opts, self.long_opts) + except getopt.error as msg: + raise PackagingArgError(msg) + + for opt, val in opts: + if len(opt) == 2 and opt[0] == '-': # it's a short option + opt = self.short2long[opt[1]] + else: + assert len(opt) > 2 and opt[:2] == '--' + opt = opt[2:] + + alias = self.alias.get(opt) + if alias: + opt = alias + + if not self.takes_arg[opt]: # boolean option? + assert val == '', "boolean option can't have value" + alias = self.negative_alias.get(opt) + if alias: + opt = alias + val = 0 + else: + val = 1 + + attr = self.attr_name[opt] + # The only repeating option at the moment is 'verbose'. + # It has a negative option -q quiet, which should set verbose = 0. + if val and self.repeat.get(attr) is not None: + val = getattr(object, attr, 0) + 1 + setattr(object, attr, val) + self.option_order.append((opt, val)) + + # for opts + if created_object: + return args, object + else: + return args + + def get_option_order(self): + """Returns the list of (option, value) tuples processed by the + previous run of 'getopt()'. Raises RuntimeError if + 'getopt()' hasn't been called yet. + """ + if self.option_order is None: + raise RuntimeError("'getopt()' hasn't been called yet") + else: + return self.option_order + + return self.option_order + + def generate_help(self, header=None): + """Generate help text (a list of strings, one per suggested line of + output) from the option table for this FancyGetopt object. + """ + # Blithely assume the option table is good: probably wouldn't call + # 'generate_help()' unless you've already called 'getopt()'. + + # First pass: determine maximum length of long option names + max_opt = 0 + for option in self.option_table: + longopt = option[0] + short = option[1] + l = len(longopt) + if longopt[-1] == '=': + l = l - 1 + if short is not None: + l = l + 5 # " (-x)" where short == 'x' + if l > max_opt: + max_opt = l + + opt_width = max_opt + 2 + 2 + 2 # room for indent + dashes + gutter + + # Typical help block looks like this: + # --foo controls foonabulation + # Help block for longest option looks like this: + # --flimflam set the flim-flam level + # and with wrapped text: + # --flimflam set the flim-flam level (must be between + # 0 and 100, except on Tuesdays) + # Options with short names will have the short name shown (but + # it doesn't contribute to max_opt): + # --foo (-f) controls foonabulation + # If adding the short option would make the left column too wide, + # we push the explanation off to the next line + # --flimflam (-l) + # set the flim-flam level + # Important parameters: + # - 2 spaces before option block start lines + # - 2 dashes for each long option name + # - min. 2 spaces between option and explanation (gutter) + # - 5 characters (incl. space) for short option name + + # Now generate lines of help text. (If 80 columns were good enough + # for Jesus, then 78 columns are good enough for me!) + line_width = 78 + text_width = line_width - opt_width + big_indent = ' ' * opt_width + if header: + lines = [header] + else: + lines = ['Option summary:'] + + for option in self.option_table: + longopt, short, help = option[:3] + text = textwrap.wrap(help, text_width) + + # Case 1: no short option at all (makes life easy) + if short is None: + if text: + lines.append(" --%-*s %s" % (max_opt, longopt, text[0])) + else: + lines.append(" --%-*s " % (max_opt, longopt)) + + # Case 2: we have a short option, so we have to include it + # just after the long option + else: + opt_names = "%s (-%s)" % (longopt, short) + if text: + lines.append(" --%-*s %s" % + (max_opt, opt_names, text[0])) + else: + lines.append(" --%-*s" % opt_names) + + for l in text[1:]: + lines.append(big_indent + l) + + return lines + + def print_help(self, header=None, file=None): + if file is None: + file = sys.stdout + for line in self.generate_help(header): + file.write(line + "\n") + + +def fancy_getopt(options, negative_opt, object, args): + parser = FancyGetopt(options) + parser.set_negative_aliases(negative_opt) + return parser.getopt(args, object) + + +class OptionDummy: + """Dummy class just used as a place to hold command-line option + values as instance attributes.""" + + def __init__(self, options=[]): + """Create a new OptionDummy instance. The attributes listed in + 'options' will be initialized to None.""" + for opt in options: + setattr(self, opt, None) diff --git a/Lib/packaging/install.py b/Lib/packaging/install.py new file mode 100644 index 0000000000..b6816e56b2 --- /dev/null +++ b/Lib/packaging/install.py @@ -0,0 +1,538 @@ +"""Building blocks for installers. + +When used as a script, this module installs a release thanks to info +obtained from an index (e.g. PyPI), with dependencies. + +This is a higher-level module built on packaging.database and +packaging.pypi. +""" +import os +import sys +import stat +import errno +import shutil +import logging +import tempfile +from sysconfig import get_config_var, get_path, is_python_build + +from packaging import logger +from packaging.dist import Distribution +from packaging.util import (_is_archive_file, ask, get_install_method, + egginfo_to_distinfo) +from packaging.pypi import wrapper +from packaging.version import get_version_predicate +from packaging.database import get_distributions, get_distribution +from packaging.depgraph import generate_graph + +from packaging.errors import (PackagingError, InstallationException, + InstallationConflict, CCompilerError) +from packaging.pypi.errors import ProjectNotFound, ReleaseNotFound +from packaging import database + + +__all__ = ['install_dists', 'install_from_infos', 'get_infos', 'remove', + 'install', 'install_local_project'] + + +def _move_files(files, destination): + """Move the list of files in the destination folder, keeping the same + structure. + + Return a list of tuple (old, new) emplacement of files + + :param files: a list of files to move. + :param destination: the destination directory to put on the files. + """ + + for old in files: + filename = os.path.split(old)[-1] + new = os.path.join(destination, filename) + # try to make the paths. + try: + os.makedirs(os.path.dirname(new)) + except OSError as e: + if e.errno != errno.EEXIST: + raise + os.rename(old, new) + yield old, new + + +def _run_distutils_install(path): + # backward compat: using setuptools or plain-distutils + cmd = '%s setup.py install --record=%s' + record_file = os.path.join(path, 'RECORD') + os.system(cmd % (sys.executable, record_file)) + if not os.path.exists(record_file): + raise ValueError('failed to install') + else: + egginfo_to_distinfo(record_file, remove_egginfo=True) + + +def _run_setuptools_install(path): + cmd = '%s setup.py install --record=%s --single-version-externally-managed' + record_file = os.path.join(path, 'RECORD') + + os.system(cmd % (sys.executable, record_file)) + if not os.path.exists(record_file): + raise ValueError('failed to install') + else: + egginfo_to_distinfo(record_file, remove_egginfo=True) + + +def _run_packaging_install(path): + # XXX check for a valid setup.cfg? + dist = Distribution() + dist.parse_config_files() + try: + dist.run_command('install_dist') + name = dist.metadata['Name'] + return database.get_distribution(name) is not None + except (IOError, os.error, PackagingError, CCompilerError) as msg: + raise ValueError("Failed to install, " + str(msg)) + + +def _install_dist(dist, path): + """Install a distribution into a path. + + This: + + * unpack the distribution + * copy the files in "path" + * determine if the distribution is packaging or distutils1. + """ + where = dist.unpack() + + if where is None: + raise ValueError('Cannot locate the unpacked archive') + + return _run_install_from_archive(where) + + +def install_local_project(path): + """Install a distribution from a source directory. + + If the source directory contains a setup.py install using distutils1. + If a setup.cfg is found, install using the install_dist command. + + Returns True on success, False on Failure. + """ + path = os.path.abspath(path) + if os.path.isdir(path): + logger.info('Installing from source directory: %r', path) + return _run_install_from_dir(path) + elif _is_archive_file(path): + logger.info('Installing from archive: %r', path) + _unpacked_dir = tempfile.mkdtemp() + try: + shutil.unpack_archive(path, _unpacked_dir) + return _run_install_from_archive(_unpacked_dir) + finally: + shutil.rmtree(_unpacked_dir) + else: + logger.warning('No project to install.') + return False + + +def _run_install_from_archive(source_dir): + # XXX need a better way + for item in os.listdir(source_dir): + fullpath = os.path.join(source_dir, item) + if os.path.isdir(fullpath): + source_dir = fullpath + break + return _run_install_from_dir(source_dir) + + +install_methods = { + 'packaging': _run_packaging_install, + 'setuptools': _run_setuptools_install, + 'distutils': _run_distutils_install} + + +def _run_install_from_dir(source_dir): + old_dir = os.getcwd() + os.chdir(source_dir) + install_method = get_install_method(source_dir) + func = install_methods[install_method] + try: + func = install_methods[install_method] + try: + func(source_dir) + return True + except ValueError as err: + # failed to install + logger.info(str(err)) + return False + finally: + os.chdir(old_dir) + + +def install_dists(dists, path, paths=None): + """Install all distributions provided in dists, with the given prefix. + + If an error occurs while installing one of the distributions, uninstall all + the installed distribution (in the context if this function). + + Return a list of installed dists. + + :param dists: distributions to install + :param path: base path to install distribution in + :param paths: list of paths (defaults to sys.path) to look for info + """ + + installed_dists = [] + for dist in dists: + logger.info('Installing %r %s...', dist.name, dist.version) + try: + _install_dist(dist, path) + installed_dists.append(dist) + except Exception as e: + logger.info('Failed: %s', e) + + # reverting + for installed_dist in installed_dists: + logger.info('Reverting %r', installed_dist) + remove(installed_dist.name, paths) + raise e + return installed_dists + + +def install_from_infos(install_path=None, install=[], remove=[], conflicts=[], + paths=None): + """Install and remove the given distributions. + + The function signature is made to be compatible with the one of get_infos. + The aim of this script is to povide a way to install/remove what's asked, + and to rollback if needed. + + So, it's not possible to be in an inconsistant state, it could be either + installed, either uninstalled, not half-installed. + + The process follow those steps: + + 1. Move all distributions that will be removed in a temporary location + 2. Install all the distributions that will be installed in a temp. loc. + 3. If the installation fails, rollback (eg. move back) those + distributions, or remove what have been installed. + 4. Else, move the distributions to the right locations, and remove for + real the distributions thats need to be removed. + + :param install_path: the installation path where we want to install the + distributions. + :param install: list of distributions that will be installed; install_path + must be provided if this list is not empty. + :param remove: list of distributions that will be removed. + :param conflicts: list of conflicting distributions, eg. that will be in + conflict once the install and remove distribution will be + processed. + :param paths: list of paths (defaults to sys.path) to look for info + """ + # first of all, if we have conflicts, stop here. + if conflicts: + raise InstallationConflict(conflicts) + + if install and not install_path: + raise ValueError("Distributions are to be installed but `install_path`" + " is not provided.") + + # before removing the files, we will start by moving them away + # then, if any error occurs, we could replace them in the good place. + temp_files = {} # contains lists of {dist: (old, new)} paths + temp_dir = None + if remove: + temp_dir = tempfile.mkdtemp() + for dist in remove: + files = dist.list_installed_files() + temp_files[dist] = _move_files(files, temp_dir) + try: + if install: + install_dists(install, install_path, paths) + except: + # if an error occurs, put back the files in the right place. + for files in temp_files.values(): + for old, new in files: + shutil.move(new, old) + if temp_dir: + shutil.rmtree(temp_dir) + # now re-raising + raise + + # we can remove them for good + for files in temp_files.values(): + for old, new in files: + os.remove(new) + if temp_dir: + shutil.rmtree(temp_dir) + + +def _get_setuptools_deps(release): + # NotImplementedError + pass + + +def get_infos(requirements, index=None, installed=None, prefer_final=True): + """Return the informations on what's going to be installed and upgraded. + + :param requirements: is a *string* containing the requirements for this + project (for instance "FooBar 1.1" or "BarBaz (<1.2)") + :param index: If an index is specified, use this one, otherwise, use + :class index.ClientWrapper: to get project metadatas. + :param installed: a list of already installed distributions. + :param prefer_final: when picking up the releases, prefer a "final" one + over a beta/alpha/etc one. + + The results are returned in a dict, containing all the operations + needed to install the given requirements:: + + >>> get_install_info("FooBar (<=1.2)") + {'install': [<FooBar 1.1>], 'remove': [], 'conflict': []} + + Conflict contains all the conflicting distributions, if there is a + conflict. + """ + # this function does several things: + # 1. get a release specified by the requirements + # 2. gather its metadata, using setuptools compatibility if needed + # 3. compare this tree with what is currently installed on the system, + # return the requirements of what is missing + # 4. do that recursively and merge back the results + # 5. return a dict containing information about what is needed to install + # or remove + + if not installed: + logger.debug('Reading installed distributions') + installed = list(get_distributions(use_egg_info=True)) + + infos = {'install': [], 'remove': [], 'conflict': []} + # Is a compatible version of the project already installed ? + predicate = get_version_predicate(requirements) + found = False + + # check that the project isn't already installed + for installed_project in installed: + # is it a compatible project ? + if predicate.name.lower() != installed_project.name.lower(): + continue + found = True + logger.info('Found %r %s', installed_project.name, + installed_project.version) + + # if we already have something installed, check it matches the + # requirements + if predicate.match(installed_project.version): + return infos + break + + if not found: + logger.debug('Project not installed') + + if not index: + index = wrapper.ClientWrapper() + + if not installed: + installed = get_distributions(use_egg_info=True) + + # Get all the releases that match the requirements + try: + release = index.get_release(requirements) + except (ReleaseNotFound, ProjectNotFound): + raise InstallationException('Release not found: %r' % requirements) + + if release is None: + logger.info('Could not find a matching project') + return infos + + metadata = release.fetch_metadata() + + # we need to build setuptools deps if any + if 'requires_dist' not in metadata: + metadata['requires_dist'] = _get_setuptools_deps(release) + + # build the dependency graph with local and required dependencies + dists = list(installed) + dists.append(release) + depgraph = generate_graph(dists) + + # Get what the missing deps are + dists = depgraph.missing[release] + if dists: + logger.info("Missing dependencies found, retrieving metadata") + # we have missing deps + for dist in dists: + _update_infos(infos, get_infos(dist, index, installed)) + + # Fill in the infos + existing = [d for d in installed if d.name == release.name] + if existing: + infos['remove'].append(existing[0]) + infos['conflict'].extend(depgraph.reverse_list[existing[0]]) + infos['install'].append(release) + return infos + + +def _update_infos(infos, new_infos): + """extends the lists contained in the `info` dict with those contained + in the `new_info` one + """ + for key, value in infos.items(): + if key in new_infos: + infos[key].extend(new_infos[key]) + + +def remove(project_name, paths=None, auto_confirm=True): + """Removes a single project from the installation. + + Returns True on success + """ + dist = get_distribution(project_name, use_egg_info=True, paths=paths) + if dist is None: + raise PackagingError('Distribution %r not found' % project_name) + files = dist.list_installed_files(local=True) + rmdirs = [] + rmfiles = [] + tmp = tempfile.mkdtemp(prefix=project_name + '-uninstall') + + def _move_file(source, target): + try: + os.rename(source, target) + except OSError as err: + return err + return None + + success = True + error = None + try: + for file_, md5, size in files: + if os.path.isfile(file_): + dirname, filename = os.path.split(file_) + tmpfile = os.path.join(tmp, filename) + try: + error = _move_file(file_, tmpfile) + if error is not None: + success = False + break + finally: + if not os.path.isfile(file_): + os.rename(tmpfile, file_) + if file_ not in rmfiles: + rmfiles.append(file_) + if dirname not in rmdirs: + rmdirs.append(dirname) + finally: + shutil.rmtree(tmp) + + if not success: + logger.info('%r cannot be removed.', project_name) + logger.info('Error: %s', error) + return False + + logger.info('Removing %r: ', project_name) + + for file_ in rmfiles: + logger.info(' %s', file_) + + # Taken from the pip project + if auto_confirm: + response = 'y' + else: + response = ask('Proceed (y/n)? ', ('y', 'n')) + + if response == 'y': + file_count = 0 + for file_ in rmfiles: + os.remove(file_) + file_count += 1 + + dir_count = 0 + for dirname in rmdirs: + if not os.path.exists(dirname): + # could + continue + + files_count = 0 + for root, dir, files in os.walk(dirname): + files_count += len(files) + + if files_count > 0: + # XXX Warning + continue + + # empty dirs with only empty dirs + if os.stat(dirname).st_mode & stat.S_IWUSR: + # XXX Add a callable in shutil.rmtree to count + # the number of deleted elements + shutil.rmtree(dirname) + dir_count += 1 + + # removing the top path + # XXX count it ? + if os.path.exists(dist.path): + shutil.rmtree(dist.path) + + logger.info('Success: removed %d files and %d dirs', + file_count, dir_count) + + return True + + +def install(project): + """Installs a project. + + Returns True on success, False on failure + """ + if is_python_build(): + # Python would try to install into the site-packages directory under + # $PREFIX, but when running from an uninstalled code checkout we don't + # want to create directories under the installation root + message = ('installing third-party projects from an uninstalled ' + 'Python is not supported') + logger.error(message) + return False + + logger.info('Checking the installation location...') + purelib_path = get_path('purelib') + + # trying to write a file there + try: + with tempfile.NamedTemporaryFile(suffix=project, + dir=purelib_path) as testfile: + testfile.write(b'test') + except OSError: + # FIXME this should check the errno, or be removed altogether (race + # condition: the directory permissions could be changed between here + # and the actual install) + logger.info('Unable to write in "%s". Do you have the permissions ?' + % purelib_path) + return False + + logger.info('Getting information about %r...', project) + try: + info = get_infos(project) + except InstallationException: + logger.info('Cound not find %r', project) + return False + + if info['install'] == []: + logger.info('Nothing to install') + return False + + install_path = get_config_var('base') + try: + install_from_infos(install_path, + info['install'], info['remove'], info['conflict']) + + except InstallationConflict as e: + if logger.isEnabledFor(logging.INFO): + projects = ('%r %s' % (p.name, p.version) for p in e.args[0]) + logger.info('%r conflicts with %s', project, ','.join(projects)) + + return True + + +def _main(**attrs): + if 'script_args' not in attrs: + attrs['requirements'] = sys.argv[1] + get_infos(**attrs) + +if __name__ == '__main__': + _main() diff --git a/Lib/packaging/manifest.py b/Lib/packaging/manifest.py new file mode 100644 index 0000000000..a3798530a5 --- /dev/null +++ b/Lib/packaging/manifest.py @@ -0,0 +1,372 @@ +"""Class representing the list of files in a distribution. + +The Manifest class can be used to: + + - read or write a MANIFEST file + - read a template file and find out the file list +""" +# XXX todo: document + add tests +import re +import os +import fnmatch + +from packaging import logger +from packaging.util import write_file, convert_path +from packaging.errors import (PackagingTemplateError, + PackagingInternalError) + +__all__ = ['Manifest'] + +# a \ followed by some spaces + EOL +_COLLAPSE_PATTERN = re.compile('\\\w*\n', re.M) +_COMMENTED_LINE = re.compile('#.*?(?=\n)|\n(?=$)', re.M | re.S) + + +class Manifest(object): + """A list of files built by on exploring the filesystem and filtered by + applying various patterns to what we find there. + """ + + def __init__(self): + self.allfiles = None + self.files = [] + + # + # Public API + # + + def findall(self, dir=os.curdir): + self.allfiles = _findall(dir) + + def append(self, item): + self.files.append(item) + + def extend(self, items): + self.files.extend(items) + + def sort(self): + # Not a strict lexical sort! + self.files = [os.path.join(*path_tuple) for path_tuple in + sorted(os.path.split(path) for path in self.files)] + + def clear(self): + """Clear all collected files.""" + self.files = [] + if self.allfiles is not None: + self.allfiles = [] + + def remove_duplicates(self): + # Assumes list has been sorted! + for i in range(len(self.files) - 1, 0, -1): + if self.files[i] == self.files[i - 1]: + del self.files[i] + + def read_template(self, path_or_file): + """Read and parse a manifest template file. + 'path' can be a path or a file-like object. + + Updates the list accordingly. + """ + if isinstance(path_or_file, str): + f = open(path_or_file) + else: + f = path_or_file + + try: + content = f.read() + # first, let's unwrap collapsed lines + content = _COLLAPSE_PATTERN.sub('', content) + # next, let's remove commented lines and empty lines + content = _COMMENTED_LINE.sub('', content) + + # now we have our cleaned up lines + lines = [line.strip() for line in content.split('\n')] + finally: + f.close() + + for line in lines: + if line == '': + continue + try: + self._process_template_line(line) + except PackagingTemplateError as msg: + logger.warning("%s, %s", path_or_file, msg) + + def write(self, path): + """Write the file list in 'self.filelist' (presumably as filled in + by 'add_defaults()' and 'read_template()') to the manifest file + named by 'self.manifest'. + """ + if os.path.isfile(path): + with open(path) as fp: + first_line = fp.readline() + + if first_line != '# file GENERATED by packaging, do NOT edit\n': + logger.info("not writing to manually maintained " + "manifest file %r", path) + return + + self.sort() + self.remove_duplicates() + content = self.files[:] + content.insert(0, '# file GENERATED by packaging, do NOT edit') + logger.info("writing manifest file %r", path) + write_file(path, content) + + def read(self, path): + """Read the manifest file (named by 'self.manifest') and use it to + fill in 'self.filelist', the list of files to include in the source + distribution. + """ + logger.info("reading manifest file %r", path) + with open(path) as manifest: + for line in manifest.readlines(): + self.append(line) + + def exclude_pattern(self, pattern, anchor=True, prefix=None, + is_regex=False): + """Remove strings (presumably filenames) from 'files' that match + 'pattern'. + + Other parameters are the same as for 'include_pattern()', above. + The list 'self.files' is modified in place. Return True if files are + found. + """ + files_found = False + pattern_re = _translate_pattern(pattern, anchor, prefix, is_regex) + for i in range(len(self.files) - 1, -1, -1): + if pattern_re.search(self.files[i]): + del self.files[i] + files_found = True + + return files_found + + # + # Private API + # + + def _parse_template_line(self, line): + words = line.split() + if len(words) == 1: + # no action given, let's use the default 'include' + words.insert(0, 'include') + + action = words[0] + patterns = dir = dir_pattern = None + + if action in ('include', 'exclude', + 'global-include', 'global-exclude'): + if len(words) < 2: + raise PackagingTemplateError( + "%r expects <pattern1> <pattern2> ..." % action) + + patterns = [convert_path(word) for word in words[1:]] + + elif action in ('recursive-include', 'recursive-exclude'): + if len(words) < 3: + raise PackagingTemplateError( + "%r expects <dir> <pattern1> <pattern2> ..." % action) + + dir = convert_path(words[1]) + patterns = [convert_path(word) for word in words[2:]] + + elif action in ('graft', 'prune'): + if len(words) != 2: + raise PackagingTemplateError( + "%r expects a single <dir_pattern>" % action) + + dir_pattern = convert_path(words[1]) + + else: + raise PackagingTemplateError("unknown action %r" % action) + + return action, patterns, dir, dir_pattern + + def _process_template_line(self, line): + # Parse the line: split it up, make sure the right number of words + # is there, and return the relevant words. 'action' is always + # defined: it's the first word of the line. Which of the other + # three are defined depends on the action; it'll be either + # patterns, (dir and patterns), or (dir_pattern). + action, patterns, dir, dir_pattern = self._parse_template_line(line) + + # OK, now we know that the action is valid and we have the + # right number of words on the line for that action -- so we + # can proceed with minimal error-checking. + if action == 'include': + for pattern in patterns: + if not self._include_pattern(pattern, anchor=True): + logger.warning("no files found matching %r", pattern) + + elif action == 'exclude': + for pattern in patterns: + if not self.exclude_pattern(pattern, anchor=True): + logger.warning("no previously-included files " + "found matching %r", pattern) + + elif action == 'global-include': + for pattern in patterns: + if not self._include_pattern(pattern, anchor=False): + logger.warning("no files found matching %r " + "anywhere in distribution", pattern) + + elif action == 'global-exclude': + for pattern in patterns: + if not self.exclude_pattern(pattern, anchor=False): + logger.warning("no previously-included files " + "matching %r found anywhere in " + "distribution", pattern) + + elif action == 'recursive-include': + for pattern in patterns: + if not self._include_pattern(pattern, prefix=dir): + logger.warning("no files found matching %r " + "under directory %r", pattern, dir) + + elif action == 'recursive-exclude': + for pattern in patterns: + if not self.exclude_pattern(pattern, prefix=dir): + logger.warning("no previously-included files " + "matching %r found under directory %r", + pattern, dir) + + elif action == 'graft': + if not self._include_pattern(None, prefix=dir_pattern): + logger.warning("no directories found matching %r", + dir_pattern) + + elif action == 'prune': + if not self.exclude_pattern(None, prefix=dir_pattern): + logger.warning("no previously-included directories found " + "matching %r", dir_pattern) + else: + raise PackagingInternalError( + "this cannot happen: invalid action %r" % action) + + def _include_pattern(self, pattern, anchor=True, prefix=None, + is_regex=False): + """Select strings (presumably filenames) from 'self.files' that + match 'pattern', a Unix-style wildcard (glob) pattern. + + Patterns are not quite the same as implemented by the 'fnmatch' + module: '*' and '?' match non-special characters, where "special" + is platform-dependent: slash on Unix; colon, slash, and backslash on + DOS/Windows; and colon on Mac OS. + + If 'anchor' is true (the default), then the pattern match is more + stringent: "*.py" will match "foo.py" but not "foo/bar.py". If + 'anchor' is false, both of these will match. + + If 'prefix' is supplied, then only filenames starting with 'prefix' + (itself a pattern) and ending with 'pattern', with anything in between + them, will match. 'anchor' is ignored in this case. + + If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and + 'pattern' is assumed to be either a string containing a regex or a + regex object -- no translation is done, the regex is just compiled + and used as-is. + + Selected strings will be added to self.files. + + Return True if files are found. + """ + files_found = False + pattern_re = _translate_pattern(pattern, anchor, prefix, is_regex) + + # delayed loading of allfiles list + if self.allfiles is None: + self.findall() + + for name in self.allfiles: + if pattern_re.search(name): + self.files.append(name) + files_found = True + + return files_found + + +# +# Utility functions +# +def _findall(dir=os.curdir): + """Find all files under 'dir' and return the list of full filenames + (relative to 'dir'). + """ + from stat import S_ISREG, S_ISDIR, S_ISLNK + + list = [] + stack = [dir] + pop = stack.pop + push = stack.append + + while stack: + dir = pop() + names = os.listdir(dir) + + for name in names: + if dir != os.curdir: # avoid the dreaded "./" syndrome + fullname = os.path.join(dir, name) + else: + fullname = name + + # Avoid excess stat calls -- just one will do, thank you! + stat = os.stat(fullname) + mode = stat.st_mode + if S_ISREG(mode): + list.append(fullname) + elif S_ISDIR(mode) and not S_ISLNK(mode): + push(fullname) + + return list + + +def _glob_to_re(pattern): + """Translate a shell-like glob pattern to a regular expression. + + Return a string containing the regex. Differs from + 'fnmatch.translate()' in that '*' does not match "special characters" + (which are platform-specific). + """ + pattern_re = fnmatch.translate(pattern) + + # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which + # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix, + # and by extension they shouldn't match such "special characters" under + # any OS. So change all non-escaped dots in the RE to match any + # character except the special characters. + # XXX currently the "special characters" are just slash -- i.e. this is + # Unix-only. + pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', r'\1[^/]', pattern_re) + + return pattern_re + + +def _translate_pattern(pattern, anchor=True, prefix=None, is_regex=False): + """Translate a shell-like wildcard pattern to a compiled regular + expression. + + Return the compiled regex. If 'is_regex' true, + then 'pattern' is directly compiled to a regex (if it's a string) + or just returned as-is (assumes it's a regex object). + """ + if is_regex: + if isinstance(pattern, str): + return re.compile(pattern) + else: + return pattern + + if pattern: + pattern_re = _glob_to_re(pattern) + else: + pattern_re = '' + + if prefix is not None: + # ditch end of pattern character + empty_pattern = _glob_to_re('') + prefix_re = _glob_to_re(prefix)[:-len(empty_pattern)] + pattern_re = "^" + os.path.join(prefix_re, ".*" + pattern_re) + else: # no prefix -- respect anchor flag + if anchor: + pattern_re = "^" + pattern_re + + return re.compile(pattern_re) diff --git a/Lib/packaging/markers.py b/Lib/packaging/markers.py new file mode 100644 index 0000000000..4bbac7eca7 --- /dev/null +++ b/Lib/packaging/markers.py @@ -0,0 +1,187 @@ +"""Parser for the environment markers micro-language defined in PEP 345.""" + +import sys +import platform +import os + +from tokenize import tokenize, NAME, OP, STRING, ENDMARKER, ENCODING +from io import BytesIO + +__all__ = ['interpret'] + + +# allowed operators +_OPERATORS = {'==': lambda x, y: x == y, + '!=': lambda x, y: x != y, + '>': lambda x, y: x > y, + '>=': lambda x, y: x >= y, + '<': lambda x, y: x < y, + '<=': lambda x, y: x <= y, + 'in': lambda x, y: x in y, + 'not in': lambda x, y: x not in y} + + +def _operate(operation, x, y): + return _OPERATORS[operation](x, y) + + +# restricted set of variables +_VARS = {'sys.platform': sys.platform, + 'python_version': sys.version[:3], + 'python_full_version': sys.version.split(' ', 1)[0], + 'os.name': os.name, + 'platform.version': platform.version(), + 'platform.machine': platform.machine(), + 'platform.python_implementation': platform.python_implementation()} + + +class _Operation: + + def __init__(self, execution_context=None): + self.left = None + self.op = None + self.right = None + if execution_context is None: + execution_context = {} + self.execution_context = execution_context + + def _get_var(self, name): + if name in self.execution_context: + return self.execution_context[name] + return _VARS[name] + + def __repr__(self): + return '%s %s %s' % (self.left, self.op, self.right) + + def _is_string(self, value): + if value is None or len(value) < 2: + return False + for delimiter in '"\'': + if value[0] == value[-1] == delimiter: + return True + return False + + def _is_name(self, value): + return value in _VARS + + def _convert(self, value): + if value in _VARS: + return self._get_var(value) + return value.strip('"\'') + + def _check_name(self, value): + if value not in _VARS: + raise NameError(value) + + def _nonsense_op(self): + msg = 'This operation is not supported : "%s"' % self + raise SyntaxError(msg) + + def __call__(self): + # make sure we do something useful + if self._is_string(self.left): + if self._is_string(self.right): + self._nonsense_op() + self._check_name(self.right) + else: + if not self._is_string(self.right): + self._nonsense_op() + self._check_name(self.left) + + if self.op not in _OPERATORS: + raise TypeError('Operator not supported "%s"' % self.op) + + left = self._convert(self.left) + right = self._convert(self.right) + return _operate(self.op, left, right) + + +class _OR: + def __init__(self, left, right=None): + self.left = left + self.right = right + + def filled(self): + return self.right is not None + + def __repr__(self): + return 'OR(%r, %r)' % (self.left, self.right) + + def __call__(self): + return self.left() or self.right() + + +class _AND: + def __init__(self, left, right=None): + self.left = left + self.right = right + + def filled(self): + return self.right is not None + + def __repr__(self): + return 'AND(%r, %r)' % (self.left, self.right) + + def __call__(self): + return self.left() and self.right() + + +def interpret(marker, execution_context=None): + """Interpret a marker and return a result depending on environment.""" + marker = marker.strip().encode() + ops = [] + op_starting = True + for token in tokenize(BytesIO(marker).readline): + # Unpack token + toktype, tokval, rowcol, line, logical_line = token + if toktype not in (NAME, OP, STRING, ENDMARKER, ENCODING): + raise SyntaxError('Type not supported "%s"' % tokval) + + if op_starting: + op = _Operation(execution_context) + if len(ops) > 0: + last = ops[-1] + if isinstance(last, (_OR, _AND)) and not last.filled(): + last.right = op + else: + ops.append(op) + else: + ops.append(op) + op_starting = False + else: + op = ops[-1] + + if (toktype == ENDMARKER or + (toktype == NAME and tokval in ('and', 'or'))): + if toktype == NAME and tokval == 'and': + ops.append(_AND(ops.pop())) + elif toktype == NAME and tokval == 'or': + ops.append(_OR(ops.pop())) + op_starting = True + continue + + if isinstance(op, (_OR, _AND)) and op.right is not None: + op = op.right + + if ((toktype in (NAME, STRING) and tokval not in ('in', 'not')) + or (toktype == OP and tokval == '.')): + if op.op is None: + if op.left is None: + op.left = tokval + else: + op.left += tokval + else: + if op.right is None: + op.right = tokval + else: + op.right += tokval + elif toktype == OP or tokval in ('in', 'not'): + if tokval == 'in' and op.op == 'not': + op.op = 'not in' + else: + op.op = tokval + + for op in ops: + if not op(): + return False + return True diff --git a/Lib/packaging/metadata.py b/Lib/packaging/metadata.py new file mode 100644 index 0000000000..104600b54e --- /dev/null +++ b/Lib/packaging/metadata.py @@ -0,0 +1,546 @@ +"""Implementation of the Metadata for Python packages PEPs. + +Supports all metadata formats (1.0, 1.1, 1.2). +""" + +import re +import logging + +from io import StringIO +from email import message_from_file +from packaging import logger +from packaging.markers import interpret +from packaging.version import (is_valid_predicate, is_valid_version, + is_valid_versions) +from packaging.errors import (MetadataMissingError, + MetadataConflictError, + MetadataUnrecognizedVersionError) + +try: + # docutils is installed + from docutils.utils import Reporter + from docutils.parsers.rst import Parser + from docutils import frontend + from docutils import nodes + + class SilentReporter(Reporter): + + def __init__(self, source, report_level, halt_level, stream=None, + debug=0, encoding='ascii', error_handler='replace'): + self.messages = [] + Reporter.__init__(self, source, report_level, halt_level, stream, + debug, encoding, error_handler) + + def system_message(self, level, message, *children, **kwargs): + self.messages.append((level, message, children, kwargs)) + + _HAS_DOCUTILS = True +except ImportError: + # docutils is not installed + _HAS_DOCUTILS = False + +# public API of this module +__all__ = ['Metadata', 'PKG_INFO_ENCODING', 'PKG_INFO_PREFERRED_VERSION'] + +# Encoding used for the PKG-INFO files +PKG_INFO_ENCODING = 'utf-8' + +# preferred version. Hopefully will be changed +# to 1.2 once PEP 345 is supported everywhere +PKG_INFO_PREFERRED_VERSION = '1.0' + +_LINE_PREFIX = re.compile('\n \|') +_241_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', + 'Summary', 'Description', + 'Keywords', 'Home-page', 'Author', 'Author-email', + 'License') + +_314_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', + 'Supported-Platform', 'Summary', 'Description', + 'Keywords', 'Home-page', 'Author', 'Author-email', + 'License', 'Classifier', 'Download-URL', 'Obsoletes', + 'Provides', 'Requires') + +_314_MARKERS = ('Obsoletes', 'Provides', 'Requires') + +_345_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', + 'Supported-Platform', 'Summary', 'Description', + 'Keywords', 'Home-page', 'Author', 'Author-email', + 'Maintainer', 'Maintainer-email', 'License', + 'Classifier', 'Download-URL', 'Obsoletes-Dist', + 'Project-URL', 'Provides-Dist', 'Requires-Dist', + 'Requires-Python', 'Requires-External') + +_345_MARKERS = ('Provides-Dist', 'Requires-Dist', 'Requires-Python', + 'Obsoletes-Dist', 'Requires-External', 'Maintainer', + 'Maintainer-email', 'Project-URL') + +_ALL_FIELDS = set() +_ALL_FIELDS.update(_241_FIELDS) +_ALL_FIELDS.update(_314_FIELDS) +_ALL_FIELDS.update(_345_FIELDS) + + +def _version2fieldlist(version): + if version == '1.0': + return _241_FIELDS + elif version == '1.1': + return _314_FIELDS + elif version == '1.2': + return _345_FIELDS + raise MetadataUnrecognizedVersionError(version) + + +def _best_version(fields): + """Detect the best version depending on the fields used.""" + def _has_marker(keys, markers): + for marker in markers: + if marker in keys: + return True + return False + + keys = list(fields) + possible_versions = ['1.0', '1.1', '1.2'] + + # first let's try to see if a field is not part of one of the version + for key in keys: + if key not in _241_FIELDS and '1.0' in possible_versions: + possible_versions.remove('1.0') + if key not in _314_FIELDS and '1.1' in possible_versions: + possible_versions.remove('1.1') + if key not in _345_FIELDS and '1.2' in possible_versions: + possible_versions.remove('1.2') + + # possible_version contains qualified versions + if len(possible_versions) == 1: + return possible_versions[0] # found ! + elif len(possible_versions) == 0: + raise MetadataConflictError('Unknown metadata set') + + # let's see if one unique marker is found + is_1_1 = '1.1' in possible_versions and _has_marker(keys, _314_MARKERS) + is_1_2 = '1.2' in possible_versions and _has_marker(keys, _345_MARKERS) + if is_1_1 and is_1_2: + raise MetadataConflictError('You used incompatible 1.1 and 1.2 fields') + + # we have the choice, either 1.0, or 1.2 + # - 1.0 has a broken Summary field but works with all tools + # - 1.1 is to avoid + # - 1.2 fixes Summary but is not widespread yet + if not is_1_1 and not is_1_2: + # we couldn't find any specific marker + if PKG_INFO_PREFERRED_VERSION in possible_versions: + return PKG_INFO_PREFERRED_VERSION + if is_1_1: + return '1.1' + + # default marker when 1.0 is disqualified + return '1.2' + + +_ATTR2FIELD = { + 'metadata_version': 'Metadata-Version', + 'name': 'Name', + 'version': 'Version', + 'platform': 'Platform', + 'supported_platform': 'Supported-Platform', + 'summary': 'Summary', + 'description': 'Description', + 'keywords': 'Keywords', + 'home_page': 'Home-page', + 'author': 'Author', + 'author_email': 'Author-email', + 'maintainer': 'Maintainer', + 'maintainer_email': 'Maintainer-email', + 'license': 'License', + 'classifier': 'Classifier', + 'download_url': 'Download-URL', + 'obsoletes_dist': 'Obsoletes-Dist', + 'provides_dist': 'Provides-Dist', + 'requires_dist': 'Requires-Dist', + 'requires_python': 'Requires-Python', + 'requires_external': 'Requires-External', + 'requires': 'Requires', + 'provides': 'Provides', + 'obsoletes': 'Obsoletes', + 'project_url': 'Project-URL', +} + +_PREDICATE_FIELDS = ('Requires-Dist', 'Obsoletes-Dist', 'Provides-Dist') +_VERSIONS_FIELDS = ('Requires-Python',) +_VERSION_FIELDS = ('Version',) +_LISTFIELDS = ('Platform', 'Classifier', 'Obsoletes', + 'Requires', 'Provides', 'Obsoletes-Dist', + 'Provides-Dist', 'Requires-Dist', 'Requires-External', + 'Project-URL', 'Supported-Platform') +_LISTTUPLEFIELDS = ('Project-URL',) + +_ELEMENTSFIELD = ('Keywords',) + +_UNICODEFIELDS = ('Author', 'Maintainer', 'Summary', 'Description') + +_MISSING = object() + + +class Metadata: + """The metadata of a release. + + Supports versions 1.0, 1.1 and 1.2 (auto-detected). You can + instantiate the class with one of these arguments (or none): + - *path*, the path to a METADATA file + - *fileobj* give a file-like object with METADATA as content + - *mapping* is a dict-like object + """ + # TODO document that execution_context and platform_dependent are used + # to filter on query, not when setting a key + # also document the mapping API and UNKNOWN default key + + def __init__(self, path=None, platform_dependent=False, + execution_context=None, fileobj=None, mapping=None): + self._fields = {} + self.requires_files = [] + self.docutils_support = _HAS_DOCUTILS + self.platform_dependent = platform_dependent + self.execution_context = execution_context + if [path, fileobj, mapping].count(None) < 2: + raise TypeError('path, fileobj and mapping are exclusive') + if path is not None: + self.read(path) + elif fileobj is not None: + self.read_file(fileobj) + elif mapping is not None: + self.update(mapping) + + def _set_best_version(self): + self._fields['Metadata-Version'] = _best_version(self._fields) + + def _write_field(self, file, name, value): + file.write('%s: %s\n' % (name, value)) + + def __getitem__(self, name): + return self.get(name) + + def __setitem__(self, name, value): + return self.set(name, value) + + def __delitem__(self, name): + field_name = self._convert_name(name) + try: + del self._fields[field_name] + except KeyError: + raise KeyError(name) + self._set_best_version() + + def __contains__(self, name): + return (name in self._fields or + self._convert_name(name) in self._fields) + + def _convert_name(self, name): + if name in _ALL_FIELDS: + return name + name = name.replace('-', '_').lower() + return _ATTR2FIELD.get(name, name) + + def _default_value(self, name): + if name in _LISTFIELDS or name in _ELEMENTSFIELD: + return [] + return 'UNKNOWN' + + def _check_rst_data(self, data): + """Return warnings when the provided data has syntax errors.""" + source_path = StringIO() + parser = Parser() + settings = frontend.OptionParser().get_default_values() + settings.tab_width = 4 + settings.pep_references = None + settings.rfc_references = None + reporter = SilentReporter(source_path, + settings.report_level, + settings.halt_level, + stream=settings.warning_stream, + debug=settings.debug, + encoding=settings.error_encoding, + error_handler=settings.error_encoding_error_handler) + + document = nodes.document(settings, reporter, source=source_path) + document.note_source(source_path, -1) + try: + parser.parse(data, document) + except AttributeError: + reporter.messages.append((-1, 'Could not finish the parsing.', + '', {})) + + return reporter.messages + + def _platform(self, value): + if not self.platform_dependent or ';' not in value: + return True, value + value, marker = value.split(';') + return interpret(marker, self.execution_context), value + + def _remove_line_prefix(self, value): + return _LINE_PREFIX.sub('\n', value) + + # + # Public API + # + def get_fullname(self): + """Return the distribution name with version""" + return '%s-%s' % (self['Name'], self['Version']) + + def is_metadata_field(self, name): + """return True if name is a valid metadata key""" + name = self._convert_name(name) + return name in _ALL_FIELDS + + def is_multi_field(self, name): + name = self._convert_name(name) + return name in _LISTFIELDS + + def read(self, filepath): + """Read the metadata values from a file path.""" + with open(filepath, 'r', encoding='utf-8') as fp: + self.read_file(fp) + + def read_file(self, fileob): + """Read the metadata values from a file object.""" + msg = message_from_file(fileob) + self._fields['Metadata-Version'] = msg['metadata-version'] + + for field in _version2fieldlist(self['Metadata-Version']): + if field in _LISTFIELDS: + # we can have multiple lines + values = msg.get_all(field) + if field in _LISTTUPLEFIELDS and values is not None: + values = [tuple(value.split(',')) for value in values] + self.set(field, values) + else: + # single line + value = msg[field] + if value is not None and value != 'UNKNOWN': + self.set(field, value) + + def write(self, filepath): + """Write the metadata fields to filepath.""" + with open(filepath, 'w', encoding='utf-8') as fp: + self.write_file(fp) + + def write_file(self, fileobject): + """Write the PKG-INFO format data to a file object.""" + self._set_best_version() + for field in _version2fieldlist(self['Metadata-Version']): + values = self.get(field) + if field in _ELEMENTSFIELD: + self._write_field(fileobject, field, ','.join(values)) + continue + if field not in _LISTFIELDS: + if field == 'Description': + values = values.replace('\n', '\n |') + values = [values] + + if field in _LISTTUPLEFIELDS: + values = [','.join(value) for value in values] + + for value in values: + self._write_field(fileobject, field, value) + + def update(self, other=None, **kwargs): + """Set metadata values from the given iterable `other` and kwargs. + + Behavior is like `dict.update`: If `other` has a ``keys`` method, + they are looped over and ``self[key]`` is assigned ``other[key]``. + Else, ``other`` is an iterable of ``(key, value)`` iterables. + + Keys that don't match a metadata field or that have an empty value are + dropped. + """ + def _set(key, value): + if key in _ATTR2FIELD and value: + self.set(self._convert_name(key), value) + + if other is None: + pass + elif hasattr(other, 'keys'): + for k in other.keys(): + _set(k, other[k]) + else: + for k, v in other: + _set(k, v) + + if kwargs: + self.update(kwargs) + + def set(self, name, value): + """Control then set a metadata field.""" + name = self._convert_name(name) + + if ((name in _ELEMENTSFIELD or name == 'Platform') and + not isinstance(value, (list, tuple))): + if isinstance(value, str): + value = [v.strip() for v in value.split(',')] + else: + value = [] + elif (name in _LISTFIELDS and + not isinstance(value, (list, tuple))): + if isinstance(value, str): + value = [value] + else: + value = [] + + if logger.isEnabledFor(logging.WARNING): + project_name = self['Name'] + + if name in _PREDICATE_FIELDS and value is not None: + for v in value: + # check that the values are valid predicates + if not is_valid_predicate(v.split(';')[0]): + logger.warning( + '%r: %r is not a valid predicate (field %r)', + project_name, v, name) + # FIXME this rejects UNKNOWN, is that right? + elif name in _VERSIONS_FIELDS and value is not None: + if not is_valid_versions(value): + logger.warning('%r: %r is not a valid version (field %r)', + project_name, value, name) + elif name in _VERSION_FIELDS and value is not None: + if not is_valid_version(value): + logger.warning('%r: %r is not a valid version (field %r)', + project_name, value, name) + + if name in _UNICODEFIELDS: + if name == 'Description': + value = self._remove_line_prefix(value) + + self._fields[name] = value + self._set_best_version() + + def get(self, name, default=_MISSING): + """Get a metadata field.""" + name = self._convert_name(name) + if name not in self._fields: + if default is _MISSING: + default = self._default_value(name) + return default + if name in _UNICODEFIELDS: + value = self._fields[name] + return value + elif name in _LISTFIELDS: + value = self._fields[name] + if value is None: + return [] + res = [] + for val in value: + valid, val = self._platform(val) + if not valid: + continue + if name not in _LISTTUPLEFIELDS: + res.append(val) + else: + # That's for Project-URL + res.append((val[0], val[1])) + return res + + elif name in _ELEMENTSFIELD: + valid, value = self._platform(self._fields[name]) + if not valid: + return [] + if isinstance(value, str): + return value.split(',') + valid, value = self._platform(self._fields[name]) + if not valid: + return None + return value + + def check(self, strict=False, restructuredtext=False): + """Check if the metadata is compliant. If strict is False then raise if + no Name or Version are provided""" + # XXX should check the versions (if the file was loaded) + missing, warnings = [], [] + + for attr in ('Name', 'Version'): # required by PEP 345 + if attr not in self: + missing.append(attr) + + if strict and missing != []: + msg = 'missing required metadata: %s' % ', '.join(missing) + raise MetadataMissingError(msg) + + for attr in ('Home-page', 'Author'): + if attr not in self: + missing.append(attr) + + if _HAS_DOCUTILS and restructuredtext: + warnings.extend(self._check_rst_data(self['Description'])) + + # checking metadata 1.2 (XXX needs to check 1.1, 1.0) + if self['Metadata-Version'] != '1.2': + return missing, warnings + + def is_valid_predicates(value): + for v in value: + if not is_valid_predicate(v.split(';')[0]): + return False + return True + + for fields, controller in ((_PREDICATE_FIELDS, is_valid_predicates), + (_VERSIONS_FIELDS, is_valid_versions), + (_VERSION_FIELDS, is_valid_version)): + for field in fields: + value = self.get(field, None) + if value is not None and not controller(value): + warnings.append('Wrong value for %r: %s' % (field, value)) + + return missing, warnings + + def todict(self): + """Return fields as a dict. + + Field names will be converted to use the underscore-lowercase style + instead of hyphen-mixed case (i.e. home_page instead of Home-page). + """ + data = { + 'metadata_version': self['Metadata-Version'], + 'name': self['Name'], + 'version': self['Version'], + 'summary': self['Summary'], + 'home_page': self['Home-page'], + 'author': self['Author'], + 'author_email': self['Author-email'], + 'license': self['License'], + 'description': self['Description'], + 'keywords': self['Keywords'], + 'platform': self['Platform'], + 'classifier': self['Classifier'], + 'download_url': self['Download-URL'], + } + + if self['Metadata-Version'] == '1.2': + data['requires_dist'] = self['Requires-Dist'] + data['requires_python'] = self['Requires-Python'] + data['requires_external'] = self['Requires-External'] + data['provides_dist'] = self['Provides-Dist'] + data['obsoletes_dist'] = self['Obsoletes-Dist'] + data['project_url'] = [','.join(url) for url in + self['Project-URL']] + + elif self['Metadata-Version'] == '1.1': + data['provides'] = self['Provides'] + data['requires'] = self['Requires'] + data['obsoletes'] = self['Obsoletes'] + + return data + + # Mapping API + + def keys(self): + return _version2fieldlist(self['Metadata-Version']) + + def __iter__(self): + for key in self.keys(): + yield key + + def values(self): + return [self[key] for key in list(self.keys())] + + def items(self): + return [(key, self[key]) for key in list(self.keys())] diff --git a/Lib/packaging/pypi/__init__.py b/Lib/packaging/pypi/__init__.py new file mode 100644 index 0000000000..5660c50ade --- /dev/null +++ b/Lib/packaging/pypi/__init__.py @@ -0,0 +1,9 @@ +"""Low-level and high-level APIs to interact with project indexes.""" + +__all__ = ['simple', + 'xmlrpc', + 'dist', + 'errors', + 'mirrors'] + +from packaging.pypi.dist import ReleaseInfo, ReleasesList, DistInfo diff --git a/Lib/packaging/pypi/base.py b/Lib/packaging/pypi/base.py new file mode 100644 index 0000000000..305fca9cc8 --- /dev/null +++ b/Lib/packaging/pypi/base.py @@ -0,0 +1,48 @@ +"""Base class for index crawlers.""" + +from packaging.pypi.dist import ReleasesList + + +class BaseClient: + """Base class containing common methods for the index crawlers/clients""" + + def __init__(self, prefer_final, prefer_source): + self._prefer_final = prefer_final + self._prefer_source = prefer_source + self._index = self + + def _get_prefer_final(self, prefer_final=None): + """Return the prefer_final internal parameter or the specified one if + provided""" + if prefer_final: + return prefer_final + else: + return self._prefer_final + + def _get_prefer_source(self, prefer_source=None): + """Return the prefer_source internal parameter or the specified one if + provided""" + if prefer_source: + return prefer_source + else: + return self._prefer_source + + def _get_project(self, project_name): + """Return an project instance, create it if necessary""" + return self._projects.setdefault(project_name.lower(), + ReleasesList(project_name, index=self._index)) + + def download_distribution(self, requirements, temp_path=None, + prefer_source=None, prefer_final=None): + """Download a distribution from the last release according to the + requirements. + + If temp_path is provided, download to this path, otherwise, create a + temporary location for the download and return it. + """ + prefer_final = self._get_prefer_final(prefer_final) + prefer_source = self._get_prefer_source(prefer_source) + release = self.get_release(requirements, prefer_final) + if release: + dist = release.get_distribution(prefer_source=prefer_source) + return dist.download(temp_path) diff --git a/Lib/packaging/pypi/dist.py b/Lib/packaging/pypi/dist.py new file mode 100644 index 0000000000..dbf6459273 --- /dev/null +++ b/Lib/packaging/pypi/dist.py @@ -0,0 +1,544 @@ +"""Classes representing releases and distributions retrieved from indexes. + +A project (= unique name) can have several releases (= versions) and +each release can have several distributions (= sdist and bdists). + +Release objects contain metadata-related information (see PEP 376); +distribution objects contain download-related information. +""" + +import re +import hashlib +import tempfile +import urllib.request +import urllib.parse +import urllib.error +import urllib.parse +from shutil import unpack_archive + +from packaging.errors import IrrationalVersionError +from packaging.version import (suggest_normalized_version, NormalizedVersion, + get_version_predicate) +from packaging.metadata import Metadata +from packaging.pypi.errors import (HashDoesNotMatch, UnsupportedHashName, + CantParseArchiveName) + + +__all__ = ['ReleaseInfo', 'DistInfo', 'ReleasesList', 'get_infos_from_url'] + +EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz .egg".split() +MD5_HASH = re.compile(r'^.*#md5=([a-f0-9]+)$') +DIST_TYPES = ['bdist', 'sdist'] + + +class IndexReference: + """Mixin used to store the index reference""" + def set_index(self, index=None): + self._index = index + + +class ReleaseInfo(IndexReference): + """Represent a release of a project (a project with a specific version). + The release contain the _metadata informations related to this specific + version, and is also a container for distribution related informations. + + See the DistInfo class for more information about distributions. + """ + + def __init__(self, name, version, metadata=None, hidden=False, + index=None, **kwargs): + """ + :param name: the name of the distribution + :param version: the version of the distribution + :param metadata: the metadata fields of the release. + :type metadata: dict + :param kwargs: optional arguments for a new distribution. + """ + self.set_index(index) + self.name = name + self._version = None + self.version = version + if metadata: + self.metadata = Metadata(mapping=metadata) + else: + self.metadata = None + self.dists = {} + self.hidden = hidden + + if 'dist_type' in kwargs: + dist_type = kwargs.pop('dist_type') + self.add_distribution(dist_type, **kwargs) + + def set_version(self, version): + try: + self._version = NormalizedVersion(version) + except IrrationalVersionError: + suggestion = suggest_normalized_version(version) + if suggestion: + self.version = suggestion + else: + raise IrrationalVersionError(version) + + def get_version(self): + return self._version + + version = property(get_version, set_version) + + def fetch_metadata(self): + """If the metadata is not set, use the indexes to get it""" + if not self.metadata: + self._index.get_metadata(self.name, str(self.version)) + return self.metadata + + @property + def is_final(self): + """proxy to version.is_final""" + return self.version.is_final + + def fetch_distributions(self): + if self.dists is None: + self._index.get_distributions(self.name, str(self.version)) + if self.dists is None: + self.dists = {} + return self.dists + + def add_distribution(self, dist_type='sdist', python_version=None, + **params): + """Add distribution informations to this release. + If distribution information is already set for this distribution type, + add the given url paths to the distribution. This can be useful while + some of them fails to download. + + :param dist_type: the distribution type (eg. "sdist", "bdist", etc.) + :param params: the fields to be passed to the distribution object + (see the :class:DistInfo constructor). + """ + if dist_type not in DIST_TYPES: + raise ValueError(dist_type) + if dist_type in self.dists: + self.dists[dist_type].add_url(**params) + else: + self.dists[dist_type] = DistInfo(self, dist_type, + index=self._index, **params) + if python_version: + self.dists[dist_type].python_version = python_version + + def get_distribution(self, dist_type=None, prefer_source=True): + """Return a distribution. + + If dist_type is set, find first for this distribution type, and just + act as an alias of __get_item__. + + If prefer_source is True, search first for source distribution, and if + not return one existing distribution. + """ + if len(self.dists) == 0: + raise LookupError + if dist_type: + return self[dist_type] + if prefer_source: + if "sdist" in self.dists: + dist = self["sdist"] + else: + dist = next(self.dists.values()) + return dist + + def unpack(self, path=None, prefer_source=True): + """Unpack the distribution to the given path. + + If not destination is given, creates a temporary location. + + Returns the location of the extracted files (root). + """ + return self.get_distribution(prefer_source=prefer_source)\ + .unpack(path=path) + + def download(self, temp_path=None, prefer_source=True): + """Download the distribution, using the requirements. + + If more than one distribution match the requirements, use the last + version. + Download the distribution, and put it in the temp_path. If no temp_path + is given, creates and return one. + + Returns the complete absolute path to the downloaded archive. + """ + return self.get_distribution(prefer_source=prefer_source)\ + .download(path=temp_path) + + def set_metadata(self, metadata): + if not self.metadata: + self.metadata = Metadata() + self.metadata.update(metadata) + + def __getitem__(self, item): + """distributions are available using release["sdist"]""" + return self.dists[item] + + def _check_is_comparable(self, other): + if not isinstance(other, ReleaseInfo): + raise TypeError("cannot compare %s and %s" + % (type(self).__name__, type(other).__name__)) + elif self.name != other.name: + raise TypeError("cannot compare %s and %s" + % (self.name, other.name)) + + def __repr__(self): + return "<%s %s>" % (self.name, self.version) + + def __eq__(self, other): + self._check_is_comparable(other) + return self.version == other.version + + def __lt__(self, other): + self._check_is_comparable(other) + return self.version < other.version + + def __ne__(self, other): + return not self.__eq__(other) + + def __gt__(self, other): + return not (self.__lt__(other) or self.__eq__(other)) + + def __le__(self, other): + return self.__eq__(other) or self.__lt__(other) + + def __ge__(self, other): + return self.__eq__(other) or self.__gt__(other) + + # See http://docs.python.org/reference/datamodel#object.__hash__ + __hash__ = object.__hash__ + + +class DistInfo(IndexReference): + """Represents a distribution retrieved from an index (sdist, bdist, ...) + """ + + def __init__(self, release, dist_type=None, url=None, hashname=None, + hashval=None, is_external=True, python_version=None, + index=None): + """Create a new instance of DistInfo. + + :param release: a DistInfo class is relative to a release. + :param dist_type: the type of the dist (eg. source, bin-*, etc.) + :param url: URL where we found this distribution + :param hashname: the name of the hash we want to use. Refer to the + hashlib.new documentation for more information. + :param hashval: the hash value. + :param is_external: we need to know if the provided url comes from + an index browsing, or from an external resource. + + """ + self.set_index(index) + self.release = release + self.dist_type = dist_type + self.python_version = python_version + self._unpacked_dir = None + # set the downloaded path to None by default. The goal here + # is to not download distributions multiple times + self.downloaded_location = None + # We store urls in dict, because we need to have a bit more infos + # than the simple URL. It will be used later to find the good url to + # use. + # We have two _url* attributes: _url and urls. urls contains a list + # of dict for the different urls, and _url contains the choosen url, in + # order to dont make the selection process multiple times. + self.urls = [] + self._url = None + self.add_url(url, hashname, hashval, is_external) + + def add_url(self, url=None, hashname=None, hashval=None, is_external=True): + """Add a new url to the list of urls""" + if hashname is not None: + try: + hashlib.new(hashname) + except ValueError: + raise UnsupportedHashName(hashname) + if url not in [u['url'] for u in self.urls]: + self.urls.append({ + 'url': url, + 'hashname': hashname, + 'hashval': hashval, + 'is_external': is_external, + }) + # reset the url selection process + self._url = None + + @property + def url(self): + """Pick up the right url for the list of urls in self.urls""" + # We return internal urls over externals. + # If there is more than one internal or external, return the first + # one. + if self._url is None: + if len(self.urls) > 1: + internals_urls = [u for u in self.urls \ + if u['is_external'] == False] + if len(internals_urls) >= 1: + self._url = internals_urls[0] + if self._url is None: + self._url = self.urls[0] + return self._url + + @property + def is_source(self): + """return if the distribution is a source one or not""" + return self.dist_type == 'sdist' + + def download(self, path=None): + """Download the distribution to a path, and return it. + + If the path is given in path, use this, otherwise, generates a new one + Return the download location. + """ + if path is None: + path = tempfile.mkdtemp() + + # if we do not have downloaded it yet, do it. + if self.downloaded_location is None: + url = self.url['url'] + archive_name = urllib.parse.urlparse(url)[2].split('/')[-1] + filename, headers = urllib.request.urlretrieve(url, + path + "/" + archive_name) + self.downloaded_location = filename + self._check_md5(filename) + return self.downloaded_location + + def unpack(self, path=None): + """Unpack the distribution to the given path. + + If not destination is given, creates a temporary location. + + Returns the location of the extracted files (root). + """ + if not self._unpacked_dir: + if path is None: + path = tempfile.mkdtemp() + + filename = self.download(path) + unpack_archive(filename, path) + self._unpacked_dir = path + + return path + + def _check_md5(self, filename): + """Check that the md5 checksum of the given file matches the one in + url param""" + hashname = self.url['hashname'] + expected_hashval = self.url['hashval'] + if None not in (expected_hashval, hashname): + with open(filename, 'rb') as f: + hashval = hashlib.new(hashname) + hashval.update(f.read()) + + if hashval.hexdigest() != expected_hashval: + raise HashDoesNotMatch("got %s instead of %s" + % (hashval.hexdigest(), expected_hashval)) + + def __repr__(self): + if self.release is None: + return "<? ? %s>" % self.dist_type + + return "<%s %s %s>" % ( + self.release.name, self.release.version, self.dist_type or "") + + +class ReleasesList(IndexReference): + """A container of Release. + + Provides useful methods and facilities to sort and filter releases. + """ + def __init__(self, name, releases=None, contains_hidden=False, index=None): + self.set_index(index) + self.releases = [] + self.name = name + self.contains_hidden = contains_hidden + if releases: + self.add_releases(releases) + + def fetch_releases(self): + self._index.get_releases(self.name) + return self.releases + + def filter(self, predicate): + """Filter and return a subset of releases matching the given predicate. + """ + return ReleasesList(self.name, [release for release in self.releases + if predicate.match(release.version)], + index=self._index) + + def get_last(self, requirements, prefer_final=None): + """Return the "last" release, that satisfy the given predicates. + + "last" is defined by the version number of the releases, you also could + set prefer_final parameter to True or False to change the order results + """ + predicate = get_version_predicate(requirements) + releases = self.filter(predicate) + if len(releases) == 0: + return None + releases.sort_releases(prefer_final, reverse=True) + return releases[0] + + def add_releases(self, releases): + """Add releases in the release list. + + :param: releases is a list of ReleaseInfo objects. + """ + for r in releases: + self.add_release(release=r) + + def add_release(self, version=None, dist_type='sdist', release=None, + **dist_args): + """Add a release to the list. + + The release can be passed in the `release` parameter, and in this case, + it will be crawled to extract the useful informations if necessary, or + the release informations can be directly passed in the `version` and + `dist_type` arguments. + + Other keywords arguments can be provided, and will be forwarded to the + distribution creation (eg. the arguments of the DistInfo constructor). + """ + if release: + if release.name.lower() != self.name.lower(): + raise ValueError("%s is not the same project as %s" % + (release.name, self.name)) + version = str(release.version) + + if version not in self.get_versions(): + # append only if not already exists + self.releases.append(release) + for dist in release.dists.values(): + for url in dist.urls: + self.add_release(version, dist.dist_type, **url) + else: + matches = [r for r in self.releases + if str(r.version) == version and r.name == self.name] + if not matches: + release = ReleaseInfo(self.name, version, index=self._index) + self.releases.append(release) + else: + release = matches[0] + + release.add_distribution(dist_type=dist_type, **dist_args) + + def sort_releases(self, prefer_final=False, reverse=True, *args, **kwargs): + """Sort the results with the given properties. + + The `prefer_final` argument can be used to specify if final + distributions (eg. not dev, bet or alpha) would be prefered or not. + + Results can be inverted by using `reverse`. + + Any other parameter provided will be forwarded to the sorted call. You + cannot redefine the key argument of "sorted" here, as it is used + internally to sort the releases. + """ + + sort_by = [] + if prefer_final: + sort_by.append("is_final") + sort_by.append("version") + + self.releases.sort( + key=lambda i: tuple(getattr(i, arg) for arg in sort_by), + reverse=reverse, *args, **kwargs) + + def get_release(self, version): + """Return a release from its version.""" + matches = [r for r in self.releases if str(r.version) == version] + if len(matches) != 1: + raise KeyError(version) + return matches[0] + + def get_versions(self): + """Return a list of releases versions contained""" + return [str(r.version) for r in self.releases] + + def __getitem__(self, key): + return self.releases[key] + + def __len__(self): + return len(self.releases) + + def __repr__(self): + string = 'Project "%s"' % self.name + if self.get_versions(): + string += ' versions: %s' % ', '.join(self.get_versions()) + return '<%s>' % string + + +def get_infos_from_url(url, probable_dist_name=None, is_external=True): + """Get useful informations from an URL. + + Return a dict of (name, version, url, hashtype, hash, is_external) + + :param url: complete url of the distribution + :param probable_dist_name: A probable name of the project. + :param is_external: Tell if the url commes from an index or from + an external URL. + """ + # if the url contains a md5 hash, get it. + md5_hash = None + match = MD5_HASH.match(url) + if match is not None: + md5_hash = match.group(1) + # remove the hash + url = url.replace("#md5=%s" % md5_hash, "") + + # parse the archive name to find dist name and version + archive_name = urllib.parse.urlparse(url)[2].split('/')[-1] + extension_matched = False + # remove the extension from the name + for ext in EXTENSIONS: + if archive_name.endswith(ext): + archive_name = archive_name[:-len(ext)] + extension_matched = True + + name, version = split_archive_name(archive_name) + if extension_matched is True: + return {'name': name, + 'version': version, + 'url': url, + 'hashname': "md5", + 'hashval': md5_hash, + 'is_external': is_external, + 'dist_type': 'sdist'} + + +def split_archive_name(archive_name, probable_name=None): + """Split an archive name into two parts: name and version. + + Return the tuple (name, version) + """ + # Try to determine wich part is the name and wich is the version using the + # "-" separator. Take the larger part to be the version number then reduce + # if this not works. + def eager_split(str, maxsplit=2): + # split using the "-" separator + splits = str.rsplit("-", maxsplit) + name = splits[0] + version = "-".join(splits[1:]) + if version.startswith("-"): + version = version[1:] + if suggest_normalized_version(version) is None and maxsplit >= 0: + # we dont get a good version number: recurse ! + return eager_split(str, maxsplit - 1) + else: + return name, version + if probable_name is not None: + probable_name = probable_name.lower() + name = None + if probable_name is not None and probable_name in archive_name: + # we get the name from probable_name, if given. + name = probable_name + version = archive_name.lstrip(name) + else: + name, version = eager_split(archive_name) + + version = suggest_normalized_version(version) + if version is not None and name != "": + return name.lower(), version + else: + raise CantParseArchiveName(archive_name) diff --git a/Lib/packaging/pypi/errors.py b/Lib/packaging/pypi/errors.py new file mode 100644 index 0000000000..2191ac100c --- /dev/null +++ b/Lib/packaging/pypi/errors.py @@ -0,0 +1,39 @@ +"""Exceptions raised by packaging.pypi code.""" + +from packaging.errors import PackagingPyPIError + + +class ProjectNotFound(PackagingPyPIError): + """Project has not been found""" + + +class DistributionNotFound(PackagingPyPIError): + """The release has not been found""" + + +class ReleaseNotFound(PackagingPyPIError): + """The release has not been found""" + + +class CantParseArchiveName(PackagingPyPIError): + """An archive name can't be parsed to find distribution name and version""" + + +class DownloadError(PackagingPyPIError): + """An error has occurs while downloading""" + + +class HashDoesNotMatch(DownloadError): + """Compared hashes does not match""" + + +class UnsupportedHashName(PackagingPyPIError): + """A unsupported hashname has been used""" + + +class UnableToDownload(PackagingPyPIError): + """All mirrors have been tried, without success""" + + +class InvalidSearchField(PackagingPyPIError): + """An invalid search field has been used""" diff --git a/Lib/packaging/pypi/mirrors.py b/Lib/packaging/pypi/mirrors.py new file mode 100644 index 0000000000..a646acff3c --- /dev/null +++ b/Lib/packaging/pypi/mirrors.py @@ -0,0 +1,52 @@ +"""Utilities related to the mirror infrastructure defined in PEP 381.""" + +from string import ascii_lowercase +import socket + +DEFAULT_MIRROR_URL = "last.pypi.python.org" + + +def get_mirrors(hostname=None): + """Return the list of mirrors from the last record found on the DNS + entry:: + + >>> from packaging.pypi.mirrors import get_mirrors + >>> get_mirrors() + ['a.pypi.python.org', 'b.pypi.python.org', 'c.pypi.python.org', + 'd.pypi.python.org'] + + """ + if hostname is None: + hostname = DEFAULT_MIRROR_URL + + # return the last mirror registered on PyPI. + try: + hostname = socket.gethostbyname_ex(hostname)[0] + except socket.gaierror: + return [] + end_letter = hostname.split(".", 1) + + # determine the list from the last one. + return ["%s.%s" % (s, end_letter[1]) for s in string_range(end_letter[0])] + + +def string_range(last): + """Compute the range of string between "a" and last. + + This works for simple "a to z" lists, but also for "a to zz" lists. + """ + for k in range(len(last)): + for x in product(ascii_lowercase, repeat=(k + 1)): + result = ''.join(x) + yield result + if result == last: + return + + +def product(*args, **kwds): + pools = [tuple(arg) for arg in args] * kwds.get('repeat', 1) + result = [[]] + for pool in pools: + result = [x + [y] for x in result for y in pool] + for prod in result: + yield tuple(prod) diff --git a/Lib/packaging/pypi/simple.py b/Lib/packaging/pypi/simple.py new file mode 100644 index 0000000000..710355d648 --- /dev/null +++ b/Lib/packaging/pypi/simple.py @@ -0,0 +1,467 @@ +"""Spider using the screen-scraping "simple" PyPI API. + +This module contains the class Crawler, a simple spider that +can be used to find and retrieve distributions from a project index +(like the Python Package Index), using its so-called simple API (see +reference implementation available at http://pypi.python.org/simple/). +""" + +import http.client +import re +import socket +import sys +import urllib.request +import urllib.parse +import urllib.error +import os + +from fnmatch import translate +from functools import wraps +from packaging import logger +from packaging.metadata import Metadata +from packaging.version import get_version_predicate +from packaging import __version__ as packaging_version +from packaging.pypi.base import BaseClient +from packaging.pypi.dist import (ReleasesList, EXTENSIONS, + get_infos_from_url, MD5_HASH) +from packaging.pypi.errors import (PackagingPyPIError, DownloadError, + UnableToDownload, CantParseArchiveName, + ReleaseNotFound, ProjectNotFound) +from packaging.pypi.mirrors import get_mirrors +from packaging.metadata import Metadata + +__all__ = ['Crawler', 'DEFAULT_SIMPLE_INDEX_URL'] + +# -- Constants ----------------------------------------------- +DEFAULT_SIMPLE_INDEX_URL = "http://a.pypi.python.org/simple/" +DEFAULT_HOSTS = ("*",) +SOCKET_TIMEOUT = 15 +USER_AGENT = "Python-urllib/%s packaging/%s" % ( + sys.version[:3], packaging_version) + +# -- Regexps ------------------------------------------------- +EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') +HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) +URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match + +# This pattern matches a character entity reference (a decimal numeric +# references, a hexadecimal numeric reference, or a named reference). +ENTITY_SUB = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub +REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) + + +def socket_timeout(timeout=SOCKET_TIMEOUT): + """Decorator to add a socket timeout when requesting pages on PyPI. + """ + def wrapper(func): + @wraps(func) + def wrapped(self, *args, **kwargs): + old_timeout = socket.getdefaulttimeout() + if hasattr(self, "_timeout"): + timeout = self._timeout + socket.setdefaulttimeout(timeout) + try: + return func(self, *args, **kwargs) + finally: + socket.setdefaulttimeout(old_timeout) + return wrapped + return wrapper + + +def with_mirror_support(): + """Decorator that makes the mirroring support easier""" + def wrapper(func): + @wraps(func) + def wrapped(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except DownloadError: + # if an error occurs, try with the next index_url + if self._mirrors_tries >= self._mirrors_max_tries: + try: + self._switch_to_next_mirror() + except KeyError: + raise UnableToDownload("Tried all mirrors") + else: + self._mirrors_tries += 1 + self._projects.clear() + return wrapped(self, *args, **kwargs) + return wrapped + return wrapper + + +class Crawler(BaseClient): + """Provides useful tools to request the Python Package Index simple API. + + You can specify both mirrors and mirrors_url, but mirrors_url will only be + used if mirrors is set to None. + + :param index_url: the url of the simple index to search on. + :param prefer_final: if the version is not mentioned, and the last + version is not a "final" one (alpha, beta, etc.), + pick up the last final version. + :param prefer_source: if the distribution type is not mentioned, pick up + the source one if available. + :param follow_externals: tell if following external links is needed or + not. Default is False. + :param hosts: a list of hosts allowed to be processed while using + follow_externals=True. Default behavior is to follow all + hosts. + :param follow_externals: tell if following external links is needed or + not. Default is False. + :param mirrors_url: the url to look on for DNS records giving mirror + addresses. + :param mirrors: a list of mirrors (see PEP 381). + :param timeout: time in seconds to consider a url has timeouted. + :param mirrors_max_tries": number of times to try requesting informations + on mirrors before switching. + """ + + def __init__(self, index_url=DEFAULT_SIMPLE_INDEX_URL, prefer_final=False, + prefer_source=True, hosts=DEFAULT_HOSTS, + follow_externals=False, mirrors_url=None, mirrors=None, + timeout=SOCKET_TIMEOUT, mirrors_max_tries=0, verbose=False): + super(Crawler, self).__init__(prefer_final, prefer_source) + self.follow_externals = follow_externals + self.verbose = verbose + + # mirroring attributes. + parsed = urllib.parse.urlparse(index_url) + self.scheme = parsed[0] + if self.scheme == 'file': + ender = os.path.sep + else: + ender = '/' + if not index_url.endswith(ender): + index_url += ender + # if no mirrors are defined, use the method described in PEP 381. + if mirrors is None: + mirrors = get_mirrors(mirrors_url) + self._mirrors = set(mirrors) + self._mirrors_used = set() + self.index_url = index_url + self._mirrors_max_tries = mirrors_max_tries + self._mirrors_tries = 0 + self._timeout = timeout + + # create a regexp to match all given hosts + self._allowed_hosts = re.compile('|'.join(map(translate, hosts))).match + + # we keep an index of pages we have processed, in order to avoid + # scanning them multple time (eg. if there is multiple pages pointing + # on one) + self._processed_urls = [] + self._projects = {} + + @with_mirror_support() + def search_projects(self, name=None, **kwargs): + """Search the index for projects containing the given name. + + Return a list of names. + """ + with self._open_url(self.index_url) as index: + if '*' in name: + name.replace('*', '.*') + else: + name = "%s%s%s" % ('*.?', name, '*.?') + name = name.replace('*', '[^<]*') # avoid matching end tag + projectname = re.compile('<a[^>]*>(%s)</a>' % name, re.I) + matching_projects = [] + + index_content = index.read() + + # FIXME should use bytes I/O and regexes instead of decoding + index_content = index_content.decode() + + for match in projectname.finditer(index_content): + project_name = match.group(1) + matching_projects.append(self._get_project(project_name)) + return matching_projects + + def get_releases(self, requirements, prefer_final=None, + force_update=False): + """Search for releases and return a ReleasesList object containing + the results. + """ + predicate = get_version_predicate(requirements) + if predicate.name.lower() in self._projects and not force_update: + return self._projects.get(predicate.name.lower()) + prefer_final = self._get_prefer_final(prefer_final) + logger.debug('Reading info on PyPI about %s', predicate.name) + self._process_index_page(predicate.name) + + if predicate.name.lower() not in self._projects: + raise ProjectNotFound + + releases = self._projects.get(predicate.name.lower()) + releases.sort_releases(prefer_final=prefer_final) + return releases + + def get_release(self, requirements, prefer_final=None): + """Return only one release that fulfill the given requirements""" + predicate = get_version_predicate(requirements) + release = self.get_releases(predicate, prefer_final)\ + .get_last(predicate) + if not release: + raise ReleaseNotFound("No release matches the given criterias") + return release + + def get_distributions(self, project_name, version): + """Return the distributions found on the index for the specific given + release""" + # as the default behavior of get_release is to return a release + # containing the distributions, just alias it. + return self.get_release("%s (%s)" % (project_name, version)) + + def get_metadata(self, project_name, version): + """Return the metadatas from the simple index. + + Currently, download one archive, extract it and use the PKG-INFO file. + """ + release = self.get_distributions(project_name, version) + if not release.metadata: + location = release.get_distribution().unpack() + pkg_info = os.path.join(location, 'PKG-INFO') + release.metadata = Metadata(pkg_info) + return release + + def _switch_to_next_mirror(self): + """Switch to the next mirror (eg. point self.index_url to the next + mirror url. + + Raise a KeyError if all mirrors have been tried. + """ + self._mirrors_used.add(self.index_url) + index_url = self._mirrors.pop() + # XXX use urllib.parse for a real check of missing scheme part + if not index_url.startswith(("http://", "https://", "file://")): + index_url = "http://%s" % index_url + + if not index_url.endswith("/simple"): + index_url = "%s/simple/" % index_url + + self.index_url = index_url + + def _is_browsable(self, url): + """Tell if the given URL can be browsed or not. + + It uses the follow_externals and the hosts list to tell if the given + url is browsable or not. + """ + # if _index_url is contained in the given URL, we are browsing the + # index, and it's always "browsable". + # local files are always considered browable resources + if self.index_url in url or urllib.parse.urlparse(url)[0] == "file": + return True + elif self.follow_externals: + if self._allowed_hosts(urllib.parse.urlparse(url)[1]): # 1 is netloc + return True + else: + return False + return False + + def _is_distribution(self, link): + """Tell if the given URL matches to a distribution name or not. + """ + #XXX find a better way to check that links are distributions + # Using a regexp ? + for ext in EXTENSIONS: + if ext in link: + return True + return False + + def _register_release(self, release=None, release_info={}): + """Register a new release. + + Both a release or a dict of release_info can be provided, the prefered + way (eg. the quicker) is the dict one. + + Return the list of existing releases for the given project. + """ + # Check if the project already has a list of releases (refering to + # the project name). If not, create a new release list. + # Then, add the release to the list. + if release: + name = release.name + else: + name = release_info['name'] + if name.lower() not in self._projects: + self._projects[name.lower()] = ReleasesList(name, index=self._index) + + if release: + self._projects[name.lower()].add_release(release=release) + else: + name = release_info.pop('name') + version = release_info.pop('version') + dist_type = release_info.pop('dist_type') + self._projects[name.lower()].add_release(version, dist_type, + **release_info) + return self._projects[name.lower()] + + def _process_url(self, url, project_name=None, follow_links=True): + """Process an url and search for distributions packages. + + For each URL found, if it's a download, creates a PyPIdistribution + object. If it's a homepage and we can follow links, process it too. + + :param url: the url to process + :param project_name: the project name we are searching for. + :param follow_links: Do not want to follow links more than from one + level. This parameter tells if we want to follow + the links we find (eg. run recursively this + method on it) + """ + with self._open_url(url) as f: + base_url = f.url + if url not in self._processed_urls: + self._processed_urls.append(url) + link_matcher = self._get_link_matcher(url) + for link, is_download in link_matcher(f.read().decode(), base_url): + if link not in self._processed_urls: + if self._is_distribution(link) or is_download: + self._processed_urls.append(link) + # it's a distribution, so create a dist object + try: + infos = get_infos_from_url(link, project_name, + is_external=self.index_url not in url) + except CantParseArchiveName as e: + if self.verbose: + logger.warning( + "version has not been parsed: %s", e) + else: + self._register_release(release_info=infos) + else: + if self._is_browsable(link) and follow_links: + self._process_url(link, project_name, + follow_links=False) + + def _get_link_matcher(self, url): + """Returns the right link matcher function of the given url + """ + if self.index_url in url: + return self._simple_link_matcher + else: + return self._default_link_matcher + + def _get_full_url(self, url, base_url): + return urllib.parse.urljoin(base_url, self._htmldecode(url)) + + def _simple_link_matcher(self, content, base_url): + """Yield all links with a rel="download" or rel="homepage". + + This matches the simple index requirements for matching links. + If follow_externals is set to False, dont yeld the external + urls. + + :param content: the content of the page we want to parse + :param base_url: the url of this page. + """ + for match in HREF.finditer(content): + url = self._get_full_url(match.group(1), base_url) + if MD5_HASH.match(url): + yield (url, True) + + for match in REL.finditer(content): + # search for rel links. + tag, rel = match.groups() + rels = [s.strip() for s in rel.lower().split(',')] + if 'homepage' in rels or 'download' in rels: + for match in HREF.finditer(tag): + url = self._get_full_url(match.group(1), base_url) + if 'download' in rels or self._is_browsable(url): + # yield a list of (url, is_download) + yield (url, 'download' in rels) + + def _default_link_matcher(self, content, base_url): + """Yield all links found on the page. + """ + for match in HREF.finditer(content): + url = self._get_full_url(match.group(1), base_url) + if self._is_browsable(url): + yield (url, False) + + @with_mirror_support() + def _process_index_page(self, name): + """Find and process a PyPI page for the given project name. + + :param name: the name of the project to find the page + """ + # Browse and index the content of the given PyPI page. + if self.scheme == 'file': + ender = os.path.sep + else: + ender = '/' + url = self.index_url + name + ender + self._process_url(url, name) + + @socket_timeout() + def _open_url(self, url): + """Open a urllib2 request, handling HTTP authentication, and local + files support. + + """ + scheme, netloc, path, params, query, frag = urllib.parse.urlparse(url) + + # authentication stuff + if scheme in ('http', 'https'): + auth, host = urllib.parse.splituser(netloc) + else: + auth = None + + # add index.html automatically for filesystem paths + if scheme == 'file': + if url.endswith(os.path.sep): + url += "index.html" + + # add authorization headers if auth is provided + if auth: + auth = "Basic " + \ + urllib.parse.unquote(auth).encode('base64').strip() + new_url = urllib.parse.urlunparse(( + scheme, host, path, params, query, frag)) + request = urllib.request.Request(new_url) + request.add_header("Authorization", auth) + else: + request = urllib.request.Request(url) + request.add_header('User-Agent', USER_AGENT) + try: + fp = urllib.request.urlopen(request) + except (ValueError, http.client.InvalidURL) as v: + msg = ' '.join([str(arg) for arg in v.args]) + raise PackagingPyPIError('%s %s' % (url, msg)) + except urllib.error.HTTPError as v: + return v + except urllib.error.URLError as v: + raise DownloadError("Download error for %s: %s" % (url, v.reason)) + except http.client.BadStatusLine as v: + raise DownloadError('%s returned a bad status line. ' + 'The server might be down, %s' % (url, v.line)) + except http.client.HTTPException as v: + raise DownloadError("Download error for %s: %s" % (url, v)) + except socket.timeout: + raise DownloadError("The server timeouted") + + if auth: + # Put authentication info back into request URL if same host, + # so that links found on the page will work + s2, h2, path2, param2, query2, frag2 = \ + urllib.parse.urlparse(fp.url) + if s2 == scheme and h2 == host: + fp.url = urllib.parse.urlunparse( + (s2, netloc, path2, param2, query2, frag2)) + return fp + + def _decode_entity(self, match): + what = match.group(1) + if what.startswith('#x'): + what = int(what[2:], 16) + elif what.startswith('#'): + what = int(what[1:]) + else: + from html.entities import name2codepoint + what = name2codepoint.get(what, match.group(0)) + return chr(what) + + def _htmldecode(self, text): + """Decode HTML entities in the given text.""" + return ENTITY_SUB(self._decode_entity, text) diff --git a/Lib/packaging/pypi/wrapper.py b/Lib/packaging/pypi/wrapper.py new file mode 100644 index 0000000000..945d08abb7 --- /dev/null +++ b/Lib/packaging/pypi/wrapper.py @@ -0,0 +1,99 @@ +"""Convenient client for all PyPI APIs. + +This module provides a ClientWrapper class which will use the "simple" +or XML-RPC API to request information or files from an index. +""" + +from packaging.pypi import simple, xmlrpc + +_WRAPPER_MAPPINGS = {'get_release': 'simple', + 'get_releases': 'simple', + 'search_projects': 'simple', + 'get_metadata': 'xmlrpc', + 'get_distributions': 'simple'} + +_WRAPPER_INDEXES = {'xmlrpc': xmlrpc.Client, + 'simple': simple.Crawler} + + +def switch_index_if_fails(func, wrapper): + """Decorator that switch of index (for instance from xmlrpc to simple) + if the first mirror return an empty list or raises an exception. + """ + def decorator(*args, **kwargs): + retry = True + exception = None + methods = [func] + for f in wrapper._indexes.values(): + if f != func.__self__ and hasattr(f, func.__name__): + methods.append(getattr(f, func.__name__)) + for method in methods: + try: + response = method(*args, **kwargs) + retry = False + except Exception as e: + exception = e + if not retry: + break + if retry and exception: + raise exception + else: + return response + return decorator + + +class ClientWrapper: + """Wrapper around simple and xmlrpc clients, + + Choose the best implementation to use depending the needs, using the given + mappings. + If one of the indexes returns an error, tries to use others indexes. + + :param index: tell which index to rely on by default. + :param index_classes: a dict of name:class to use as indexes. + :param indexes: a dict of name:index already instantiated + :param mappings: the mappings to use for this wrapper + """ + + def __init__(self, default_index='simple', index_classes=_WRAPPER_INDEXES, + indexes={}, mappings=_WRAPPER_MAPPINGS): + self._projects = {} + self._mappings = mappings + self._indexes = indexes + self._default_index = default_index + + # instantiate the classes and set their _project attribute to the one + # of the wrapper. + for name, cls in index_classes.items(): + obj = self._indexes.setdefault(name, cls()) + obj._projects = self._projects + obj._index = self + + def __getattr__(self, method_name): + """When asking for methods of the wrapper, return the implementation of + the wrapped classes, depending the mapping. + + Decorate the methods to switch of implementation if an error occurs + """ + real_method = None + if method_name in _WRAPPER_MAPPINGS: + obj = self._indexes[_WRAPPER_MAPPINGS[method_name]] + real_method = getattr(obj, method_name) + else: + # the method is not defined in the mappings, so we try first to get + # it via the default index, and rely on others if needed. + try: + real_method = getattr(self._indexes[self._default_index], + method_name) + except AttributeError: + other_indexes = [i for i in self._indexes + if i != self._default_index] + for index in other_indexes: + real_method = getattr(self._indexes[index], method_name, + None) + if real_method: + break + if real_method: + return switch_index_if_fails(real_method, self) + else: + raise AttributeError("No index have attribute '%s'" % method_name) diff --git a/Lib/packaging/pypi/xmlrpc.py b/Lib/packaging/pypi/xmlrpc.py new file mode 100644 index 0000000000..befdf6dbbb --- /dev/null +++ b/Lib/packaging/pypi/xmlrpc.py @@ -0,0 +1,200 @@ +"""Spider using the XML-RPC PyPI API. + +This module contains the class Client, a spider that can be used to find +and retrieve distributions from a project index (like the Python Package +Index), using its XML-RPC API (see documentation of the reference +implementation at http://wiki.python.org/moin/PyPiXmlRpc). +""" + +import xmlrpc.client + +from packaging import logger +from packaging.errors import IrrationalVersionError +from packaging.version import get_version_predicate +from packaging.pypi.base import BaseClient +from packaging.pypi.errors import (ProjectNotFound, InvalidSearchField, + ReleaseNotFound) +from packaging.pypi.dist import ReleaseInfo + +__all__ = ['Client', 'DEFAULT_XMLRPC_INDEX_URL'] + +DEFAULT_XMLRPC_INDEX_URL = 'http://python.org/pypi' + +_SEARCH_FIELDS = ['name', 'version', 'author', 'author_email', 'maintainer', + 'maintainer_email', 'home_page', 'license', 'summary', + 'description', 'keywords', 'platform', 'download_url'] + + +class Client(BaseClient): + """Client to query indexes using XML-RPC method calls. + + If no server_url is specified, use the default PyPI XML-RPC URL, + defined in the DEFAULT_XMLRPC_INDEX_URL constant:: + + >>> client = Client() + >>> client.server_url == DEFAULT_XMLRPC_INDEX_URL + True + + >>> client = Client("http://someurl/") + >>> client.server_url + 'http://someurl/' + """ + + def __init__(self, server_url=DEFAULT_XMLRPC_INDEX_URL, prefer_final=False, + prefer_source=True): + super(Client, self).__init__(prefer_final, prefer_source) + self.server_url = server_url + self._projects = {} + + def get_release(self, requirements, prefer_final=False): + """Return a release with all complete metadata and distribution + related informations. + """ + prefer_final = self._get_prefer_final(prefer_final) + predicate = get_version_predicate(requirements) + releases = self.get_releases(predicate.name) + release = releases.get_last(predicate, prefer_final) + self.get_metadata(release.name, str(release.version)) + self.get_distributions(release.name, str(release.version)) + return release + + def get_releases(self, requirements, prefer_final=None, show_hidden=True, + force_update=False): + """Return the list of existing releases for a specific project. + + Cache the results from one call to another. + + If show_hidden is True, return the hidden releases too. + If force_update is True, reprocess the index to update the + informations (eg. make a new XML-RPC call). + :: + + >>> client = Client() + >>> client.get_releases('Foo') + ['1.1', '1.2', '1.3'] + + If no such project exists, raise a ProjectNotFound exception:: + + >>> client.get_project_versions('UnexistingProject') + ProjectNotFound: UnexistingProject + + """ + def get_versions(project_name, show_hidden): + return self.proxy.package_releases(project_name, show_hidden) + + predicate = get_version_predicate(requirements) + prefer_final = self._get_prefer_final(prefer_final) + project_name = predicate.name + if not force_update and (project_name.lower() in self._projects): + project = self._projects[project_name.lower()] + if not project.contains_hidden and show_hidden: + # if hidden releases are requested, and have an existing + # list of releases that does not contains hidden ones + all_versions = get_versions(project_name, show_hidden) + existing_versions = project.get_versions() + hidden_versions = set(all_versions) - set(existing_versions) + for version in hidden_versions: + project.add_release(release=ReleaseInfo(project_name, + version, index=self._index)) + else: + versions = get_versions(project_name, show_hidden) + if not versions: + raise ProjectNotFound(project_name) + project = self._get_project(project_name) + project.add_releases([ReleaseInfo(project_name, version, + index=self._index) + for version in versions]) + project = project.filter(predicate) + if len(project) == 0: + raise ReleaseNotFound("%s" % predicate) + project.sort_releases(prefer_final) + return project + + + def get_distributions(self, project_name, version): + """Grab informations about distributions from XML-RPC. + + Return a ReleaseInfo object, with distribution-related informations + filled in. + """ + url_infos = self.proxy.release_urls(project_name, version) + project = self._get_project(project_name) + if version not in project.get_versions(): + project.add_release(release=ReleaseInfo(project_name, version, + index=self._index)) + release = project.get_release(version) + for info in url_infos: + packagetype = info['packagetype'] + dist_infos = {'url': info['url'], + 'hashval': info['md5_digest'], + 'hashname': 'md5', + 'is_external': False, + 'python_version': info['python_version']} + release.add_distribution(packagetype, **dist_infos) + return release + + def get_metadata(self, project_name, version): + """Retrieve project metadata. + + Return a ReleaseInfo object, with metadata informations filled in. + """ + # to be case-insensitive, get the informations from the XMLRPC API + projects = [d['name'] for d in + self.proxy.search({'name': project_name}) + if d['name'].lower() == project_name] + if len(projects) > 0: + project_name = projects[0] + + metadata = self.proxy.release_data(project_name, version) + project = self._get_project(project_name) + if version not in project.get_versions(): + project.add_release(release=ReleaseInfo(project_name, version, + index=self._index)) + release = project.get_release(version) + release.set_metadata(metadata) + return release + + def search_projects(self, name=None, operator="or", **kwargs): + """Find using the keys provided in kwargs. + + You can set operator to "and" or "or". + """ + for key in kwargs: + if key not in _SEARCH_FIELDS: + raise InvalidSearchField(key) + if name: + kwargs["name"] = name + projects = self.proxy.search(kwargs, operator) + for p in projects: + project = self._get_project(p['name']) + try: + project.add_release(release=ReleaseInfo(p['name'], + p['version'], metadata={'summary': p['summary']}, + index=self._index)) + except IrrationalVersionError as e: + logger.warning("Irrational version error found: %s", e) + return [self._projects[p['name'].lower()] for p in projects] + + def get_all_projects(self): + """Return the list of all projects registered in the package index""" + projects = self.proxy.list_packages() + for name in projects: + self.get_releases(name, show_hidden=True) + + return [self._projects[name.lower()] for name in set(projects)] + + @property + def proxy(self): + """Property used to return the XMLRPC server proxy. + + If no server proxy is defined yet, creates a new one:: + + >>> client = Client() + >>> client.proxy() + <ServerProxy for python.org/pypi> + + """ + if not hasattr(self, '_server_proxy'): + self._server_proxy = xmlrpc.client.ServerProxy(self.server_url) + + return self._server_proxy diff --git a/Lib/packaging/run.py b/Lib/packaging/run.py new file mode 100644 index 0000000000..ef20f35865 --- /dev/null +++ b/Lib/packaging/run.py @@ -0,0 +1,669 @@ +"""Main command line parser. Implements the pysetup script.""" + +import os +import re +import sys +import getopt +import logging + +from packaging import logger +from packaging.dist import Distribution +from packaging.util import _is_archive_file, generate_setup_py +from packaging.command import get_command_class, STANDARD_COMMANDS +from packaging.install import install, install_local_project, remove +from packaging.database import get_distribution, get_distributions +from packaging.depgraph import generate_graph +from packaging.fancy_getopt import FancyGetopt +from packaging.errors import (PackagingArgError, PackagingError, + PackagingModuleError, PackagingClassError, + CCompilerError) + + +command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$') + +common_usage = """\ +Actions: +%(actions)s + +To get more help on an action, use: + + pysetup action --help +""" + +global_options = [ + # The fourth entry for verbose means that it can be repeated. + ('verbose', 'v', "run verbosely (default)", True), + ('quiet', 'q', "run quietly (turns verbosity off)"), + ('dry-run', 'n', "don't actually do anything"), + ('help', 'h', "show detailed help message"), + ('no-user-cfg', None, 'ignore pydistutils.cfg in your home directory'), + ('version', None, 'Display the version'), +] + +negative_opt = {'quiet': 'verbose'} + +display_options = [ + ('help-commands', None, "list all available commands"), +] + +display_option_names = [x[0].replace('-', '_') for x in display_options] + + +def _parse_args(args, options, long_options): + """Transform sys.argv input into a dict. + + :param args: the args to parse (i.e sys.argv) + :param options: the list of options to pass to getopt + :param long_options: the list of string with the names of the long options + to be passed to getopt. + + The function returns a dict with options/long_options as keys and matching + values as values. + """ + optlist, args = getopt.gnu_getopt(args, options, long_options) + optdict = {} + optdict['args'] = args + for k, v in optlist: + k = k.lstrip('-') + if k not in optdict: + optdict[k] = [] + if v: + optdict[k].append(v) + else: + optdict[k].append(v) + return optdict + + +class action_help: + """Prints a help message when the standard help flags: -h and --help + are used on the commandline. + """ + + def __init__(self, help_msg): + self.help_msg = help_msg + + def __call__(self, f): + def wrapper(*args, **kwargs): + f_args = args[1] + if '--help' in f_args or '-h' in f_args: + print(self.help_msg) + return + return f(*args, **kwargs) + return wrapper + + +@action_help("""\ +Usage: pysetup create + or: pysetup create --help + +Create a new Python project. +""") +def _create(distpatcher, args, **kw): + from packaging.create import main + return main() + + +@action_help("""\ +Usage: pysetup generate-setup + or: pysetup generate-setup --help + +Generate a setup.py script for backward-compatibility purposes. +""") +def _generate(distpatcher, args, **kw): + generate_setup_py() + logger.info('The setup.py was generated') + + +@action_help("""\ +Usage: pysetup graph dist + or: pysetup graph --help + +Print dependency graph for the distribution. + +positional arguments: + dist installed distribution name +""") +def _graph(dispatcher, args, **kw): + name = args[1] + dist = get_distribution(name, use_egg_info=True) + if dist is None: + logger.warning('Distribution not found.') + return 1 + else: + dists = get_distributions(use_egg_info=True) + graph = generate_graph(dists) + print(graph.repr_node(dist)) + + +@action_help("""\ +Usage: pysetup install [dist] + or: pysetup install [archive] + or: pysetup install [src_dir] + or: pysetup install --help + +Install a Python distribution from the indexes, source directory, or sdist. + +positional arguments: + archive path to source distribution (zip, tar.gz) + dist distribution name to install from the indexes + scr_dir path to source directory +""") +def _install(dispatcher, args, **kw): + # first check if we are in a source directory + if len(args) < 2: + # are we inside a project dir? + if os.path.isfile('setup.cfg') or os.path.isfile('setup.py'): + args.insert(1, os.getcwd()) + else: + logger.warning('No project to install.') + return 1 + + target = args[1] + # installing from a source dir or archive file? + if os.path.isdir(target) or _is_archive_file(target): + return not install_local_project(target) + else: + # download from PyPI + return not install(target) + + +@action_help("""\ +Usage: pysetup metadata [dist] + or: pysetup metadata [dist] [-f field ...] + or: pysetup metadata --help + +Print metadata for the distribution. + +positional arguments: + dist installed distribution name + +optional arguments: + -f metadata field to print; omit to get all fields +""") +def _metadata(dispatcher, args, **kw): + opts = _parse_args(args[1:], 'f:', []) + if opts['args']: + name = opts['args'][0] + dist = get_distribution(name, use_egg_info=True) + if dist is None: + logger.warning('%r not installed', name) + return 1 + elif os.path.isfile('setup.cfg'): + logger.info('searching local dir for metadata') + dist = Distribution() # XXX use config module + dist.parse_config_files() + else: + logger.warning('no argument given and no local setup.cfg found') + return 1 + + metadata = dist.metadata + + if 'f' in opts: + keys = (k for k in opts['f'] if k in metadata) + else: + keys = metadata.keys() + + for key in keys: + if key in metadata: + print(metadata._convert_name(key) + ':') + value = metadata[key] + if isinstance(value, list): + for v in value: + print(' ', v) + else: + print(' ', value.replace('\n', '\n ')) + + +@action_help("""\ +Usage: pysetup remove dist [-y] + or: pysetup remove --help + +Uninstall a Python distribution. + +positional arguments: + dist installed distribution name + +optional arguments: + -y auto confirm distribution removal +""") +def _remove(distpatcher, args, **kw): + opts = _parse_args(args[1:], 'y', []) + if 'y' in opts: + auto_confirm = True + else: + auto_confirm = False + + retcode = 0 + for dist in set(opts['args']): + try: + remove(dist, auto_confirm=auto_confirm) + except PackagingError: + logger.warning('%r not installed', dist) + retcode = 1 + + return retcode + + +@action_help("""\ +Usage: pysetup run [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...] + or: pysetup run --help + or: pysetup run --list-commands + or: pysetup run cmd --help +""") +def _run(dispatcher, args, **kw): + parser = dispatcher.parser + args = args[1:] + + commands = STANDARD_COMMANDS # + extra commands + + if args == ['--list-commands']: + print('List of available commands:') + cmds = sorted(commands) + + for cmd in cmds: + cls = dispatcher.cmdclass.get(cmd) or get_command_class(cmd) + desc = getattr(cls, 'description', + '(no description available)') + print(' %s: %s' % (cmd, desc)) + return + + while args: + args = dispatcher._parse_command_opts(parser, args) + if args is None: + return + + # create the Distribution class + # need to feed setup.cfg here ! + dist = Distribution() + + # Find and parse the config file(s): they will override options from + # the setup script, but be overridden by the command line. + + # XXX still need to be extracted from Distribution + dist.parse_config_files() + + for cmd in dispatcher.commands: + dist.run_command(cmd, dispatcher.command_options[cmd]) + + # XXX this is crappy + return dist + + +@action_help("""\ +Usage: pysetup list dist [dist ...] + or: pysetup list --help + or: pysetup list --all + +Print name, version and location for the matching installed distributions. + +positional arguments: + dist installed distribution name + +optional arguments: + --all list all installed distributions +""") +def _list(dispatcher, args, **kw): + opts = _parse_args(args[1:], '', ['all']) + dists = get_distributions(use_egg_info=True) + if 'all' in opts or opts['args'] == []: + results = dists + listall = True + else: + results = (d for d in dists if d.name.lower() in opts['args']) + listall = False + + number = 0 + for dist in results: + print('%r %s (from %r)' % (dist.name, dist.version, dist.path)) + number += 1 + + if number == 0: + if listall: + logger.info('Nothing seems to be installed.') + else: + logger.warning('No matching distribution found.') + return 1 + else: + logger.info('Found %d projects installed.', number) + + +@action_help("""\ +Usage: pysetup search [project] [--simple [url]] [--xmlrpc [url] [--fieldname value ...] --operator or|and] + or: pysetup search --help + +Search the indexes for the matching projects. + +positional arguments: + project the project pattern to search for + +optional arguments: + --xmlrpc [url] whether to use the xmlrpc index or not. If an url is + specified, it will be used rather than the default one. + + --simple [url] whether to use the simple index or not. If an url is + specified, it will be used rather than the default one. + + --fieldname value Make a search on this field. Can only be used if + --xmlrpc has been selected or is the default index. + + --operator or|and Defines what is the operator to use when doing xmlrpc + searchs with multiple fieldnames. Can only be used if + --xmlrpc has been selected or is the default index. +""") +def _search(dispatcher, args, **kw): + """The search action. + + It is able to search for a specific index (specified with --index), using + the simple or xmlrpc index types (with --type xmlrpc / --type simple) + """ + #opts = _parse_args(args[1:], '', ['simple', 'xmlrpc']) + # 1. what kind of index is requested ? (xmlrpc / simple) + logger.error('not implemented') + return 1 + + +actions = [ + ('run', 'Run one or several commands', _run), + ('metadata', 'Display the metadata of a project', _metadata), + ('install', 'Install a project', _install), + ('remove', 'Remove a project', _remove), + ('search', 'Search for a project in the indexes', _search), + ('list', 'List installed releases', _list), + ('graph', 'Display a graph', _graph), + ('create', 'Create a project', _create), + ('generate-setup', 'Generate a backward-comptatible setup.py', _generate), +] + + +class Dispatcher: + """Reads the command-line options + """ + def __init__(self, args=None): + self.verbose = 1 + self.dry_run = False + self.help = False + self.cmdclass = {} + self.commands = [] + self.command_options = {} + + for attr in display_option_names: + setattr(self, attr, False) + + self.parser = FancyGetopt(global_options + display_options) + self.parser.set_negative_aliases(negative_opt) + # FIXME this parses everything, including command options (e.g. "run + # build -i" errors with "option -i not recognized") + args = self.parser.getopt(args=args, object=self) + + # if first arg is "run", we have some commands + if len(args) == 0: + self.action = None + else: + self.action = args[0] + + allowed = [action[0] for action in actions] + [None] + if self.action not in allowed: + msg = 'Unrecognized action "%s"' % self.action + raise PackagingArgError(msg) + + self._set_logger() + self.args = args + + # for display options we return immediately + if self.help or self.action is None: + self._show_help(self.parser, display_options_=False) + + def _set_logger(self): + # setting up the logging level from the command-line options + # -q gets warning, error and critical + if self.verbose == 0: + level = logging.WARNING + # default level or -v gets info too + # XXX there's a bug somewhere: the help text says that -v is default + # (and verbose is set to 1 above), but when the user explicitly gives + # -v on the command line, self.verbose is incremented to 2! Here we + # compensate for that (I tested manually). On a related note, I think + # it's a good thing to use -q/nothing/-v/-vv on the command line + # instead of logging constants; it will be easy to add support for + # logging configuration in setup.cfg for advanced users. --merwok + elif self.verbose in (1, 2): + level = logging.INFO + else: # -vv and more for debug + level = logging.DEBUG + + # setting up the stream handler + handler = logging.StreamHandler(sys.stderr) + handler.setLevel(level) + logger.addHandler(handler) + logger.setLevel(level) + + def _parse_command_opts(self, parser, args): + # Pull the current command from the head of the command line + command = args[0] + if not command_re.match(command): + raise SystemExit("invalid command name %r" % (command,)) + self.commands.append(command) + + # Dig up the command class that implements this command, so we + # 1) know that it's a valid command, and 2) know which options + # it takes. + try: + cmd_class = get_command_class(command) + except PackagingModuleError as msg: + raise PackagingArgError(msg) + + # XXX We want to push this in packaging.command + # + # Require that the command class be derived from Command -- want + # to be sure that the basic "command" interface is implemented. + for meth in ('initialize_options', 'finalize_options', 'run'): + if hasattr(cmd_class, meth): + continue + raise PackagingClassError( + 'command %r must implement %r' % (cmd_class, meth)) + + # Also make sure that the command object provides a list of its + # known options. + if not (hasattr(cmd_class, 'user_options') and + isinstance(cmd_class.user_options, list)): + raise PackagingClassError( + "command class %s must provide " + "'user_options' attribute (a list of tuples)" % cmd_class) + + # If the command class has a list of negative alias options, + # merge it in with the global negative aliases. + _negative_opt = negative_opt.copy() + + if hasattr(cmd_class, 'negative_opt'): + _negative_opt.update(cmd_class.negative_opt) + + # Check for help_options in command class. They have a different + # format (tuple of four) so we need to preprocess them here. + if (hasattr(cmd_class, 'help_options') and + isinstance(cmd_class.help_options, list)): + help_options = cmd_class.help_options[:] + else: + help_options = [] + + # All commands support the global options too, just by adding + # in 'global_options'. + parser.set_option_table(global_options + + cmd_class.user_options + + help_options) + parser.set_negative_aliases(_negative_opt) + args, opts = parser.getopt(args[1:]) + + if hasattr(opts, 'help') and opts.help: + self._show_command_help(cmd_class) + return + + if (hasattr(cmd_class, 'help_options') and + isinstance(cmd_class.help_options, list)): + help_option_found = False + for help_option, short, desc, func in cmd_class.help_options: + if hasattr(opts, help_option.replace('-', '_')): + help_option_found = True + if hasattr(func, '__call__'): + func() + else: + raise PackagingClassError( + "invalid help function %r for help option %r: " + "must be a callable object (function, etc.)" + % (func, help_option)) + + if help_option_found: + return + + # Put the options from the command line into their official + # holding pen, the 'command_options' dictionary. + opt_dict = self.get_option_dict(command) + for name, value in vars(opts).items(): + opt_dict[name] = ("command line", value) + + return args + + def get_option_dict(self, command): + """Get the option dictionary for a given command. If that + command's option dictionary hasn't been created yet, then create it + and return the new dictionary; otherwise, return the existing + option dictionary. + """ + d = self.command_options.get(command) + if d is None: + d = self.command_options[command] = {} + return d + + def show_help(self): + self._show_help(self.parser) + + def print_usage(self, parser): + parser.set_option_table(global_options) + + actions_ = [' %s: %s' % (name, desc) for name, desc, __ in actions] + usage = common_usage % {'actions': '\n'.join(actions_)} + + parser.print_help(usage + "\nGlobal options:") + + def _show_help(self, parser, global_options_=True, display_options_=True, + commands=[]): + # late import because of mutual dependence between these modules + from packaging.command.cmd import Command + + print('Usage: pysetup [options] action [action_options]') + print() + if global_options_: + self.print_usage(self.parser) + print() + + if display_options_: + parser.set_option_table(display_options) + parser.print_help( + "Information display options (just display " + + "information, ignore any commands)") + print() + + for command in commands: + if isinstance(command, type) and issubclass(command, Command): + cls = command + else: + cls = get_command_class(command) + if (hasattr(cls, 'help_options') and + isinstance(cls.help_options, list)): + parser.set_option_table(cls.user_options + cls.help_options) + else: + parser.set_option_table(cls.user_options) + + parser.print_help("Options for %r command:" % cls.__name__) + print() + + def _show_command_help(self, command): + if isinstance(command, str): + command = get_command_class(command) + + desc = getattr(command, 'description', '(no description available)') + print('Description:', desc) + print() + + if (hasattr(command, 'help_options') and + isinstance(command.help_options, list)): + self.parser.set_option_table(command.user_options + + command.help_options) + else: + self.parser.set_option_table(command.user_options) + + self.parser.print_help("Options:") + print() + + def _get_command_groups(self): + """Helper function to retrieve all the command class names divided + into standard commands (listed in + packaging.command.STANDARD_COMMANDS) and extra commands (given in + self.cmdclass and not standard commands). + """ + extra_commands = [cmd for cmd in self.cmdclass + if cmd not in STANDARD_COMMANDS] + return STANDARD_COMMANDS, extra_commands + + def print_commands(self): + """Print out a help message listing all available commands with a + description of each. The list is divided into standard commands + (listed in packaging.command.STANDARD_COMMANDS) and extra commands + (given in self.cmdclass and not standard commands). The + descriptions come from the command class attribute + 'description'. + """ + std_commands, extra_commands = self._get_command_groups() + max_length = max(len(command) + for commands in (std_commands, extra_commands) + for command in commands) + + self.print_command_list(std_commands, "Standard commands", max_length) + if extra_commands: + print() + self.print_command_list(extra_commands, "Extra commands", + max_length) + + def print_command_list(self, commands, header, max_length): + """Print a subset of the list of all commands -- used by + 'print_commands()'. + """ + print(header + ":") + + for cmd in commands: + cls = self.cmdclass.get(cmd) or get_command_class(cmd) + description = getattr(cls, 'description', + '(no description available)') + + print(" %-*s %s" % (max_length, cmd, description)) + + def __call__(self): + if self.action is None: + return + + for action, desc, func in actions: + if action == self.action: + return func(self, self.args) + return -1 + + +def main(args=None): + old_level = logger.level + old_handlers = list(logger.handlers) + try: + dispatcher = Dispatcher(args) + if dispatcher.action is None: + return + return dispatcher() + except KeyboardInterrupt: + logger.info('interrupted') + return 1 + except (IOError, os.error, PackagingError, CCompilerError) as exc: + logger.exception(exc) + return 1 + finally: + logger.setLevel(old_level) + logger.handlers[:] = old_handlers + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/Lib/packaging/tests/LONG_DESC.txt b/Lib/packaging/tests/LONG_DESC.txt new file mode 100644 index 0000000000..2b4358aae6 --- /dev/null +++ b/Lib/packaging/tests/LONG_DESC.txt @@ -0,0 +1,44 @@ +CLVault +======= + +CLVault uses Keyring to provide a command-line utility to safely store +and retrieve passwords. + +Install it using pip or the setup.py script:: + + $ python setup.py install + + $ pip install clvault + +Once it's installed, you will have three scripts installed in your +Python scripts folder, you can use to list, store and retrieve passwords:: + + $ clvault-set blog + Set your password: + Set the associated username (can be blank): tarek + Set a description (can be blank): My blog password + Password set. + + $ clvault-get blog + The username is "tarek" + The password has been copied in your clipboard + + $ clvault-list + Registered services: + blog My blog password + + +*clvault-set* takes a service name then prompt you for a password, and some +optional information about your service. The password is safely stored in +a keyring while the description is saved in a ``.clvault`` file in your +home directory. This file is created automatically the first time the command +is used. + +*clvault-get* copies the password for a given service in your clipboard, and +displays the associated user if any. + +*clvault-list* lists all registered services, with their description when +given. + + +Project page: http://bitbucket.org/tarek/clvault diff --git a/Lib/packaging/tests/PKG-INFO b/Lib/packaging/tests/PKG-INFO new file mode 100644 index 0000000000..f48546e5a8 --- /dev/null +++ b/Lib/packaging/tests/PKG-INFO @@ -0,0 +1,57 @@ +Metadata-Version: 1.2 +Name: CLVault +Version: 0.5 +Summary: Command-Line utility to store and retrieve passwords +Home-page: http://bitbucket.org/tarek/clvault +Author: Tarek Ziade +Author-email: tarek@ziade.org +License: PSF +Keywords: keyring,password,crypt +Requires-Dist: foo; sys.platform == 'okook' +Requires-Dist: bar; sys.platform == '%s' +Platform: UNKNOWN +Description: CLVault + |======= + | + |CLVault uses Keyring to provide a command-line utility to safely store + |and retrieve passwords. + | + |Install it using pip or the setup.py script:: + | + | $ python setup.py install + | + | $ pip install clvault + | + |Once it's installed, you will have three scripts installed in your + |Python scripts folder, you can use to list, store and retrieve passwords:: + | + | $ clvault-set blog + | Set your password: + | Set the associated username (can be blank): tarek + | Set a description (can be blank): My blog password + | Password set. + | + | $ clvault-get blog + | The username is "tarek" + | The password has been copied in your clipboard + | + | $ clvault-list + | Registered services: + | blog My blog password + | + | + |*clvault-set* takes a service name then prompt you for a password, and some + |optional information about your service. The password is safely stored in + |a keyring while the description is saved in a ``.clvault`` file in your + |home directory. This file is created automatically the first time the command + |is used. + | + |*clvault-get* copies the password for a given service in your clipboard, and + |displays the associated user if any. + | + |*clvault-list* lists all registered services, with their description when + |given. + | + | + |Project page: http://bitbucket.org/tarek/clvault + | diff --git a/Lib/packaging/tests/SETUPTOOLS-PKG-INFO b/Lib/packaging/tests/SETUPTOOLS-PKG-INFO new file mode 100644 index 0000000000..dff8d00567 --- /dev/null +++ b/Lib/packaging/tests/SETUPTOOLS-PKG-INFO @@ -0,0 +1,182 @@ +Metadata-Version: 1.0 +Name: setuptools +Version: 0.6c9 +Summary: Download, build, install, upgrade, and uninstall Python packages -- easily! +Home-page: http://pypi.python.org/pypi/setuptools +Author: Phillip J. Eby +Author-email: distutils-sig@python.org +License: PSF or ZPL +Description: =============================== + Installing and Using Setuptools + =============================== + + .. contents:: **Table of Contents** + + + ------------------------- + Installation Instructions + ------------------------- + + Windows + ======= + + Install setuptools using the provided ``.exe`` installer. If you've previously + installed older versions of setuptools, please delete all ``setuptools*.egg`` + and ``setuptools.pth`` files from your system's ``site-packages`` directory + (and any other ``sys.path`` directories) FIRST. + + If you are upgrading a previous version of setuptools that was installed using + an ``.exe`` installer, please be sure to also *uninstall that older version* + via your system's "Add/Remove Programs" feature, BEFORE installing the newer + version. + + Once installation is complete, you will find an ``easy_install.exe`` program in + your Python ``Scripts`` subdirectory. Be sure to add this directory to your + ``PATH`` environment variable, if you haven't already done so. + + + RPM-Based Systems + ================= + + Install setuptools using the provided source RPM. The included ``.spec`` file + assumes you are installing using the default ``python`` executable, and is not + specific to a particular Python version. The ``easy_install`` executable will + be installed to a system ``bin`` directory such as ``/usr/bin``. + + If you wish to install to a location other than the default Python + installation's default ``site-packages`` directory (and ``$prefix/bin`` for + scripts), please use the ``.egg``-based installation approach described in the + following section. + + + Cygwin, Mac OS X, Linux, Other + ============================== + + 1. Download the appropriate egg for your version of Python (e.g. + ``setuptools-0.6c9-py2.4.egg``). Do NOT rename it. + + 2. Run it as if it were a shell script, e.g. ``sh setuptools-0.6c9-py2.4.egg``. + Setuptools will install itself using the matching version of Python (e.g. + ``python2.4``), and will place the ``easy_install`` executable in the + default location for installing Python scripts (as determined by the + standard distutils configuration files, or by the Python installation). + + If you want to install setuptools to somewhere other than ``site-packages`` or + your default distutils installation locations for libraries and scripts, you + may include EasyInstall command-line options such as ``--prefix``, + ``--install-dir``, and so on, following the ``.egg`` filename on the same + command line. For example:: + + sh setuptools-0.6c9-py2.4.egg --prefix=~ + + You can use ``--help`` to get a full options list, but we recommend consulting + the `EasyInstall manual`_ for detailed instructions, especially `the section + on custom installation locations`_. + + .. _EasyInstall manual: http://peak.telecommunity.com/DevCenter/EasyInstall + .. _the section on custom installation locations: http://peak.telecommunity.com/DevCenter/EasyInstall#custom-installation-locations + + + Cygwin Note + ----------- + + If you are trying to install setuptools for the **Windows** version of Python + (as opposed to the Cygwin version that lives in ``/usr/bin``), you must make + sure that an appropriate executable (``python2.3``, ``python2.4``, or + ``python2.5``) is on your **Cygwin** ``PATH`` when invoking the egg. For + example, doing the following at a Cygwin bash prompt will install setuptools + for the **Windows** Python found at ``C:\\Python24``:: + + ln -s /cygdrive/c/Python24/python.exe python2.4 + PATH=.:$PATH sh setuptools-0.6c9-py2.4.egg + rm python2.4 + + + Downloads + ========= + + All setuptools downloads can be found at `the project's home page in the Python + Package Index`_. Scroll to the very bottom of the page to find the links. + + .. _the project's home page in the Python Package Index: http://pypi.python.org/pypi/setuptools + + In addition to the PyPI downloads, the development version of ``setuptools`` + is available from the `Python SVN sandbox`_, and in-development versions of the + `0.6 branch`_ are available as well. + + .. _0.6 branch: http://svn.python.org/projects/sandbox/branches/setuptools-0.6/#egg=setuptools-dev06 + + .. _Python SVN sandbox: http://svn.python.org/projects/sandbox/trunk/setuptools/#egg=setuptools-dev + + -------------------------------- + Using Setuptools and EasyInstall + -------------------------------- + + Here are some of the available manuals, tutorials, and other resources for + learning about Setuptools, Python Eggs, and EasyInstall: + + * `The EasyInstall user's guide and reference manual`_ + * `The setuptools Developer's Guide`_ + * `The pkg_resources API reference`_ + * `Package Compatibility Notes`_ (user-maintained) + * `The Internal Structure of Python Eggs`_ + + Questions, comments, and bug reports should be directed to the `distutils-sig + mailing list`_. If you have written (or know of) any tutorials, documentation, + plug-ins, or other resources for setuptools users, please let us know about + them there, so this reference list can be updated. If you have working, + *tested* patches to correct problems or add features, you may submit them to + the `setuptools bug tracker`_. + + .. _setuptools bug tracker: http://bugs.python.org/setuptools/ + .. _Package Compatibility Notes: http://peak.telecommunity.com/DevCenter/PackageNotes + .. _The Internal Structure of Python Eggs: http://peak.telecommunity.com/DevCenter/EggFormats + .. _The setuptools Developer's Guide: http://peak.telecommunity.com/DevCenter/setuptools + .. _The pkg_resources API reference: http://peak.telecommunity.com/DevCenter/PkgResources + .. _The EasyInstall user's guide and reference manual: http://peak.telecommunity.com/DevCenter/EasyInstall + .. _distutils-sig mailing list: http://mail.python.org/pipermail/distutils-sig/ + + + ------- + Credits + ------- + + * The original design for the ``.egg`` format and the ``pkg_resources`` API was + co-created by Phillip Eby and Bob Ippolito. Bob also implemented the first + version of ``pkg_resources``, and supplied the OS X operating system version + compatibility algorithm. + + * Ian Bicking implemented many early "creature comfort" features of + easy_install, including support for downloading via Sourceforge and + Subversion repositories. Ian's comments on the Web-SIG about WSGI + application deployment also inspired the concept of "entry points" in eggs, + and he has given talks at PyCon and elsewhere to inform and educate the + community about eggs and setuptools. + + * Jim Fulton contributed time and effort to build automated tests of various + aspects of ``easy_install``, and supplied the doctests for the command-line + ``.exe`` wrappers on Windows. + + * Phillip J. Eby is the principal author and maintainer of setuptools, and + first proposed the idea of an importable binary distribution format for + Python application plug-ins. + + * Significant parts of the implementation of setuptools were funded by the Open + Source Applications Foundation, to provide a plug-in infrastructure for the + Chandler PIM application. In addition, many OSAF staffers (such as Mike + "Code Bear" Taylor) contributed their time and stress as guinea pigs for the + use of eggs and setuptools, even before eggs were "cool". (Thanks, guys!) + + +Keywords: CPAN PyPI distutils eggs package management +Platform: UNKNOWN +Classifier: Development Status :: 3 - Alpha +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Python Software Foundation License +Classifier: License :: OSI Approved :: Zope Public License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: System :: Archiving :: Packaging +Classifier: Topic :: System :: Systems Administration +Classifier: Topic :: Utilities diff --git a/Lib/packaging/tests/SETUPTOOLS-PKG-INFO2 b/Lib/packaging/tests/SETUPTOOLS-PKG-INFO2 new file mode 100644 index 0000000000..4b3906a41b --- /dev/null +++ b/Lib/packaging/tests/SETUPTOOLS-PKG-INFO2 @@ -0,0 +1,183 @@ +Metadata-Version: 1.1 +Name: setuptools +Version: 0.6c9 +Summary: Download, build, install, upgrade, and uninstall Python packages -- easily! +Home-page: http://pypi.python.org/pypi/setuptools +Author: Phillip J. Eby +Author-email: distutils-sig@python.org +License: PSF or ZPL +Description: =============================== + Installing and Using Setuptools + =============================== + + .. contents:: **Table of Contents** + + + ------------------------- + Installation Instructions + ------------------------- + + Windows + ======= + + Install setuptools using the provided ``.exe`` installer. If you've previously + installed older versions of setuptools, please delete all ``setuptools*.egg`` + and ``setuptools.pth`` files from your system's ``site-packages`` directory + (and any other ``sys.path`` directories) FIRST. + + If you are upgrading a previous version of setuptools that was installed using + an ``.exe`` installer, please be sure to also *uninstall that older version* + via your system's "Add/Remove Programs" feature, BEFORE installing the newer + version. + + Once installation is complete, you will find an ``easy_install.exe`` program in + your Python ``Scripts`` subdirectory. Be sure to add this directory to your + ``PATH`` environment variable, if you haven't already done so. + + + RPM-Based Systems + ================= + + Install setuptools using the provided source RPM. The included ``.spec`` file + assumes you are installing using the default ``python`` executable, and is not + specific to a particular Python version. The ``easy_install`` executable will + be installed to a system ``bin`` directory such as ``/usr/bin``. + + If you wish to install to a location other than the default Python + installation's default ``site-packages`` directory (and ``$prefix/bin`` for + scripts), please use the ``.egg``-based installation approach described in the + following section. + + + Cygwin, Mac OS X, Linux, Other + ============================== + + 1. Download the appropriate egg for your version of Python (e.g. + ``setuptools-0.6c9-py2.4.egg``). Do NOT rename it. + + 2. Run it as if it were a shell script, e.g. ``sh setuptools-0.6c9-py2.4.egg``. + Setuptools will install itself using the matching version of Python (e.g. + ``python2.4``), and will place the ``easy_install`` executable in the + default location for installing Python scripts (as determined by the + standard distutils configuration files, or by the Python installation). + + If you want to install setuptools to somewhere other than ``site-packages`` or + your default distutils installation locations for libraries and scripts, you + may include EasyInstall command-line options such as ``--prefix``, + ``--install-dir``, and so on, following the ``.egg`` filename on the same + command line. For example:: + + sh setuptools-0.6c9-py2.4.egg --prefix=~ + + You can use ``--help`` to get a full options list, but we recommend consulting + the `EasyInstall manual`_ for detailed instructions, especially `the section + on custom installation locations`_. + + .. _EasyInstall manual: http://peak.telecommunity.com/DevCenter/EasyInstall + .. _the section on custom installation locations: http://peak.telecommunity.com/DevCenter/EasyInstall#custom-installation-locations + + + Cygwin Note + ----------- + + If you are trying to install setuptools for the **Windows** version of Python + (as opposed to the Cygwin version that lives in ``/usr/bin``), you must make + sure that an appropriate executable (``python2.3``, ``python2.4``, or + ``python2.5``) is on your **Cygwin** ``PATH`` when invoking the egg. For + example, doing the following at a Cygwin bash prompt will install setuptools + for the **Windows** Python found at ``C:\\Python24``:: + + ln -s /cygdrive/c/Python24/python.exe python2.4 + PATH=.:$PATH sh setuptools-0.6c9-py2.4.egg + rm python2.4 + + + Downloads + ========= + + All setuptools downloads can be found at `the project's home page in the Python + Package Index`_. Scroll to the very bottom of the page to find the links. + + .. _the project's home page in the Python Package Index: http://pypi.python.org/pypi/setuptools + + In addition to the PyPI downloads, the development version of ``setuptools`` + is available from the `Python SVN sandbox`_, and in-development versions of the + `0.6 branch`_ are available as well. + + .. _0.6 branch: http://svn.python.org/projects/sandbox/branches/setuptools-0.6/#egg=setuptools-dev06 + + .. _Python SVN sandbox: http://svn.python.org/projects/sandbox/trunk/setuptools/#egg=setuptools-dev + + -------------------------------- + Using Setuptools and EasyInstall + -------------------------------- + + Here are some of the available manuals, tutorials, and other resources for + learning about Setuptools, Python Eggs, and EasyInstall: + + * `The EasyInstall user's guide and reference manual`_ + * `The setuptools Developer's Guide`_ + * `The pkg_resources API reference`_ + * `Package Compatibility Notes`_ (user-maintained) + * `The Internal Structure of Python Eggs`_ + + Questions, comments, and bug reports should be directed to the `distutils-sig + mailing list`_. If you have written (or know of) any tutorials, documentation, + plug-ins, or other resources for setuptools users, please let us know about + them there, so this reference list can be updated. If you have working, + *tested* patches to correct problems or add features, you may submit them to + the `setuptools bug tracker`_. + + .. _setuptools bug tracker: http://bugs.python.org/setuptools/ + .. _Package Compatibility Notes: http://peak.telecommunity.com/DevCenter/PackageNotes + .. _The Internal Structure of Python Eggs: http://peak.telecommunity.com/DevCenter/EggFormats + .. _The setuptools Developer's Guide: http://peak.telecommunity.com/DevCenter/setuptools + .. _The pkg_resources API reference: http://peak.telecommunity.com/DevCenter/PkgResources + .. _The EasyInstall user's guide and reference manual: http://peak.telecommunity.com/DevCenter/EasyInstall + .. _distutils-sig mailing list: http://mail.python.org/pipermail/distutils-sig/ + + + ------- + Credits + ------- + + * The original design for the ``.egg`` format and the ``pkg_resources`` API was + co-created by Phillip Eby and Bob Ippolito. Bob also implemented the first + version of ``pkg_resources``, and supplied the OS X operating system version + compatibility algorithm. + + * Ian Bicking implemented many early "creature comfort" features of + easy_install, including support for downloading via Sourceforge and + Subversion repositories. Ian's comments on the Web-SIG about WSGI + application deployment also inspired the concept of "entry points" in eggs, + and he has given talks at PyCon and elsewhere to inform and educate the + community about eggs and setuptools. + + * Jim Fulton contributed time and effort to build automated tests of various + aspects of ``easy_install``, and supplied the doctests for the command-line + ``.exe`` wrappers on Windows. + + * Phillip J. Eby is the principal author and maintainer of setuptools, and + first proposed the idea of an importable binary distribution format for + Python application plug-ins. + + * Significant parts of the implementation of setuptools were funded by the Open + Source Applications Foundation, to provide a plug-in infrastructure for the + Chandler PIM application. In addition, many OSAF staffers (such as Mike + "Code Bear" Taylor) contributed their time and stress as guinea pigs for the + use of eggs and setuptools, even before eggs were "cool". (Thanks, guys!) + + +Keywords: CPAN PyPI distutils eggs package management +Platform: UNKNOWN +Classifier: Development Status :: 3 - Alpha +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Python Software Foundation License +Classifier: License :: OSI Approved :: Zope Public License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: System :: Archiving :: Packaging +Classifier: Topic :: System :: Systems Administration +Classifier: Topic :: Utilities +Requires: Foo diff --git a/Lib/packaging/tests/__init__.py b/Lib/packaging/tests/__init__.py new file mode 100644 index 0000000000..0b0e3c5543 --- /dev/null +++ b/Lib/packaging/tests/__init__.py @@ -0,0 +1,133 @@ +"""Test suite for packaging. + +This test suite consists of a collection of test modules in the +packaging.tests package. Each test module has a name starting with +'test' and contains a function test_suite(). The function is expected +to return an initialized unittest.TestSuite instance. + +Utility code is included in packaging.tests.support. +""" + +# Put this text back for the backport +#Always import unittest from this module, it will be the right version +#(standard library unittest for 3.2 and higher, third-party unittest2 +#elease for older versions). + +import os +import sys +import unittest +from test.support import TESTFN + +# XXX move helpers to support, add tests for them, remove things that +# duplicate test.support (or keep them for the backport; needs thinking) + +here = os.path.dirname(__file__) or os.curdir +verbose = 1 + +def test_suite(): + suite = unittest.TestSuite() + for fn in os.listdir(here): + if fn.startswith("test") and fn.endswith(".py"): + modname = "packaging.tests." + fn[:-3] + __import__(modname) + module = sys.modules[modname] + suite.addTest(module.test_suite()) + return suite + + +class Error(Exception): + """Base class for regression test exceptions.""" + + +class TestFailed(Error): + """Test failed.""" + + +class BasicTestRunner: + def run(self, test): + result = unittest.TestResult() + test(result) + return result + + +def _run_suite(suite, verbose_=1): + """Run tests from a unittest.TestSuite-derived class.""" + global verbose + verbose = verbose_ + if verbose_: + runner = unittest.TextTestRunner(sys.stdout, verbosity=2) + else: + runner = BasicTestRunner() + + result = runner.run(suite) + if not result.wasSuccessful(): + if len(result.errors) == 1 and not result.failures: + err = result.errors[0][1] + elif len(result.failures) == 1 and not result.errors: + err = result.failures[0][1] + else: + err = "errors occurred; run in verbose mode for details" + raise TestFailed(err) + + +def run_unittest(classes, verbose_=1): + """Run tests from unittest.TestCase-derived classes. + + Originally extracted from stdlib test.test_support and modified to + support unittest2. + """ + valid_types = (unittest.TestSuite, unittest.TestCase) + suite = unittest.TestSuite() + for cls in classes: + if isinstance(cls, str): + if cls in sys.modules: + suite.addTest(unittest.findTestCases(sys.modules[cls])) + else: + raise ValueError("str arguments must be keys in sys.modules") + elif isinstance(cls, valid_types): + suite.addTest(cls) + else: + suite.addTest(unittest.makeSuite(cls)) + _run_suite(suite, verbose_) + + +def reap_children(): + """Use this function at the end of test_main() whenever sub-processes + are started. This will help ensure that no extra children (zombies) + stick around to hog resources and create problems when looking + for refleaks. + + Extracted from stdlib test.support. + """ + + # Reap all our dead child processes so we don't leave zombies around. + # These hog resources and might be causing some of the buildbots to die. + if hasattr(os, 'waitpid'): + any_process = -1 + while True: + try: + # This will raise an exception on Windows. That's ok. + pid, status = os.waitpid(any_process, os.WNOHANG) + if pid == 0: + break + except: + break + + +def captured_stdout(func, *args, **kw): + import io + orig_stdout = getattr(sys, 'stdout') + setattr(sys, 'stdout', io.StringIO()) + try: + res = func(*args, **kw) + sys.stdout.seek(0) + return res, sys.stdout.read() + finally: + setattr(sys, 'stdout', orig_stdout) + + +def unload(name): + try: + del sys.modules[name] + except KeyError: + pass diff --git a/Lib/packaging/tests/__main__.py b/Lib/packaging/tests/__main__.py new file mode 100644 index 0000000000..0f175cf961 --- /dev/null +++ b/Lib/packaging/tests/__main__.py @@ -0,0 +1,23 @@ +"""Packaging test suite runner.""" + +# Ripped from importlib tests, thanks Brett! + +import os +import sys +import unittest +from test.support import run_unittest, reap_children, reap_threads + + +@reap_threads +def test_main(): + try: + start_dir = os.path.dirname(__file__) + top_dir = os.path.dirname(os.path.dirname(start_dir)) + test_loader = unittest.TestLoader() + run_unittest(test_loader.discover(start_dir, top_level_dir=top_dir)) + finally: + reap_children() + + +if __name__ == '__main__': + test_main() diff --git a/Lib/email/test/__init__.py b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/INSTALLER index e69de29bb2..e69de29bb2 100644 --- a/Lib/email/test/__init__.py +++ b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/INSTALLER diff --git a/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/METADATA b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/METADATA new file mode 100644 index 0000000000..65e839a01f --- /dev/null +++ b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/METADATA @@ -0,0 +1,4 @@ +Metadata-version: 1.2 +Name: babar +Version: 0.1 +Author: FELD Boris
\ No newline at end of file diff --git a/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/RECORD b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/RECORD new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/RECORD diff --git a/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/REQUESTED b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/REQUESTED new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/REQUESTED diff --git a/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/RESOURCES b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/RESOURCES new file mode 100644 index 0000000000..5d0da494a5 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/RESOURCES @@ -0,0 +1,2 @@ +babar.png,babar.png +babar.cfg,babar.cfg
\ No newline at end of file diff --git a/Lib/packaging/tests/fake_dists/babar.cfg b/Lib/packaging/tests/fake_dists/babar.cfg new file mode 100644 index 0000000000..ecd6efe9a6 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/babar.cfg @@ -0,0 +1 @@ +Config
\ No newline at end of file diff --git a/Lib/packaging/tests/fake_dists/babar.png b/Lib/packaging/tests/fake_dists/babar.png new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/babar.png diff --git a/Lib/packaging/tests/fake_dists/bacon-0.1.egg-info/PKG-INFO b/Lib/packaging/tests/fake_dists/bacon-0.1.egg-info/PKG-INFO new file mode 100644 index 0000000000..a176dfdfde --- /dev/null +++ b/Lib/packaging/tests/fake_dists/bacon-0.1.egg-info/PKG-INFO @@ -0,0 +1,6 @@ +Metadata-Version: 1.2 +Name: bacon +Version: 0.1 +Provides-Dist: truffles (2.0) +Provides-Dist: bacon (0.1) +Obsoletes-Dist: truffles (>=0.9,<=1.5) diff --git a/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/PKG-INFO b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/PKG-INFO new file mode 100644 index 0000000000..a7e118a88f --- /dev/null +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/PKG-INFO @@ -0,0 +1,18 @@ +Metadata-Version: 1.0 +Name: banana +Version: 0.4 +Summary: A yellow fruit +Home-page: http://en.wikipedia.org/wiki/Banana +Author: Josip Djolonga +Author-email: foo@nbar.com +License: BSD +Description: A fruit +Keywords: foo bar +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Scientific/Engineering :: GIS diff --git a/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/SOURCES.txt b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/SOURCES.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/SOURCES.txt diff --git a/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/dependency_links.txt b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/dependency_links.txt new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/entry_points.txt b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/entry_points.txt new file mode 100644 index 0000000000..5d3e5f68c7 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/entry_points.txt @@ -0,0 +1,3 @@ + + # -*- Entry points: -*- +
\ No newline at end of file diff --git a/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/not-zip-safe b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/not-zip-safe new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/not-zip-safe @@ -0,0 +1 @@ + diff --git a/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/requires.txt b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/requires.txt new file mode 100644 index 0000000000..4354305659 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/requires.txt @@ -0,0 +1,6 @@ +# this should be ignored + +strawberry >=0.5 + +[section ignored] +foo ==0.5 diff --git a/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/top_level.txt b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/top_level.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/top_level.txt diff --git a/Lib/packaging/tests/fake_dists/cheese-2.0.2.egg-info b/Lib/packaging/tests/fake_dists/cheese-2.0.2.egg-info new file mode 100644 index 0000000000..27cbe3014d --- /dev/null +++ b/Lib/packaging/tests/fake_dists/cheese-2.0.2.egg-info @@ -0,0 +1,5 @@ +Metadata-Version: 1.2 +Name: cheese +Version: 2.0.2 +Provides-Dist: truffles (1.0.2) +Obsoletes-Dist: truffles (!=1.2,<=2.0) diff --git a/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/INSTALLER b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/INSTALLER new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/INSTALLER diff --git a/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/METADATA b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/METADATA new file mode 100644 index 0000000000..418929ecca --- /dev/null +++ b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/METADATA @@ -0,0 +1,9 @@ +Metadata-Version: 1.2 +Name: choxie +Version: 2.0.0.9 +Summary: Chocolate with a kick! +Requires-Dist: towel-stuff (0.1) +Requires-Dist: nut +Provides-Dist: truffles (1.0) +Obsoletes-Dist: truffles (<=0.8,>=0.5) +Obsoletes-Dist: truffles (<=0.9,>=0.6) diff --git a/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/RECORD b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/RECORD new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/RECORD diff --git a/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/REQUESTED b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/REQUESTED new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/REQUESTED diff --git a/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/__init__.py b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/__init__.py new file mode 100644 index 0000000000..40a96afc6f --- /dev/null +++ b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/chocolate.py b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/chocolate.py new file mode 100644 index 0000000000..c4027f36c1 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/chocolate.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +from towel_stuff import Towel + +class Chocolate(object): + """A piece of chocolate.""" + + def wrap_with_towel(self): + towel = Towel() + towel.wrap(self) + return towel diff --git a/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/truffles.py b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/truffles.py new file mode 100644 index 0000000000..342b8ea851 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/truffles.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +from choxie.chocolate import Chocolate + +class Truffle(Chocolate): + """A truffle.""" diff --git a/Lib/packaging/tests/fake_dists/coconuts-aster-10.3.egg-info/PKG-INFO b/Lib/packaging/tests/fake_dists/coconuts-aster-10.3.egg-info/PKG-INFO new file mode 100644 index 0000000000..499a083e40 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/coconuts-aster-10.3.egg-info/PKG-INFO @@ -0,0 +1,5 @@ +Metadata-Version: 1.2 +Name: coconuts-aster +Version: 10.3 +Provides-Dist: strawberry (0.6) +Provides-Dist: banana (0.4) diff --git a/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/INSTALLER b/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/INSTALLER new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/INSTALLER diff --git a/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/METADATA b/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/METADATA new file mode 100644 index 0000000000..0b99f5249a --- /dev/null +++ b/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/METADATA @@ -0,0 +1,5 @@ +Metadata-Version: 1.2 +Name: grammar +Version: 1.0a4 +Requires-Dist: truffles (>=1.2) +Author: Sherlock Holmes diff --git a/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/RECORD b/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/RECORD new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/RECORD diff --git a/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/REQUESTED b/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/REQUESTED new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/REQUESTED diff --git a/Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/__init__.py b/Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/__init__.py new file mode 100644 index 0000000000..40a96afc6f --- /dev/null +++ b/Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/utils.py b/Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/utils.py new file mode 100644 index 0000000000..66ba796c3d --- /dev/null +++ b/Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/utils.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +from random import randint + +def is_valid_grammar(sentence): + if randint(0, 10) < 2: + return False + else: + return True diff --git a/Lib/packaging/tests/fake_dists/nut-funkyversion.egg-info b/Lib/packaging/tests/fake_dists/nut-funkyversion.egg-info new file mode 100644 index 0000000000..0c58ec1ce9 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/nut-funkyversion.egg-info @@ -0,0 +1,3 @@ +Metadata-Version: 1.2 +Name: nut +Version: funkyversion diff --git a/Lib/packaging/tests/fake_dists/strawberry-0.6.egg b/Lib/packaging/tests/fake_dists/strawberry-0.6.egg Binary files differnew file mode 100644 index 0000000000..6d160e8b16 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/strawberry-0.6.egg diff --git a/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/INSTALLER b/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/INSTALLER new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/INSTALLER diff --git a/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/METADATA b/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/METADATA new file mode 100644 index 0000000000..ca46d0a4ea --- /dev/null +++ b/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/METADATA @@ -0,0 +1,7 @@ +Metadata-Version: 1.2 +Name: towel-stuff +Version: 0.1 +Provides-Dist: truffles (1.1.2) +Provides-Dist: towel-stuff (0.1) +Obsoletes-Dist: truffles (!=0.8,<1.0) +Requires-Dist: bacon (<=0.2) diff --git a/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/RECORD b/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/RECORD new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/RECORD diff --git a/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/REQUESTED b/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/REQUESTED new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/REQUESTED diff --git a/Lib/packaging/tests/fake_dists/towel_stuff-0.1/towel_stuff/__init__.py b/Lib/packaging/tests/fake_dists/towel_stuff-0.1/towel_stuff/__init__.py new file mode 100644 index 0000000000..191f895b0a --- /dev/null +++ b/Lib/packaging/tests/fake_dists/towel_stuff-0.1/towel_stuff/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +class Towel(object): + """A towel, that one should never be without.""" + + def __init__(self, color='tie-dye'): + self.color = color + self.wrapped_obj = None + + def wrap(self, obj): + """Wrap an object up in our towel.""" + self.wrapped_obj = obj + + def unwrap(self): + """Unwrap whatever is in our towel and return whatever it is.""" + obj = self.wrapped_obj + self.wrapped_obj = None + return obj diff --git a/Lib/packaging/tests/fake_dists/truffles-5.0.egg-info b/Lib/packaging/tests/fake_dists/truffles-5.0.egg-info new file mode 100644 index 0000000000..45f0cf8b39 --- /dev/null +++ b/Lib/packaging/tests/fake_dists/truffles-5.0.egg-info @@ -0,0 +1,3 @@ +Metadata-Version: 1.2 +Name: truffles +Version: 5.0 diff --git a/Lib/packaging/tests/fixer/__init__.py b/Lib/packaging/tests/fixer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/fixer/__init__.py diff --git a/Lib/packaging/tests/fixer/fix_idioms.py b/Lib/packaging/tests/fixer/fix_idioms.py new file mode 100644 index 0000000000..64f5ea0516 --- /dev/null +++ b/Lib/packaging/tests/fixer/fix_idioms.py @@ -0,0 +1,134 @@ +"""Adjust some old Python 2 idioms to their modern counterparts. + +* Change some type comparisons to isinstance() calls: + type(x) == T -> isinstance(x, T) + type(x) is T -> isinstance(x, T) + type(x) != T -> not isinstance(x, T) + type(x) is not T -> not isinstance(x, T) + +* Change "while 1:" into "while True:". + +* Change both + + v = list(EXPR) + v.sort() + foo(v) + +and the more general + + v = EXPR + v.sort() + foo(v) + +into + + v = sorted(EXPR) + foo(v) +""" +# Author: Jacques Frechet, Collin Winter + +# Local imports +from lib2to3 import fixer_base +from lib2to3.fixer_util import Call, Comma, Name, Node, syms + +CMP = "(n='!=' | '==' | 'is' | n=comp_op< 'is' 'not' >)" +TYPE = "power< 'type' trailer< '(' x=any ')' > >" + +class FixIdioms(fixer_base.BaseFix): + + explicit = False # The user must ask for this fixer + + PATTERN = r""" + isinstance=comparison< %s %s T=any > + | + isinstance=comparison< T=any %s %s > + | + while_stmt< 'while' while='1' ':' any+ > + | + sorted=any< + any* + simple_stmt< + expr_stmt< id1=any '=' + power< list='list' trailer< '(' (not arglist<any+>) any ')' > > + > + '\n' + > + sort= + simple_stmt< + power< id2=any + trailer< '.' 'sort' > trailer< '(' ')' > + > + '\n' + > + next=any* + > + | + sorted=any< + any* + simple_stmt< expr_stmt< id1=any '=' expr=any > '\n' > + sort= + simple_stmt< + power< id2=any + trailer< '.' 'sort' > trailer< '(' ')' > + > + '\n' + > + next=any* + > + """ % (TYPE, CMP, CMP, TYPE) + + def match(self, node): + r = super(FixIdioms, self).match(node) + # If we've matched one of the sort/sorted subpatterns above, we + # want to reject matches where the initial assignment and the + # subsequent .sort() call involve different identifiers. + if r and "sorted" in r: + if r["id1"] == r["id2"]: + return r + return None + return r + + def transform(self, node, results): + if "isinstance" in results: + return self.transform_isinstance(node, results) + elif "while" in results: + return self.transform_while(node, results) + elif "sorted" in results: + return self.transform_sort(node, results) + else: + raise RuntimeError("Invalid match") + + def transform_isinstance(self, node, results): + x = results["x"].clone() # The thing inside of type() + T = results["T"].clone() # The type being compared against + x.prefix = "" + T.prefix = " " + test = Call(Name("isinstance"), [x, Comma(), T]) + if "n" in results: + test.prefix = " " + test = Node(syms.not_test, [Name("not"), test]) + test.prefix = node.prefix + return test + + def transform_while(self, node, results): + one = results["while"] + one.replace(Name("True", prefix=one.prefix)) + + def transform_sort(self, node, results): + sort_stmt = results["sort"] + next_stmt = results["next"] + list_call = results.get("list") + simple_expr = results.get("expr") + + if list_call: + list_call.replace(Name("sorted", prefix=list_call.prefix)) + elif simple_expr: + new = simple_expr.clone() + new.prefix = "" + simple_expr.replace(Call(Name("sorted"), [new], + prefix=simple_expr.prefix)) + else: + raise RuntimeError("should not have reached here") + sort_stmt.remove() + if next_stmt: + next_stmt[0].prefix = sort_stmt._prefix diff --git a/Lib/packaging/tests/pypi_server.py b/Lib/packaging/tests/pypi_server.py new file mode 100644 index 0000000000..41bf9c42a4 --- /dev/null +++ b/Lib/packaging/tests/pypi_server.py @@ -0,0 +1,448 @@ +"""Mock PyPI Server implementation, to use in tests. + +This module also provides a simple test case to extend if you need to use +the PyPIServer all along your test case. Be sure to read the documentation +before any use. + +XXX TODO: + +The mock server can handle simple HTTP request (to simulate a simple index) or +XMLRPC requests, over HTTP. Both does not have the same intergface to deal +with, and I think it's a pain. + +A good idea could be to re-think a bit the way dstributions are handled in the +mock server. As it should return malformed HTML pages, we need to keep the +static behavior. + +I think of something like that: + + >>> server = PyPIMockServer() + >>> server.startHTTP() + >>> server.startXMLRPC() + +Then, the server must have only one port to rely on, eg. + + >>> server.fulladdress() + "http://ip:port/" + +It could be simple to have one HTTP server, relaying the requests to the two +implementations (static HTTP and XMLRPC over HTTP). +""" + +import os +import queue +import select +import threading +import socketserver +from functools import wraps +from http.server import HTTPServer, SimpleHTTPRequestHandler +from xmlrpc.server import SimpleXMLRPCServer + +from packaging.tests import unittest + +PYPI_DEFAULT_STATIC_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'pypiserver') + + +def use_xmlrpc_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = True + return use_pypi_server(*server_args, **server_kwargs) + + +def use_http_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = False + return use_pypi_server(*server_args, **server_kwargs) + + +def use_pypi_server(*server_args, **server_kwargs): + """Decorator to make use of the PyPIServer for test methods, + just when needed, and not for the entire duration of the testcase. + """ + def wrapper(func): + @wraps(func) + def wrapped(*args, **kwargs): + server = PyPIServer(*server_args, **server_kwargs) + server.start() + try: + func(server=server, *args, **kwargs) + finally: + server.stop() + return wrapped + return wrapper + + +class PyPIServerTestCase(unittest.TestCase): + + def setUp(self): + super(PyPIServerTestCase, self).setUp() + self.pypi = PyPIServer() + self.pypi.start() + self.addCleanup(self.pypi.stop) + + +class PyPIServer(threading.Thread): + """PyPI Mocked server. + Provides a mocked version of the PyPI API's, to ease tests. + + Support serving static content and serving previously given text. + """ + + def __init__(self, test_static_path=None, + static_filesystem_paths=None, + static_uri_paths=["simple", "packages"], serve_xmlrpc=False): + """Initialize the server. + + Default behavior is to start the HTTP server. You can either start the + xmlrpc server by setting xmlrpc to True. Caution: Only one server will + be started. + + static_uri_paths and static_base_path are parameters used to provides + respectively the http_paths to serve statically, and where to find the + matching files on the filesystem. + """ + # we want to launch the server in a new dedicated thread, to not freeze + # tests. + threading.Thread.__init__(self) + self._run = True + self._serve_xmlrpc = serve_xmlrpc + if static_filesystem_paths is None: + static_filesystem_paths = ["default"] + + #TODO allow to serve XMLRPC and HTTP static files at the same time. + if not self._serve_xmlrpc: + self.server = HTTPServer(('127.0.0.1', 0), PyPIRequestHandler) + self.server.RequestHandlerClass.pypi_server = self + + self.request_queue = queue.Queue() + self._requests = [] + self.default_response_status = 404 + self.default_response_headers = [('Content-type', 'text/plain')] + self.default_response_data = "The page does not exists" + + # initialize static paths / filesystems + self.static_uri_paths = static_uri_paths + + # append the static paths defined locally + if test_static_path is not None: + static_filesystem_paths.append(test_static_path) + self.static_filesystem_paths = [ + PYPI_DEFAULT_STATIC_PATH + "/" + path + for path in static_filesystem_paths] + else: + # XMLRPC server + self.server = PyPIXMLRPCServer(('127.0.0.1', 0)) + self.xmlrpc = XMLRPCMockIndex() + # register the xmlrpc methods + self.server.register_introspection_functions() + self.server.register_instance(self.xmlrpc) + + self.address = ('127.0.0.1', self.server.server_port) + # to not have unwanted outputs. + self.server.RequestHandlerClass.log_request = lambda *_: None + + def run(self): + # loop because we can't stop it otherwise, for python < 2.6 + while self._run: + r, w, e = select.select([self.server], [], [], 0.5) + if r: + self.server.handle_request() + + def stop(self): + """self shutdown is not supported for python < 2.6""" + self._run = False + if self.is_alive(): + self.join() + self.server.server_close() + + def get_next_response(self): + return (self.default_response_status, + self.default_response_headers, + self.default_response_data) + + @property + def requests(self): + """Use this property to get all requests that have been made + to the server + """ + while True: + try: + self._requests.append(self.request_queue.get_nowait()) + except queue.Empty: + break + return self._requests + + @property + def full_address(self): + return "http://%s:%s" % self.address + + +class PyPIRequestHandler(SimpleHTTPRequestHandler): + # we need to access the pypi server while serving the content + pypi_server = None + + def serve_request(self): + """Serve the content. + + Also record the requests to be accessed later. If trying to access an + url matching a static uri, serve static content, otherwise serve + what is provided by the `get_next_response` method. + + If nothing is defined there, return a 404 header. + """ + # record the request. Read the input only on PUT or POST requests + if self.command in ("PUT", "POST"): + if 'content-length' in self.headers: + request_data = self.rfile.read( + int(self.headers['content-length'])) + else: + request_data = self.rfile.read() + + elif self.command in ("GET", "DELETE"): + request_data = '' + + self.pypi_server.request_queue.put((self, request_data)) + + # serve the content from local disc if we request an URL beginning + # by a pattern defined in `static_paths` + url_parts = self.path.split("/") + if (len(url_parts) > 1 and + url_parts[1] in self.pypi_server.static_uri_paths): + data = None + # always take the last first. + fs_paths = [] + fs_paths.extend(self.pypi_server.static_filesystem_paths) + fs_paths.reverse() + relative_path = self.path + for fs_path in fs_paths: + try: + if self.path.endswith("/"): + relative_path += "index.html" + + if relative_path.endswith('.tar.gz'): + with open(fs_path + relative_path, 'br') as file: + data = file.read() + headers = [('Content-type', 'application/x-gtar')] + else: + with open(fs_path + relative_path) as file: + data = file.read().encode() + headers = [('Content-type', 'text/html')] + + headers.append(('Content-Length', len(data))) + self.make_response(data, headers=headers) + + except IOError: + pass + + if data is None: + self.make_response("Not found", 404) + + # otherwise serve the content from get_next_response + else: + # send back a response + status, headers, data = self.pypi_server.get_next_response() + self.make_response(data, status, headers) + + do_POST = do_GET = do_DELETE = do_PUT = serve_request + + def make_response(self, data, status=200, + headers=[('Content-type', 'text/html')]): + """Send the response to the HTTP client""" + if not isinstance(status, int): + try: + status = int(status) + except ValueError: + # we probably got something like YYY Codename. + # Just get the first 3 digits + status = int(status[:3]) + + self.send_response(status) + for header, value in headers: + self.send_header(header, value) + self.end_headers() + + if type(data) is str: + data = data.encode() + + self.wfile.write(data) + + +class PyPIXMLRPCServer(SimpleXMLRPCServer): + def server_bind(self): + """Override server_bind to store the server name.""" + socketserver.TCPServer.server_bind(self) + host, port = self.socket.getsockname()[:2] + self.server_port = port + + +class MockDist: + """Fake distribution, used in the Mock PyPI Server""" + + def __init__(self, name, version="1.0", hidden=False, url="http://url/", + type="sdist", filename="", size=10000, + digest="123456", downloads=7, has_sig=False, + python_version="source", comment="comment", + author="John Doe", author_email="john@doe.name", + maintainer="Main Tayner", maintainer_email="maintainer_mail", + project_url="http://project_url/", homepage="http://homepage/", + keywords="", platform="UNKNOWN", classifiers=[], licence="", + description="Description", summary="Summary", stable_version="", + ordering="", documentation_id="", code_kwalitee_id="", + installability_id="", obsoletes=[], obsoletes_dist=[], + provides=[], provides_dist=[], requires=[], requires_dist=[], + requires_external=[], requires_python=""): + + # basic fields + self.name = name + self.version = version + self.hidden = hidden + + # URL infos + self.url = url + self.digest = digest + self.downloads = downloads + self.has_sig = has_sig + self.python_version = python_version + self.comment = comment + self.type = type + + # metadata + self.author = author + self.author_email = author_email + self.maintainer = maintainer + self.maintainer_email = maintainer_email + self.project_url = project_url + self.homepage = homepage + self.keywords = keywords + self.platform = platform + self.classifiers = classifiers + self.licence = licence + self.description = description + self.summary = summary + self.stable_version = stable_version + self.ordering = ordering + self.cheesecake_documentation_id = documentation_id + self.cheesecake_code_kwalitee_id = code_kwalitee_id + self.cheesecake_installability_id = installability_id + + self.obsoletes = obsoletes + self.obsoletes_dist = obsoletes_dist + self.provides = provides + self.provides_dist = provides_dist + self.requires = requires + self.requires_dist = requires_dist + self.requires_external = requires_external + self.requires_python = requires_python + + def url_infos(self): + return { + 'url': self.url, + 'packagetype': self.type, + 'filename': 'filename.tar.gz', + 'size': '6000', + 'md5_digest': self.digest, + 'downloads': self.downloads, + 'has_sig': self.has_sig, + 'python_version': self.python_version, + 'comment_text': self.comment, + } + + def metadata(self): + return { + 'maintainer': self.maintainer, + 'project_url': [self.project_url], + 'maintainer_email': self.maintainer_email, + 'cheesecake_code_kwalitee_id': self.cheesecake_code_kwalitee_id, + 'keywords': self.keywords, + 'obsoletes_dist': self.obsoletes_dist, + 'requires_external': self.requires_external, + 'author': self.author, + 'author_email': self.author_email, + 'download_url': self.url, + 'platform': self.platform, + 'version': self.version, + 'obsoletes': self.obsoletes, + 'provides': self.provides, + 'cheesecake_documentation_id': self.cheesecake_documentation_id, + '_pypi_hidden': self.hidden, + 'description': self.description, + '_pypi_ordering': 19, + 'requires_dist': self.requires_dist, + 'requires_python': self.requires_python, + 'classifiers': [], + 'name': self.name, + 'licence': self.licence, + 'summary': self.summary, + 'home_page': self.homepage, + 'stable_version': self.stable_version, + 'provides_dist': self.provides_dist or "%s (%s)" % (self.name, + self.version), + 'requires': self.requires, + 'cheesecake_installability_id': self.cheesecake_installability_id, + } + + def search_result(self): + return { + '_pypi_ordering': 0, + 'version': self.version, + 'name': self.name, + 'summary': self.summary, + } + + +class XMLRPCMockIndex: + """Mock XMLRPC server""" + + def __init__(self, dists=[]): + self._dists = dists + self._search_result = [] + + def add_distributions(self, dists): + for dist in dists: + self._dists.append(MockDist(**dist)) + + def set_distributions(self, dists): + self._dists = [] + self.add_distributions(dists) + + def set_search_result(self, result): + """set a predefined search result""" + self._search_result = result + + def _get_search_results(self): + results = [] + for name in self._search_result: + found_dist = [d for d in self._dists if d.name == name] + if found_dist: + results.append(found_dist[0]) + else: + dist = MockDist(name) + results.append(dist) + self._dists.append(dist) + return [r.search_result() for r in results] + + def list_packages(self): + return [d.name for d in self._dists] + + def package_releases(self, package_name, show_hidden=False): + if show_hidden: + # return all + return [d.version for d in self._dists if d.name == package_name] + else: + # return only un-hidden + return [d.version for d in self._dists if d.name == package_name + and not d.hidden] + + def release_urls(self, package_name, version): + return [d.url_infos() for d in self._dists + if d.name == package_name and d.version == version] + + def release_data(self, package_name, version): + release = [d for d in self._dists + if d.name == package_name and d.version == version] + if release: + return release[0].metadata() + else: + return {} + + def search(self, spec, operator="and"): + return self._get_search_results() diff --git a/Lib/packaging/tests/pypi_test_server.py b/Lib/packaging/tests/pypi_test_server.py new file mode 100644 index 0000000000..8c8c641eb0 --- /dev/null +++ b/Lib/packaging/tests/pypi_test_server.py @@ -0,0 +1,59 @@ +"""Test PyPI Server implementation at testpypi.python.org, to use in tests. + +This is a drop-in replacement for the mock pypi server for testing against a +real pypi server hosted by python.org especially for testing against. +""" + +import unittest + +PYPI_DEFAULT_STATIC_PATH = None + + +def use_xmlrpc_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = True + return use_pypi_server(*server_args, **server_kwargs) + + +def use_http_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = False + return use_pypi_server(*server_args, **server_kwargs) + + +def use_pypi_server(*server_args, **server_kwargs): + """Decorator to make use of the PyPIServer for test methods, + just when needed, and not for the entire duration of the testcase. + """ + def wrapper(func): + def wrapped(*args, **kwargs): + server = PyPIServer(*server_args, **server_kwargs) + func(server=server, *args, **kwargs) + return wrapped + return wrapper + + +class PyPIServerTestCase(unittest.TestCase): + + def setUp(self): + super(PyPIServerTestCase, self).setUp() + self.pypi = PyPIServer() + self.pypi.start() + self.addCleanup(self.pypi.stop) + + +class PyPIServer: + """Shim to access testpypi.python.org, for testing a real server.""" + + def __init__(self, test_static_path=None, + static_filesystem_paths=["default"], + static_uri_paths=["simple"], serve_xmlrpc=False): + self.address = ('testpypi.python.org', '80') + + def start(self): + pass + + def stop(self): + pass + + @property + def full_address(self): + return "http://%s:%s" % self.address diff --git a/Lib/packaging/tests/pypiserver/downloads_with_md5/packages/source/f/foobar/foobar-0.1.tar.gz b/Lib/packaging/tests/pypiserver/downloads_with_md5/packages/source/f/foobar/foobar-0.1.tar.gz Binary files differnew file mode 100644 index 0000000000..333961eb18 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/downloads_with_md5/packages/source/f/foobar/foobar-0.1.tar.gz diff --git a/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/badmd5/badmd5-0.1.tar.gz b/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/badmd5/badmd5-0.1.tar.gz new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/badmd5/badmd5-0.1.tar.gz diff --git a/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/badmd5/index.html b/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/badmd5/index.html new file mode 100644 index 0000000000..b89f1bdb84 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/badmd5/index.html @@ -0,0 +1,3 @@ +<html><body> +<a href="badmd5-0.1.tar.gz#md5=3e3d86693d6564c807272b11b3069dfe" rel="download">badmd5-0.1.tar.gz</a><br/> +</body></html> diff --git a/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/foobar/index.html b/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/foobar/index.html new file mode 100644 index 0000000000..9e42b16d23 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/foobar/index.html @@ -0,0 +1,3 @@ +<html><body> +<a href="foobar-0.1.tar.gz#md5=fe18804c5b722ff024cabdf514924fc4" rel="download">foobar-0.1.tar.gz</a><br/> +</body></html> diff --git a/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/index.html b/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/index.html new file mode 100644 index 0000000000..9baee0479e --- /dev/null +++ b/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/index.html @@ -0,0 +1,2 @@ +<a href="foobar/">foobar/</a> +<a href="badmd5/">badmd5/</a> diff --git a/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/bar/index.html b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/bar/index.html new file mode 100644 index 0000000000..c3d42c5692 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/bar/index.html @@ -0,0 +1,6 @@ +<html><head><title>Links for bar</title></head><body><h1>Links for bar</h1> +<a rel="download" href="../../packages/source/F/bar/bar-1.0.tar.gz">bar-1.0.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/bar/bar-1.0.1.tar.gz">bar-1.0.1.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/bar/bar-2.0.tar.gz">bar-2.0.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/bar/bar-2.0.1.tar.gz">bar-2.0.1.tar.gz</a><br/> +</body></html> diff --git a/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/baz/index.html b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/baz/index.html new file mode 100644 index 0000000000..4f34312a30 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/baz/index.html @@ -0,0 +1,6 @@ +<html><head><title>Links for baz</title></head><body><h1>Links for baz</h1> +<a rel="download" href="../../packages/source/F/baz/baz-1.0.tar.gz">baz-1.0.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/baz/baz-1.0.1.tar.gz">baz-1.0.1.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/baz/baz-2.0.tar.gz">baz-2.0.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/baz/baz-2.0.1.tar.gz">baz-2.0.1.tar.gz</a><br/> +</body></html> diff --git a/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/foo/index.html b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/foo/index.html new file mode 100644 index 0000000000..0565e11bdd --- /dev/null +++ b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/foo/index.html @@ -0,0 +1,6 @@ +<html><head><title>Links for foo</title></head><body><h1>Links for foo</h1> +<a rel="download" href="../../packages/source/F/foo/foo-1.0.tar.gz">foo-1.0.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/foo/foo-1.0.1.tar.gz">foo-1.0.1.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/foo/foo-2.0.tar.gz">foo-2.0.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/foo/foo-2.0.1.tar.gz">foo-2.0.1.tar.gz</a><br/> +</body></html> diff --git a/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/index.html b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/index.html new file mode 100644 index 0000000000..a70cfd345f --- /dev/null +++ b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/index.html @@ -0,0 +1,3 @@ +<a href="foo/">foo/</a> +<a href="bar/">bar/</a> +<a href="baz/">baz/</a> diff --git a/Lib/packaging/tests/pypiserver/project_list/simple/index.html b/Lib/packaging/tests/pypiserver/project_list/simple/index.html new file mode 100644 index 0000000000..b36d728a0b --- /dev/null +++ b/Lib/packaging/tests/pypiserver/project_list/simple/index.html @@ -0,0 +1,5 @@ +<a class="test" href="yeah">FooBar-bar</a> +<a class="test" href="yeah">Foobar-baz</a> +<a class="test" href="yeah">Baz-FooBar</a> +<a class="test" href="yeah">Baz</a> +<a class="test" href="yeah">Foo</a> diff --git a/Lib/packaging/tests/pypiserver/test_found_links/simple/foobar/index.html b/Lib/packaging/tests/pypiserver/test_found_links/simple/foobar/index.html new file mode 100644 index 0000000000..a282a4ea31 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/test_found_links/simple/foobar/index.html @@ -0,0 +1,6 @@ +<html><head><title>Links for Foobar</title></head><body><h1>Links for Foobar</h1> +<a rel="download" href="../../packages/source/F/Foobar/Foobar-1.0.tar.gz#md5=98fa833fdabcdd78d00245aead66c174">Foobar-1.0.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/Foobar/Foobar-1.0.1.tar.gz#md5=2351efb20f6b7b5d9ce80fa4cb1bd9ca">Foobar-1.0.1.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/Foobar/Foobar-2.0.tar.gz#md5=98fa833fdabcdd78d00245aead66c274">Foobar-2.0.tar.gz</a><br/> +<a rel="download" href="../../packages/source/F/Foobar/Foobar-2.0.1.tar.gz#md5=2352efb20f6b7b5d9ce80fa4cb2bd9ca">Foobar-2.0.1.tar.gz</a><br/> +</body></html> diff --git a/Lib/packaging/tests/pypiserver/test_found_links/simple/index.html b/Lib/packaging/tests/pypiserver/test_found_links/simple/index.html new file mode 100644 index 0000000000..a1a7bb7282 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/test_found_links/simple/index.html @@ -0,0 +1 @@ +<a href="foobar/">foobar/</a> diff --git a/Lib/packaging/tests/pypiserver/test_pypi_server/external/index.html b/Lib/packaging/tests/pypiserver/test_pypi_server/external/index.html new file mode 100644 index 0000000000..265ee0af95 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/test_pypi_server/external/index.html @@ -0,0 +1 @@ +index.html from external server diff --git a/Lib/packaging/tests/pypiserver/test_pypi_server/simple/index.html b/Lib/packaging/tests/pypiserver/test_pypi_server/simple/index.html new file mode 100644 index 0000000000..6f976676f5 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/test_pypi_server/simple/index.html @@ -0,0 +1 @@ +Yeah diff --git a/Lib/packaging/tests/pypiserver/with_externals/external/external.html b/Lib/packaging/tests/pypiserver/with_externals/external/external.html new file mode 100644 index 0000000000..92e4702f63 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/with_externals/external/external.html @@ -0,0 +1,3 @@ +<html><body> +<a href="/foobar-0.1.tar.gz#md5=1__bad_md5___">bad old link</a> +</body></html> diff --git a/Lib/packaging/tests/pypiserver/with_externals/simple/foobar/index.html b/Lib/packaging/tests/pypiserver/with_externals/simple/foobar/index.html new file mode 100644 index 0000000000..b100a26542 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/with_externals/simple/foobar/index.html @@ -0,0 +1,4 @@ +<html><body> +<a rel ="download" href="/foobar-0.1.tar.gz#md5=12345678901234567">foobar-0.1.tar.gz</a><br/> +<a href="../../external/external.html" rel="homepage">external homepage</a><br/> +</body></html> diff --git a/Lib/packaging/tests/pypiserver/with_externals/simple/index.html b/Lib/packaging/tests/pypiserver/with_externals/simple/index.html new file mode 100644 index 0000000000..a1a7bb7282 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/with_externals/simple/index.html @@ -0,0 +1 @@ +<a href="foobar/">foobar/</a> diff --git a/Lib/packaging/tests/pypiserver/with_norel_links/external/homepage.html b/Lib/packaging/tests/pypiserver/with_norel_links/external/homepage.html new file mode 100644 index 0000000000..1cc0c32f1b --- /dev/null +++ b/Lib/packaging/tests/pypiserver/with_norel_links/external/homepage.html @@ -0,0 +1,7 @@ +<html> +<body> +<p>a rel=homepage HTML page</p> +<a href="/foobar-2.0.tar.gz">foobar 2.0</a> +</body> +</html> + diff --git a/Lib/packaging/tests/pypiserver/with_norel_links/external/nonrel.html b/Lib/packaging/tests/pypiserver/with_norel_links/external/nonrel.html new file mode 100644 index 0000000000..f6ace22054 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/with_norel_links/external/nonrel.html @@ -0,0 +1 @@ +A page linked without rel="download" or rel="homepage" link. diff --git a/Lib/packaging/tests/pypiserver/with_norel_links/simple/foobar/index.html b/Lib/packaging/tests/pypiserver/with_norel_links/simple/foobar/index.html new file mode 100644 index 0000000000..171df9360c --- /dev/null +++ b/Lib/packaging/tests/pypiserver/with_norel_links/simple/foobar/index.html @@ -0,0 +1,6 @@ +<html><body> +<a rel="download" href="/foobar-0.1.tar.gz" rel="download">foobar-0.1.tar.gz</a><br/> +<a href="../../external/homepage.html" rel="homepage">external homepage</a><br/> +<a href="../../external/nonrel.html">unrelated link</a><br/> +<a href="/unrelated-0.2.tar.gz">unrelated download</a></br/> +</body></html> diff --git a/Lib/packaging/tests/pypiserver/with_norel_links/simple/index.html b/Lib/packaging/tests/pypiserver/with_norel_links/simple/index.html new file mode 100644 index 0000000000..a1a7bb7282 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/with_norel_links/simple/index.html @@ -0,0 +1 @@ +<a href="foobar/">foobar/</a> diff --git a/Lib/packaging/tests/pypiserver/with_real_externals/simple/foobar/index.html b/Lib/packaging/tests/pypiserver/with_real_externals/simple/foobar/index.html new file mode 100644 index 0000000000..b2885ae384 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/with_real_externals/simple/foobar/index.html @@ -0,0 +1,4 @@ +<html><body> +<a rel="download" href="/foobar-0.1.tar.gz#md5=0_correct_md5">foobar-0.1.tar.gz</a><br/> +<a href="http://a-really-external-website/external/external.html" rel="homepage">external homepage</a><br/> +</body></html> diff --git a/Lib/packaging/tests/pypiserver/with_real_externals/simple/index.html b/Lib/packaging/tests/pypiserver/with_real_externals/simple/index.html new file mode 100644 index 0000000000..a1a7bb7282 --- /dev/null +++ b/Lib/packaging/tests/pypiserver/with_real_externals/simple/index.html @@ -0,0 +1 @@ +<a href="foobar/">foobar/</a> diff --git a/Lib/packaging/tests/support.py b/Lib/packaging/tests/support.py new file mode 100644 index 0000000000..33d59668b3 --- /dev/null +++ b/Lib/packaging/tests/support.py @@ -0,0 +1,358 @@ +"""Support code for packaging test cases. + +*This module should not be considered public: its content and API may +change in incompatible ways.* + +A few helper classes are provided: LoggingCatcher, TempdirManager and +EnvironRestorer. They are written to be used as mixins:: + + from packaging.tests import unittest + from packaging.tests.support import LoggingCatcher + + class SomeTestCase(LoggingCatcher, unittest.TestCase): + ... + +If you need to define a setUp method on your test class, you have to +call the mixin class' setUp method or it won't work (same thing for +tearDown): + + def setUp(self): + super(SomeTestCase, self).setUp() + ... # other setup code + +Also provided is a DummyCommand class, useful to mock commands in the +tests of another command that needs them, for example to fake +compilation in build_ext (this requires that the mock build_ext command +be injected into the distribution object's command_obj dictionary). + +For tests that need to compile an extension module, use the +copy_xxmodule_c and fixup_build_ext functions. + +Each class or function has a docstring to explain its purpose and usage. +Existing tests should also be used as examples. +""" + +import os +import sys +import shutil +import logging +import weakref +import tempfile +import sysconfig + +from packaging.dist import Distribution +from packaging.tests import unittest +from test.support import requires_zlib, unlink + +# define __all__ to make pydoc more useful +__all__ = [ + # TestCase mixins + 'LoggingCatcher', 'TempdirManager', 'EnvironRestorer', + # mocks + 'DummyCommand', 'TestDistribution', + # misc. functions and decorators + 'fake_dec', 'create_distribution', 'copy_xxmodule_c', 'fixup_build_ext', + # imported from this module for backport purposes + 'unittest', 'requires_zlib', 'skip_unless_symlink', +] + + +logger = logging.getLogger('packaging') +logger2to3 = logging.getLogger('RefactoringTool') + + +class _TestHandler(logging.handlers.BufferingHandler): + # stolen and adapted from test.support + + def __init__(self): + logging.handlers.BufferingHandler.__init__(self, 0) + self.setLevel(logging.DEBUG) + + def shouldFlush(self): + return False + + def emit(self, record): + self.buffer.append(record) + + +class LoggingCatcher: + """TestCase-compatible mixin to receive logging calls. + + Upon setUp, instances of this classes get a BufferingHandler that's + configured to record all messages logged to the 'packaging' logger. + + Use get_logs to retrieve messages and self.loghandler.flush to discard + them. get_logs automatically flushes the logs; if you test code that + generates logging messages but don't use get_logs, you have to flush + manually before doing other checks on logging message, otherwise you + will get irrelevant results. See example in test_command_check. + """ + + def setUp(self): + super(LoggingCatcher, self).setUp() + self.loghandler = handler = _TestHandler() + self._old_levels = logger.level, logger2to3.level + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) # we want all messages + logger2to3.setLevel(logging.CRITICAL) # we don't want 2to3 messages + + def tearDown(self): + handler = self.loghandler + # All this is necessary to properly shut down the logging system and + # avoid a regrtest complaint. Thanks to Vinay Sajip for the help. + handler.close() + logger.removeHandler(handler) + for ref in weakref.getweakrefs(handler): + logging._removeHandlerRef(ref) + del self.loghandler + logger.setLevel(self._old_levels[0]) + logger2to3.setLevel(self._old_levels[1]) + super(LoggingCatcher, self).tearDown() + + def get_logs(self, *levels): + """Return all log messages with level in *levels*. + + Without explicit levels given, returns all messages. *levels* defaults + to all levels. For log calls with arguments (i.e. + logger.info('bla bla %r', arg)), the messages will be formatted before + being returned (e.g. "bla bla 'thing'"). + + Returns a list. Automatically flushes the loghandler after being + called. + + Example: self.get_logs(logging.WARN, logging.DEBUG). + """ + if not levels: + messages = [log.getMessage() for log in self.loghandler.buffer] + else: + messages = [log.getMessage() for log in self.loghandler.buffer + if log.levelno in levels] + self.loghandler.flush() + return messages + + +class TempdirManager: + """TestCase-compatible mixin to create temporary directories and files. + + Directories and files created in a test_* method will be removed after it + has run. + """ + + def setUp(self): + super(TempdirManager, self).setUp() + self._olddir = os.getcwd() + self._basetempdir = tempfile.mkdtemp() + self._files = [] + + def tearDown(self): + for handle, name in self._files: + handle.close() + unlink(name) + + os.chdir(self._olddir) + shutil.rmtree(self._basetempdir) + super(TempdirManager, self).tearDown() + + def mktempfile(self): + """Create a read-write temporary file and return it.""" + fd, fn = tempfile.mkstemp(dir=self._basetempdir) + os.close(fd) + fp = open(fn, 'w+') + self._files.append((fp, fn)) + return fp + + def mkdtemp(self): + """Create a temporary directory and return its path.""" + d = tempfile.mkdtemp(dir=self._basetempdir) + return d + + def write_file(self, path, content='xxx', encoding=None): + """Write a file at the given path. + + path can be a string, a tuple or a list; if it's a tuple or list, + os.path.join will be used to produce a path. + """ + if isinstance(path, (list, tuple)): + path = os.path.join(*path) + with open(path, 'w', encoding=encoding) as f: + f.write(content) + + def create_dist(self, **kw): + """Create a stub distribution object and files. + + This function creates a Distribution instance (use keyword arguments + to customize it) and a temporary directory with a project structure + (currently an empty directory). + + It returns the path to the directory and the Distribution instance. + You can use self.write_file to write any file in that + directory, e.g. setup scripts or Python modules. + """ + if 'name' not in kw: + kw['name'] = 'foo' + tmp_dir = self.mkdtemp() + project_dir = os.path.join(tmp_dir, kw['name']) + os.mkdir(project_dir) + dist = Distribution(attrs=kw) + return project_dir, dist + + def assertIsFile(self, *args): + path = os.path.join(*args) + dirname = os.path.dirname(path) + file = os.path.basename(path) + if os.path.isdir(dirname): + files = os.listdir(dirname) + msg = "%s not found in %s: %s" % (file, dirname, files) + assert os.path.isfile(path), msg + else: + raise AssertionError( + '%s not found. %s does not exist' % (file, dirname)) + + def assertIsNotFile(self, *args): + path = os.path.join(*args) + self.assertFalse(os.path.isfile(path), "%r exists" % path) + + +class EnvironRestorer: + """TestCase-compatible mixin to restore or delete environment variables. + + The variables to restore (or delete if they were not originally present) + must be explicitly listed in self.restore_environ. It's better to be + aware of what we're modifying instead of saving and restoring the whole + environment. + """ + + def setUp(self): + super(EnvironRestorer, self).setUp() + self._saved = [] + self._added = [] + for key in self.restore_environ: + if key in os.environ: + self._saved.append((key, os.environ[key])) + else: + self._added.append(key) + + def tearDown(self): + for key, value in self._saved: + os.environ[key] = value + for key in self._added: + os.environ.pop(key, None) + super(EnvironRestorer, self).tearDown() + + +class DummyCommand: + """Class to store options for retrieval via set_undefined_options(). + + Useful for mocking one dependency command in the tests for another + command, see e.g. the dummy build command in test_build_scripts. + """ + # XXX does not work with dist.get_reinitialized_command, which typechecks + # and wants a finalized attribute + + def __init__(self, **kwargs): + for kw, val in kwargs.items(): + setattr(self, kw, val) + + def ensure_finalized(self): + pass + + +class TestDistribution(Distribution): + """Distribution subclasses that avoids the default search for + configuration files. + + The ._config_files attribute must be set before + .parse_config_files() is called. + """ + + def find_config_files(self): + return self._config_files + + +def create_distribution(configfiles=()): + """Prepares a distribution with given config files parsed.""" + d = TestDistribution() + d.config.find_config_files = d.find_config_files + d._config_files = configfiles + d.parse_config_files() + d.parse_command_line() + return d + + +def fake_dec(*args, **kw): + """Fake decorator""" + def _wrap(func): + def __wrap(*args, **kw): + return func(*args, **kw) + return __wrap + return _wrap + + +def copy_xxmodule_c(directory): + """Helper for tests that need the xxmodule.c source file. + + Example use: + + def test_compile(self): + copy_xxmodule_c(self.tmpdir) + self.assertIn('xxmodule.c', os.listdir(self.tmpdir)) + + If the source file can be found, it will be copied to *directory*. If not, + the test will be skipped. Errors during copy are not caught. + """ + filename = _get_xxmodule_path() + if filename is None: + raise unittest.SkipTest('cannot find xxmodule.c (test must run in ' + 'the python build dir)') + shutil.copy(filename, directory) + + +def _get_xxmodule_path(): + srcdir = sysconfig.get_config_var('srcdir') + candidates = [ + # use installed copy if available + os.path.join(os.path.dirname(__file__), 'xxmodule.c'), + # otherwise try using copy from build directory + os.path.join(srcdir, 'Modules', 'xxmodule.c'), + ] + for path in candidates: + if os.path.exists(path): + return path + + +def fixup_build_ext(cmd): + """Function needed to make build_ext tests pass. + + When Python was built with --enable-shared on Unix, -L. is not enough to + find libpython<blah>.so, because regrtest runs in a tempdir, not in the + source directory where the .so lives. + + When Python was built with in debug mode on Windows, build_ext commands + need their debug attribute set, and it is not done automatically for + some reason. + + This function handles both of these things. Example use: + + cmd = build_ext(dist) + support.fixup_build_ext(cmd) + cmd.ensure_finalized() + """ + if os.name == 'nt': + cmd.debug = sys.executable.endswith('_d.exe') + elif sysconfig.get_config_var('Py_ENABLE_SHARED'): + # To further add to the shared builds fun on Unix, we can't just add + # library_dirs to the Extension() instance because that doesn't get + # plumbed through to the final compiler command. + runshared = sysconfig.get_config_var('RUNSHARED') + if runshared is None: + cmd.library_dirs = ['.'] + else: + name, equals, value = runshared.partition('=') + cmd.library_dirs = value.split(os.pathsep) + + +try: + from test.support import skip_unless_symlink +except ImportError: + skip_unless_symlink = unittest.skip( + 'requires test.support.skip_unless_symlink') diff --git a/Lib/packaging/tests/test_ccompiler.py b/Lib/packaging/tests/test_ccompiler.py new file mode 100644 index 0000000000..dd4bdd9d95 --- /dev/null +++ b/Lib/packaging/tests/test_ccompiler.py @@ -0,0 +1,15 @@ +"""Tests for distutils.compiler.ccompiler.""" + +from packaging.compiler import ccompiler +from packaging.tests import unittest, support + + +class CCompilerTestCase(unittest.TestCase): + pass # XXX need some tests on CCompiler + + +def test_suite(): + return unittest.makeSuite(CCompilerTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_bdist.py b/Lib/packaging/tests/test_command_bdist.py new file mode 100644 index 0000000000..dd10188391 --- /dev/null +++ b/Lib/packaging/tests/test_command_bdist.py @@ -0,0 +1,58 @@ +"""Tests for distutils.command.bdist.""" +import os +from packaging.command.bdist import bdist, show_formats +from packaging.tests import unittest, support, captured_stdout + + +class BuildTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_formats(self): + # let's create a command and make sure + # we can set the format + dist = self.create_dist()[1] + cmd = bdist(dist) + cmd.formats = ['msi'] + cmd.ensure_finalized() + self.assertEqual(cmd.formats, ['msi']) + + # what formats does bdist offer? + # XXX hard-coded lists are not the best way to find available bdist_* + # commands; we should add a registry + formats = ['bztar', 'gztar', 'msi', 'tar', 'wininst', 'zip'] + found = sorted(cmd.format_command) + self.assertEqual(found, formats) + + def test_skip_build(self): + # bug #10946: bdist --skip-build should trickle down to subcommands + dist = self.create_dist()[1] + cmd = bdist(dist) + cmd.skip_build = True + cmd.ensure_finalized() + dist.command_obj['bdist'] = cmd + + names = ['bdist_dumb', 'bdist_wininst'] + if os.name == 'nt': + names.append('bdist_msi') + + for name in names: + subcmd = cmd.get_finalized_command(name) + self.assertTrue(subcmd.skip_build, + '%s should take --skip-build from bdist' % name) + + def test_show_formats(self): + __, stdout = captured_stdout(show_formats) + + # the output should be a header line + one line per format + num_formats = len(bdist.format_commands) + output = [line for line in stdout.split('\n') + if line.strip().startswith('--formats=')] + self.assertEqual(len(output), num_formats) + + +def test_suite(): + return unittest.makeSuite(BuildTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_command_bdist_dumb.py b/Lib/packaging/tests/test_command_bdist_dumb.py new file mode 100644 index 0000000000..cc03fa53c6 --- /dev/null +++ b/Lib/packaging/tests/test_command_bdist_dumb.py @@ -0,0 +1,82 @@ +"""Tests for distutils.command.bdist_dumb.""" + +import os +import packaging.util + +from packaging.dist import Distribution +from packaging.command.bdist_dumb import bdist_dumb +from packaging.tests import unittest, support +from packaging.tests.support import requires_zlib + + +class BuildDumbTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def setUp(self): + super(BuildDumbTestCase, self).setUp() + self.old_location = os.getcwd() + + def tearDown(self): + os.chdir(self.old_location) + packaging.util._path_created.clear() + super(BuildDumbTestCase, self).tearDown() + + @requires_zlib + def test_simple_built(self): + + # let's create a simple package + tmp_dir = self.mkdtemp() + pkg_dir = os.path.join(tmp_dir, 'foo') + os.mkdir(pkg_dir) + self.write_file((pkg_dir, 'foo.py'), '#') + self.write_file((pkg_dir, 'MANIFEST.in'), 'include foo.py') + self.write_file((pkg_dir, 'README'), '') + + dist = Distribution({'name': 'foo', 'version': '0.1', + 'py_modules': ['foo'], + 'home-page': 'xxx', 'author': 'xxx', + 'author_email': 'xxx'}) + os.chdir(pkg_dir) + cmd = bdist_dumb(dist) + + # so the output is the same no matter + # what is the platform + cmd.format = 'zip' + + cmd.ensure_finalized() + cmd.run() + + # see what we have + dist_created = os.listdir(os.path.join(pkg_dir, 'dist')) + base = "%s.%s" % (dist.get_fullname(), cmd.plat_name) + if os.name == 'os2': + base = base.replace(':', '-') + + wanted = ['%s.zip' % base] + self.assertEqual(dist_created, wanted) + + # now let's check what we have in the zip file + # XXX to be done + + def test_finalize_options(self): + pkg_dir, dist = self.create_dist() + os.chdir(pkg_dir) + cmd = bdist_dumb(dist) + self.assertEqual(cmd.bdist_dir, None) + cmd.finalize_options() + + # bdist_dir is initialized to bdist_base/dumb if not set + base = cmd.get_finalized_command('bdist').bdist_base + self.assertEqual(cmd.bdist_dir, os.path.join(base, 'dumb')) + + # the format is set to a default value depending on the os.name + default = cmd.default_format[os.name] + self.assertEqual(cmd.format, default) + + +def test_suite(): + return unittest.makeSuite(BuildDumbTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_command_bdist_msi.py b/Lib/packaging/tests/test_command_bdist_msi.py new file mode 100644 index 0000000000..fded962dbf --- /dev/null +++ b/Lib/packaging/tests/test_command_bdist_msi.py @@ -0,0 +1,25 @@ +"""Tests for distutils.command.bdist_msi.""" +import sys + +from packaging.tests import unittest, support + + +class BDistMSITestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") + def test_minimal(self): + # minimal test XXX need more tests + from packaging.command.bdist_msi import bdist_msi + pkg_pth, dist = self.create_dist() + cmd = bdist_msi(dist) + cmd.ensure_finalized() + + +def test_suite(): + return unittest.makeSuite(BDistMSITestCase) + + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_command_bdist_wininst.py b/Lib/packaging/tests/test_command_bdist_wininst.py new file mode 100644 index 0000000000..09bdaadfc9 --- /dev/null +++ b/Lib/packaging/tests/test_command_bdist_wininst.py @@ -0,0 +1,32 @@ +"""Tests for distutils.command.bdist_wininst.""" + +from packaging.command.bdist_wininst import bdist_wininst +from packaging.tests import unittest, support + + +class BuildWinInstTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_get_exe_bytes(self): + + # issue5731: command was broken on non-windows platforms + # this test makes sure it works now for every platform + # let's create a command + pkg_pth, dist = self.create_dist() + cmd = bdist_wininst(dist) + cmd.ensure_finalized() + + # let's run the code that finds the right wininst*.exe file + # and make sure it finds it and returns its content + # no matter what platform we have + exe_file = cmd.get_exe_bytes() + self.assertGreater(len(exe_file), 10) + + +def test_suite(): + return unittest.makeSuite(BuildWinInstTestCase) + + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_command_build.py b/Lib/packaging/tests/test_command_build.py new file mode 100644 index 0000000000..91fbe42a0d --- /dev/null +++ b/Lib/packaging/tests/test_command_build.py @@ -0,0 +1,55 @@ +"""Tests for distutils.command.build.""" +import os +import sys + +from packaging.command.build import build +from sysconfig import get_platform +from packaging.tests import unittest, support + + +class BuildTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_finalize_options(self): + pkg_dir, dist = self.create_dist() + cmd = build(dist) + cmd.finalize_options() + + # if not specified, plat_name gets the current platform + self.assertEqual(cmd.plat_name, get_platform()) + + # build_purelib is build + lib + wanted = os.path.join(cmd.build_base, 'lib') + self.assertEqual(cmd.build_purelib, wanted) + + # build_platlib is 'build/lib.platform-x.x[-pydebug]' + # examples: + # build/lib.macosx-10.3-i386-2.7 + plat_spec = '.%s-%s' % (cmd.plat_name, sys.version[0:3]) + if hasattr(sys, 'gettotalrefcount'): + self.assertTrue(cmd.build_platlib.endswith('-pydebug')) + plat_spec += '-pydebug' + wanted = os.path.join(cmd.build_base, 'lib' + plat_spec) + self.assertEqual(cmd.build_platlib, wanted) + + # by default, build_lib = build_purelib + self.assertEqual(cmd.build_lib, cmd.build_purelib) + + # build_temp is build/temp.<plat> + wanted = os.path.join(cmd.build_base, 'temp' + plat_spec) + self.assertEqual(cmd.build_temp, wanted) + + # build_scripts is build/scripts-x.x + wanted = os.path.join(cmd.build_base, 'scripts-' + sys.version[0:3]) + self.assertEqual(cmd.build_scripts, wanted) + + # executable is os.path.normpath(sys.executable) + self.assertEqual(cmd.executable, os.path.normpath(sys.executable)) + + +def test_suite(): + return unittest.makeSuite(BuildTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_build_clib.py b/Lib/packaging/tests/test_command_build_clib.py new file mode 100644 index 0000000000..a2a8583b0f --- /dev/null +++ b/Lib/packaging/tests/test_command_build_clib.py @@ -0,0 +1,141 @@ +"""Tests for distutils.command.build_clib.""" +import os +import sys + +from packaging.util import find_executable +from packaging.command.build_clib import build_clib +from packaging.errors import PackagingSetupError +from packaging.tests import unittest, support + + +class BuildCLibTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_check_library_dist(self): + pkg_dir, dist = self.create_dist() + cmd = build_clib(dist) + + # 'libraries' option must be a list + self.assertRaises(PackagingSetupError, cmd.check_library_list, 'foo') + + # each element of 'libraries' must a 2-tuple + self.assertRaises(PackagingSetupError, cmd.check_library_list, + ['foo1', 'foo2']) + + # first element of each tuple in 'libraries' + # must be a string (the library name) + self.assertRaises(PackagingSetupError, cmd.check_library_list, + [(1, 'foo1'), ('name', 'foo2')]) + + # library name may not contain directory separators + self.assertRaises(PackagingSetupError, cmd.check_library_list, + [('name', 'foo1'), + ('another/name', 'foo2')]) + + # second element of each tuple must be a dictionary (build info) + self.assertRaises(PackagingSetupError, cmd.check_library_list, + [('name', {}), + ('another', 'foo2')]) + + # those work + libs = [('name', {}), ('name', {'ok': 'good'})] + cmd.check_library_list(libs) + + def test_get_source_files(self): + pkg_dir, dist = self.create_dist() + cmd = build_clib(dist) + + # "in 'libraries' option 'sources' must be present and must be + # a list of source filenames + cmd.libraries = [('name', {})] + self.assertRaises(PackagingSetupError, cmd.get_source_files) + + cmd.libraries = [('name', {'sources': 1})] + self.assertRaises(PackagingSetupError, cmd.get_source_files) + + cmd.libraries = [('name', {'sources': ['a', 'b']})] + self.assertEqual(cmd.get_source_files(), ['a', 'b']) + + cmd.libraries = [('name', {'sources': ('a', 'b')})] + self.assertEqual(cmd.get_source_files(), ['a', 'b']) + + cmd.libraries = [('name', {'sources': ('a', 'b')}), + ('name2', {'sources': ['c', 'd']})] + self.assertEqual(cmd.get_source_files(), ['a', 'b', 'c', 'd']) + + def test_build_libraries(self): + pkg_dir, dist = self.create_dist() + cmd = build_clib(dist) + + class FakeCompiler: + def compile(*args, **kw): + pass + create_static_lib = compile + + cmd.compiler = FakeCompiler() + + # build_libraries is also doing a bit of type checking + lib = [('name', {'sources': 'notvalid'})] + self.assertRaises(PackagingSetupError, cmd.build_libraries, lib) + + lib = [('name', {'sources': []})] + cmd.build_libraries(lib) + + lib = [('name', {'sources': ()})] + cmd.build_libraries(lib) + + def test_finalize_options(self): + pkg_dir, dist = self.create_dist() + cmd = build_clib(dist) + + cmd.include_dirs = 'one-dir' + cmd.finalize_options() + self.assertEqual(cmd.include_dirs, ['one-dir']) + + cmd.include_dirs = None + cmd.finalize_options() + self.assertEqual(cmd.include_dirs, []) + + cmd.distribution.libraries = 'WONTWORK' + self.assertRaises(PackagingSetupError, cmd.finalize_options) + + @unittest.skipIf(sys.platform == 'win32', 'disabled on win32') + def test_run(self): + pkg_dir, dist = self.create_dist() + cmd = build_clib(dist) + + foo_c = os.path.join(pkg_dir, 'foo.c') + self.write_file(foo_c, 'int main(void) { return 1;}\n') + cmd.libraries = [('foo', {'sources': [foo_c]})] + + build_temp = os.path.join(pkg_dir, 'build') + os.mkdir(build_temp) + cmd.build_temp = build_temp + cmd.build_clib = build_temp + + # before we run the command, we want to make sure + # all commands are present on the system + # by creating a compiler and checking its executables + from packaging.compiler import new_compiler, customize_compiler + + compiler = new_compiler() + customize_compiler(compiler) + for ccmd in compiler.executables.values(): + if ccmd is None: + continue + if find_executable(ccmd[0]) is None: + raise unittest.SkipTest("can't test") + + # this should work + cmd.run() + + # let's check the result + self.assertIn('libfoo.a', os.listdir(build_temp)) + + +def test_suite(): + return unittest.makeSuite(BuildCLibTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_build_ext.py b/Lib/packaging/tests/test_command_build_ext.py new file mode 100644 index 0000000000..59d09bf710 --- /dev/null +++ b/Lib/packaging/tests/test_command_build_ext.py @@ -0,0 +1,416 @@ +import os +import sys +import site +import sysconfig +import textwrap +from io import StringIO +from packaging.dist import Distribution +from packaging.errors import (UnknownFileError, CompileError, + PackagingPlatformError) +from packaging.command.build_ext import build_ext +from packaging.compiler.extension import Extension +from test.script_helper import assert_python_ok + +from packaging.tests import support, unittest, verbose + + +class BuildExtTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + def setUp(self): + # Create a simple test environment + # Note that we're making changes to sys.path + super(BuildExtTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.old_user_base = site.USER_BASE + site.USER_BASE = self.mkdtemp() + + def tearDown(self): + # Get everything back to normal + if sys.version > "2.6": + site.USER_BASE = self.old_user_base + + super(BuildExtTestCase, self).tearDown() + + def test_build_ext(self): + support.copy_xxmodule_c(self.tmp_dir) + xx_c = os.path.join(self.tmp_dir, 'xxmodule.c') + xx_ext = Extension('xx', [xx_c]) + dist = Distribution({'name': 'xx', 'ext_modules': [xx_ext]}) + dist.package_dir = self.tmp_dir + cmd = build_ext(dist) + support.fixup_build_ext(cmd) + cmd.build_lib = self.tmp_dir + cmd.build_temp = self.tmp_dir + + old_stdout = sys.stdout + if not verbose: + # silence compiler output + sys.stdout = StringIO() + try: + cmd.ensure_finalized() + cmd.run() + finally: + sys.stdout = old_stdout + + code = """if 1: + import sys + sys.path.insert(0, %r) + + import xx + + for attr in ('error', 'foo', 'new', 'roj'): + assert hasattr(xx, attr) + + assert xx.foo(2, 5) == 7 + assert xx.foo(13, 15) == 28 + assert xx.new().demo() is None + doc = 'This is a template module just for instruction.' + assert xx.__doc__ == doc + assert isinstance(xx.Null(), xx.Null) + assert isinstance(xx.Str(), xx.Str)""" + code = code % self.tmp_dir + assert_python_ok('-c', code) + + def test_solaris_enable_shared(self): + dist = Distribution({'name': 'xx'}) + cmd = build_ext(dist) + old = sys.platform + + sys.platform = 'sunos' # fooling finalize_options + from sysconfig import _CONFIG_VARS + + old_var = _CONFIG_VARS.get('Py_ENABLE_SHARED') + _CONFIG_VARS['Py_ENABLE_SHARED'] = 1 + try: + cmd.ensure_finalized() + finally: + sys.platform = old + if old_var is None: + del _CONFIG_VARS['Py_ENABLE_SHARED'] + else: + _CONFIG_VARS['Py_ENABLE_SHARED'] = old_var + + # make sure we get some library dirs under solaris + self.assertGreater(len(cmd.library_dirs), 0) + + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') + def test_user_site(self): + dist = Distribution({'name': 'xx'}) + cmd = build_ext(dist) + + # making sure the user option is there + options = [name for name, short, label in + cmd.user_options] + self.assertIn('user', options) + + # setting a value + cmd.user = True + + # setting user based lib and include + lib = os.path.join(site.USER_BASE, 'lib') + incl = os.path.join(site.USER_BASE, 'include') + os.mkdir(lib) + os.mkdir(incl) + + # let's run finalize + cmd.ensure_finalized() + + # see if include_dirs and library_dirs + # were set + self.assertIn(lib, cmd.library_dirs) + self.assertIn(lib, cmd.rpath) + self.assertIn(incl, cmd.include_dirs) + + def test_optional_extension(self): + + # this extension will fail, but let's ignore this failure + # with the optional argument. + modules = [Extension('foo', ['xxx'], optional=False)] + dist = Distribution({'name': 'xx', 'ext_modules': modules}) + cmd = build_ext(dist) + cmd.ensure_finalized() + self.assertRaises((UnknownFileError, CompileError), + cmd.run) # should raise an error + + modules = [Extension('foo', ['xxx'], optional=True)] + dist = Distribution({'name': 'xx', 'ext_modules': modules}) + cmd = build_ext(dist) + cmd.ensure_finalized() + cmd.run() # should pass + + def test_finalize_options(self): + # Make sure Python's include directories (for Python.h, pyconfig.h, + # etc.) are in the include search path. + modules = [Extension('foo', ['xxx'], optional=False)] + dist = Distribution({'name': 'xx', 'ext_modules': modules}) + cmd = build_ext(dist) + cmd.finalize_options() + + py_include = sysconfig.get_path('include') + self.assertIn(py_include, cmd.include_dirs) + + plat_py_include = sysconfig.get_path('platinclude') + self.assertIn(plat_py_include, cmd.include_dirs) + + # make sure cmd.libraries is turned into a list + # if it's a string + cmd = build_ext(dist) + cmd.libraries = 'my_lib' + cmd.finalize_options() + self.assertEqual(cmd.libraries, ['my_lib']) + + # make sure cmd.library_dirs is turned into a list + # if it's a string + cmd = build_ext(dist) + cmd.library_dirs = 'my_lib_dir' + cmd.finalize_options() + self.assertIn('my_lib_dir', cmd.library_dirs) + + # make sure rpath is turned into a list + # if it's a list of os.pathsep's paths + cmd = build_ext(dist) + cmd.rpath = os.pathsep.join(['one', 'two']) + cmd.finalize_options() + self.assertEqual(cmd.rpath, ['one', 'two']) + + # XXX more tests to perform for win32 + + # make sure define is turned into 2-tuples + # strings if they are ','-separated strings + cmd = build_ext(dist) + cmd.define = 'one,two' + cmd.finalize_options() + self.assertEqual(cmd.define, [('one', '1'), ('two', '1')]) + + # make sure undef is turned into a list of + # strings if they are ','-separated strings + cmd = build_ext(dist) + cmd.undef = 'one,two' + cmd.finalize_options() + self.assertEqual(cmd.undef, ['one', 'two']) + + # make sure swig_opts is turned into a list + cmd = build_ext(dist) + cmd.swig_opts = None + cmd.finalize_options() + self.assertEqual(cmd.swig_opts, []) + + cmd = build_ext(dist) + cmd.swig_opts = '1 2' + cmd.finalize_options() + self.assertEqual(cmd.swig_opts, ['1', '2']) + + def test_get_source_files(self): + modules = [Extension('foo', ['xxx'], optional=False)] + dist = Distribution({'name': 'xx', 'ext_modules': modules}) + cmd = build_ext(dist) + cmd.ensure_finalized() + self.assertEqual(cmd.get_source_files(), ['xxx']) + + def test_compiler_option(self): + # cmd.compiler is an option and + # should not be overriden by a compiler instance + # when the command is run + dist = Distribution() + cmd = build_ext(dist) + cmd.compiler = 'unix' + cmd.ensure_finalized() + cmd.run() + self.assertEqual(cmd.compiler, 'unix') + + def test_get_outputs(self): + tmp_dir = self.mkdtemp() + c_file = os.path.join(tmp_dir, 'foo.c') + self.write_file(c_file, 'void PyInit_foo(void) {}\n') + ext = Extension('foo', [c_file], optional=False) + dist = Distribution({'name': 'xx', + 'ext_modules': [ext]}) + cmd = build_ext(dist) + support.fixup_build_ext(cmd) + cmd.ensure_finalized() + self.assertEqual(len(cmd.get_outputs()), 1) + + cmd.build_lib = os.path.join(self.tmp_dir, 'build') + cmd.build_temp = os.path.join(self.tmp_dir, 'tempt') + + # issue #5977 : distutils build_ext.get_outputs + # returns wrong result with --inplace + other_tmp_dir = os.path.realpath(self.mkdtemp()) + old_wd = os.getcwd() + os.chdir(other_tmp_dir) + try: + cmd.inplace = True + cmd.run() + so_file = cmd.get_outputs()[0] + finally: + os.chdir(old_wd) + self.assertTrue(os.path.exists(so_file)) + so_ext = sysconfig.get_config_var('SO') + self.assertTrue(so_file.endswith(so_ext)) + so_dir = os.path.dirname(so_file) + self.assertEqual(so_dir, other_tmp_dir) + + cmd.inplace = False + cmd.run() + so_file = cmd.get_outputs()[0] + self.assertTrue(os.path.exists(so_file)) + self.assertTrue(so_file.endswith(so_ext)) + so_dir = os.path.dirname(so_file) + self.assertEqual(so_dir, cmd.build_lib) + + # inplace = False, cmd.package = 'bar' + build_py = cmd.get_finalized_command('build_py') + build_py.package_dir = 'bar' + path = cmd.get_ext_fullpath('foo') + # checking that the last directory is the build_dir + path = os.path.split(path)[0] + self.assertEqual(path, cmd.build_lib) + + # inplace = True, cmd.package = 'bar' + cmd.inplace = True + other_tmp_dir = os.path.realpath(self.mkdtemp()) + old_wd = os.getcwd() + os.chdir(other_tmp_dir) + try: + path = cmd.get_ext_fullpath('foo') + finally: + os.chdir(old_wd) + # checking that the last directory is bar + path = os.path.split(path)[0] + lastdir = os.path.split(path)[-1] + self.assertEqual(lastdir, 'bar') + + def test_ext_fullpath(self): + ext = sysconfig.get_config_vars()['SO'] + # building lxml.etree inplace + #etree_c = os.path.join(self.tmp_dir, 'lxml.etree.c') + #etree_ext = Extension('lxml.etree', [etree_c]) + #dist = Distribution({'name': 'lxml', 'ext_modules': [etree_ext]}) + dist = Distribution() + cmd = build_ext(dist) + cmd.inplace = True + cmd.distribution.package_dir = 'src' + cmd.distribution.packages = ['lxml', 'lxml.html'] + curdir = os.getcwd() + wanted = os.path.join(curdir, 'src', 'lxml', 'etree' + ext) + path = cmd.get_ext_fullpath('lxml.etree') + self.assertEqual(wanted, path) + + # building lxml.etree not inplace + cmd.inplace = False + cmd.build_lib = os.path.join(curdir, 'tmpdir') + wanted = os.path.join(curdir, 'tmpdir', 'lxml', 'etree' + ext) + path = cmd.get_ext_fullpath('lxml.etree') + self.assertEqual(wanted, path) + + # building twisted.runner.portmap not inplace + build_py = cmd.get_finalized_command('build_py') + build_py.package_dir = None + cmd.distribution.packages = ['twisted', 'twisted.runner.portmap'] + path = cmd.get_ext_fullpath('twisted.runner.portmap') + wanted = os.path.join(curdir, 'tmpdir', 'twisted', 'runner', + 'portmap' + ext) + self.assertEqual(wanted, path) + + # building twisted.runner.portmap inplace + cmd.inplace = True + path = cmd.get_ext_fullpath('twisted.runner.portmap') + wanted = os.path.join(curdir, 'twisted', 'runner', 'portmap' + ext) + self.assertEqual(wanted, path) + + @unittest.skipUnless(sys.platform == 'darwin', + 'test only relevant for Mac OS X') + def test_deployment_target_default(self): + # Issue 9516: Test that, in the absence of the environment variable, + # an extension module is compiled with the same deployment target as + # the interpreter. + self._try_compile_deployment_target('==', None) + + @unittest.skipUnless(sys.platform == 'darwin', + 'test only relevant for Mac OS X') + def test_deployment_target_too_low(self): + # Issue 9516: Test that an extension module is not allowed to be + # compiled with a deployment target less than that of the interpreter. + self.assertRaises(PackagingPlatformError, + self._try_compile_deployment_target, '>', '10.1') + + @unittest.skipUnless(sys.platform == 'darwin', + 'test only relevant for Mac OS X') + def test_deployment_target_higher_ok(self): + # Issue 9516: Test that an extension module can be compiled with a + # deployment target higher than that of the interpreter: the ext + # module may depend on some newer OS feature. + deptarget = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if deptarget: + # increment the minor version number (i.e. 10.6 -> 10.7) + deptarget = [int(x) for x in deptarget.split('.')] + deptarget[-1] += 1 + deptarget = '.'.join(str(i) for i in deptarget) + self._try_compile_deployment_target('<', deptarget) + + def _try_compile_deployment_target(self, operator, target): + orig_environ = os.environ + os.environ = orig_environ.copy() + self.addCleanup(setattr, os, 'environ', orig_environ) + + if target is None: + if os.environ.get('MACOSX_DEPLOYMENT_TARGET'): + del os.environ['MACOSX_DEPLOYMENT_TARGET'] + else: + os.environ['MACOSX_DEPLOYMENT_TARGET'] = target + + deptarget_c = os.path.join(self.tmp_dir, 'deptargetmodule.c') + + with open(deptarget_c, 'w') as fp: + fp.write(textwrap.dedent('''\ + #include <AvailabilityMacros.h> + + int dummy; + + #if TARGET %s MAC_OS_X_VERSION_MIN_REQUIRED + #else + #error "Unexpected target" + #endif + + ''' % operator)) + + # get the deployment target that the interpreter was built with + target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + target = tuple(map(int, target.split('.'))) + target = '%02d%01d0' % target + + deptarget_ext = Extension( + 'deptarget', + [deptarget_c], + extra_compile_args=['-DTARGET=%s' % (target,)], + ) + dist = Distribution({ + 'name': 'deptarget', + 'ext_modules': [deptarget_ext], + }) + dist.package_dir = self.tmp_dir + cmd = build_ext(dist) + cmd.build_lib = self.tmp_dir + cmd.build_temp = self.tmp_dir + + try: + old_stdout = sys.stdout + if not verbose: + # silence compiler output + sys.stdout = StringIO() + try: + cmd.ensure_finalized() + cmd.run() + finally: + sys.stdout = old_stdout + + except CompileError: + self.fail("Wrong deployment target during compilation") + + +def test_suite(): + return unittest.makeSuite(BuildExtTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_command_build_py.py b/Lib/packaging/tests/test_command_build_py.py new file mode 100644 index 0000000000..243a863a60 --- /dev/null +++ b/Lib/packaging/tests/test_command_build_py.py @@ -0,0 +1,124 @@ +"""Tests for distutils.command.build_py.""" + +import os +import sys + +from packaging.command.build_py import build_py +from packaging.dist import Distribution +from packaging.errors import PackagingFileError + +from packaging.tests import unittest, support + + +class BuildPyTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_package_data(self): + sources = self.mkdtemp() + pkg_dir = os.path.join(sources, 'pkg') + os.mkdir(pkg_dir) + f = open(os.path.join(pkg_dir, "__init__.py"), "w") + try: + f.write("# Pretend this is a package.") + finally: + f.close() + f = open(os.path.join(pkg_dir, "README.txt"), "w") + try: + f.write("Info about this package") + finally: + f.close() + + destination = self.mkdtemp() + + dist = Distribution({"packages": ["pkg"], + "package_dir": sources}) + + dist.command_obj["build"] = support.DummyCommand( + force=False, + build_lib=destination, + use_2to3_fixers=None, + convert_2to3_doctests=None, + use_2to3=False) + dist.packages = ["pkg"] + dist.package_data = {"pkg": ["README.txt"]} + dist.package_dir = sources + + cmd = build_py(dist) + cmd.compile = True + cmd.ensure_finalized() + self.assertEqual(cmd.package_data, dist.package_data) + + cmd.run() + + # This makes sure the list of outputs includes byte-compiled + # files for Python modules but not for package data files + # (there shouldn't *be* byte-code files for those!). + # + self.assertEqual(len(cmd.get_outputs()), 3) + pkgdest = os.path.join(destination, "pkg") + files = os.listdir(pkgdest) + self.assertIn("__init__.py", files) + self.assertIn("README.txt", files) + # XXX even with -O, distutils writes pyc, not pyo; bug? + if sys.dont_write_bytecode: + self.assertNotIn("__init__.pyc", files) + else: + self.assertIn("__init__.pyc", files) + + def test_empty_package_dir(self): + # See SF 1668596/1720897. + cwd = os.getcwd() + + # create the distribution files. + sources = self.mkdtemp() + pkg = os.path.join(sources, 'pkg') + os.mkdir(pkg) + open(os.path.join(pkg, "__init__.py"), "wb").close() + testdir = os.path.join(pkg, "doc") + os.mkdir(testdir) + open(os.path.join(testdir, "testfile"), "wb").close() + + os.chdir(sources) + old_stdout = sys.stdout + #sys.stdout = StringIO.StringIO() + + try: + dist = Distribution({"packages": ["pkg"], + "package_dir": sources, + "package_data": {"pkg": ["doc/*"]}}) + dist.script_args = ["build"] + dist.parse_command_line() + + try: + dist.run_commands() + except PackagingFileError: + self.fail("failed package_data test when package_dir is ''") + finally: + # Restore state. + os.chdir(cwd) + sys.stdout = old_stdout + + @unittest.skipUnless(hasattr(sys, 'dont_write_bytecode'), + 'sys.dont_write_bytecode not supported') + def test_dont_write_bytecode(self): + # makes sure byte_compile is not used + pkg_dir, dist = self.create_dist() + cmd = build_py(dist) + cmd.compile = True + cmd.optimize = 1 + + old_dont_write_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + cmd.byte_compile([]) + finally: + sys.dont_write_bytecode = old_dont_write_bytecode + + self.assertIn('byte-compiling is disabled', self.get_logs()[0]) + +def test_suite(): + return unittest.makeSuite(BuildPyTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_build_scripts.py b/Lib/packaging/tests/test_command_build_scripts.py new file mode 100644 index 0000000000..fd3ac246a6 --- /dev/null +++ b/Lib/packaging/tests/test_command_build_scripts.py @@ -0,0 +1,109 @@ +"""Tests for distutils.command.build_scripts.""" + +import os +import sys +import sysconfig +from packaging.dist import Distribution +from packaging.command.build_scripts import build_scripts + +from packaging.tests import unittest, support + + +class BuildScriptsTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_default_settings(self): + cmd = self.get_build_scripts_cmd("/foo/bar", []) + self.assertFalse(cmd.force) + self.assertIs(cmd.build_dir, None) + + cmd.finalize_options() + + self.assertTrue(cmd.force) + self.assertEqual(cmd.build_dir, "/foo/bar") + + def test_build(self): + source = self.mkdtemp() + target = self.mkdtemp() + expected = self.write_sample_scripts(source) + + cmd = self.get_build_scripts_cmd(target, + [os.path.join(source, fn) + for fn in expected]) + cmd.finalize_options() + cmd.run() + + built = os.listdir(target) + for name in expected: + self.assertIn(name, built) + + def get_build_scripts_cmd(self, target, scripts): + dist = Distribution() + dist.scripts = scripts + dist.command_obj["build"] = support.DummyCommand( + build_scripts=target, + force=True, + executable=sys.executable, + use_2to3=False, + use_2to3_fixers=None, + convert_2to3_doctests=None + ) + return build_scripts(dist) + + def write_sample_scripts(self, dir): + expected = [] + expected.append("script1.py") + self.write_script(dir, "script1.py", + ("#! /usr/bin/env python2.3\n" + "# bogus script w/ Python sh-bang\n" + "pass\n")) + expected.append("script2.py") + self.write_script(dir, "script2.py", + ("#!/usr/bin/python\n" + "# bogus script w/ Python sh-bang\n" + "pass\n")) + expected.append("shell.sh") + self.write_script(dir, "shell.sh", + ("#!/bin/sh\n" + "# bogus shell script w/ sh-bang\n" + "exit 0\n")) + return expected + + def write_script(self, dir, name, text): + with open(os.path.join(dir, name), "w") as f: + f.write(text) + + def test_version_int(self): + source = self.mkdtemp() + target = self.mkdtemp() + expected = self.write_sample_scripts(source) + + + cmd = self.get_build_scripts_cmd(target, + [os.path.join(source, fn) + for fn in expected]) + cmd.finalize_options() + + # http://bugs.python.org/issue4524 + # + # On linux-g++-32 with command line `./configure --enable-ipv6 + # --with-suffix=3`, python is compiled okay but the build scripts + # failed when writing the name of the executable + old = sysconfig.get_config_vars().get('VERSION') + sysconfig._CONFIG_VARS['VERSION'] = 4 + try: + cmd.run() + finally: + if old is not None: + sysconfig._CONFIG_VARS['VERSION'] = old + + built = os.listdir(target) + for name in expected: + self.assertIn(name, built) + +def test_suite(): + return unittest.makeSuite(BuildScriptsTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_check.py b/Lib/packaging/tests/test_command_check.py new file mode 100644 index 0000000000..271e457961 --- /dev/null +++ b/Lib/packaging/tests/test_command_check.py @@ -0,0 +1,143 @@ +"""Tests for distutils.command.check.""" + +import logging +from packaging.command.check import check +from packaging.metadata import _HAS_DOCUTILS +from packaging.errors import PackagingSetupError, MetadataMissingError +from packaging.tests import unittest, support + + +class CheckTestCase(support.LoggingCatcher, + support.TempdirManager, + unittest.TestCase): + + def _run(self, metadata=None, **options): + if metadata is None: + metadata = {'name': 'xxx', 'version': '1.2'} + pkg_info, dist = self.create_dist(**metadata) + cmd = check(dist) + cmd.initialize_options() + for name, value in options.items(): + setattr(cmd, name, value) + cmd.ensure_finalized() + cmd.run() + return cmd + + def test_check_metadata(self): + # let's run the command with no metadata at all + # by default, check is checking the metadata + # should have some warnings + cmd = self._run() + # trick: using assertNotEqual with an empty list will give us a more + # useful error message than assertGreater(.., 0) when the code change + # and the test fails + self.assertNotEqual([], self.get_logs(logging.WARNING)) + + # now let's add the required fields + # and run it again, to make sure we don't get + # any warning anymore + metadata = {'home_page': 'xxx', 'author': 'xxx', + 'author_email': 'xxx', + 'name': 'xxx', 'version': '4.2', + } + cmd = self._run(metadata) + self.assertEqual([], self.get_logs(logging.WARNING)) + + # now with the strict mode, we should + # get an error if there are missing metadata + self.assertRaises(MetadataMissingError, self._run, {}, **{'strict': 1}) + self.assertRaises(PackagingSetupError, self._run, + {'name': 'xxx', 'version': 'xxx'}, **{'strict': 1}) + + # clear warnings from the previous calls + self.loghandler.flush() + + # and of course, no error when all metadata fields are present + cmd = self._run(metadata, strict=True) + self.assertEqual([], self.get_logs(logging.WARNING)) + + def test_check_metadata_1_2(self): + # let's run the command with no metadata at all + # by default, check is checking the metadata + # should have some warnings + cmd = self._run() + self.assertNotEqual([], self.get_logs(logging.WARNING)) + + # now let's add the required fields and run it again, to make sure we + # don't get any warning anymore let's use requires_python as a marker + # to enforce Metadata-Version 1.2 + metadata = {'home_page': 'xxx', 'author': 'xxx', + 'author_email': 'xxx', + 'name': 'xxx', 'version': '4.2', + 'requires_python': '2.4', + } + cmd = self._run(metadata) + self.assertEqual([], self.get_logs(logging.WARNING)) + + # now with the strict mode, we should + # get an error if there are missing metadata + self.assertRaises(MetadataMissingError, self._run, {}, **{'strict': 1}) + self.assertRaises(PackagingSetupError, self._run, + {'name': 'xxx', 'version': 'xxx'}, **{'strict': 1}) + + # complain about version format + metadata['version'] = 'xxx' + self.assertRaises(PackagingSetupError, self._run, metadata, + **{'strict': 1}) + + # clear warnings from the previous calls + self.loghandler.flush() + + # now with correct version format again + metadata['version'] = '4.2' + cmd = self._run(metadata, strict=True) + self.assertEqual([], self.get_logs(logging.WARNING)) + + @unittest.skipUnless(_HAS_DOCUTILS, "requires docutils") + def test_check_restructuredtext(self): + # let's see if it detects broken rest in long_description + broken_rest = 'title\n===\n\ntest' + pkg_info, dist = self.create_dist(description=broken_rest) + cmd = check(dist) + cmd.check_restructuredtext() + self.assertEqual(len(self.get_logs(logging.WARNING)), 1) + + pkg_info, dist = self.create_dist(description='title\n=====\n\ntest') + cmd = check(dist) + cmd.check_restructuredtext() + self.assertEqual([], self.get_logs(logging.WARNING)) + + def test_check_all(self): + self.assertRaises(PackagingSetupError, self._run, + {'name': 'xxx', 'version': 'xxx'}, **{'strict': 1, + 'all': 1}) + self.assertRaises(MetadataMissingError, self._run, + {}, **{'strict': 1, + 'all': 1}) + + def test_check_hooks(self): + pkg_info, dist = self.create_dist() + dist.command_options['install_dist'] = { + 'pre_hook': ('file', {"a": 'some.nonextistant.hook.ghrrraarrhll'}), + } + cmd = check(dist) + cmd.check_hooks_resolvable() + self.assertEqual(len(self.get_logs(logging.WARNING)), 1) + + def test_warn(self): + _, dist = self.create_dist() + cmd = check(dist) + self.assertEqual([], self.get_logs()) + cmd.warn('hello') + self.assertEqual(['check: hello'], self.get_logs()) + cmd.warn('hello %s', 'world') + self.assertEqual(['check: hello world'], self.get_logs()) + cmd.warn('hello %s %s', 'beautiful', 'world') + self.assertEqual(['check: hello beautiful world'], self.get_logs()) + + +def test_suite(): + return unittest.makeSuite(CheckTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_clean.py b/Lib/packaging/tests/test_command_clean.py new file mode 100644 index 0000000000..8d29e4dcdc --- /dev/null +++ b/Lib/packaging/tests/test_command_clean.py @@ -0,0 +1,48 @@ +"""Tests for distutils.command.clean.""" +import os + +from packaging.command.clean import clean +from packaging.tests import unittest, support + + +class cleanTestCase(support.TempdirManager, support.LoggingCatcher, + unittest.TestCase): + + def test_simple_run(self): + pkg_dir, dist = self.create_dist() + cmd = clean(dist) + + # let's add some elements clean should remove + dirs = [(d, os.path.join(pkg_dir, d)) + for d in ('build_temp', 'build_lib', 'bdist_base', + 'build_scripts', 'build_base')] + + for name, path in dirs: + os.mkdir(path) + setattr(cmd, name, path) + if name == 'build_base': + continue + for f in ('one', 'two', 'three'): + self.write_file(os.path.join(path, f)) + + # let's run the command + cmd.all = True + cmd.ensure_finalized() + cmd.run() + + # make sure the files where removed + for name, path in dirs: + self.assertFalse(os.path.exists(path), + '%r was not removed' % path) + + # let's run the command again (should spit warnings but succeed) + cmd.all = True + cmd.ensure_finalized() + cmd.run() + + +def test_suite(): + return unittest.makeSuite(cleanTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_cmd.py b/Lib/packaging/tests/test_command_cmd.py new file mode 100644 index 0000000000..8ac9dce66d --- /dev/null +++ b/Lib/packaging/tests/test_command_cmd.py @@ -0,0 +1,101 @@ +"""Tests for distutils.cmd.""" +import os + +from packaging.command.cmd import Command +from packaging.dist import Distribution +from packaging.errors import PackagingOptionError +from packaging.tests import support, unittest + + +class MyCmd(Command): + def initialize_options(self): + pass + + +class CommandTestCase(support.LoggingCatcher, + unittest.TestCase): + + def setUp(self): + super(CommandTestCase, self).setUp() + dist = Distribution() + self.cmd = MyCmd(dist) + + def test_make_file(self): + cmd = self.cmd + + # making sure it raises when infiles is not a string or a list/tuple + self.assertRaises(TypeError, cmd.make_file, + infiles=1, outfile='', func='func', args=()) + + # making sure execute gets called properly + def _execute(func, args, exec_msg, level): + self.assertEqual(exec_msg, 'generating out from in') + cmd.force = True + cmd.execute = _execute + cmd.make_file(infiles='in', outfile='out', func='func', args=()) + + def test_dump_options(self): + cmd = self.cmd + cmd.option1 = 1 + cmd.option2 = 1 + cmd.user_options = [('option1', '', ''), ('option2', '', '')] + cmd.dump_options() + + wanted = ["command options for 'MyCmd':", ' option1 = 1', + ' option2 = 1'] + msgs = self.get_logs() + self.assertEqual(msgs, wanted) + + def test_ensure_string(self): + cmd = self.cmd + cmd.option1 = 'ok' + cmd.ensure_string('option1') + + cmd.option2 = None + cmd.ensure_string('option2', 'xxx') + self.assertTrue(hasattr(cmd, 'option2')) + + cmd.option3 = 1 + self.assertRaises(PackagingOptionError, cmd.ensure_string, 'option3') + + def test_ensure_string_list(self): + cmd = self.cmd + cmd.option1 = 'ok,dok' + cmd.ensure_string_list('option1') + self.assertEqual(cmd.option1, ['ok', 'dok']) + + cmd.yes_string_list = ['one', 'two', 'three'] + cmd.yes_string_list2 = 'ok' + cmd.ensure_string_list('yes_string_list') + cmd.ensure_string_list('yes_string_list2') + self.assertEqual(cmd.yes_string_list, ['one', 'two', 'three']) + self.assertEqual(cmd.yes_string_list2, ['ok']) + + cmd.not_string_list = ['one', 2, 'three'] + cmd.not_string_list2 = object() + self.assertRaises(PackagingOptionError, + cmd.ensure_string_list, 'not_string_list') + + self.assertRaises(PackagingOptionError, + cmd.ensure_string_list, 'not_string_list2') + + def test_ensure_filename(self): + cmd = self.cmd + cmd.option1 = __file__ + cmd.ensure_filename('option1') + cmd.option2 = 'xxx' + self.assertRaises(PackagingOptionError, cmd.ensure_filename, 'option2') + + def test_ensure_dirname(self): + cmd = self.cmd + cmd.option1 = os.path.dirname(__file__) or os.curdir + cmd.ensure_dirname('option1') + cmd.option2 = 'xxx' + self.assertRaises(PackagingOptionError, cmd.ensure_dirname, 'option2') + + +def test_suite(): + return unittest.makeSuite(CommandTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_command_config.py b/Lib/packaging/tests/test_command_config.py new file mode 100644 index 0000000000..6d780c5fd5 --- /dev/null +++ b/Lib/packaging/tests/test_command_config.py @@ -0,0 +1,76 @@ +"""Tests for distutils.command.config.""" +import os +import sys +import logging + +from packaging.command.config import dump_file, config +from packaging.tests import unittest, support + + +class ConfigTestCase(support.LoggingCatcher, + support.TempdirManager, + unittest.TestCase): + + def test_dump_file(self): + this_file = __file__.rstrip('co') + with open(this_file) as f: + numlines = len(f.readlines()) + + dump_file(this_file, 'I am the header') + + logs = [] + for log in self.get_logs(logging.INFO): + logs.extend(line for line in log.split('\n')) + self.assertEqual(len(logs), numlines + 2) + + @unittest.skipIf(sys.platform == 'win32', 'disabled on win32') + def test_search_cpp(self): + pkg_dir, dist = self.create_dist() + cmd = config(dist) + + # simple pattern searches + match = cmd.search_cpp(pattern='xxx', body='// xxx') + self.assertEqual(match, 0) + + match = cmd.search_cpp(pattern='_configtest', body='// xxx') + self.assertEqual(match, 1) + + def test_finalize_options(self): + # finalize_options does a bit of transformation + # on options + pkg_dir, dist = self.create_dist() + cmd = config(dist) + cmd.include_dirs = 'one%stwo' % os.pathsep + cmd.libraries = 'one' + cmd.library_dirs = 'three%sfour' % os.pathsep + cmd.ensure_finalized() + + self.assertEqual(cmd.include_dirs, ['one', 'two']) + self.assertEqual(cmd.libraries, ['one']) + self.assertEqual(cmd.library_dirs, ['three', 'four']) + + def test_clean(self): + # _clean removes files + tmp_dir = self.mkdtemp() + f1 = os.path.join(tmp_dir, 'one') + f2 = os.path.join(tmp_dir, 'two') + + self.write_file(f1, 'xxx') + self.write_file(f2, 'xxx') + + for f in (f1, f2): + self.assertTrue(os.path.exists(f)) + + pkg_dir, dist = self.create_dist() + cmd = config(dist) + cmd._clean(f1, f2) + + for f in (f1, f2): + self.assertFalse(os.path.exists(f)) + + +def test_suite(): + return unittest.makeSuite(ConfigTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_install_data.py b/Lib/packaging/tests/test_command_install_data.py new file mode 100644 index 0000000000..0486427a2a --- /dev/null +++ b/Lib/packaging/tests/test_command_install_data.py @@ -0,0 +1,87 @@ +"""Tests for packaging.command.install_data.""" +import os +import sysconfig +from sysconfig import _get_default_scheme +from packaging.tests import unittest, support +from packaging.command.install_data import install_data + + +class InstallDataTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_simple_run(self): + scheme = _get_default_scheme() + old_items = sysconfig._SCHEMES.items(scheme) + def restore(): + sysconfig._SCHEMES.remove_section(scheme) + sysconfig._SCHEMES.add_section(scheme) + for option, value in old_items: + sysconfig._SCHEMES.set(scheme, option, value) + self.addCleanup(restore) + + pkg_dir, dist = self.create_dist() + cmd = install_data(dist) + cmd.install_dir = inst = os.path.join(pkg_dir, 'inst') + + sysconfig._SCHEMES.set(scheme, 'inst', + os.path.join(pkg_dir, 'inst')) + sysconfig._SCHEMES.set(scheme, 'inst2', + os.path.join(pkg_dir, 'inst2')) + + one = os.path.join(pkg_dir, 'one') + self.write_file(one, 'xxx') + inst2 = os.path.join(pkg_dir, 'inst2') + two = os.path.join(pkg_dir, 'two') + self.write_file(two, 'xxx') + + cmd.data_files = {one: '{inst}/one', two: '{inst2}/two'} + self.assertCountEqual(cmd.get_inputs(), [one, two]) + + # let's run the command + cmd.ensure_finalized() + cmd.run() + + # let's check the result + self.assertEqual(len(cmd.get_outputs()), 2) + rtwo = os.path.split(two)[-1] + self.assertTrue(os.path.exists(os.path.join(inst2, rtwo))) + rone = os.path.split(one)[-1] + self.assertTrue(os.path.exists(os.path.join(inst, rone))) + cmd.outfiles = [] + + # let's try with warn_dir one + cmd.warn_dir = True + cmd.ensure_finalized() + cmd.run() + + # let's check the result + self.assertEqual(len(cmd.get_outputs()), 2) + self.assertTrue(os.path.exists(os.path.join(inst2, rtwo))) + self.assertTrue(os.path.exists(os.path.join(inst, rone))) + cmd.outfiles = [] + + # now using root and empty dir + cmd.root = os.path.join(pkg_dir, 'root') + three = os.path.join(cmd.install_dir, 'three') + self.write_file(three, 'xx') + + sysconfig._SCHEMES.set(scheme, 'inst3', + cmd.install_dir) + + cmd.data_files = {one: '{inst}/one', two: '{inst2}/two', + three: '{inst3}/three'} + cmd.ensure_finalized() + cmd.run() + + # let's check the result + self.assertEqual(len(cmd.get_outputs()), 3) + self.assertTrue(os.path.exists(os.path.join(inst2, rtwo))) + self.assertTrue(os.path.exists(os.path.join(inst, rone))) + + +def test_suite(): + return unittest.makeSuite(InstallDataTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_install_dist.py b/Lib/packaging/tests/test_command_install_dist.py new file mode 100644 index 0000000000..8f90aff97e --- /dev/null +++ b/Lib/packaging/tests/test_command_install_dist.py @@ -0,0 +1,240 @@ +"""Tests for packaging.command.install.""" + +import os +import sys +from sysconfig import (get_scheme_names, get_config_vars, + _SCHEMES, get_config_var, get_path) + +from packaging.command.build_ext import build_ext +from packaging.command.install_dist import install_dist +from packaging.compiler.extension import Extension +from packaging.dist import Distribution +from packaging.errors import PackagingOptionError + +from packaging.tests import unittest, support + + +_CONFIG_VARS = get_config_vars() + + +def _make_ext_name(modname): + if os.name == 'nt' and sys.executable.endswith('_d.exe'): + modname += '_d' + return modname + get_config_var('SO') + + +class InstallTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_home_installation_scheme(self): + # This ensure two things: + # - that --home generates the desired set of directory names + # - test --home is supported on all platforms + builddir = self.mkdtemp() + destination = os.path.join(builddir, "installation") + + dist = Distribution({"name": "foopkg"}) + dist.command_obj["build"] = support.DummyCommand( + build_base=builddir, + build_lib=os.path.join(builddir, "lib"), + ) + + old_posix_prefix = _SCHEMES.get('posix_prefix', 'platinclude') + old_posix_home = _SCHEMES.get('posix_home', 'platinclude') + + new_path = '{platbase}/include/python{py_version_short}' + _SCHEMES.set('posix_prefix', 'platinclude', new_path) + _SCHEMES.set('posix_home', 'platinclude', '{platbase}/include/python') + + try: + cmd = install_dist(dist) + cmd.home = destination + cmd.ensure_finalized() + finally: + _SCHEMES.set('posix_prefix', 'platinclude', old_posix_prefix) + _SCHEMES.set('posix_home', 'platinclude', old_posix_home) + + self.assertEqual(cmd.install_base, destination) + self.assertEqual(cmd.install_platbase, destination) + + def check_path(got, expected): + got = os.path.normpath(got) + expected = os.path.normpath(expected) + self.assertEqual(got, expected) + + libdir = os.path.join(destination, "lib", "python") + check_path(cmd.install_lib, libdir) + check_path(cmd.install_platlib, libdir) + check_path(cmd.install_purelib, libdir) + check_path(cmd.install_headers, + os.path.join(destination, "include", "python", "foopkg")) + check_path(cmd.install_scripts, os.path.join(destination, "bin")) + check_path(cmd.install_data, destination) + + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') + def test_user_site(self): + # test install with --user + # preparing the environment for the test + self.old_user_base = get_config_var('userbase') + self.old_user_site = get_path('purelib', '%s_user' % os.name) + self.tmpdir = self.mkdtemp() + self.user_base = os.path.join(self.tmpdir, 'B') + self.user_site = os.path.join(self.tmpdir, 'S') + _CONFIG_VARS['userbase'] = self.user_base + scheme = '%s_user' % os.name + _SCHEMES.set(scheme, 'purelib', self.user_site) + + def _expanduser(path): + if path[0] == '~': + path = os.path.normpath(self.tmpdir) + path[1:] + return path + + self.old_expand = os.path.expanduser + os.path.expanduser = _expanduser + + try: + # this is the actual test + self._test_user_site() + finally: + _CONFIG_VARS['userbase'] = self.old_user_base + _SCHEMES.set(scheme, 'purelib', self.old_user_site) + os.path.expanduser = self.old_expand + + def _test_user_site(self): + schemes = get_scheme_names() + for key in ('nt_user', 'posix_user', 'os2_home'): + self.assertIn(key, schemes) + + dist = Distribution({'name': 'xx'}) + cmd = install_dist(dist) + # making sure the user option is there + options = [name for name, short, lable in + cmd.user_options] + self.assertIn('user', options) + + # setting a value + cmd.user = True + + # user base and site shouldn't be created yet + self.assertFalse(os.path.exists(self.user_base)) + self.assertFalse(os.path.exists(self.user_site)) + + # let's run finalize + cmd.ensure_finalized() + + # now they should + self.assertTrue(os.path.exists(self.user_base)) + self.assertTrue(os.path.exists(self.user_site)) + + self.assertIn('userbase', cmd.config_vars) + self.assertIn('usersite', cmd.config_vars) + + def test_handle_extra_path(self): + dist = Distribution({'name': 'xx', 'extra_path': 'path,dirs'}) + cmd = install_dist(dist) + + # two elements + cmd.handle_extra_path() + self.assertEqual(cmd.extra_path, ['path', 'dirs']) + self.assertEqual(cmd.extra_dirs, 'dirs') + self.assertEqual(cmd.path_file, 'path') + + # one element + cmd.extra_path = ['path'] + cmd.handle_extra_path() + self.assertEqual(cmd.extra_path, ['path']) + self.assertEqual(cmd.extra_dirs, 'path') + self.assertEqual(cmd.path_file, 'path') + + # none + dist.extra_path = cmd.extra_path = None + cmd.handle_extra_path() + self.assertEqual(cmd.extra_path, None) + self.assertEqual(cmd.extra_dirs, '') + self.assertEqual(cmd.path_file, None) + + # three elements (no way !) + cmd.extra_path = 'path,dirs,again' + self.assertRaises(PackagingOptionError, cmd.handle_extra_path) + + def test_finalize_options(self): + dist = Distribution({'name': 'xx'}) + cmd = install_dist(dist) + + # must supply either prefix/exec-prefix/home or + # install-base/install-platbase -- not both + cmd.prefix = 'prefix' + cmd.install_base = 'base' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + # must supply either home or prefix/exec-prefix -- not both + cmd.install_base = None + cmd.home = 'home' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + if sys.version >= '2.6': + # can't combine user with with prefix/exec_prefix/home or + # install_(plat)base + cmd.prefix = None + cmd.user = 'user' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + def test_old_record(self): + # test pre-PEP 376 --record option (outside dist-info dir) + install_dir = self.mkdtemp() + project_dir, dist = self.create_dist(scripts=['hello']) + os.chdir(project_dir) + self.write_file('hello', "print('o hai')") + + cmd = install_dist(dist) + dist.command_obj['install_dist'] = cmd + cmd.root = install_dir + cmd.record = os.path.join(project_dir, 'filelist') + cmd.ensure_finalized() + cmd.run() + + with open(cmd.record) as f: + content = f.read() + + found = [os.path.basename(line) for line in content.splitlines()] + expected = ['hello', 'METADATA', 'INSTALLER', 'REQUESTED', 'RECORD'] + self.assertEqual(found, expected) + + # XXX test that fancy_getopt is okay with options named + # record and no-record but unrelated + + def test_old_record_extensions(self): + # test pre-PEP 376 --record option with ext modules + install_dir = self.mkdtemp() + project_dir, dist = self.create_dist(ext_modules=[ + Extension('xx', ['xxmodule.c'])]) + os.chdir(project_dir) + support.copy_xxmodule_c(project_dir) + + buildextcmd = build_ext(dist) + support.fixup_build_ext(buildextcmd) + buildextcmd.ensure_finalized() + + cmd = install_dist(dist) + dist.command_obj['install_dist'] = cmd + dist.command_obj['build_ext'] = buildextcmd + cmd.root = install_dir + cmd.record = os.path.join(project_dir, 'filelist') + cmd.ensure_finalized() + cmd.run() + + with open(cmd.record) as f: + content = f.read() + + found = [os.path.basename(line) for line in content.splitlines()] + expected = [_make_ext_name('xx'), + 'METADATA', 'INSTALLER', 'REQUESTED', 'RECORD'] + self.assertEqual(found, expected) + + +def test_suite(): + return unittest.makeSuite(InstallTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_install_distinfo.py b/Lib/packaging/tests/test_command_install_distinfo.py new file mode 100644 index 0000000000..31636fd9c3 --- /dev/null +++ b/Lib/packaging/tests/test_command_install_distinfo.py @@ -0,0 +1,249 @@ +"""Tests for ``packaging.command.install_distinfo``. """ + +import os +import csv +import hashlib +import sysconfig + +from packaging.command.install_distinfo import install_distinfo +from packaging.command.cmd import Command +from packaging.compiler.extension import Extension +from packaging.metadata import Metadata +from packaging.tests import unittest, support + + +class DummyInstallCmd(Command): + + def __init__(self, dist=None): + self.outputs = [] + self.distribution = dist + + def __getattr__(self, name): + return None + + def ensure_finalized(self): + pass + + def get_outputs(self): + return (self.outputs + + self.get_finalized_command('install_distinfo').get_outputs()) + + +class InstallDistinfoTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + checkLists = lambda self, x, y: self.assertListEqual(sorted(x), sorted(y)) + + def test_empty_install(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install_dist'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.distinfo_dir = install_dir + cmd.ensure_finalized() + cmd.run() + + self.checkLists(os.listdir(install_dir), ['foo-1.0.dist-info']) + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'RECORD', 'REQUESTED', 'INSTALLER']) + with open(os.path.join(dist_info, 'INSTALLER')) as fp: + self.assertEqual(fp.read(), 'distutils') + with open(os.path.join(dist_info, 'REQUESTED')) as fp: + self.assertEqual(fp.read(), '') + meta_path = os.path.join(dist_info, 'METADATA') + self.assertTrue(Metadata(path=meta_path).check()) + + def test_installer(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install_dist'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.distinfo_dir = install_dir + cmd.installer = 'bacon-python' + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + with open(os.path.join(dist_info, 'INSTALLER')) as fp: + self.assertEqual(fp.read(), 'bacon-python') + + def test_requested(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install_dist'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.distinfo_dir = install_dir + cmd.requested = False + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'RECORD', 'INSTALLER']) + + def test_no_record(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install_dist'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.distinfo_dir = install_dir + cmd.no_record = True + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'REQUESTED', 'INSTALLER']) + + def test_record_basic(self): + install_dir = self.mkdtemp() + modules_dest = os.path.join(install_dir, 'lib') + scripts_dest = os.path.join(install_dir, 'bin') + project_dir, dist = self.create_dist( + name='Spamlib', version='0.1', + py_modules=['spam'], scripts=['spamd'], + ext_modules=[Extension('_speedspam', ['_speedspam.c'])]) + + # using a real install_dist command is too painful, so we use a mock + # class that's only a holder for options to be used by install_distinfo + # and we create placeholder files manually instead of using build_*. + # the install_* commands will still be consulted by install_distinfo. + os.chdir(project_dir) + self.write_file('spam', '# Python module') + self.write_file('spamd', '# Python script') + extmod = '_speedspam' + sysconfig.get_config_var('SO') + self.write_file(extmod, '') + + install = DummyInstallCmd(dist) + install.outputs = ['spam', 'spamd', extmod] + install.install_lib = modules_dest + install.install_scripts = scripts_dest + dist.command_obj['install_dist'] = install + + cmd = install_distinfo(dist) + cmd.ensure_finalized() + dist.command_obj['install_distinfo'] = cmd + cmd.run() + + # checksum and size are not hard-coded for METADATA as it is + # platform-dependent (line endings) + metadata = os.path.join(modules_dest, 'Spamlib-0.1.dist-info', + 'METADATA') + with open(metadata, 'rb') as fp: + content = fp.read() + + metadata_size = str(len(content)) + metadata_md5 = hashlib.md5(content).hexdigest() + + record = os.path.join(modules_dest, 'Spamlib-0.1.dist-info', 'RECORD') + with open(record, encoding='utf-8') as fp: + content = fp.read() + + found = [] + for line in content.splitlines(): + filename, checksum, size = line.split(',') + filename = os.path.basename(filename) + found.append((filename, checksum, size)) + + expected = [ + ('spam', '6ab2f288ef2545868effe68757448b45', '15'), + ('spamd', 'd13e6156ce78919a981e424b2fdcd974', '15'), + (extmod, 'd41d8cd98f00b204e9800998ecf8427e', '0'), + ('METADATA', metadata_md5, metadata_size), + ('INSTALLER', '44e3fde05f3f537ed85831969acf396d', '9'), + ('REQUESTED', 'd41d8cd98f00b204e9800998ecf8427e', '0'), + ('RECORD', '', ''), + ] + self.assertEqual(found, expected) + + def test_record(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install_dist'] = install + + fake_dists = os.path.join(os.path.dirname(__file__), 'fake_dists') + fake_dists = os.path.realpath(fake_dists) + + # for testing, we simply add all files from _backport's fake_dists + dirs = [] + for dir in os.listdir(fake_dists): + full_path = os.path.join(fake_dists, dir) + if (not dir.endswith('.egg') or dir.endswith('.egg-info') or + dir.endswith('.dist-info')) and os.path.isdir(full_path): + dirs.append(full_path) + + for dir in dirs: + for path, subdirs, files in os.walk(dir): + install.outputs += [os.path.join(path, f) for f in files] + install.outputs += [os.path.join('path', f + 'c') + for f in files if f.endswith('.py')] + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.distinfo_dir = install_dir + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + + expected = [] + for f in install.get_outputs(): + if (f.endswith(('.pyc', '.pyo')) or f == os.path.join( + install_dir, 'foo-1.0.dist-info', 'RECORD')): + expected.append([f, '', '']) + else: + size = os.path.getsize(f) + md5 = hashlib.md5() + with open(f, 'rb') as fp: + md5.update(fp.read()) + hash = md5.hexdigest() + expected.append([f, hash, str(size)]) + + parsed = [] + with open(os.path.join(dist_info, 'RECORD'), 'r') as f: + reader = csv.reader(f, delimiter=',', + lineterminator=os.linesep, + quotechar='"') + parsed = list(reader) + + self.maxDiff = None + self.checkLists(parsed, expected) + + +def test_suite(): + return unittest.makeSuite(InstallDistinfoTestCase) + + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_install_headers.py b/Lib/packaging/tests/test_command_install_headers.py new file mode 100644 index 0000000000..f2906a7e18 --- /dev/null +++ b/Lib/packaging/tests/test_command_install_headers.py @@ -0,0 +1,38 @@ +"""Tests for packaging.command.install_headers.""" +import os + +from packaging.command.install_headers import install_headers +from packaging.tests import unittest, support + + +class InstallHeadersTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_simple_run(self): + # we have two headers + header_list = self.mkdtemp() + header1 = os.path.join(header_list, 'header1') + header2 = os.path.join(header_list, 'header2') + self.write_file(header1) + self.write_file(header2) + headers = [header1, header2] + + pkg_dir, dist = self.create_dist(headers=headers) + cmd = install_headers(dist) + self.assertEqual(cmd.get_inputs(), headers) + + # let's run the command + cmd.install_dir = os.path.join(pkg_dir, 'inst') + cmd.ensure_finalized() + cmd.run() + + # let's check the results + self.assertEqual(len(cmd.get_outputs()), 2) + + +def test_suite(): + return unittest.makeSuite(InstallHeadersTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_install_lib.py b/Lib/packaging/tests/test_command_install_lib.py new file mode 100644 index 0000000000..b46f3bd2bb --- /dev/null +++ b/Lib/packaging/tests/test_command_install_lib.py @@ -0,0 +1,113 @@ +"""Tests for packaging.command.install_data.""" +import sys +import os + +from packaging.tests import unittest, support +from packaging.command.install_lib import install_lib +from packaging.compiler.extension import Extension +from packaging.errors import PackagingOptionError + +try: + no_bytecode = sys.dont_write_bytecode + bytecode_support = True +except AttributeError: + no_bytecode = False + bytecode_support = False + + +class InstallLibTestCase(support.TempdirManager, + support.LoggingCatcher, + support.EnvironRestorer, + unittest.TestCase): + + restore_environ = ['PYTHONPATH'] + + def test_finalize_options(self): + pkg_dir, dist = self.create_dist() + cmd = install_lib(dist) + + cmd.finalize_options() + self.assertTrue(cmd.compile) + self.assertEqual(cmd.optimize, 0) + + # optimize must be 0, 1, or 2 + cmd.optimize = 'foo' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + cmd.optimize = '4' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + cmd.optimize = '2' + cmd.finalize_options() + self.assertEqual(cmd.optimize, 2) + + @unittest.skipIf(no_bytecode, 'byte-compile not supported') + def test_byte_compile(self): + pkg_dir, dist = self.create_dist() + cmd = install_lib(dist) + cmd.compile = True + cmd.optimize = 1 + + f = os.path.join(pkg_dir, 'foo.py') + self.write_file(f, '# python file') + cmd.byte_compile([f]) + self.assertTrue(os.path.exists(os.path.join(pkg_dir, 'foo.pyc'))) + self.assertTrue(os.path.exists(os.path.join(pkg_dir, 'foo.pyo'))) + + def test_get_outputs(self): + pkg_dir, dist = self.create_dist() + cmd = install_lib(dist) + + # setting up a dist environment + cmd.compile = True + cmd.optimize = 1 + cmd.install_dir = pkg_dir + f = os.path.join(pkg_dir, '__init__.py') + self.write_file(f, '# python package') + cmd.distribution.ext_modules = [Extension('foo', ['xxx'])] + cmd.distribution.packages = [pkg_dir] + + # make sure the build_lib is set the temp dir + build_dir = os.path.split(pkg_dir)[0] + cmd.get_finalized_command('build_py').build_lib = build_dir + + # get_output should return 4 elements + self.assertEqual(len(cmd.get_outputs()), 4) + + def test_get_inputs(self): + pkg_dir, dist = self.create_dist() + cmd = install_lib(dist) + + # setting up a dist environment + cmd.compile = True + cmd.optimize = 1 + cmd.install_dir = pkg_dir + f = os.path.join(pkg_dir, '__init__.py') + self.write_file(f, '# python package') + cmd.distribution.ext_modules = [Extension('foo', ['xxx'])] + cmd.distribution.packages = [pkg_dir] + + # get_input should return 2 elements + self.assertEqual(len(cmd.get_inputs()), 2) + + @unittest.skipUnless(bytecode_support, + 'sys.dont_write_bytecode not supported') + def test_dont_write_bytecode(self): + # makes sure byte_compile is not used + pkg_dir, dist = self.create_dist() + cmd = install_lib(dist) + cmd.compile = True + cmd.optimize = 1 + + self.addCleanup(setattr, sys, 'dont_write_bytecode', + sys.dont_write_bytecode) + sys.dont_write_bytecode = True + cmd.byte_compile([]) + + self.assertIn('byte-compiling is disabled', self.get_logs()[0]) + + +def test_suite(): + return unittest.makeSuite(InstallLibTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_install_scripts.py b/Lib/packaging/tests/test_command_install_scripts.py new file mode 100644 index 0000000000..6452a3455a --- /dev/null +++ b/Lib/packaging/tests/test_command_install_scripts.py @@ -0,0 +1,75 @@ +"""Tests for packaging.command.install_scripts.""" +import os + +from packaging.tests import unittest, support +from packaging.command.install_scripts import install_scripts +from packaging.dist import Distribution + + +class InstallScriptsTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_default_settings(self): + dist = Distribution() + dist.command_obj["build"] = support.DummyCommand( + build_scripts="/foo/bar") + dist.command_obj["install_dist"] = support.DummyCommand( + install_scripts="/splat/funk", + force=True, + skip_build=True, + ) + cmd = install_scripts(dist) + self.assertFalse(cmd.force) + self.assertFalse(cmd.skip_build) + self.assertIs(cmd.build_dir, None) + self.assertIs(cmd.install_dir, None) + + cmd.finalize_options() + + self.assertTrue(cmd.force) + self.assertTrue(cmd.skip_build) + self.assertEqual(cmd.build_dir, "/foo/bar") + self.assertEqual(cmd.install_dir, "/splat/funk") + + def test_installation(self): + source = self.mkdtemp() + expected = [] + + def write_script(name, text): + expected.append(name) + with open(os.path.join(source, name), "w") as f: + f.write(text) + + write_script("script1.py", ("#! /usr/bin/env python2.3\n" + "# bogus script w/ Python sh-bang\n" + "pass\n")) + write_script("script2.py", ("#!/usr/bin/python\n" + "# bogus script w/ Python sh-bang\n" + "pass\n")) + write_script("shell.sh", ("#!/bin/sh\n" + "# bogus shell script w/ sh-bang\n" + "exit 0\n")) + + target = self.mkdtemp() + dist = Distribution() + dist.command_obj["build"] = support.DummyCommand(build_scripts=source) + dist.command_obj["install_dist"] = support.DummyCommand( + install_scripts=target, + force=True, + skip_build=True, + ) + cmd = install_scripts(dist) + cmd.finalize_options() + cmd.run() + + installed = os.listdir(target) + for name in expected: + self.assertIn(name, installed) + + +def test_suite(): + return unittest.makeSuite(InstallScriptsTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_register.py b/Lib/packaging/tests/test_command_register.py new file mode 100644 index 0000000000..9c64e2dd29 --- /dev/null +++ b/Lib/packaging/tests/test_command_register.py @@ -0,0 +1,259 @@ +"""Tests for packaging.command.register.""" +import os +import getpass +import urllib.request +import urllib.error +import urllib.parse + +try: + import docutils + DOCUTILS_SUPPORT = True +except ImportError: + DOCUTILS_SUPPORT = False + +from packaging.tests import unittest, support +from packaging.command import register as register_module +from packaging.command.register import register +from packaging.errors import PackagingSetupError + + +PYPIRC_NOPASSWORD = """\ +[distutils] + +index-servers = + server1 + +[server1] +username:me +""" + +WANTED_PYPIRC = """\ +[distutils] +index-servers = + pypi + +[pypi] +username:tarek +password:password +""" + + +class Inputs: + """Fakes user inputs.""" + def __init__(self, *answers): + self.answers = answers + self.index = 0 + + def __call__(self, prompt=''): + try: + return self.answers[self.index] + finally: + self.index += 1 + + +class FakeOpener: + """Fakes a PyPI server""" + def __init__(self): + self.reqs = [] + + def __call__(self, *args): + return self + + def open(self, req): + self.reqs.append(req) + return self + + def read(self): + return 'xxx' + + +class RegisterTestCase(support.TempdirManager, + support.EnvironRestorer, + support.LoggingCatcher, + unittest.TestCase): + + restore_environ = ['HOME'] + + def setUp(self): + super(RegisterTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.rc = os.path.join(self.tmp_dir, '.pypirc') + os.environ['HOME'] = self.tmp_dir + + # patching the password prompt + self._old_getpass = getpass.getpass + + def _getpass(prompt): + return 'password' + + getpass.getpass = _getpass + self.old_opener = urllib.request.build_opener + self.conn = urllib.request.build_opener = FakeOpener() + + def tearDown(self): + getpass.getpass = self._old_getpass + urllib.request.build_opener = self.old_opener + if hasattr(register_module, 'input'): + del register_module.input + super(RegisterTestCase, self).tearDown() + + def _get_cmd(self, metadata=None): + if metadata is None: + metadata = {'home-page': 'xxx', 'author': 'xxx', + 'author_email': 'xxx', + 'name': 'xxx', 'version': 'xxx'} + pkg_info, dist = self.create_dist(**metadata) + return register(dist) + + def test_create_pypirc(self): + # this test makes sure a .pypirc file + # is created when requested. + + # let's create a register instance + cmd = self._get_cmd() + + # we shouldn't have a .pypirc file yet + self.assertFalse(os.path.exists(self.rc)) + + # patching input and getpass.getpass + # so register gets happy + # Here's what we are faking : + # use your existing login (choice 1.) + # Username : 'tarek' + # Password : 'password' + # Save your login (y/N)? : 'y' + inputs = Inputs('1', 'tarek', 'y') + register_module.input = inputs + cmd.ensure_finalized() + cmd.run() + + # we should have a brand new .pypirc file + self.assertTrue(os.path.exists(self.rc)) + + # with the content similar to WANTED_PYPIRC + with open(self.rc) as fp: + content = fp.read() + self.assertEqual(content, WANTED_PYPIRC) + + # now let's make sure the .pypirc file generated + # really works : we shouldn't be asked anything + # if we run the command again + def _no_way(prompt=''): + raise AssertionError(prompt) + + register_module.input = _no_way + cmd.show_response = True + cmd.ensure_finalized() + cmd.run() + + # let's see what the server received : we should + # have 2 similar requests + self.assertEqual(len(self.conn.reqs), 2) + req1 = dict(self.conn.reqs[0].headers) + req2 = dict(self.conn.reqs[1].headers) + self.assertEqual(req2['Content-length'], req1['Content-length']) + self.assertIn(b'xxx', self.conn.reqs[1].data) + + def test_password_not_in_file(self): + + self.write_file(self.rc, PYPIRC_NOPASSWORD) + cmd = self._get_cmd() + cmd.finalize_options() + cmd._set_config() + cmd.send_metadata() + + # dist.password should be set + # therefore used afterwards by other commands + self.assertEqual(cmd.distribution.password, 'password') + + def test_registration(self): + # this test runs choice 2 + cmd = self._get_cmd() + inputs = Inputs('2', 'tarek', 'tarek@ziade.org') + register_module.input = inputs + # let's run the command + # FIXME does this send a real request? use a mock server + cmd.ensure_finalized() + cmd.run() + + # we should have send a request + self.assertEqual(len(self.conn.reqs), 1) + req = self.conn.reqs[0] + headers = dict(req.headers) + self.assertEqual(headers['Content-length'], '628') + self.assertIn(b'tarek', req.data) + + def test_password_reset(self): + # this test runs choice 3 + cmd = self._get_cmd() + inputs = Inputs('3', 'tarek@ziade.org') + register_module.input = inputs + cmd.ensure_finalized() + cmd.run() + + # we should have send a request + self.assertEqual(len(self.conn.reqs), 1) + req = self.conn.reqs[0] + headers = dict(req.headers) + self.assertEqual(headers['Content-length'], '298') + self.assertIn(b'tarek', req.data) + + @unittest.skipUnless(DOCUTILS_SUPPORT, 'needs docutils') + def test_strict(self): + # testing the script option + # when on, the register command stops if + # the metadata is incomplete or if + # long_description is not reSt compliant + + # empty metadata + cmd = self._get_cmd({'name': 'xxx', 'version': 'xxx'}) + cmd.ensure_finalized() + cmd.strict = True + inputs = Inputs('1', 'tarek', 'y') + register_module.input = inputs + self.assertRaises(PackagingSetupError, cmd.run) + + # metadata is OK but long_description is broken + metadata = {'home_page': 'xxx', 'author': 'xxx', + 'author_email': 'éxéxé', + 'name': 'xxx', 'version': 'xxx', + 'description': 'title\n==\n\ntext'} + + cmd = self._get_cmd(metadata) + cmd.ensure_finalized() + cmd.strict = True + + self.assertRaises(PackagingSetupError, cmd.run) + + # now something that works + metadata['description'] = 'title\n=====\n\ntext' + cmd = self._get_cmd(metadata) + cmd.ensure_finalized() + cmd.strict = True + inputs = Inputs('1', 'tarek', 'y') + register_module.input = inputs + cmd.ensure_finalized() + cmd.run() + + # strict is not by default + cmd = self._get_cmd() + cmd.ensure_finalized() + inputs = Inputs('1', 'tarek', 'y') + register_module.input = inputs + cmd.ensure_finalized() + cmd.run() + + def test_register_pep345(self): + cmd = self._get_cmd({}) + cmd.ensure_finalized() + cmd.distribution.metadata['Requires-Dist'] = ['lxml'] + data = cmd.build_post_data('submit') + self.assertEqual(data['metadata_version'], '1.2') + self.assertEqual(data['requires_dist'], ['lxml']) + + +def test_suite(): + return unittest.makeSuite(RegisterTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_sdist.py b/Lib/packaging/tests/test_command_sdist.py new file mode 100644 index 0000000000..ddc6bf7039 --- /dev/null +++ b/Lib/packaging/tests/test_command_sdist.py @@ -0,0 +1,399 @@ +"""Tests for packaging.command.sdist.""" +import os +import zipfile +import tarfile +import logging + +from packaging.tests.support import requires_zlib + +try: + import grp + import pwd + UID_GID_SUPPORT = True +except ImportError: + UID_GID_SUPPORT = False + +from os.path import join +from packaging.tests import captured_stdout +from packaging.command.sdist import sdist +from packaging.command.sdist import show_formats +from packaging.dist import Distribution +from packaging.tests import unittest +from packaging.errors import PackagingOptionError +from packaging.util import find_executable +from packaging.tests import support +from shutil import get_archive_formats + + +MANIFEST = """\ +# file GENERATED by packaging, do NOT edit +inroot.txt +setup.cfg +data%(sep)sdata.dt +scripts%(sep)sscript.py +some%(sep)sfile.txt +some%(sep)sother_file.txt +somecode%(sep)s__init__.py +somecode%(sep)sdoc.dat +somecode%(sep)sdoc.txt +""" + + +def builder(dist, filelist): + filelist.append('bah') + + +class SDistTestCase(support.TempdirManager, + support.LoggingCatcher, + support.EnvironRestorer, + unittest.TestCase): + + restore_environ = ['HOME'] + + def setUp(self): + super(SDistTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + os.environ['HOME'] = self.tmp_dir + # setting up an environment + self.old_path = os.getcwd() + os.mkdir(join(self.tmp_dir, 'somecode')) + os.mkdir(join(self.tmp_dir, 'dist')) + # a package, and a README + self.write_file((self.tmp_dir, 'README'), 'xxx') + self.write_file((self.tmp_dir, 'somecode', '__init__.py'), '#') + os.chdir(self.tmp_dir) + + def tearDown(self): + # back to normal + os.chdir(self.old_path) + super(SDistTestCase, self).tearDown() + + def get_cmd(self, metadata=None): + """Returns a cmd""" + if metadata is None: + metadata = {'name': 'fake', 'version': '1.0', + 'home_page': 'xxx', 'author': 'xxx', + 'author_email': 'xxx'} + dist = Distribution(metadata) + dist.packages = ['somecode'] + dist.include_package_data = True + cmd = sdist(dist) + cmd.dist_dir = 'dist' + return dist, cmd + + @requires_zlib + def test_prune_file_list(self): + # this test creates a package with some vcs dirs in it + # and launch sdist to make sure they get pruned + # on all systems + + # creating VCS directories with some files in them + os.mkdir(join(self.tmp_dir, 'somecode', '.svn')) + + self.write_file((self.tmp_dir, 'somecode', '.svn', 'ok.py'), 'xxx') + + os.mkdir(join(self.tmp_dir, 'somecode', '.hg')) + self.write_file((self.tmp_dir, 'somecode', '.hg', + 'ok'), 'xxx') + + os.mkdir(join(self.tmp_dir, 'somecode', '.git')) + self.write_file((self.tmp_dir, 'somecode', '.git', + 'ok'), 'xxx') + + # now building a sdist + dist, cmd = self.get_cmd() + + # zip is available universally + # (tar might not be installed under win32) + cmd.formats = ['zip'] + + cmd.ensure_finalized() + cmd.run() + + # now let's check what we have + dist_folder = join(self.tmp_dir, 'dist') + files = os.listdir(dist_folder) + self.assertEqual(files, ['fake-1.0.zip']) + + with zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip')) as zip_file: + content = zip_file.namelist() + + # making sure everything has been pruned correctly + self.assertEqual(len(content), 2) + + @requires_zlib + @unittest.skipIf(find_executable('tar') is None or + find_executable('gzip') is None, + 'requires tar and gzip programs') + def test_make_distribution(self): + # building a sdist + dist, cmd = self.get_cmd() + + # creating a gztar then a tar + cmd.formats = ['gztar', 'tar'] + cmd.ensure_finalized() + cmd.run() + + # making sure we have two files + dist_folder = join(self.tmp_dir, 'dist') + result = sorted(os.listdir(dist_folder)) + self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz']) + + os.remove(join(dist_folder, 'fake-1.0.tar')) + os.remove(join(dist_folder, 'fake-1.0.tar.gz')) + + # now trying a tar then a gztar + cmd.formats = ['tar', 'gztar'] + + cmd.ensure_finalized() + cmd.run() + + result = sorted(os.listdir(dist_folder)) + self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz']) + + @requires_zlib + def test_add_defaults(self): + + # http://bugs.python.org/issue2279 + + # add_default should also include + # data_files and package_data + dist, cmd = self.get_cmd() + + # filling data_files by pointing files + # in package_data + dist.package_data = {'': ['*.cfg', '*.dat'], + 'somecode': ['*.txt']} + self.write_file((self.tmp_dir, 'setup.cfg'), '#') + self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#') + self.write_file((self.tmp_dir, 'somecode', 'doc.dat'), '#') + + # adding some data in data_files + data_dir = join(self.tmp_dir, 'data') + os.mkdir(data_dir) + self.write_file((data_dir, 'data.dt'), '#') + some_dir = join(self.tmp_dir, 'some') + os.mkdir(some_dir) + self.write_file((self.tmp_dir, 'inroot.txt'), '#') + self.write_file((some_dir, 'file.txt'), '#') + self.write_file((some_dir, 'other_file.txt'), '#') + + dist.data_files = {'data/data.dt': '{appdata}/data.dt', + 'inroot.txt': '{appdata}/inroot.txt', + 'some/file.txt': '{appdata}/file.txt', + 'some/other_file.txt': '{appdata}/other_file.txt'} + + # adding a script + script_dir = join(self.tmp_dir, 'scripts') + os.mkdir(script_dir) + self.write_file((script_dir, 'script.py'), '#') + dist.scripts = [join('scripts', 'script.py')] + + cmd.formats = ['zip'] + cmd.use_defaults = True + + cmd.ensure_finalized() + cmd.run() + + # now let's check what we have + dist_folder = join(self.tmp_dir, 'dist') + files = os.listdir(dist_folder) + self.assertEqual(files, ['fake-1.0.zip']) + + with zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip')) as zip_file: + content = zip_file.namelist() + + # Making sure everything was added. This includes 8 code and data + # files in addition to PKG-INFO and setup.cfg + self.assertEqual(len(content), 10) + + # Checking the MANIFEST + with open(join(self.tmp_dir, 'MANIFEST')) as fp: + manifest = fp.read() + self.assertEqual(manifest, MANIFEST % {'sep': os.sep}) + + @requires_zlib + def test_metadata_check_option(self): + # testing the `check-metadata` option + dist, cmd = self.get_cmd(metadata={'name': 'xxx', 'version': 'xxx'}) + + # this should raise some warnings + # with the check subcommand + cmd.ensure_finalized() + cmd.run() + warnings = self.get_logs(logging.WARN) + self.assertEqual(len(warnings), 4) + + # trying with a complete set of metadata + self.loghandler.flush() + dist, cmd = self.get_cmd() + cmd.ensure_finalized() + cmd.metadata_check = False + cmd.run() + warnings = self.get_logs(logging.WARN) + # removing manifest generated warnings + warnings = [warn for warn in warnings if + not warn.endswith('-- skipping')] + # the remaining warnings are about the use of the default file list and + # the absence of setup.cfg + self.assertEqual(len(warnings), 2) + + def test_show_formats(self): + __, stdout = captured_stdout(show_formats) + + # the output should be a header line + one line per format + num_formats = len(get_archive_formats()) + output = [line for line in stdout.split('\n') + if line.strip().startswith('--formats=')] + self.assertEqual(len(output), num_formats) + + def test_finalize_options(self): + + dist, cmd = self.get_cmd() + cmd.finalize_options() + + # default options set by finalize + self.assertEqual(cmd.manifest, 'MANIFEST') + self.assertEqual(cmd.dist_dir, 'dist') + + # formats has to be a string splitable on (' ', ',') or + # a stringlist + cmd.formats = 1 + self.assertRaises(PackagingOptionError, cmd.finalize_options) + cmd.formats = ['zip'] + cmd.finalize_options() + + # formats has to be known + cmd.formats = 'supazipa' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + @requires_zlib + @unittest.skipUnless(UID_GID_SUPPORT, "requires grp and pwd support") + @unittest.skipIf(find_executable('tar') is None or + find_executable('gzip') is None, + 'requires tar and gzip programs') + def test_make_distribution_owner_group(self): + # building a sdist + dist, cmd = self.get_cmd() + + # creating a gztar and specifying the owner+group + cmd.formats = ['gztar'] + cmd.owner = pwd.getpwuid(0)[0] + cmd.group = grp.getgrgid(0)[0] + cmd.ensure_finalized() + cmd.run() + + # making sure we have the good rights + archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz') + with tarfile.open(archive_name) as archive: + for member in archive.getmembers(): + self.assertEqual(member.uid, 0) + self.assertEqual(member.gid, 0) + + # building a sdist again + dist, cmd = self.get_cmd() + + # creating a gztar + cmd.formats = ['gztar'] + cmd.ensure_finalized() + cmd.run() + + # making sure we have the good rights + archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz') + with tarfile.open(archive_name) as archive: + + # note that we are not testing the group ownership here + # because, depending on the platforms and the container + # rights (see #7408) + for member in archive.getmembers(): + self.assertEqual(member.uid, os.getuid()) + + @requires_zlib + def test_get_file_list(self): + # make sure MANIFEST is recalculated + dist, cmd = self.get_cmd() + # filling data_files by pointing files in package_data + dist.package_data = {'somecode': ['*.txt']} + self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#') + cmd.ensure_finalized() + cmd.run() + + # Should produce four lines. Those lines are one comment, one default + # (README) and two package files. + with open(cmd.manifest) as f: + manifest = [line.strip() for line in f.read().split('\n') + if line.strip() != ''] + self.assertEqual(len(manifest), 3) + + # Adding a file + self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#') + + # make sure build_py is reinitialized, like a fresh run + build_py = dist.get_command_obj('build_py') + build_py.finalized = False + build_py.ensure_finalized() + + cmd.run() + + with open(cmd.manifest) as f: + manifest2 = [line.strip() for line in f.read().split('\n') + if line.strip() != ''] + + # Do we have the new file in MANIFEST? + self.assertEqual(len(manifest2), 4) + self.assertIn('doc2.txt', manifest2[-1]) + + @requires_zlib + def test_manifest_marker(self): + # check that autogenerated MANIFESTs have a marker + dist, cmd = self.get_cmd() + cmd.ensure_finalized() + cmd.run() + + with open(cmd.manifest) as f: + manifest = [line.strip() for line in f.read().split('\n') + if line.strip() != ''] + + self.assertEqual(manifest[0], + '# file GENERATED by packaging, do NOT edit') + + @requires_zlib + def test_manual_manifest(self): + # check that a MANIFEST without a marker is left alone + dist, cmd = self.get_cmd() + cmd.ensure_finalized() + self.write_file((self.tmp_dir, cmd.manifest), 'README.manual') + cmd.run() + + with open(cmd.manifest) as f: + manifest = [line.strip() for line in f.read().split('\n') + if line.strip() != ''] + + self.assertEqual(manifest, ['README.manual']) + + @requires_zlib + def test_template(self): + dist, cmd = self.get_cmd() + dist.extra_files = ['include yeah'] + cmd.ensure_finalized() + self.write_file((self.tmp_dir, 'yeah'), 'xxx') + cmd.run() + with open(cmd.manifest) as f: + content = f.read() + + self.assertIn('yeah', content) + + @requires_zlib + def test_manifest_builder(self): + dist, cmd = self.get_cmd() + cmd.manifest_builders = 'packaging.tests.test_command_sdist.builder' + cmd.ensure_finalized() + cmd.run() + self.assertIn('bah', cmd.filelist.files) + + +def test_suite(): + return unittest.makeSuite(SDistTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_test.py b/Lib/packaging/tests/test_command_test.py new file mode 100644 index 0000000000..f780723970 --- /dev/null +++ b/Lib/packaging/tests/test_command_test.py @@ -0,0 +1,224 @@ +import os +import re +import sys +import shutil +import logging +import unittest as ut1 +import packaging.database + +from os.path import join +from operator import getitem, setitem, delitem +from packaging.command.build import build +from packaging.tests import unittest +from packaging.tests.support import (TempdirManager, EnvironRestorer, + LoggingCatcher) +from packaging.command.test import test +from packaging.command import set_command +from packaging.dist import Distribution + + +EXPECTED_OUTPUT_RE = r'''FAIL: test_blah \(myowntestmodule.SomeTest\) +---------------------------------------------------------------------- +Traceback \(most recent call last\): + File ".+/myowntestmodule.py", line \d+, in test_blah + self.fail\("horribly"\) +AssertionError: horribly +''' + +here = os.path.dirname(os.path.abspath(__file__)) + + +class MockBuildCmd(build): + build_lib = "mock build lib" + command_name = 'build' + plat_name = 'whatever' + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + self._record.append("build has run") + + +class TestTest(TempdirManager, + EnvironRestorer, + LoggingCatcher, + unittest.TestCase): + + restore_environ = ['PYTHONPATH'] + + def setUp(self): + super(TestTest, self).setUp() + self.addCleanup(packaging.database.clear_cache) + new_pythonpath = os.path.dirname(os.path.dirname(here)) + pythonpath = os.environ.get('PYTHONPATH') + if pythonpath is not None: + new_pythonpath = os.pathsep.join((new_pythonpath, pythonpath)) + os.environ['PYTHONPATH'] = new_pythonpath + + def assert_re_match(self, pattern, string): + def quote(s): + lines = ['## ' + line for line in s.split('\n')] + sep = ["#" * 60] + return [''] + sep + lines + sep + msg = quote(pattern) + ["didn't match"] + quote(string) + msg = "\n".join(msg) + if not re.search(pattern, string): + self.fail(msg) + + def prepare_dist(self, dist_name): + pkg_dir = join(os.path.dirname(__file__), "dists", dist_name) + temp_pkg_dir = join(self.mkdtemp(), dist_name) + shutil.copytree(pkg_dir, temp_pkg_dir) + return temp_pkg_dir + + def safely_replace(self, obj, attr, + new_val=None, delete=False, dictionary=False): + """Replace a object's attribute returning to its original state at the + end of the test run. Creates the attribute if not present before + (deleting afterwards). When delete=True, makes sure the value is del'd + for the test run. If dictionary is set to True, operates of its items + rather than attributes.""" + if dictionary: + _setattr, _getattr, _delattr = setitem, getitem, delitem + + def _hasattr(_dict, value): + return value in _dict + else: + _setattr, _getattr, _delattr, _hasattr = (setattr, getattr, + delattr, hasattr) + + orig_has_attr = _hasattr(obj, attr) + if orig_has_attr: + orig_val = _getattr(obj, attr) + + if delete is False: + _setattr(obj, attr, new_val) + elif orig_has_attr: + _delattr(obj, attr) + + def do_cleanup(): + if orig_has_attr: + _setattr(obj, attr, orig_val) + elif _hasattr(obj, attr): + _delattr(obj, attr) + + self.addCleanup(do_cleanup) + + def test_runs_unittest(self): + module_name, a_module = self.prepare_a_module() + record = [] + a_module.recorder = lambda *args: record.append("suite") + + class MockTextTestRunner: + def __init__(*_, **__): + pass + + def run(_self, suite): + record.append("run") + + self.safely_replace(ut1, "TextTestRunner", MockTextTestRunner) + + dist = Distribution() + cmd = test(dist) + cmd.suite = "%s.recorder" % module_name + cmd.run() + self.assertEqual(record, ["suite", "run"]) + + def test_builds_before_running_tests(self): + self.addCleanup(set_command, 'packaging.command.build.build') + set_command('packaging.tests.test_command_test.MockBuildCmd') + + dist = Distribution() + dist.get_command_obj('build')._record = record = [] + cmd = test(dist) + cmd.runner = self.prepare_named_function(lambda: None) + cmd.ensure_finalized() + cmd.run() + self.assertEqual(['build has run'], record) + + def _test_works_with_2to3(self): + pass + + def test_checks_requires(self): + dist = Distribution() + cmd = test(dist) + phony_project = 'ohno_ohno-impossible_1234-name_stop-that!' + cmd.tests_require = [phony_project] + cmd.ensure_finalized() + logs = self.get_logs(logging.WARNING) + self.assertIn(phony_project, logs[-1]) + + def prepare_a_module(self): + tmp_dir = self.mkdtemp() + sys.path.append(tmp_dir) + self.addCleanup(sys.path.remove, tmp_dir) + + self.write_file((tmp_dir, 'packaging_tests_a.py'), '') + import packaging_tests_a as a_module + return "packaging_tests_a", a_module + + def prepare_named_function(self, func): + module_name, a_module = self.prepare_a_module() + a_module.recorder = func + return "%s.recorder" % module_name + + def test_custom_runner(self): + dist = Distribution() + cmd = test(dist) + record = [] + cmd.runner = self.prepare_named_function( + lambda: record.append("runner called")) + cmd.ensure_finalized() + cmd.run() + self.assertEqual(["runner called"], record) + + def prepare_mock_ut2(self): + class MockUTClass: + def __init__(*_, **__): + pass + + def discover(self): + pass + + def run(self, _): + pass + + class MockUTModule: + TestLoader = MockUTClass + TextTestRunner = MockUTClass + + mock_ut2 = MockUTModule() + self.safely_replace(sys.modules, "unittest2", + mock_ut2, dictionary=True) + return mock_ut2 + + def test_gets_unittest_discovery(self): + mock_ut2 = self.prepare_mock_ut2() + dist = Distribution() + cmd = test(dist) + self.safely_replace(ut1.TestLoader, "discover", lambda: None) + self.assertEqual(cmd.get_ut_with_discovery(), ut1) + + del ut1.TestLoader.discover + self.assertEqual(cmd.get_ut_with_discovery(), mock_ut2) + + def test_calls_discover(self): + self.safely_replace(ut1.TestLoader, "discover", delete=True) + mock_ut2 = self.prepare_mock_ut2() + record = [] + mock_ut2.TestLoader.discover = lambda self, path: record.append(path) + dist = Distribution() + cmd = test(dist) + cmd.run() + self.assertEqual([os.curdir], record) + + +def test_suite(): + return unittest.makeSuite(TestTest) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_upload.py b/Lib/packaging/tests/test_command_upload.py new file mode 100644 index 0000000000..dbb4db784b --- /dev/null +++ b/Lib/packaging/tests/test_command_upload.py @@ -0,0 +1,163 @@ +"""Tests for packaging.command.upload.""" +import os +import sys + +from packaging.command.upload import upload +from packaging.dist import Distribution +from packaging.errors import PackagingOptionError + +from packaging.tests import unittest, support +try: + import threading + from packaging.tests.pypi_server import PyPIServerTestCase +except ImportError: + threading = None + PyPIServerTestCase = unittest.TestCase + + +PYPIRC_NOPASSWORD = """\ +[distutils] + +index-servers = + server1 + +[server1] +username:me +""" + +PYPIRC = """\ +[distutils] + +index-servers = + server1 + server2 + +[server1] +username:me +password:secret + +[server2] +username:meagain +password: secret +realm:acme +repository:http://another.pypi/ +""" + + +@unittest.skipIf(threading is None, 'needs threading') +class UploadTestCase(support.TempdirManager, support.EnvironRestorer, + support.LoggingCatcher, PyPIServerTestCase): + + restore_environ = ['HOME'] + + def setUp(self): + super(UploadTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.rc = os.path.join(self.tmp_dir, '.pypirc') + os.environ['HOME'] = self.tmp_dir + + def test_finalize_options(self): + # new format + self.write_file(self.rc, PYPIRC) + dist = Distribution() + cmd = upload(dist) + cmd.finalize_options() + for attr, expected in (('username', 'me'), ('password', 'secret'), + ('realm', 'pypi'), + ('repository', 'http://pypi.python.org/pypi')): + self.assertEqual(getattr(cmd, attr), expected) + + def test_finalize_options_unsigned_identity_raises_exception(self): + self.write_file(self.rc, PYPIRC) + dist = Distribution() + cmd = upload(dist) + cmd.identity = True + cmd.sign = False + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + def test_saved_password(self): + # file with no password + self.write_file(self.rc, PYPIRC_NOPASSWORD) + + # make sure it passes + dist = Distribution() + cmd = upload(dist) + cmd.ensure_finalized() + self.assertEqual(cmd.password, None) + + # make sure we get it as well, if another command + # initialized it at the dist level + dist.password = 'xxx' + cmd = upload(dist) + cmd.finalize_options() + self.assertEqual(cmd.password, 'xxx') + + def test_upload_without_files_raises_exception(self): + dist = Distribution() + cmd = upload(dist) + self.assertRaises(PackagingOptionError, cmd.run) + + def test_upload(self): + path = os.path.join(self.tmp_dir, 'xxx') + self.write_file(path) + command, pyversion, filename = 'xxx', '3.3', path + dist_files = [(command, pyversion, filename)] + + # lets run it + pkg_dir, dist = self.create_dist(dist_files=dist_files, author='dédé') + cmd = upload(dist) + cmd.ensure_finalized() + cmd.repository = self.pypi.full_address + cmd.run() + + # what did we send ? + handler, request_data = self.pypi.requests[-1] + headers = handler.headers + #self.assertIn('dédé', str(request_data)) + self.assertIn(b'xxx', request_data) + + self.assertEqual(int(headers['content-length']), len(request_data)) + self.assertLess(int(headers['content-length']), 2500) + self.assertTrue(headers['content-type'].startswith('multipart/form-data')) + self.assertEqual(handler.command, 'POST') + self.assertNotIn('\n', headers['authorization']) + + def test_upload_docs(self): + path = os.path.join(self.tmp_dir, 'xxx') + self.write_file(path) + command, pyversion, filename = 'xxx', '3.3', path + dist_files = [(command, pyversion, filename)] + docs_path = os.path.join(self.tmp_dir, "build", "docs") + os.makedirs(docs_path) + self.write_file(os.path.join(docs_path, "index.html"), "yellow") + self.write_file(self.rc, PYPIRC) + + # lets run it + pkg_dir, dist = self.create_dist(dist_files=dist_files, author='dédé') + + cmd = upload(dist) + cmd.get_finalized_command("build").run() + cmd.upload_docs = True + cmd.ensure_finalized() + cmd.repository = self.pypi.full_address + prev_dir = os.getcwd() + try: + os.chdir(self.tmp_dir) + cmd.run() + finally: + os.chdir(prev_dir) + + handler, request_data = self.pypi.requests[-1] + action, name, content = request_data.split( + "----------------GHSKFJDLGDS7543FJKLFHRE75642756743254" + .encode())[1:4] + + self.assertIn(b'name=":action"', action) + self.assertIn(b'doc_upload', action) + + +def test_suite(): + return unittest.makeSuite(UploadTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_command_upload_docs.py b/Lib/packaging/tests/test_command_upload_docs.py new file mode 100644 index 0000000000..4070279b7a --- /dev/null +++ b/Lib/packaging/tests/test_command_upload_docs.py @@ -0,0 +1,192 @@ +"""Tests for packaging.command.upload_docs.""" +import os +import sys +import shutil +import zipfile +try: + import _ssl +except ImportError: + _ssl = None + +from packaging.command import upload_docs as upload_docs_mod +from packaging.command.upload_docs import upload_docs, zip_dir +from packaging.dist import Distribution +from packaging.errors import PackagingFileError, PackagingOptionError + +from packaging.tests import unittest, support +try: + import threading + from packaging.tests.pypi_server import PyPIServerTestCase +except ImportError: + threading = None + PyPIServerTestCase = object + + +PYPIRC = """\ +[distutils] +index-servers = server1 + +[server1] +repository = %s +username = real_slim_shady +password = long_island +""" + + +@unittest.skipIf(threading is None, "Needs threading") +class UploadDocsTestCase(support.TempdirManager, + support.EnvironRestorer, + support.LoggingCatcher, + PyPIServerTestCase): + + restore_environ = ['HOME'] + + def setUp(self): + super(UploadDocsTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.rc = os.path.join(self.tmp_dir, '.pypirc') + os.environ['HOME'] = self.tmp_dir + self.dist = Distribution() + self.dist.metadata['Name'] = "distr-name" + self.cmd = upload_docs(self.dist) + + def test_default_uploaddir(self): + sandbox = self.mkdtemp() + previous = os.getcwd() + os.chdir(sandbox) + try: + os.mkdir("build") + self.prepare_sample_dir("build") + self.cmd.ensure_finalized() + self.assertEqual(self.cmd.upload_dir, os.path.join("build", "docs")) + finally: + os.chdir(previous) + + def test_default_uploaddir_looks_for_doc_also(self): + sandbox = self.mkdtemp() + previous = os.getcwd() + os.chdir(sandbox) + try: + os.mkdir("build") + self.prepare_sample_dir("build") + os.rename(os.path.join("build", "docs"), os.path.join("build", "doc")) + self.cmd.ensure_finalized() + self.assertEqual(self.cmd.upload_dir, os.path.join("build", "doc")) + finally: + os.chdir(previous) + + def prepare_sample_dir(self, sample_dir=None): + if sample_dir is None: + sample_dir = self.mkdtemp() + os.mkdir(os.path.join(sample_dir, "docs")) + self.write_file(os.path.join(sample_dir, "docs", "index.html"), "Ce mortel ennui") + self.write_file(os.path.join(sample_dir, "index.html"), "Oh la la") + return sample_dir + + def test_zip_dir(self): + source_dir = self.prepare_sample_dir() + compressed = zip_dir(source_dir) + + zip_f = zipfile.ZipFile(compressed) + self.assertEqual(zip_f.namelist(), ['index.html', 'docs/index.html']) + + def prepare_command(self): + self.cmd.upload_dir = self.prepare_sample_dir() + self.cmd.ensure_finalized() + self.cmd.repository = self.pypi.full_address + self.cmd.username = "username" + self.cmd.password = "password" + + def test_upload(self): + self.prepare_command() + self.cmd.run() + + self.assertEqual(len(self.pypi.requests), 1) + handler, request_data = self.pypi.requests[-1] + self.assertIn(b"content", request_data) + self.assertIn("Basic", handler.headers['authorization']) + self.assertTrue(handler.headers['content-type'] + .startswith('multipart/form-data;')) + + action, name, version, content =\ + request_data.split("----------------GHSKFJDLGDS7543FJKLFHRE75642756743254".encode())[1:5] + + + # check that we picked the right chunks + self.assertIn(b'name=":action"', action) + self.assertIn(b'name="name"', name) + self.assertIn(b'name="version"', version) + self.assertIn(b'name="content"', content) + + # check their contents + self.assertIn(b'doc_upload', action) + self.assertIn(b'distr-name', name) + self.assertIn(b'docs/index.html', content) + self.assertIn(b'Ce mortel ennui', content) + + @unittest.skipIf(_ssl is None, 'Needs SSL support') + def test_https_connection(self): + https_called = False + + orig_https = upload_docs_mod.http.client.HTTPSConnection + + def https_conn_wrapper(*args): + nonlocal https_called + https_called = True + # the testing server is http + return upload_docs_mod.http.client.HTTPConnection(*args) + + upload_docs_mod.http.client.HTTPSConnection = https_conn_wrapper + try: + self.prepare_command() + self.cmd.run() + self.assertFalse(https_called) + + self.cmd.repository = self.cmd.repository.replace("http", "https") + self.cmd.run() + self.assertTrue(https_called) + finally: + upload_docs_mod.http.client.HTTPSConnection = orig_https + + def test_handling_response(self): + self.pypi.default_response_status = '403 Forbidden' + self.prepare_command() + self.cmd.run() + self.assertIn('Upload failed (403): Forbidden', self.get_logs()[-1]) + + self.pypi.default_response_status = '301 Moved Permanently' + self.pypi.default_response_headers.append(("Location", "brand_new_location")) + self.cmd.run() + self.assertIn('brand_new_location', self.get_logs()[-1]) + + def test_reads_pypirc_data(self): + self.write_file(self.rc, PYPIRC % self.pypi.full_address) + self.cmd.repository = self.pypi.full_address + self.cmd.upload_dir = self.prepare_sample_dir() + self.cmd.ensure_finalized() + self.assertEqual(self.cmd.username, "real_slim_shady") + self.assertEqual(self.cmd.password, "long_island") + + def test_checks_index_html_presence(self): + self.cmd.upload_dir = self.prepare_sample_dir() + os.remove(os.path.join(self.cmd.upload_dir, "index.html")) + self.assertRaises(PackagingFileError, self.cmd.ensure_finalized) + + def test_checks_upload_dir(self): + self.cmd.upload_dir = self.prepare_sample_dir() + shutil.rmtree(os.path.join(self.cmd.upload_dir)) + self.assertRaises(PackagingOptionError, self.cmd.ensure_finalized) + + def test_show_response(self): + self.prepare_command() + self.cmd.show_response = True + self.cmd.run() + record = self.get_logs()[-1] + self.assertTrue(record, "should report the response") + self.assertIn(self.pypi.default_response_data, record) + +def test_suite(): + return unittest.makeSuite(UploadDocsTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_compiler.py b/Lib/packaging/tests/test_compiler.py new file mode 100644 index 0000000000..2c620cb513 --- /dev/null +++ b/Lib/packaging/tests/test_compiler.py @@ -0,0 +1,66 @@ +"""Tests for distutils.compiler.""" +import os + +from packaging.compiler import (get_default_compiler, customize_compiler, + gen_lib_options) +from packaging.tests import unittest, support + + +class FakeCompiler: + + name = 'fake' + description = 'Fake' + + def library_dir_option(self, dir): + return "-L" + dir + + def runtime_library_dir_option(self, dir): + return ["-cool", "-R" + dir] + + def find_library_file(self, dirs, lib, debug=False): + return 'found' + + def library_option(self, lib): + return "-l" + lib + + +class CompilerTestCase(support.EnvironRestorer, unittest.TestCase): + + restore_environ = ['AR', 'ARFLAGS'] + + @unittest.skipUnless(get_default_compiler() == 'unix', + 'irrelevant if default compiler is not unix') + def test_customize_compiler(self): + + os.environ['AR'] = 'my_ar' + os.environ['ARFLAGS'] = '-arflags' + + # make sure AR gets caught + class compiler: + name = 'unix' + + def set_executables(self, **kw): + self.exes = kw + + comp = compiler() + customize_compiler(comp) + self.assertEqual(comp.exes['archiver'], 'my_ar -arflags') + + def test_gen_lib_options(self): + compiler = FakeCompiler() + libdirs = ['lib1', 'lib2'] + runlibdirs = ['runlib1'] + libs = [os.path.join('dir', 'name'), 'name2'] + + opts = gen_lib_options(compiler, libdirs, runlibdirs, libs) + wanted = ['-Llib1', '-Llib2', '-cool', '-Rrunlib1', 'found', + '-lname2'] + self.assertEqual(opts, wanted) + + +def test_suite(): + return unittest.makeSuite(CompilerTestCase) + + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_config.py b/Lib/packaging/tests/test_config.py new file mode 100644 index 0000000000..bcb55fb727 --- /dev/null +++ b/Lib/packaging/tests/test_config.py @@ -0,0 +1,506 @@ +"""Tests for packaging.config.""" +import os +import sys +import logging +from io import StringIO + +from packaging import command +from packaging.dist import Distribution +from packaging.errors import PackagingFileError, PackagingOptionError +from packaging.compiler import new_compiler, _COMPILERS +from packaging.command.sdist import sdist + +from packaging.tests import unittest, support +from packaging.tests.support import requires_zlib + + +SETUP_CFG = """ +[metadata] +name = RestingParrot +version = 0.6.4 +author = Carl Meyer +author_email = carl@oddbird.net +maintainer = Éric Araujo +maintainer_email = merwok@netwok.org +summary = A sample project demonstrating packaging +description-file = %(description-file)s +keywords = packaging, sample project + +classifier = + Development Status :: 4 - Beta + Environment :: Console (Text Based) + Environment :: X11 Applications :: GTK; python_version < '3' + License :: OSI Approved :: MIT License + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 3 + +requires_python = >=2.4, <3.2 + +requires_dist = + PetShoppe + MichaelPalin (> 1.1) + pywin32; sys.platform == 'win32' + pysqlite2; python_version < '2.5' + inotify (0.0.1); sys.platform == 'linux2' + +requires_external = libxml2 + +provides_dist = packaging-sample-project (0.2) + unittest2-sample-project + +project_url = + Main repository, http://bitbucket.org/carljm/sample-distutils2-project + Fork in progress, http://bitbucket.org/Merwok/sample-distutils2-project + +[files] +packages_root = src + +packages = one + two + three + +modules = haven + +scripts = + script1.py + scripts/find-coconuts + bin/taunt + +package_data = + cheese = data/templates/* + +extra_files = %(extra-files)s + +# Replaces MANIFEST.in +sdist_extra = + include THANKS HACKING + recursive-include examples *.txt *.py + prune examples/sample?/build + +resources= + bm/ {b1,b2}.gif = {icon} + Cf*/ *.CFG = {config}/baBar/ + init_script = {script}/JunGle/ + +[global] +commands = + packaging.tests.test_config.FooBarBazTest + +compilers = + packaging.tests.test_config.DCompiler + +setup_hooks = %(setup-hooks)s + + + +[install_dist] +sub_commands = foo +""" + +# Can not be merged with SETUP_CFG else install_dist +# command will fail when trying to compile C sources +# TODO use a DummyCommand to mock build_ext +EXT_SETUP_CFG = """ +[files] +packages = one + two + parent.undeclared + +[extension:one.speed_coconuts] +sources = c_src/speed_coconuts.c +extra_link_args = "`gcc -print-file-name=libgcc.a`" -shared +define_macros = HAVE_CAIRO HAVE_GTK2 +libraries = gecodeint gecodekernel -- sys.platform != 'win32' + GecodeInt GecodeKernel -- sys.platform == 'win32' + +[extension: two.fast_taunt] +sources = cxx_src/utils_taunt.cxx + cxx_src/python_module.cxx +include_dirs = /usr/include/gecode + /usr/include/blitz +extra_compile_args = -fPIC -O2 + -DGECODE_VERSION=$(./gecode_version) -- sys.platform != 'win32' + /DGECODE_VERSION='win32' -- sys.platform == 'win32' +language = cxx + +# corner case: if the parent package of an extension is declared but +# not its grandparent, it's legal +[extension: parent.undeclared._speed] +sources = parent/undeclared/_speed.c +""" + +EXT_SETUP_CFG_BUGGY_1 = """ +[extension: realname] +name = crash_here +""" + +EXT_SETUP_CFG_BUGGY_2 = """ +[files] +packages = ham + +[extension: spam.eggs] +""" + +EXT_SETUP_CFG_BUGGY_3 = """ +[files] +packages = ok + ok.works + +[extension: ok.works.breaks._ext] +""" + +HOOKS_MODULE = """ +import logging + +logger = logging.getLogger('packaging') + +def logging_hook(config): + logger.warning('logging_hook called') +""" + + +class DCompiler: + name = 'd' + description = 'D Compiler' + + def __init__(self, *args): + pass + + +def version_hook(config): + config['metadata']['version'] += '.dev1' + + +def first_hook(config): + config['files']['modules'] += '\n first' + + +def third_hook(config): + config['files']['modules'] += '\n third' + + +class FooBarBazTest: + + def __init__(self, dist): + self.distribution = dist + + @classmethod + def get_command_name(cls): + return 'foo' + + def run(self): + self.distribution.foo_was_here = True + + def nothing(self): + pass + + def get_source_files(self): + return [] + + ensure_finalized = finalize_options = initialize_options = nothing + + +class ConfigTestCase(support.TempdirManager, + support.EnvironRestorer, + support.LoggingCatcher, + unittest.TestCase): + + restore_environ = ['PLAT'] + + def setUp(self): + super(ConfigTestCase, self).setUp() + self.addCleanup(setattr, sys, 'stdout', sys.stdout) + self.addCleanup(setattr, sys, 'stderr', sys.stderr) + sys.stdout = StringIO() + sys.stderr = StringIO() + + self.addCleanup(os.chdir, os.getcwd()) + tempdir = self.mkdtemp() + self.working_dir = os.getcwd() + os.chdir(tempdir) + self.tempdir = tempdir + + def tearDown(self): + os.chdir(self.working_dir) + super(ConfigTestCase, self).tearDown() + + def write_setup(self, kwargs=None): + opts = {'description-file': 'README', 'extra-files': '', + 'setup-hooks': 'packaging.tests.test_config.version_hook'} + if kwargs: + opts.update(kwargs) + self.write_file('setup.cfg', SETUP_CFG % opts, encoding='utf-8') + + def get_dist(self): + dist = Distribution() + dist.parse_config_files() + return dist + + def test_config(self): + self.write_setup() + self.write_file('README', 'yeah') + os.mkdir('bm') + self.write_file(('bm', 'b1.gif'), '') + self.write_file(('bm', 'b2.gif'), '') + os.mkdir('Cfg') + self.write_file(('Cfg', 'data.CFG'), '') + self.write_file('init_script', '') + + # try to load the metadata now + dist = self.get_dist() + + # check what was done + self.assertEqual(dist.metadata['Author'], 'Carl Meyer') + self.assertEqual(dist.metadata['Author-Email'], 'carl@oddbird.net') + + # the hook adds .dev1 + self.assertEqual(dist.metadata['Version'], '0.6.4.dev1') + + wanted = [ + 'Development Status :: 4 - Beta', + 'Environment :: Console (Text Based)', + "Environment :: X11 Applications :: GTK; python_version < '3'", + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3'] + self.assertEqual(dist.metadata['Classifier'], wanted) + + wanted = ['packaging', 'sample project'] + self.assertEqual(dist.metadata['Keywords'], wanted) + + self.assertEqual(dist.metadata['Requires-Python'], '>=2.4, <3.2') + + wanted = ['PetShoppe', + 'MichaelPalin (> 1.1)', + "pywin32; sys.platform == 'win32'", + "pysqlite2; python_version < '2.5'", + "inotify (0.0.1); sys.platform == 'linux2'"] + + self.assertEqual(dist.metadata['Requires-Dist'], wanted) + urls = [('Main repository', + 'http://bitbucket.org/carljm/sample-distutils2-project'), + ('Fork in progress', + 'http://bitbucket.org/Merwok/sample-distutils2-project')] + self.assertEqual(dist.metadata['Project-Url'], urls) + + self.assertEqual(dist.packages, ['one', 'two', 'three']) + self.assertEqual(dist.py_modules, ['haven']) + self.assertEqual(dist.package_data, {'cheese': 'data/templates/*'}) + self.assertEqual( + {'bm/b1.gif': '{icon}/b1.gif', + 'bm/b2.gif': '{icon}/b2.gif', + 'Cfg/data.CFG': '{config}/baBar/data.CFG', + 'init_script': '{script}/JunGle/init_script'}, + dist.data_files) + + self.assertEqual(dist.package_dir, 'src') + + # Make sure we get the foo command loaded. We use a string comparison + # instead of assertIsInstance because the class is not the same when + # this test is run directly: foo is packaging.tests.test_config.Foo + # because get_command_class uses the full name, but a bare "Foo" in + # this file would be __main__.Foo when run as "python test_config.py". + # The name FooBarBazTest should be unique enough to prevent + # collisions. + self.assertEqual('FooBarBazTest', + dist.get_command_obj('foo').__class__.__name__) + + # did the README got loaded ? + self.assertEqual(dist.metadata['description'], 'yeah') + + # do we have the D Compiler enabled ? + self.assertIn('d', _COMPILERS) + d = new_compiler(compiler='d') + self.assertEqual(d.description, 'D Compiler') + + def test_multiple_description_file(self): + self.write_setup({'description-file': 'README CHANGES'}) + self.write_file('README', 'yeah') + self.write_file('CHANGES', 'changelog2') + dist = self.get_dist() + self.assertEqual(dist.metadata.requires_files, ['README', 'CHANGES']) + + def test_multiline_description_file(self): + self.write_setup({'description-file': 'README\n CHANGES'}) + self.write_file('README', 'yeah') + self.write_file('CHANGES', 'changelog') + dist = self.get_dist() + self.assertEqual(dist.metadata['description'], 'yeah\nchangelog') + self.assertEqual(dist.metadata.requires_files, ['README', 'CHANGES']) + + def test_parse_extensions_in_config(self): + self.write_file('setup.cfg', EXT_SETUP_CFG) + dist = self.get_dist() + + ext_modules = dict((mod.name, mod) for mod in dist.ext_modules) + self.assertEqual(len(ext_modules), 3) + ext = ext_modules.get('one.speed_coconuts') + self.assertEqual(ext.sources, ['c_src/speed_coconuts.c']) + self.assertEqual(ext.define_macros, ['HAVE_CAIRO', 'HAVE_GTK2']) + libs = ['gecodeint', 'gecodekernel'] + if sys.platform == 'win32': + libs = ['GecodeInt', 'GecodeKernel'] + self.assertEqual(ext.libraries, libs) + self.assertEqual(ext.extra_link_args, + ['`gcc -print-file-name=libgcc.a`', '-shared']) + + ext = ext_modules.get('two.fast_taunt') + self.assertEqual(ext.sources, + ['cxx_src/utils_taunt.cxx', 'cxx_src/python_module.cxx']) + self.assertEqual(ext.include_dirs, + ['/usr/include/gecode', '/usr/include/blitz']) + cargs = ['-fPIC', '-O2'] + if sys.platform == 'win32': + cargs.append("/DGECODE_VERSION=win32") + else: + cargs.append('-DGECODE_VERSION=$(./gecode_version)') + self.assertEqual(ext.extra_compile_args, cargs) + self.assertEqual(ext.language, 'cxx') + + self.write_file('setup.cfg', EXT_SETUP_CFG_BUGGY_1) + self.assertRaises(PackagingOptionError, self.get_dist) + + self.write_file('setup.cfg', EXT_SETUP_CFG_BUGGY_2) + self.assertRaises(PackagingOptionError, self.get_dist) + + self.write_file('setup.cfg', EXT_SETUP_CFG_BUGGY_3) + self.assertRaises(PackagingOptionError, self.get_dist) + + def test_project_setup_hook_works(self): + # Bug #11637: ensure the project directory is on sys.path to allow + # project-specific hooks + self.write_setup({'setup-hooks': 'hooks.logging_hook'}) + self.write_file('README', 'yeah') + self.write_file('hooks.py', HOOKS_MODULE) + self.get_dist() + logs = self.get_logs(logging.WARNING) + self.assertEqual(['logging_hook called'], logs) + self.assertIn('hooks', sys.modules) + + def test_missing_setup_hook_warns(self): + self.write_setup({'setup-hooks': 'this.does._not.exist'}) + self.write_file('README', 'yeah') + self.get_dist() + logs = self.get_logs(logging.WARNING) + self.assertEqual(1, len(logs)) + self.assertIn('cannot find setup hook', logs[0]) + + def test_multiple_setup_hooks(self): + self.write_setup({ + 'setup-hooks': '\n packaging.tests.test_config.first_hook' + '\n packaging.tests.test_config.missing_hook' + '\n packaging.tests.test_config.third_hook', + }) + self.write_file('README', 'yeah') + dist = self.get_dist() + + self.assertEqual(['haven', 'first', 'third'], dist.py_modules) + logs = self.get_logs(logging.WARNING) + self.assertEqual(1, len(logs)) + self.assertIn('cannot find setup hook', logs[0]) + + def test_metadata_requires_description_files_missing(self): + self.write_setup({'description-file': 'README README2'}) + self.write_file('README', 'yeah') + self.write_file('README2', 'yeah') + os.mkdir('src') + self.write_file(('src', 'haven.py'), '#') + self.write_file('script1.py', '#') + os.mkdir('scripts') + self.write_file(('scripts', 'find-coconuts'), '#') + os.mkdir('bin') + self.write_file(('bin', 'taunt'), '#') + + for pkg in ('one', 'two', 'three'): + pkg = os.path.join('src', pkg) + os.mkdir(pkg) + self.write_file((pkg, '__init__.py'), '#') + + dist = self.get_dist() + cmd = sdist(dist) + cmd.finalize_options() + cmd.get_file_list() + self.assertRaises(PackagingFileError, cmd.make_distribution) + + @requires_zlib + def test_metadata_requires_description_files(self): + # Create the following file structure: + # README + # README2 + # script1.py + # scripts/ + # find-coconuts + # bin/ + # taunt + # src/ + # haven.py + # one/__init__.py + # two/__init__.py + # three/__init__.py + + self.write_setup({'description-file': 'README\n README2', + 'extra-files': '\n README3'}) + self.write_file('README', 'yeah 1') + self.write_file('README2', 'yeah 2') + self.write_file('README3', 'yeah 3') + os.mkdir('src') + self.write_file(('src', 'haven.py'), '#') + self.write_file('script1.py', '#') + os.mkdir('scripts') + self.write_file(('scripts', 'find-coconuts'), '#') + os.mkdir('bin') + self.write_file(('bin', 'taunt'), '#') + + for pkg in ('one', 'two', 'three'): + pkg = os.path.join('src', pkg) + os.mkdir(pkg) + self.write_file((pkg, '__init__.py'), '#') + + dist = self.get_dist() + self.assertIn('yeah 1\nyeah 2', dist.metadata['description']) + + cmd = sdist(dist) + cmd.finalize_options() + cmd.get_file_list() + self.assertRaises(PackagingFileError, cmd.make_distribution) + + self.write_setup({'description-file': 'README\n README2', + 'extra-files': '\n README2\n README'}) + dist = self.get_dist() + cmd = sdist(dist) + cmd.finalize_options() + cmd.get_file_list() + cmd.make_distribution() + with open('MANIFEST') as fp: + self.assertIn('README\nREADME2\n', fp.read()) + + def test_sub_commands(self): + self.write_setup() + self.write_file('README', 'yeah') + os.mkdir('src') + self.write_file(('src', 'haven.py'), '#') + self.write_file('script1.py', '#') + os.mkdir('scripts') + self.write_file(('scripts', 'find-coconuts'), '#') + os.mkdir('bin') + self.write_file(('bin', 'taunt'), '#') + + for pkg in ('one', 'two', 'three'): + pkg = os.path.join('src', pkg) + os.mkdir(pkg) + self.write_file((pkg, '__init__.py'), '#') + + # try to run the install command to see if foo is called + dist = self.get_dist() + self.assertIn('foo', command.get_command_names()) + self.assertEqual('FooBarBazTest', + dist.get_command_obj('foo').__class__.__name__) + + +def test_suite(): + return unittest.makeSuite(ConfigTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_create.py b/Lib/packaging/tests/test_create.py new file mode 100644 index 0000000000..a82ab43254 --- /dev/null +++ b/Lib/packaging/tests/test_create.py @@ -0,0 +1,243 @@ +"""Tests for packaging.create.""" +import io +import os +import sys +import sysconfig +from textwrap import dedent +from packaging.create import MainProgram, ask_yn, ask, main + +from packaging.tests import support, unittest + + +class CreateTestCase(support.TempdirManager, + support.EnvironRestorer, + unittest.TestCase): + + maxDiff = None + restore_environ = ['PLAT'] + + def setUp(self): + super(CreateTestCase, self).setUp() + self._stdin = sys.stdin # TODO use Inputs + self._stdout = sys.stdout + sys.stdin = io.StringIO() + sys.stdout = io.StringIO() + self._cwd = os.getcwd() + self.wdir = self.mkdtemp() + os.chdir(self.wdir) + # patch sysconfig + self._old_get_paths = sysconfig.get_paths + sysconfig.get_paths = lambda *args, **kwargs: { + 'man': sys.prefix + '/share/man', + 'doc': sys.prefix + '/share/doc/pyxfoil', } + + def tearDown(self): + sys.stdin = self._stdin + sys.stdout = self._stdout + os.chdir(self._cwd) + sysconfig.get_paths = self._old_get_paths + super(CreateTestCase, self).tearDown() + + def test_ask_yn(self): + sys.stdin.write('y\n') + sys.stdin.seek(0) + self.assertEqual('y', ask_yn('is this a test')) + + def test_ask(self): + sys.stdin.write('a\n') + sys.stdin.write('b\n') + sys.stdin.seek(0) + self.assertEqual('a', ask('is this a test')) + self.assertEqual('b', ask(str(list(range(0, 70))), default='c', + lengthy=True)) + + def test_set_multi(self): + mainprogram = MainProgram() + sys.stdin.write('aaaaa\n') + sys.stdin.seek(0) + mainprogram.data['author'] = [] + mainprogram._set_multi('_set_multi test', 'author') + self.assertEqual(['aaaaa'], mainprogram.data['author']) + + def test_find_files(self): + # making sure we scan a project dir correctly + mainprogram = MainProgram() + + # building the structure + tempdir = self.wdir + dirs = ['pkg1', 'data', 'pkg2', 'pkg2/sub'] + files = [ + 'README', + 'data/data1', + 'foo.py', + 'pkg1/__init__.py', + 'pkg1/bar.py', + 'pkg2/__init__.py', + 'pkg2/sub/__init__.py', + ] + + for dir_ in dirs: + os.mkdir(os.path.join(tempdir, dir_)) + + for file_ in files: + path = os.path.join(tempdir, file_) + self.write_file(path, 'xxx') + + mainprogram._find_files() + mainprogram.data['packages'].sort() + + # do we have what we want? + self.assertEqual(mainprogram.data['packages'], + ['pkg1', 'pkg2', 'pkg2.sub']) + self.assertEqual(mainprogram.data['modules'], ['foo']) + data_fn = os.path.join('data', 'data1') + self.assertEqual(mainprogram.data['extra_files'], + ['README', data_fn]) + + def test_convert_setup_py_to_cfg(self): + self.write_file((self.wdir, 'setup.py'), + dedent(""" + # coding: utf-8 + from distutils.core import setup + + long_description = '''My super Death-scription + barbar is now on the public domain, + ho, baby !''' + + setup(name='pyxfoil', + version='0.2', + description='Python bindings for the Xfoil engine', + long_description=long_description, + maintainer='André Espaze', + maintainer_email='andre.espaze@logilab.fr', + url='http://www.python-science.org/project/pyxfoil', + license='GPLv2', + packages=['pyxfoil', 'babar', 'me'], + data_files=[ + ('share/doc/pyxfoil', ['README.rst']), + ('share/man', ['pyxfoil.1']), + ], + py_modules=['my_lib', 'mymodule'], + package_dir={ + 'babar': '', + 'me': 'Martinique/Lamentin', + }, + package_data={ + 'babar': ['Pom', 'Flora', 'Alexander'], + 'me': ['dady', 'mumy', 'sys', 'bro'], + '': ['setup.py', 'README'], + 'pyxfoil': ['fengine.so'], + }, + scripts=['my_script', 'bin/run'], + ) + """), encoding='utf-8') + sys.stdin.write('y\n') + sys.stdin.seek(0) + main() + + with open(os.path.join(self.wdir, 'setup.cfg'), encoding='utf-8') as fp: + contents = fp.read() + + self.assertEqual(contents, dedent("""\ + [metadata] + name = pyxfoil + version = 0.2 + summary = Python bindings for the Xfoil engine + download_url = UNKNOWN + home_page = http://www.python-science.org/project/pyxfoil + maintainer = André Espaze + maintainer_email = andre.espaze@logilab.fr + description = My super Death-scription + |barbar is now on the public domain, + |ho, baby ! + + [files] + packages = pyxfoil + babar + me + modules = my_lib + mymodule + scripts = my_script + bin/run + extra_files = Martinique/Lamentin/dady + Martinique/Lamentin/mumy + Martinique/Lamentin/sys + Martinique/Lamentin/bro + setup.py + README + Pom + Flora + Alexander + pyxfoil/fengine.so + + resources = + README.rst = {doc} + pyxfoil.1 = {man} + + """)) + + def test_convert_setup_py_to_cfg_with_description_in_readme(self): + self.write_file((self.wdir, 'setup.py'), + dedent(""" + # coding: utf-8 + from distutils.core import setup + with open('README.txt') as fp: + long_description = fp.read() + + setup(name='pyxfoil', + version='0.2', + description='Python bindings for the Xfoil engine', + long_description=long_description, + maintainer='André Espaze', + maintainer_email='andre.espaze@logilab.fr', + url='http://www.python-science.org/project/pyxfoil', + license='GPLv2', + packages=['pyxfoil'], + package_data={'pyxfoil': ['fengine.so', 'babar.so']}, + data_files=[ + ('share/doc/pyxfoil', ['README.rst']), + ('share/man', ['pyxfoil.1']), + ], + ) + """), encoding='utf-8') + self.write_file((self.wdir, 'README.txt'), + dedent(''' +My super Death-scription +barbar is now in the public domain, +ho, baby! + ''')) + sys.stdin.write('y\n') + sys.stdin.seek(0) + # FIXME Out of memory error. + main() + with open(os.path.join(self.wdir, 'setup.cfg'), encoding='utf-8') as fp: + contents = fp.read() + + self.assertEqual(contents, dedent("""\ + [metadata] + name = pyxfoil + version = 0.2 + summary = Python bindings for the Xfoil engine + download_url = UNKNOWN + home_page = http://www.python-science.org/project/pyxfoil + maintainer = André Espaze + maintainer_email = andre.espaze@logilab.fr + description-file = README.txt + + [files] + packages = pyxfoil + extra_files = pyxfoil/fengine.so + pyxfoil/babar.so + + resources = + README.rst = {doc} + pyxfoil.1 = {man} + + """)) + + +def test_suite(): + return unittest.makeSuite(CreateTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_cygwinccompiler.py b/Lib/packaging/tests/test_cygwinccompiler.py new file mode 100644 index 0000000000..17c43cd28a --- /dev/null +++ b/Lib/packaging/tests/test_cygwinccompiler.py @@ -0,0 +1,88 @@ +"""Tests for packaging.cygwinccompiler.""" +import os +import sys +import sysconfig +from packaging.compiler.cygwinccompiler import ( + check_config_h, get_msvcr, + CONFIG_H_OK, CONFIG_H_NOTOK, CONFIG_H_UNCERTAIN) + +from packaging.tests import unittest, support + + +class CygwinCCompilerTestCase(support.TempdirManager, + unittest.TestCase): + + def setUp(self): + super(CygwinCCompilerTestCase, self).setUp() + self.version = sys.version + self.python_h = os.path.join(self.mkdtemp(), 'python.h') + self.old_get_config_h_filename = sysconfig.get_config_h_filename + sysconfig.get_config_h_filename = self._get_config_h_filename + + def tearDown(self): + sys.version = self.version + sysconfig.get_config_h_filename = self.old_get_config_h_filename + super(CygwinCCompilerTestCase, self).tearDown() + + def _get_config_h_filename(self): + return self.python_h + + def test_check_config_h(self): + # check_config_h looks for "GCC" in sys.version first + # returns CONFIG_H_OK if found + sys.version = ('2.6.1 (r261:67515, Dec 6 2008, 16:42:21) \n[GCC ' + '4.0.1 (Apple Computer, Inc. build 5370)]') + + self.assertEqual(check_config_h()[0], CONFIG_H_OK) + + # then it tries to see if it can find "__GNUC__" in pyconfig.h + sys.version = 'something without the *CC word' + + # if the file doesn't exist it returns CONFIG_H_UNCERTAIN + self.assertEqual(check_config_h()[0], CONFIG_H_UNCERTAIN) + + # if it exists but does not contain __GNUC__, it returns CONFIG_H_NOTOK + self.write_file(self.python_h, 'xxx') + self.assertEqual(check_config_h()[0], CONFIG_H_NOTOK) + + # and CONFIG_H_OK if __GNUC__ is found + self.write_file(self.python_h, 'xxx __GNUC__ xxx') + self.assertEqual(check_config_h()[0], CONFIG_H_OK) + + def test_get_msvcr(self): + # none + sys.version = ('2.6.1 (r261:67515, Dec 6 2008, 16:42:21) ' + '\n[GCC 4.0.1 (Apple Computer, Inc. build 5370)]') + self.assertEqual(get_msvcr(), None) + + # MSVC 7.0 + sys.version = ('2.5.1 (r251:54863, Apr 18 2007, 08:51:08) ' + '[MSC v.1300 32 bits (Intel)]') + self.assertEqual(get_msvcr(), ['msvcr70']) + + # MSVC 7.1 + sys.version = ('2.5.1 (r251:54863, Apr 18 2007, 08:51:08) ' + '[MSC v.1310 32 bits (Intel)]') + self.assertEqual(get_msvcr(), ['msvcr71']) + + # VS2005 / MSVC 8.0 + sys.version = ('2.5.1 (r251:54863, Apr 18 2007, 08:51:08) ' + '[MSC v.1400 32 bits (Intel)]') + self.assertEqual(get_msvcr(), ['msvcr80']) + + # VS2008 / MSVC 9.0 + sys.version = ('2.5.1 (r251:54863, Apr 18 2007, 08:51:08) ' + '[MSC v.1500 32 bits (Intel)]') + self.assertEqual(get_msvcr(), ['msvcr90']) + + # unknown + sys.version = ('2.5.1 (r251:54863, Apr 18 2007, 08:51:08) ' + '[MSC v.1999 32 bits (Intel)]') + self.assertRaises(ValueError, get_msvcr) + + +def test_suite(): + return unittest.makeSuite(CygwinCCompilerTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_database.py b/Lib/packaging/tests/test_database.py new file mode 100644 index 0000000000..9baf3960c6 --- /dev/null +++ b/Lib/packaging/tests/test_database.py @@ -0,0 +1,675 @@ +import os +import io +import csv +import sys +import shutil +import tempfile +from os.path import relpath # separate import for backport concerns +from hashlib import md5 +from textwrap import dedent + +from packaging.tests.test_util import GlobTestCaseBase +from packaging.tests.support import requires_zlib + +from packaging.config import get_resources_dests +from packaging.errors import PackagingError +from packaging.metadata import Metadata +from packaging.tests import unittest, support +from packaging.database import ( + Distribution, EggInfoDistribution, get_distribution, get_distributions, + provides_distribution, obsoletes_distribution, get_file_users, + enable_cache, disable_cache, distinfo_dirname, _yield_distributions, + get_file, get_file_path) + +# TODO Add a test for getting a distribution provided by another distribution +# TODO Add a test for absolute pathed RECORD items (e.g. /etc/myapp/config.ini) +# TODO Add tests from the former pep376 project (zipped site-packages, etc.) + + +def get_hexdigest(filename): + with open(filename, 'rb') as file: + checksum = md5(file.read()) + return checksum.hexdigest() + + +def record_pieces(file): + path = relpath(file, sys.prefix) + digest = get_hexdigest(file) + size = os.path.getsize(file) + return [path, digest, size] + + +class FakeDistsMixin: + + def setUp(self): + super(FakeDistsMixin, self).setUp() + self.addCleanup(enable_cache) + disable_cache() + + # make a copy that we can write into for our fake installed + # distributions + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) + self.fake_dists_path = os.path.join(tmpdir, 'fake_dists') + fake_dists_src = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'fake_dists')) + shutil.copytree(fake_dists_src, self.fake_dists_path) + # XXX ugly workaround: revert copystat calls done by shutil behind our + # back (to avoid getting a read-only copy of a read-only file). we + # could pass a custom copy_function to change the mode of files, but + # shutil gives no control over the mode of directories :( + for root, dirs, files in os.walk(self.fake_dists_path): + os.chmod(root, 0o755) + for f in files: + os.chmod(os.path.join(root, f), 0o644) + for d in dirs: + os.chmod(os.path.join(root, d), 0o755) + + +class CommonDistributionTests(FakeDistsMixin): + """Mixin used to test the interface common to both Distribution classes. + + Derived classes define cls, sample_dist, dirs and records. These + attributes are used in test methods. See source code for details. + """ + + def test_instantiation(self): + # check that useful attributes are here + name, version, distdir = self.sample_dist + here = os.path.abspath(os.path.dirname(__file__)) + dist_path = os.path.join(here, 'fake_dists', distdir) + + dist = self.dist = self.cls(dist_path) + self.assertEqual(dist.path, dist_path) + self.assertEqual(dist.name, name) + self.assertEqual(dist.metadata['Name'], name) + self.assertIsInstance(dist.metadata, Metadata) + self.assertEqual(dist.version, version) + self.assertEqual(dist.metadata['Version'], version) + + @requires_zlib + def test_repr(self): + dist = self.cls(self.dirs[0]) + # just check that the class name is in the repr + self.assertIn(self.cls.__name__, repr(dist)) + + @requires_zlib + def test_comparison(self): + # tests for __eq__ and __hash__ + dist = self.cls(self.dirs[0]) + dist2 = self.cls(self.dirs[0]) + dist3 = self.cls(self.dirs[1]) + self.assertIn(dist, {dist: True}) + self.assertEqual(dist, dist) + + self.assertIsNot(dist, dist2) + self.assertEqual(dist, dist2) + self.assertNotEqual(dist, dist3) + self.assertNotEqual(dist, ()) + + def test_list_installed_files(self): + for dir_ in self.dirs: + dist = self.cls(dir_) + for path, md5_, size in dist.list_installed_files(): + record_data = self.records[dist.path] + self.assertIn(path, record_data) + self.assertEqual(md5_, record_data[path][0]) + self.assertEqual(size, record_data[path][1]) + + +class TestDistribution(CommonDistributionTests, unittest.TestCase): + + cls = Distribution + sample_dist = 'choxie', '2.0.0.9', 'choxie-2.0.0.9.dist-info' + + def setUp(self): + super(TestDistribution, self).setUp() + self.dirs = [os.path.join(self.fake_dists_path, f) + for f in os.listdir(self.fake_dists_path) + if f.endswith('.dist-info')] + + self.records = {} + for distinfo_dir in self.dirs: + + record_file = os.path.join(distinfo_dir, 'RECORD') + with open(record_file, 'w') as file: + record_writer = csv.writer( + file, delimiter=',', quoting=csv.QUOTE_NONE, + lineterminator='\n') + + dist_location = distinfo_dir.replace('.dist-info', '') + + for path, dirs, files in os.walk(dist_location): + for f in files: + record_writer.writerow(record_pieces( + os.path.join(path, f))) + for file in ('INSTALLER', 'METADATA', 'REQUESTED'): + record_writer.writerow(record_pieces( + os.path.join(distinfo_dir, file))) + record_writer.writerow([relpath(record_file, sys.prefix)]) + + with open(record_file) as file: + record_reader = csv.reader(file, lineterminator='\n') + record_data = {} + for row in record_reader: + if row == []: + continue + path, md5_, size = (row[:] + + [None for i in range(len(row), 3)]) + record_data[path] = md5_, size + self.records[distinfo_dir] = record_data + + def test_instantiation(self): + super(TestDistribution, self).test_instantiation() + self.assertIsInstance(self.dist.requested, bool) + + def test_uses(self): + # Test to determine if a distribution uses a specified file. + # Criteria to test against + distinfo_name = 'grammar-1.0a4' + distinfo_dir = os.path.join(self.fake_dists_path, + distinfo_name + '.dist-info') + true_path = [self.fake_dists_path, distinfo_name, + 'grammar', 'utils.py'] + true_path = relpath(os.path.join(*true_path), sys.prefix) + false_path = [self.fake_dists_path, 'towel_stuff-0.1', 'towel_stuff', + '__init__.py'] + false_path = relpath(os.path.join(*false_path), sys.prefix) + + # Test if the distribution uses the file in question + dist = Distribution(distinfo_dir) + self.assertTrue(dist.uses(true_path)) + self.assertFalse(dist.uses(false_path)) + + def test_get_distinfo_file(self): + # Test the retrieval of dist-info file objects. + distinfo_name = 'choxie-2.0.0.9' + other_distinfo_name = 'grammar-1.0a4' + distinfo_dir = os.path.join(self.fake_dists_path, + distinfo_name + '.dist-info') + dist = Distribution(distinfo_dir) + # Test for known good file matches + distinfo_files = [ + # Relative paths + 'INSTALLER', 'METADATA', + # Absolute paths + os.path.join(distinfo_dir, 'RECORD'), + os.path.join(distinfo_dir, 'REQUESTED'), + ] + + for distfile in distinfo_files: + with dist.get_distinfo_file(distfile) as value: + self.assertIsInstance(value, io.TextIOWrapper) + # Is it the correct file? + self.assertEqual(value.name, + os.path.join(distinfo_dir, distfile)) + + # Test an absolute path that is part of another distributions dist-info + other_distinfo_file = os.path.join( + self.fake_dists_path, other_distinfo_name + '.dist-info', + 'REQUESTED') + self.assertRaises(PackagingError, dist.get_distinfo_file, + other_distinfo_file) + # Test for a file that should not exist + self.assertRaises(PackagingError, dist.get_distinfo_file, + 'MAGICFILE') + + def test_list_distinfo_files(self): + # Test for the iteration of RECORD path entries. + distinfo_name = 'towel_stuff-0.1' + distinfo_dir = os.path.join(self.fake_dists_path, + distinfo_name + '.dist-info') + dist = Distribution(distinfo_dir) + # Test for the iteration of the raw path + distinfo_record_paths = self.records[distinfo_dir].keys() + found = dist.list_distinfo_files() + self.assertEqual(sorted(found), sorted(distinfo_record_paths)) + # Test for the iteration of local absolute paths + distinfo_record_paths = [os.path.join(sys.prefix, path) + for path in self.records[distinfo_dir]] + found = dist.list_distinfo_files(local=True) + self.assertEqual(sorted(found), sorted(distinfo_record_paths)) + + def test_get_resources_path(self): + distinfo_name = 'babar-0.1' + distinfo_dir = os.path.join(self.fake_dists_path, + distinfo_name + '.dist-info') + dist = Distribution(distinfo_dir) + resource_path = dist.get_resource_path('babar.png') + self.assertEqual(resource_path, 'babar.png') + self.assertRaises(KeyError, dist.get_resource_path, 'notexist') + + +class TestEggInfoDistribution(CommonDistributionTests, + support.LoggingCatcher, + unittest.TestCase): + + cls = EggInfoDistribution + sample_dist = 'bacon', '0.1', 'bacon-0.1.egg-info' + + def setUp(self): + super(TestEggInfoDistribution, self).setUp() + + self.dirs = [os.path.join(self.fake_dists_path, f) + for f in os.listdir(self.fake_dists_path) + if f.endswith('.egg') or f.endswith('.egg-info')] + + self.records = {} + + @unittest.skip('not implemented yet') + def test_list_installed_files(self): + # EggInfoDistribution defines list_installed_files but there is no + # test for it yet; someone with setuptools expertise needs to add a + # file with the list of installed files for one of the egg fake dists + # and write the support code to populate self.records (and then delete + # this method) + pass + + +class TestDatabase(support.LoggingCatcher, + FakeDistsMixin, + unittest.TestCase): + + def setUp(self): + super(TestDatabase, self).setUp() + sys.path.insert(0, self.fake_dists_path) + self.addCleanup(sys.path.remove, self.fake_dists_path) + + def test_distinfo_dirname(self): + # Given a name and a version, we expect the distinfo_dirname function + # to return a standard distribution information directory name. + + items = [ + # (name, version, standard_dirname) + # Test for a very simple single word name and decimal version + # number + ('docutils', '0.5', 'docutils-0.5.dist-info'), + # Test for another except this time with a '-' in the name, which + # needs to be transformed during the name lookup + ('python-ldap', '2.5', 'python_ldap-2.5.dist-info'), + # Test for both '-' in the name and a funky version number + ('python-ldap', '2.5 a---5', 'python_ldap-2.5 a---5.dist-info'), + ] + + # Loop through the items to validate the results + for name, version, standard_dirname in items: + dirname = distinfo_dirname(name, version) + self.assertEqual(dirname, standard_dirname) + + @requires_zlib + def test_get_distributions(self): + # Lookup all distributions found in the ``sys.path``. + # This test could potentially pick up other installed distributions + fake_dists = [('grammar', '1.0a4'), ('choxie', '2.0.0.9'), + ('towel-stuff', '0.1'), ('babar', '0.1')] + found_dists = [] + + # Verify the fake dists have been found. + dists = [dist for dist in get_distributions()] + for dist in dists: + self.assertIsInstance(dist, Distribution) + if (dist.name in dict(fake_dists) and + dist.path.startswith(self.fake_dists_path)): + found_dists.append((dist.name, dist.version)) + else: + # check that it doesn't find anything more than this + self.assertFalse(dist.path.startswith(self.fake_dists_path)) + # otherwise we don't care what other distributions are found + + # Finally, test that we found all that we were looking for + self.assertEqual(sorted(found_dists), sorted(fake_dists)) + + # Now, test if the egg-info distributions are found correctly as well + fake_dists += [('bacon', '0.1'), ('cheese', '2.0.2'), + ('coconuts-aster', '10.3'), + ('banana', '0.4'), ('strawberry', '0.6'), + ('truffles', '5.0'), ('nut', 'funkyversion')] + found_dists = [] + + dists = [dist for dist in get_distributions(use_egg_info=True)] + for dist in dists: + self.assertIsInstance(dist, (Distribution, EggInfoDistribution)) + if (dist.name in dict(fake_dists) and + dist.path.startswith(self.fake_dists_path)): + found_dists.append((dist.name, dist.version)) + else: + self.assertFalse(dist.path.startswith(self.fake_dists_path)) + + self.assertEqual(sorted(fake_dists), sorted(found_dists)) + + @requires_zlib + def test_get_distribution(self): + # Test for looking up a distribution by name. + # Test the lookup of the towel-stuff distribution + name = 'towel-stuff' # Note: This is different from the directory name + + # Lookup the distribution + dist = get_distribution(name) + self.assertIsInstance(dist, Distribution) + self.assertEqual(dist.name, name) + + # Verify that an unknown distribution returns None + self.assertIsNone(get_distribution('bogus')) + + # Verify partial name matching doesn't work + self.assertIsNone(get_distribution('towel')) + + # Verify that it does not find egg-info distributions, when not + # instructed to + self.assertIsNone(get_distribution('bacon')) + self.assertIsNone(get_distribution('cheese')) + self.assertIsNone(get_distribution('strawberry')) + self.assertIsNone(get_distribution('banana')) + + # Now check that it works well in both situations, when egg-info + # is a file and directory respectively. + dist = get_distribution('cheese', use_egg_info=True) + self.assertIsInstance(dist, EggInfoDistribution) + self.assertEqual(dist.name, 'cheese') + + dist = get_distribution('bacon', use_egg_info=True) + self.assertIsInstance(dist, EggInfoDistribution) + self.assertEqual(dist.name, 'bacon') + + dist = get_distribution('banana', use_egg_info=True) + self.assertIsInstance(dist, EggInfoDistribution) + self.assertEqual(dist.name, 'banana') + + dist = get_distribution('strawberry', use_egg_info=True) + self.assertIsInstance(dist, EggInfoDistribution) + self.assertEqual(dist.name, 'strawberry') + + def test_get_file_users(self): + # Test the iteration of distributions that use a file. + name = 'towel_stuff-0.1' + path = os.path.join(self.fake_dists_path, name, + 'towel_stuff', '__init__.py') + for dist in get_file_users(path): + self.assertIsInstance(dist, Distribution) + self.assertEqual(dist.name, name) + + @requires_zlib + def test_provides(self): + # Test for looking up distributions by what they provide + checkLists = lambda x, y: self.assertEqual(sorted(x), sorted(y)) + + l = [dist.name for dist in provides_distribution('truffles')] + checkLists(l, ['choxie', 'towel-stuff']) + + l = [dist.name for dist in provides_distribution('truffles', '1.0')] + checkLists(l, ['choxie']) + + l = [dist.name for dist in provides_distribution('truffles', '1.0', + use_egg_info=True)] + checkLists(l, ['choxie', 'cheese']) + + l = [dist.name for dist in provides_distribution('truffles', '1.1.2')] + checkLists(l, ['towel-stuff']) + + l = [dist.name for dist in provides_distribution('truffles', '1.1')] + checkLists(l, ['towel-stuff']) + + l = [dist.name for dist in provides_distribution('truffles', + '!=1.1,<=2.0')] + checkLists(l, ['choxie']) + + l = [dist.name for dist in provides_distribution('truffles', + '!=1.1,<=2.0', + use_egg_info=True)] + checkLists(l, ['choxie', 'bacon', 'cheese']) + + l = [dist.name for dist in provides_distribution('truffles', '>1.0')] + checkLists(l, ['towel-stuff']) + + l = [dist.name for dist in provides_distribution('truffles', '>1.5')] + checkLists(l, []) + + l = [dist.name for dist in provides_distribution('truffles', '>1.5', + use_egg_info=True)] + checkLists(l, ['bacon']) + + l = [dist.name for dist in provides_distribution('truffles', '>=1.0')] + checkLists(l, ['choxie', 'towel-stuff']) + + l = [dist.name for dist in provides_distribution('strawberry', '0.6', + use_egg_info=True)] + checkLists(l, ['coconuts-aster']) + + l = [dist.name for dist in provides_distribution('strawberry', '>=0.5', + use_egg_info=True)] + checkLists(l, ['coconuts-aster']) + + l = [dist.name for dist in provides_distribution('strawberry', '>0.6', + use_egg_info=True)] + checkLists(l, []) + + l = [dist.name for dist in provides_distribution('banana', '0.4', + use_egg_info=True)] + checkLists(l, ['coconuts-aster']) + + l = [dist.name for dist in provides_distribution('banana', '>=0.3', + use_egg_info=True)] + checkLists(l, ['coconuts-aster']) + + l = [dist.name for dist in provides_distribution('banana', '!=0.4', + use_egg_info=True)] + checkLists(l, []) + + @requires_zlib + def test_obsoletes(self): + # Test looking for distributions based on what they obsolete + checkLists = lambda x, y: self.assertEqual(sorted(x), sorted(y)) + + l = [dist.name for dist in obsoletes_distribution('truffles', '1.0')] + checkLists(l, []) + + l = [dist.name for dist in obsoletes_distribution('truffles', '1.0', + use_egg_info=True)] + checkLists(l, ['cheese', 'bacon']) + + l = [dist.name for dist in obsoletes_distribution('truffles', '0.8')] + checkLists(l, ['choxie']) + + l = [dist.name for dist in obsoletes_distribution('truffles', '0.8', + use_egg_info=True)] + checkLists(l, ['choxie', 'cheese']) + + l = [dist.name for dist in obsoletes_distribution('truffles', '0.9.6')] + checkLists(l, ['choxie', 'towel-stuff']) + + l = [dist.name for dist in obsoletes_distribution('truffles', + '0.5.2.3')] + checkLists(l, ['choxie', 'towel-stuff']) + + l = [dist.name for dist in obsoletes_distribution('truffles', '0.2')] + checkLists(l, ['towel-stuff']) + + @requires_zlib + def test_yield_distribution(self): + # tests the internal function _yield_distributions + checkLists = lambda x, y: self.assertEqual(sorted(x), sorted(y)) + + eggs = [('bacon', '0.1'), ('banana', '0.4'), ('strawberry', '0.6'), + ('truffles', '5.0'), ('cheese', '2.0.2'), + ('coconuts-aster', '10.3'), ('nut', 'funkyversion')] + dists = [('choxie', '2.0.0.9'), ('grammar', '1.0a4'), + ('towel-stuff', '0.1'), ('babar', '0.1')] + + checkLists([], _yield_distributions(False, False, sys.path)) + + found = [(dist.name, dist.version) + for dist in _yield_distributions(False, True, sys.path) + if dist.path.startswith(self.fake_dists_path)] + checkLists(eggs, found) + + found = [(dist.name, dist.version) + for dist in _yield_distributions(True, False, sys.path) + if dist.path.startswith(self.fake_dists_path)] + checkLists(dists, found) + + found = [(dist.name, dist.version) + for dist in _yield_distributions(True, True, sys.path) + if dist.path.startswith(self.fake_dists_path)] + checkLists(dists + eggs, found) + + +class DataFilesTestCase(GlobTestCaseBase): + + def assertRulesMatch(self, rules, spec): + tempdir = self.build_files_tree(spec) + expected = self.clean_tree(spec) + result = get_resources_dests(tempdir, rules) + self.assertEqual(expected, result) + + def clean_tree(self, spec): + files = {} + for path, value in spec.items(): + if value is not None: + files[path] = value + return files + + def test_simple_glob(self): + rules = [('', '*.tpl', '{data}')] + spec = {'coucou.tpl': '{data}/coucou.tpl', + 'Donotwant': None} + self.assertRulesMatch(rules, spec) + + def test_multiple_match(self): + rules = [('scripts', '*.bin', '{appdata}'), + ('scripts', '*', '{appscript}')] + spec = {'scripts/script.bin': '{appscript}/script.bin', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_set_match(self): + rules = [('scripts', '*.{bin,sh}', '{appscript}')] + spec = {'scripts/script.bin': '{appscript}/script.bin', + 'scripts/babar.sh': '{appscript}/babar.sh', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_set_match_multiple(self): + rules = [('scripts', 'script{s,}.{bin,sh}', '{appscript}')] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'scripts/script.sh': '{appscript}/script.sh', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_set_match_exclude(self): + rules = [('scripts', '*', '{appscript}'), + ('', os.path.join('**', '*.sh'), None)] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'scripts/script.sh': None, + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_glob_in_base(self): + rules = [('scrip*', '*.bin', '{appscript}')] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'scripouille/babar.bin': '{appscript}/babar.bin', + 'scriptortu/lotus.bin': '{appscript}/lotus.bin', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_recursive_glob(self): + rules = [('', os.path.join('**', '*.bin'), '{binary}')] + spec = {'binary0.bin': '{binary}/binary0.bin', + 'scripts/binary1.bin': '{binary}/scripts/binary1.bin', + 'scripts/bin/binary2.bin': '{binary}/scripts/bin/binary2.bin', + 'you/kill/pandabear.guy': None} + self.assertRulesMatch(rules, spec) + + def test_final_exemple_glob(self): + rules = [ + ('mailman/database/schemas/', '*', '{appdata}/schemas'), + ('', os.path.join('**', '*.tpl'), '{appdata}/templates'), + ('', os.path.join('developer-docs', '**', '*.txt'), '{doc}'), + ('', 'README', '{doc}'), + ('mailman/etc/', '*', '{config}'), + ('mailman/foo/', os.path.join('**', 'bar', '*.cfg'), + '{config}/baz'), + ('mailman/foo/', os.path.join('**', '*.cfg'), '{config}/hmm'), + ('', 'some-new-semantic.sns', '{funky-crazy-category}'), + ] + spec = { + 'README': '{doc}/README', + 'some.tpl': '{appdata}/templates/some.tpl', + 'some-new-semantic.sns': + '{funky-crazy-category}/some-new-semantic.sns', + 'mailman/database/mailman.db': None, + 'mailman/database/schemas/blah.schema': + '{appdata}/schemas/blah.schema', + 'mailman/etc/my.cnf': '{config}/my.cnf', + 'mailman/foo/some/path/bar/my.cfg': + '{config}/hmm/some/path/bar/my.cfg', + 'mailman/foo/some/path/other.cfg': + '{config}/hmm/some/path/other.cfg', + 'developer-docs/index.txt': '{doc}/developer-docs/index.txt', + 'developer-docs/api/toc.txt': '{doc}/developer-docs/api/toc.txt', + } + self.maxDiff = None + self.assertRulesMatch(rules, spec) + + def test_get_file(self): + # Create a fake dist + temp_site_packages = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, temp_site_packages) + + dist_name = 'test' + dist_info = os.path.join(temp_site_packages, 'test-0.1.dist-info') + os.mkdir(dist_info) + + metadata_path = os.path.join(dist_info, 'METADATA') + resources_path = os.path.join(dist_info, 'RESOURCES') + + with open(metadata_path, 'w') as fp: + fp.write(dedent("""\ + Metadata-Version: 1.2 + Name: test + Version: 0.1 + Summary: test + Author: me + """)) + + test_path = 'test.cfg' + + fd, test_resource_path = tempfile.mkstemp() + os.close(fd) + self.addCleanup(os.remove, test_resource_path) + + with open(test_resource_path, 'w') as fp: + fp.write('Config') + + with open(resources_path, 'w') as fp: + fp.write('%s,%s' % (test_path, test_resource_path)) + + # Add fake site-packages to sys.path to retrieve fake dist + self.addCleanup(sys.path.remove, temp_site_packages) + sys.path.insert(0, temp_site_packages) + + # Force packaging.database to rescan the sys.path + self.addCleanup(enable_cache) + disable_cache() + + # Try to retrieve resources paths and files + self.assertEqual(get_file_path(dist_name, test_path), + test_resource_path) + self.assertRaises(KeyError, get_file_path, dist_name, 'i-dont-exist') + + with get_file(dist_name, test_path) as fp: + self.assertEqual(fp.read(), 'Config') + self.assertRaises(KeyError, get_file, dist_name, 'i-dont-exist') + + +def test_suite(): + suite = unittest.TestSuite() + load = unittest.defaultTestLoader.loadTestsFromTestCase + suite.addTest(load(TestDistribution)) + suite.addTest(load(TestEggInfoDistribution)) + suite.addTest(load(TestDatabase)) + suite.addTest(load(DataFilesTestCase)) + return suite + + +if __name__ == "__main__": + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_depgraph.py b/Lib/packaging/tests/test_depgraph.py new file mode 100644 index 0000000000..64c22eb175 --- /dev/null +++ b/Lib/packaging/tests/test_depgraph.py @@ -0,0 +1,309 @@ +"""Tests for packaging.depgraph """ +import io +import os +import re +import sys +import packaging.database +from packaging import depgraph + +from packaging.tests import unittest, support +from packaging.tests.support import requires_zlib + + +class DepGraphTestCase(support.LoggingCatcher, + unittest.TestCase): + + DISTROS_DIST = ('choxie', 'grammar', 'towel-stuff') + DISTROS_EGG = ('bacon', 'banana', 'strawberry', 'cheese') + BAD_EGGS = ('nut',) + + EDGE = re.compile( + r'"(?P<from>.*)" -> "(?P<to>.*)" \[label="(?P<label>.*)"\]') + + def checkLists(self, l1, l2): + """ Compare two lists without taking the order into consideration """ + self.assertListEqual(sorted(l1), sorted(l2)) + + def setUp(self): + super(DepGraphTestCase, self).setUp() + path = os.path.join(os.path.dirname(__file__), 'fake_dists') + path = os.path.abspath(path) + sys.path.insert(0, path) + self.addCleanup(sys.path.remove, path) + self.addCleanup(packaging.database.enable_cache) + packaging.database.disable_cache() + + def test_generate_graph(self): + dists = [] + for name in self.DISTROS_DIST: + dist = packaging.database.get_distribution(name) + self.assertNotEqual(dist, None) + dists.append(dist) + + choxie, grammar, towel = dists + + graph = depgraph.generate_graph(dists) + + deps = [(x.name, y) for x, y in graph.adjacency_list[choxie]] + self.checkLists([('towel-stuff', 'towel-stuff (0.1)')], deps) + self.assertIn(choxie, graph.reverse_list[towel]) + self.checkLists(graph.missing[choxie], ['nut']) + + deps = [(x.name, y) for x, y in graph.adjacency_list[grammar]] + self.checkLists([], deps) + self.checkLists(graph.missing[grammar], ['truffles (>=1.2)']) + + deps = [(x.name, y) for x, y in graph.adjacency_list[towel]] + self.checkLists([], deps) + self.checkLists(graph.missing[towel], ['bacon (<=0.2)']) + + @requires_zlib + def test_generate_graph_egg(self): + dists = [] + for name in self.DISTROS_DIST + self.DISTROS_EGG: + dist = packaging.database.get_distribution(name, use_egg_info=True) + self.assertNotEqual(dist, None) + dists.append(dist) + + choxie, grammar, towel, bacon, banana, strawberry, cheese = dists + + graph = depgraph.generate_graph(dists) + + deps = [(x.name, y) for x, y in graph.adjacency_list[choxie]] + self.checkLists([('towel-stuff', 'towel-stuff (0.1)')], deps) + self.assertIn(choxie, graph.reverse_list[towel]) + self.checkLists(graph.missing[choxie], ['nut']) + + deps = [(x.name, y) for x, y in graph.adjacency_list[grammar]] + self.checkLists([('bacon', 'truffles (>=1.2)')], deps) + self.checkLists(graph.missing[grammar], []) + self.assertIn(grammar, graph.reverse_list[bacon]) + + deps = [(x.name, y) for x, y in graph.adjacency_list[towel]] + self.checkLists([('bacon', 'bacon (<=0.2)')], deps) + self.checkLists(graph.missing[towel], []) + self.assertIn(towel, graph.reverse_list[bacon]) + + deps = [(x.name, y) for x, y in graph.adjacency_list[bacon]] + self.checkLists([], deps) + self.checkLists(graph.missing[bacon], []) + + deps = [(x.name, y) for x, y in graph.adjacency_list[banana]] + self.checkLists([('strawberry', 'strawberry (>=0.5)')], deps) + self.checkLists(graph.missing[banana], []) + self.assertIn(banana, graph.reverse_list[strawberry]) + + deps = [(x.name, y) for x, y in graph.adjacency_list[strawberry]] + self.checkLists([], deps) + self.checkLists(graph.missing[strawberry], []) + + deps = [(x.name, y) for x, y in graph.adjacency_list[cheese]] + self.checkLists([], deps) + self.checkLists(graph.missing[cheese], []) + + def test_dependent_dists(self): + dists = [] + for name in self.DISTROS_DIST: + dist = packaging.database.get_distribution(name) + self.assertNotEqual(dist, None) + dists.append(dist) + + choxie, grammar, towel = dists + + deps = [d.name for d in depgraph.dependent_dists(dists, choxie)] + self.checkLists([], deps) + + deps = [d.name for d in depgraph.dependent_dists(dists, grammar)] + self.checkLists([], deps) + + deps = [d.name for d in depgraph.dependent_dists(dists, towel)] + self.checkLists(['choxie'], deps) + + @requires_zlib + def test_dependent_dists_egg(self): + dists = [] + for name in self.DISTROS_DIST + self.DISTROS_EGG: + dist = packaging.database.get_distribution(name, use_egg_info=True) + self.assertNotEqual(dist, None) + dists.append(dist) + + choxie, grammar, towel, bacon, banana, strawberry, cheese = dists + + deps = [d.name for d in depgraph.dependent_dists(dists, choxie)] + self.checkLists([], deps) + + deps = [d.name for d in depgraph.dependent_dists(dists, grammar)] + self.checkLists([], deps) + + deps = [d.name for d in depgraph.dependent_dists(dists, towel)] + self.checkLists(['choxie'], deps) + + deps = [d.name for d in depgraph.dependent_dists(dists, bacon)] + self.checkLists(['choxie', 'towel-stuff', 'grammar'], deps) + + deps = [d.name for d in depgraph.dependent_dists(dists, strawberry)] + self.checkLists(['banana'], deps) + + deps = [d.name for d in depgraph.dependent_dists(dists, cheese)] + self.checkLists([], deps) + + @requires_zlib + def test_graph_to_dot(self): + expected = ( + ('towel-stuff', 'bacon', 'bacon (<=0.2)'), + ('grammar', 'bacon', 'truffles (>=1.2)'), + ('choxie', 'towel-stuff', 'towel-stuff (0.1)'), + ('banana', 'strawberry', 'strawberry (>=0.5)'), + ) + + dists = [] + for name in self.DISTROS_DIST + self.DISTROS_EGG: + dist = packaging.database.get_distribution(name, use_egg_info=True) + self.assertNotEqual(dist, None) + dists.append(dist) + + graph = depgraph.generate_graph(dists) + buf = io.StringIO() + depgraph.graph_to_dot(graph, buf) + buf.seek(0) + matches = [] + lines = buf.readlines() + for line in lines[1:-1]: # skip the first and the last lines + if line[-1] == '\n': + line = line[:-1] + match = self.EDGE.match(line.strip()) + self.assertIsNot(match, None) + matches.append(match.groups()) + + self.checkLists(matches, expected) + + @requires_zlib + def test_graph_disconnected_to_dot(self): + dependencies_expected = ( + ('towel-stuff', 'bacon', 'bacon (<=0.2)'), + ('grammar', 'bacon', 'truffles (>=1.2)'), + ('choxie', 'towel-stuff', 'towel-stuff (0.1)'), + ('banana', 'strawberry', 'strawberry (>=0.5)'), + ) + disconnected_expected = ('cheese', 'bacon', 'strawberry') + + dists = [] + for name in self.DISTROS_DIST + self.DISTROS_EGG: + dist = packaging.database.get_distribution(name, use_egg_info=True) + self.assertNotEqual(dist, None) + dists.append(dist) + + graph = depgraph.generate_graph(dists) + buf = io.StringIO() + depgraph.graph_to_dot(graph, buf, skip_disconnected=False) + buf.seek(0) + lines = buf.readlines() + + dependencies_lines = [] + disconnected_lines = [] + + # First sort output lines into dependencies and disconnected lines. + # We also skip the attribute lines, and don't include the "{" and "}" + # lines. + disconnected_active = False + for line in lines[1:-1]: # Skip first and last line + if line.startswith('subgraph disconnected'): + disconnected_active = True + continue + if line.startswith('}') and disconnected_active: + disconnected_active = False + continue + + if disconnected_active: + # Skip the 'label = "Disconnected"', etc. attribute lines. + if ' = ' not in line: + disconnected_lines.append(line) + else: + dependencies_lines.append(line) + + dependencies_matches = [] + for line in dependencies_lines: + if line[-1] == '\n': + line = line[:-1] + match = self.EDGE.match(line.strip()) + self.assertIsNot(match, None) + dependencies_matches.append(match.groups()) + + disconnected_matches = [] + for line in disconnected_lines: + if line[-1] == '\n': + line = line[:-1] + line = line.strip('"') + disconnected_matches.append(line) + + self.checkLists(dependencies_matches, dependencies_expected) + self.checkLists(disconnected_matches, disconnected_expected) + + @requires_zlib + def test_graph_bad_version_to_dot(self): + expected = ( + ('towel-stuff', 'bacon', 'bacon (<=0.2)'), + ('grammar', 'bacon', 'truffles (>=1.2)'), + ('choxie', 'towel-stuff', 'towel-stuff (0.1)'), + ('banana', 'strawberry', 'strawberry (>=0.5)'), + ) + + dists = [] + for name in self.DISTROS_DIST + self.DISTROS_EGG + self.BAD_EGGS: + dist = packaging.database.get_distribution(name, use_egg_info=True) + self.assertNotEqual(dist, None) + dists.append(dist) + + graph = depgraph.generate_graph(dists) + buf = io.StringIO() + depgraph.graph_to_dot(graph, buf) + buf.seek(0) + matches = [] + lines = buf.readlines() + for line in lines[1:-1]: # skip the first and the last lines + if line[-1] == '\n': + line = line[:-1] + match = self.EDGE.match(line.strip()) + self.assertIsNot(match, None) + matches.append(match.groups()) + + self.checkLists(matches, expected) + + @requires_zlib + def test_repr(self): + dists = [] + for name in self.DISTROS_DIST + self.DISTROS_EGG + self.BAD_EGGS: + dist = packaging.database.get_distribution(name, use_egg_info=True) + self.assertNotEqual(dist, None) + dists.append(dist) + + graph = depgraph.generate_graph(dists) + self.assertTrue(repr(graph)) + + @requires_zlib + def test_main(self): + tempout = io.StringIO() + old = sys.stdout + sys.stdout = tempout + oldargv = sys.argv[:] + sys.argv[:] = ['script.py'] + try: + try: + depgraph.main() + except SystemExit: + pass + finally: + sys.stdout = old + sys.argv[:] = oldargv + + # checks what main did XXX could do more here + tempout.seek(0) + res = tempout.read() + self.assertIn('towel', res) + + +def test_suite(): + return unittest.makeSuite(DepGraphTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_dist.py b/Lib/packaging/tests/test_dist.py new file mode 100644 index 0000000000..1d78fa9b98 --- /dev/null +++ b/Lib/packaging/tests/test_dist.py @@ -0,0 +1,454 @@ +"""Tests for packaging.dist.""" +import os +import io +import sys +import logging +import textwrap +import sysconfig +import packaging.dist + +from packaging.dist import Distribution +from packaging.command import set_command +from packaging.command.cmd import Command +from packaging.metadata import Metadata +from packaging.errors import PackagingModuleError, PackagingOptionError +from packaging.tests import TESTFN, captured_stdout +from packaging.tests import support, unittest +from packaging.tests.support import create_distribution +from test.support import unload + + +class test_dist(Command): + """Sample packaging extension command.""" + + user_options = [ + ("sample-option=", "S", "help text"), + ] + + def initialize_options(self): + self.sample_option = None + + def finalize_options(self): + pass + + +class DistributionTestCase(support.TempdirManager, + support.LoggingCatcher, + support.EnvironRestorer, + unittest.TestCase): + + restore_environ = ['HOME', 'PLAT'] + + def setUp(self): + super(DistributionTestCase, self).setUp() + self.argv = sys.argv, sys.argv[:] + del sys.argv[1:] + + def tearDown(self): + sys.argv = self.argv[0] + sys.argv[:] = self.argv[1] + super(DistributionTestCase, self).tearDown() + + def test_debug_mode(self): + self.addCleanup(os.unlink, TESTFN) + with open(TESTFN, "w") as f: + f.write("[global]\n") + f.write("command_packages = foo.bar, splat") + + files = [TESTFN] + sys.argv.append("build") + __, stdout = captured_stdout(create_distribution, files) + self.assertEqual(stdout, '') + packaging.dist.DEBUG = True + try: + __, stdout = captured_stdout(create_distribution, files) + self.assertEqual(stdout, '') + finally: + packaging.dist.DEBUG = False + + def test_write_pkg_file(self): + # Check Metadata handling of Unicode fields + tmp_dir = self.mkdtemp() + my_file = os.path.join(tmp_dir, 'f') + cls = Distribution + + dist = cls(attrs={'author': 'Mister Café', + 'name': 'my.package', + 'maintainer': 'Café Junior', + 'summary': 'Café torréfié', + 'description': 'Héhéhé'}) + + # let's make sure the file can be written + # with Unicode fields. they are encoded with + # PKG_INFO_ENCODING + with open(my_file, 'w', encoding='utf-8') as fp: + dist.metadata.write_file(fp) + + # regular ascii is of course always usable + dist = cls(attrs={'author': 'Mister Cafe', + 'name': 'my.package', + 'maintainer': 'Cafe Junior', + 'summary': 'Cafe torrefie', + 'description': 'Hehehe'}) + + with open(my_file, 'w') as fp: + dist.metadata.write_file(fp) + + def test_bad_attr(self): + Distribution(attrs={'author': 'xxx', + 'name': 'xxx', + 'version': '1.2', + 'home-page': 'xxxx', + 'badoptname': 'xxx'}) + logs = self.get_logs(logging.WARNING) + self.assertEqual(1, len(logs)) + self.assertIn('unknown argument', logs[0]) + + def test_bad_version(self): + Distribution(attrs={'author': 'xxx', + 'name': 'xxx', + 'version': 'xxx', + 'home-page': 'xxxx'}) + logs = self.get_logs(logging.WARNING) + self.assertEqual(1, len(logs)) + self.assertIn('not a valid version', logs[0]) + + def test_empty_options(self): + # an empty options dictionary should not stay in the + # list of attributes + Distribution(attrs={'author': 'xxx', + 'name': 'xxx', + 'version': '1.2', + 'home-page': 'xxxx', + 'options': {}}) + + self.assertEqual([], self.get_logs(logging.WARNING)) + + def test_non_empty_options(self): + # TODO: how to actually use options is not documented except + # for a few cryptic comments in dist.py. If this is to stay + # in the public API, it deserves some better documentation. + + # Here is an example of how it's used out there: + # http://svn.pythonmac.org/py2app/py2app/trunk/doc/ + # index.html#specifying-customizations + dist = Distribution(attrs={'author': 'xxx', + 'name': 'xxx', + 'version': 'xxx', + 'home-page': 'xxxx', + 'options': {'sdist': {'owner': 'root'}}}) + + self.assertIn('owner', dist.get_option_dict('sdist')) + + def test_finalize_options(self): + + attrs = {'keywords': 'one,two', + 'platform': 'one,two'} + + dist = Distribution(attrs=attrs) + dist.finalize_options() + + # finalize_option splits platforms and keywords + self.assertEqual(dist.metadata['platform'], ['one', 'two']) + self.assertEqual(dist.metadata['keywords'], ['one', 'two']) + + def test_find_config_files_disable(self): + # Bug #1180: Allow users to disable their own config file. + temp_home = self.mkdtemp() + if os.name == 'posix': + user_filename = os.path.join(temp_home, ".pydistutils.cfg") + else: + user_filename = os.path.join(temp_home, "pydistutils.cfg") + + with open(user_filename, 'w') as f: + f.write('[distutils2]\n') + + def _expander(path): + return temp_home + + old_expander = os.path.expanduser + os.path.expanduser = _expander + try: + d = packaging.dist.Distribution() + all_files = d.find_config_files() + + d = packaging.dist.Distribution(attrs={'script_args': + ['--no-user-cfg']}) + files = d.find_config_files() + finally: + os.path.expanduser = old_expander + + # make sure --no-user-cfg disables the user cfg file + self.assertEqual((len(all_files) - 1), len(files)) + + def test_special_hooks_parsing(self): + temp_home = self.mkdtemp() + config_files = [os.path.join(temp_home, "config1.cfg"), + os.path.join(temp_home, "config2.cfg")] + + # Store two aliased hooks in config files + self.write_file((temp_home, "config1.cfg"), + '[test_dist]\npre-hook.a = type') + self.write_file((temp_home, "config2.cfg"), + '[test_dist]\npre-hook.b = type') + + set_command('packaging.tests.test_dist.test_dist') + dist = create_distribution(config_files) + cmd = dist.get_command_obj("test_dist") + self.assertEqual(cmd.pre_hook, {"a": 'type', "b": 'type'}) + + def test_hooks_get_run(self): + temp_home = self.mkdtemp() + module_name = os.path.split(temp_home)[-1] + pyname = '%s.py' % module_name + config_file = os.path.join(temp_home, "config1.cfg") + hooks_module = os.path.join(temp_home, pyname) + + self.write_file(config_file, textwrap.dedent('''\ + [test_dist] + pre-hook.test = %(modname)s.log_pre_call + post-hook.test = %(modname)s.log_post_call''' + % {'modname': module_name})) + + self.write_file(hooks_module, textwrap.dedent('''\ + record = [] + + def log_pre_call(cmd): + record.append('pre-%s' % cmd.get_command_name()) + + def log_post_call(cmd): + record.append('post-%s' % cmd.get_command_name()) + ''')) + + set_command('packaging.tests.test_dist.test_dist') + d = create_distribution([config_file]) + cmd = d.get_command_obj("test_dist") + + # prepare the call recorders + sys.path.append(temp_home) + self.addCleanup(sys.path.remove, temp_home) + self.addCleanup(unload, module_name) + record = __import__(module_name).record + + cmd.run = lambda: record.append('run') + cmd.finalize_options = lambda: record.append('finalize') + d.run_command('test_dist') + + self.assertEqual(record, ['finalize', + 'pre-test_dist', + 'run', + 'post-test_dist']) + + def test_hooks_importable(self): + temp_home = self.mkdtemp() + config_file = os.path.join(temp_home, "config1.cfg") + + self.write_file(config_file, textwrap.dedent('''\ + [test_dist] + pre-hook.test = nonexistent.dotted.name''')) + + set_command('packaging.tests.test_dist.test_dist') + d = create_distribution([config_file]) + cmd = d.get_command_obj("test_dist") + cmd.ensure_finalized() + + self.assertRaises(PackagingModuleError, d.run_command, 'test_dist') + + def test_hooks_callable(self): + temp_home = self.mkdtemp() + config_file = os.path.join(temp_home, "config1.cfg") + + self.write_file(config_file, textwrap.dedent('''\ + [test_dist] + pre-hook.test = packaging.tests.test_dist.__doc__''')) + + set_command('packaging.tests.test_dist.test_dist') + d = create_distribution([config_file]) + cmd = d.get_command_obj("test_dist") + cmd.ensure_finalized() + + self.assertRaises(PackagingOptionError, d.run_command, 'test_dist') + + +class MetadataTestCase(support.TempdirManager, + support.LoggingCatcher, + support.EnvironRestorer, + unittest.TestCase): + + restore_environ = ['HOME'] + + def setUp(self): + super(MetadataTestCase, self).setUp() + self.argv = sys.argv, sys.argv[:] + + def tearDown(self): + sys.argv = self.argv[0] + sys.argv[:] = self.argv[1] + super(MetadataTestCase, self).tearDown() + + def test_simple_metadata(self): + attrs = {"name": "package", + "version": "1.0"} + dist = Distribution(attrs) + meta = self.format_metadata(dist) + self.assertIn("Metadata-Version: 1.0", meta) + self.assertNotIn("provides:", meta.lower()) + self.assertNotIn("requires:", meta.lower()) + self.assertNotIn("obsoletes:", meta.lower()) + + def test_provides_dist(self): + attrs = {"name": "package", + "version": "1.0", + "provides_dist": ["package", "package.sub"]} + dist = Distribution(attrs) + self.assertEqual(dist.metadata['Provides-Dist'], + ["package", "package.sub"]) + meta = self.format_metadata(dist) + self.assertIn("Metadata-Version: 1.2", meta) + self.assertNotIn("requires:", meta.lower()) + self.assertNotIn("obsoletes:", meta.lower()) + + def _test_provides_illegal(self): + # XXX to do: check the versions + self.assertRaises(ValueError, Distribution, + {"name": "package", + "version": "1.0", + "provides_dist": ["my.pkg (splat)"]}) + + def test_requires_dist(self): + attrs = {"name": "package", + "version": "1.0", + "requires_dist": ["other", "another (==1.0)"]} + dist = Distribution(attrs) + self.assertEqual(dist.metadata['Requires-Dist'], + ["other", "another (==1.0)"]) + meta = self.format_metadata(dist) + self.assertIn("Metadata-Version: 1.2", meta) + self.assertNotIn("provides:", meta.lower()) + self.assertIn("Requires-Dist: other", meta) + self.assertIn("Requires-Dist: another (==1.0)", meta) + self.assertNotIn("obsoletes:", meta.lower()) + + def _test_requires_illegal(self): + # XXX + self.assertRaises(ValueError, Distribution, + {"name": "package", + "version": "1.0", + "requires": ["my.pkg (splat)"]}) + + def test_obsoletes_dist(self): + attrs = {"name": "package", + "version": "1.0", + "obsoletes_dist": ["other", "another (<1.0)"]} + dist = Distribution(attrs) + self.assertEqual(dist.metadata['Obsoletes-Dist'], + ["other", "another (<1.0)"]) + meta = self.format_metadata(dist) + self.assertIn("Metadata-Version: 1.2", meta) + self.assertNotIn("provides:", meta.lower()) + self.assertNotIn("requires:", meta.lower()) + self.assertIn("Obsoletes-Dist: other", meta) + self.assertIn("Obsoletes-Dist: another (<1.0)", meta) + + def _test_obsoletes_illegal(self): + # XXX + self.assertRaises(ValueError, Distribution, + {"name": "package", + "version": "1.0", + "obsoletes": ["my.pkg (splat)"]}) + + def format_metadata(self, dist): + sio = io.StringIO() + dist.metadata.write_file(sio) + return sio.getvalue() + + def test_custom_pydistutils(self): + # fixes #2166 + # make sure pydistutils.cfg is found + if os.name == 'posix': + user_filename = ".pydistutils.cfg" + else: + user_filename = "pydistutils.cfg" + + temp_dir = self.mkdtemp() + user_filename = os.path.join(temp_dir, user_filename) + with open(user_filename, 'w') as f: + f.write('.') + + dist = Distribution() + + # linux-style + if sys.platform in ('linux', 'darwin'): + os.environ['HOME'] = temp_dir + files = dist.find_config_files() + self.assertIn(user_filename, files) + + # win32-style + if sys.platform == 'win32': + # home drive should be found + os.environ['HOME'] = temp_dir + files = dist.find_config_files() + self.assertIn(user_filename, files) + + def test_show_help(self): + # smoke test, just makes sure some help is displayed + dist = Distribution() + sys.argv = [] + dist.help = True + dist.script_name = os.path.join(sysconfig.get_path('scripts'), + 'pysetup') + __, stdout = captured_stdout(dist.parse_command_line) + output = [line for line in stdout.split('\n') + if line.strip() != ''] + self.assertGreater(len(output), 0) + + def test_description(self): + desc = textwrap.dedent("""\ + example:: + We start here + and continue here + and end here.""") + attrs = {"name": "package", + "version": "1.0", + "description": desc} + + dist = packaging.dist.Distribution(attrs) + meta = self.format_metadata(dist) + meta = meta.replace('\n' + 7 * ' ' + '|', '\n') + self.assertIn(desc, meta) + + def test_read_metadata(self): + attrs = {"name": "package", + "version": "1.0", + "description": "desc", + "summary": "xxx", + "download_url": "http://example.com", + "keywords": ['one', 'two'], + "requires_dist": ['foo']} + + dist = Distribution(attrs) + PKG_INFO = io.StringIO() + dist.metadata.write_file(PKG_INFO) + PKG_INFO.seek(0) + + metadata = Metadata() + metadata.read_file(PKG_INFO) + + self.assertEqual(metadata['name'], "package") + self.assertEqual(metadata['version'], "1.0") + self.assertEqual(metadata['summary'], "xxx") + self.assertEqual(metadata['download_url'], 'http://example.com') + self.assertEqual(metadata['keywords'], ['one', 'two']) + self.assertEqual(metadata['platform'], []) + self.assertEqual(metadata['obsoletes'], []) + self.assertEqual(metadata['requires-dist'], ['foo']) + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(DistributionTestCase)) + suite.addTest(unittest.makeSuite(MetadataTestCase)) + return suite + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_extension.py b/Lib/packaging/tests/test_extension.py new file mode 100644 index 0000000000..41182e5f7c --- /dev/null +++ b/Lib/packaging/tests/test_extension.py @@ -0,0 +1,15 @@ +"""Tests for packaging.extension.""" +import os + +from packaging.compiler.extension import Extension +from packaging.tests import unittest + +class ExtensionTestCase(unittest.TestCase): + + pass + +def test_suite(): + return unittest.makeSuite(ExtensionTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_install.py b/Lib/packaging/tests/test_install.py new file mode 100644 index 0000000000..cf0817ca4d --- /dev/null +++ b/Lib/packaging/tests/test_install.py @@ -0,0 +1,391 @@ +"""Tests for the packaging.install module.""" +import os +import logging +from sysconfig import is_python_build +from tempfile import mkstemp + +from packaging import install +from packaging.pypi.xmlrpc import Client +from packaging.metadata import Metadata +from packaging.tests.support import (LoggingCatcher, TempdirManager, unittest, + fake_dec) +try: + import threading + from packaging.tests.pypi_server import use_xmlrpc_server +except ImportError: + threading = None + use_xmlrpc_server = fake_dec + + +class InstalledDist: + """Distribution object, represent distributions currently installed on the + system""" + def __init__(self, name, version, deps): + self.metadata = Metadata() + self.name = name + self.version = version + self.metadata['Name'] = name + self.metadata['Version'] = version + self.metadata['Requires-Dist'] = deps + + def __repr__(self): + return '<InstalledDist %r>' % self.metadata['Name'] + + +class ToInstallDist: + """Distribution that will be installed""" + + def __init__(self, files=False): + self._files = files + self.install_called = False + self.install_called_with = {} + self.uninstall_called = False + self._real_files = [] + self.name = "fake" + self.version = "fake" + if files: + for f in range(0, 3): + fp, fn = mkstemp() + os.close(fp) + self._real_files.append(fn) + + def _unlink_installed_files(self): + if self._files: + for fn in self._real_files: + os.unlink(fn) + + def list_installed_files(self, **args): + if self._files: + return self._real_files + + def get_install(self, **args): + return self.list_installed_files() + + +class MagicMock: + def __init__(self, return_value=None, raise_exception=False): + self.called = False + self._times_called = 0 + self._called_with = [] + self._return_value = return_value + self._raise = raise_exception + + def __call__(self, *args, **kwargs): + self.called = True + self._times_called = self._times_called + 1 + self._called_with.append((args, kwargs)) + iterable = hasattr(self._raise, '__iter__') + if self._raise: + if ((not iterable and self._raise) + or self._raise[self._times_called - 1]): + raise Exception + return self._return_value + + def called_with(self, *args, **kwargs): + return (args, kwargs) in self._called_with + + +def get_installed_dists(dists): + """Return a list of fake installed dists. + The list is name, version, deps""" + objects = [] + for name, version, deps in dists: + objects.append(InstalledDist(name, version, deps)) + return objects + + +class TestInstall(LoggingCatcher, TempdirManager, unittest.TestCase): + def _get_client(self, server, *args, **kwargs): + return Client(server.full_address, *args, **kwargs) + + def _get_results(self, output): + """return a list of results""" + installed = [(o.name, str(o.version)) for o in output['install']] + remove = [(o.name, str(o.version)) for o in output['remove']] + conflict = [(o.name, str(o.version)) for o in output['conflict']] + return installed, remove, conflict + + @unittest.skipIf(threading is None, 'needs threading') + @use_xmlrpc_server() + def test_existing_deps(self, server): + # Test that the installer get the dependencies from the metadatas + # and ask the index for this dependencies. + # In this test case, we have choxie that is dependent from towel-stuff + # 0.1, which is in-turn dependent on bacon <= 0.2: + # choxie -> towel-stuff -> bacon. + # Each release metadata is not provided in metadata 1.2. + client = self._get_client(server) + archive_path = '%s/distribution.tar.gz' % server.full_address + server.xmlrpc.set_distributions([ + {'name': 'choxie', + 'version': '2.0.0.9', + 'requires_dist': ['towel-stuff (0.1)'], + 'url': archive_path}, + {'name': 'towel-stuff', + 'version': '0.1', + 'requires_dist': ['bacon (<= 0.2)'], + 'url': archive_path}, + {'name': 'bacon', + 'version': '0.1', + 'requires_dist': [], + 'url': archive_path}, + ]) + installed = get_installed_dists([('bacon', '0.1', [])]) + output = install.get_infos("choxie", index=client, + installed=installed) + + # we don't have installed bacon as it's already installed system-wide + self.assertEqual(0, len(output['remove'])) + self.assertEqual(2, len(output['install'])) + readable_output = [(o.name, str(o.version)) + for o in output['install']] + self.assertIn(('towel-stuff', '0.1'), readable_output) + self.assertIn(('choxie', '2.0.0.9'), readable_output) + + @unittest.skipIf(threading is None, 'needs threading') + @use_xmlrpc_server() + def test_upgrade_existing_deps(self, server): + client = self._get_client(server) + archive_path = '%s/distribution.tar.gz' % server.full_address + server.xmlrpc.set_distributions([ + {'name': 'choxie', + 'version': '2.0.0.9', + 'requires_dist': ['towel-stuff (0.1)'], + 'url': archive_path}, + {'name': 'towel-stuff', + 'version': '0.1', + 'requires_dist': ['bacon (>= 0.2)'], + 'url': archive_path}, + {'name': 'bacon', + 'version': '0.2', + 'requires_dist': [], + 'url': archive_path}, + ]) + + output = install.get_infos("choxie", index=client, + installed=get_installed_dists([('bacon', '0.1', [])])) + installed = [(o.name, str(o.version)) for o in output['install']] + + # we need bacon 0.2, but 0.1 is installed. + # So we expect to remove 0.1 and to install 0.2 instead. + remove = [(o.name, str(o.version)) for o in output['remove']] + self.assertIn(('choxie', '2.0.0.9'), installed) + self.assertIn(('towel-stuff', '0.1'), installed) + self.assertIn(('bacon', '0.2'), installed) + self.assertIn(('bacon', '0.1'), remove) + self.assertEqual(0, len(output['conflict'])) + + @unittest.skipIf(threading is None, 'needs threading') + @use_xmlrpc_server() + def test_conflicts(self, server): + # Tests that conflicts are detected + client = self._get_client(server) + archive_path = '%s/distribution.tar.gz' % server.full_address + + # choxie depends on towel-stuff, which depends on bacon. + server.xmlrpc.set_distributions([ + {'name': 'choxie', + 'version': '2.0.0.9', + 'requires_dist': ['towel-stuff (0.1)'], + 'url': archive_path}, + {'name': 'towel-stuff', + 'version': '0.1', + 'requires_dist': ['bacon (>= 0.2)'], + 'url': archive_path}, + {'name': 'bacon', + 'version': '0.2', + 'requires_dist': [], + 'url': archive_path}, + ]) + + # name, version, deps. + already_installed = [('bacon', '0.1', []), + ('chicken', '1.1', ['bacon (0.1)'])] + output = install.get_infos( + 'choxie', index=client, + installed=get_installed_dists(already_installed)) + + # we need bacon 0.2, but 0.1 is installed. + # So we expect to remove 0.1 and to install 0.2 instead. + installed, remove, conflict = self._get_results(output) + self.assertIn(('choxie', '2.0.0.9'), installed) + self.assertIn(('towel-stuff', '0.1'), installed) + self.assertIn(('bacon', '0.2'), installed) + self.assertIn(('bacon', '0.1'), remove) + self.assertIn(('chicken', '1.1'), conflict) + + @unittest.skipIf(threading is None, 'needs threading') + @use_xmlrpc_server() + def test_installation_unexisting_project(self, server): + # Test that the isntalled raises an exception if the project does not + # exists. + client = self._get_client(server) + self.assertRaises(install.InstallationException, + install.get_infos, + 'unexisting project', index=client) + + def test_move_files(self): + # test that the files are really moved, and that the new path is + # returned. + path = self.mkdtemp() + newpath = self.mkdtemp() + files = [os.path.join(path, str(x)) for x in range(1, 20)] + for f in files: + open(f, 'ab+').close() + output = [o for o in install._move_files(files, newpath)] + + # check that output return the list of old/new places + for file_ in files: + name = os.path.split(file_)[-1] + newloc = os.path.join(newpath, name) + self.assertIn((file_, newloc), output) + + # remove the files + for f in [o[1] for o in output]: # o[1] is the new place + os.remove(f) + + def test_update_infos(self): + tests = [[ + {'foo': ['foobar', 'foo', 'baz'], 'baz': ['foo', 'foo']}, + {'foo': ['additional_content', 'yeah'], 'baz': ['test', 'foo']}, + {'foo': ['foobar', 'foo', 'baz', 'additional_content', 'yeah'], + 'baz': ['foo', 'foo', 'test', 'foo']}, + ]] + + for dict1, dict2, expect in tests: + install._update_infos(dict1, dict2) + for key in expect: + self.assertEqual(expect[key], dict1[key]) + + def test_install_dists_rollback(self): + # if one of the distribution installation fails, call uninstall on all + # installed distributions. + + old_install_dist = install._install_dist + old_uninstall = getattr(install, 'uninstall', None) + + install._install_dist = MagicMock(return_value=[], + raise_exception=(False, True)) + install.remove = MagicMock() + try: + d1 = ToInstallDist() + d2 = ToInstallDist() + path = self.mkdtemp() + self.assertRaises(Exception, install.install_dists, [d1, d2], path) + self.assertTrue(install._install_dist.called_with(d1, path)) + self.assertTrue(install.remove.called) + finally: + install._install_dist = old_install_dist + install.remove = old_uninstall + + def test_install_dists_success(self): + old_install_dist = install._install_dist + install._install_dist = MagicMock(return_value=[]) + try: + # test that the install method is called on each distributions + d1 = ToInstallDist() + d2 = ToInstallDist() + + # should call install + path = self.mkdtemp() + install.install_dists([d1, d2], path) + for dist in (d1, d2): + self.assertTrue(install._install_dist.called_with(dist, path)) + finally: + install._install_dist = old_install_dist + + def test_install_from_infos_conflict(self): + # assert conflicts raise an exception + self.assertRaises(install.InstallationConflict, + install.install_from_infos, + conflicts=[ToInstallDist()]) + + def test_install_from_infos_remove_success(self): + old_install_dists = install.install_dists + install.install_dists = lambda x, y=None: None + try: + dists = [] + for i in range(2): + dists.append(ToInstallDist(files=True)) + install.install_from_infos(remove=dists) + + # assert that the files have been removed + for dist in dists: + for f in dist.list_installed_files(): + self.assertFalse(os.path.exists(f)) + finally: + install.install_dists = old_install_dists + + def test_install_from_infos_remove_rollback(self): + old_install_dist = install._install_dist + old_uninstall = getattr(install, 'uninstall', None) + + install._install_dist = MagicMock(return_value=[], + raise_exception=(False, True)) + install.uninstall = MagicMock() + try: + # assert that if an error occurs, the removed files are restored. + remove = [] + for i in range(2): + remove.append(ToInstallDist(files=True)) + to_install = [ToInstallDist(), ToInstallDist()] + temp_dir = self.mkdtemp() + + self.assertRaises(Exception, install.install_from_infos, + install_path=temp_dir, install=to_install, + remove=remove) + # assert that the files are in the same place + # assert that the files have been removed + for dist in remove: + for f in dist.list_installed_files(): + self.assertTrue(os.path.exists(f)) + dist._unlink_installed_files() + finally: + install._install_dist = old_install_dist + install.uninstall = old_uninstall + + def test_install_from_infos_install_succes(self): + old_install_dist = install._install_dist + install._install_dist = MagicMock([]) + try: + # assert that the distribution can be installed + install_path = "my_install_path" + to_install = [ToInstallDist(), ToInstallDist()] + + install.install_from_infos(install_path, install=to_install) + for dist in to_install: + install._install_dist.called_with(install_path) + finally: + install._install_dist = old_install_dist + + def test_install_permission_denied(self): + # if we don't have access to the installation path, we should abort + # immediately + project = os.path.join(os.path.dirname(__file__), 'package.tgz') + + # when running from an uninstalled build, a warning is emitted and the + # installation is not attempted + if is_python_build(): + self.assertFalse(install.install(project)) + self.assertEqual(1, len(self.get_logs(logging.ERROR))) + return + + install_path = self.mkdtemp() + old_get_path = install.get_path + install.get_path = lambda path: install_path + old_mod = os.stat(install_path).st_mode + os.chmod(install_path, 0) + try: + self.assertFalse(install.install(project)) + finally: + os.chmod(install_path, old_mod) + install.get_path = old_get_path + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestInstall)) + return suite + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_manifest.py b/Lib/packaging/tests/test_manifest.py new file mode 100644 index 0000000000..e0bcbbc3e9 --- /dev/null +++ b/Lib/packaging/tests/test_manifest.py @@ -0,0 +1,77 @@ +"""Tests for packaging.manifest.""" +import os +import logging +from io import StringIO +from packaging.manifest import Manifest + +from packaging.tests import unittest, support + +_MANIFEST = """\ +recursive-include foo *.py # ok +# nothing here + +# + +recursive-include bar \\ + *.dat *.txt +""" + +_MANIFEST2 = """\ +README +file1 +""" + + +class ManifestTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def setUp(self): + super(ManifestTestCase, self).setUp() + self.cwd = os.getcwd() + + def tearDown(self): + os.chdir(self.cwd) + super(ManifestTestCase, self).tearDown() + + def test_manifest_reader(self): + tmpdir = self.mkdtemp() + MANIFEST = os.path.join(tmpdir, 'MANIFEST.in') + with open(MANIFEST, 'w') as f: + f.write(_MANIFEST) + + manifest = Manifest() + manifest.read_template(MANIFEST) + + warnings = self.get_logs(logging.WARNING) + # the manifest should have been read and 3 warnings issued + # (we didn't provide the files) + self.assertEqual(3, len(warnings)) + for warning in warnings: + self.assertIn('no files found matching', warning) + + # manifest also accepts file-like objects + with open(MANIFEST) as f: + manifest.read_template(f) + + # the manifest should have been read and 3 warnings issued + # (we didn't provide the files) + self.assertEqual(3, len(warnings)) + + def test_default_actions(self): + tmpdir = self.mkdtemp() + self.addCleanup(os.chdir, os.getcwd()) + os.chdir(tmpdir) + self.write_file('README', 'xxx') + self.write_file('file1', 'xxx') + content = StringIO(_MANIFEST2) + manifest = Manifest() + manifest.read_template(content) + self.assertEqual(['README', 'file1'], manifest.files) + + +def test_suite(): + return unittest.makeSuite(ManifestTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_markers.py b/Lib/packaging/tests/test_markers.py new file mode 100644 index 0000000000..dec0429762 --- /dev/null +++ b/Lib/packaging/tests/test_markers.py @@ -0,0 +1,71 @@ +"""Tests for packaging.markers.""" +import os +import sys +import platform +from packaging.markers import interpret + +from packaging.tests import unittest +from packaging.tests.support import LoggingCatcher + + +class MarkersTestCase(LoggingCatcher, + unittest.TestCase): + + def test_interpret(self): + sys_platform = sys.platform + version = sys.version.split()[0] + os_name = os.name + platform_version = platform.version() + platform_machine = platform.machine() + platform_python_implementation = platform.python_implementation() + + self.assertTrue(interpret("sys.platform == '%s'" % sys_platform)) + self.assertTrue(interpret( + "sys.platform == '%s' or python_version == '2.4'" % sys_platform)) + self.assertTrue(interpret( + "sys.platform == '%s' and python_full_version == '%s'" % + (sys_platform, version))) + self.assertTrue(interpret("'%s' == sys.platform" % sys_platform)) + self.assertTrue(interpret('os.name == "%s"' % os_name)) + self.assertTrue(interpret( + 'platform.version == "%s" and platform.machine == "%s"' % + (platform_version, platform_machine))) + self.assertTrue(interpret('platform.python_implementation == "%s"' % + platform_python_implementation)) + + # stuff that need to raise a syntax error + ops = ('os.name == os.name', 'os.name == 2', "'2' == '2'", + 'okpjonon', '', 'os.name ==', 'python_version == 2.4') + for op in ops: + self.assertRaises(SyntaxError, interpret, op) + + # combined operations + OP = 'os.name == "%s"' % os_name + AND = ' and ' + OR = ' or ' + self.assertTrue(interpret(OP + AND + OP)) + self.assertTrue(interpret(OP + AND + OP + AND + OP)) + self.assertTrue(interpret(OP + OR + OP)) + self.assertTrue(interpret(OP + OR + OP + OR + OP)) + + # other operators + self.assertTrue(interpret("os.name != 'buuuu'")) + self.assertTrue(interpret("python_version > '1.0'")) + self.assertTrue(interpret("python_version < '5.0'")) + self.assertTrue(interpret("python_version <= '5.0'")) + self.assertTrue(interpret("python_version >= '1.0'")) + self.assertTrue(interpret("'%s' in os.name" % os_name)) + self.assertTrue(interpret("'buuuu' not in os.name")) + self.assertTrue(interpret( + "'buuuu' not in os.name and '%s' in os.name" % os_name)) + + # execution context + self.assertTrue(interpret('python_version == "0.1"', + {'python_version': '0.1'})) + + +def test_suite(): + return unittest.makeSuite(MarkersTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_metadata.py b/Lib/packaging/tests/test_metadata.py new file mode 100644 index 0000000000..904019c34e --- /dev/null +++ b/Lib/packaging/tests/test_metadata.py @@ -0,0 +1,279 @@ +"""Tests for packaging.metadata.""" +import os +import sys +import logging +from io import StringIO + +from packaging.errors import (MetadataConflictError, MetadataMissingError, + MetadataUnrecognizedVersionError) +from packaging.metadata import Metadata, PKG_INFO_PREFERRED_VERSION + +from packaging.tests import unittest +from packaging.tests.support import LoggingCatcher + + +class MetadataTestCase(LoggingCatcher, + unittest.TestCase): + + def test_instantiation(self): + PKG_INFO = os.path.join(os.path.dirname(__file__), 'PKG-INFO') + with open(PKG_INFO, 'r', encoding='utf-8') as f: + contents = f.read() + fp = StringIO(contents) + + m = Metadata() + self.assertRaises(MetadataUnrecognizedVersionError, m.items) + + m = Metadata(PKG_INFO) + self.assertEqual(len(m.items()), 22) + + m = Metadata(fileobj=fp) + self.assertEqual(len(m.items()), 22) + + m = Metadata(mapping=dict(name='Test', version='1.0')) + self.assertEqual(len(m.items()), 11) + + d = dict(m.items()) + self.assertRaises(TypeError, Metadata, + PKG_INFO, fileobj=fp) + self.assertRaises(TypeError, Metadata, + PKG_INFO, mapping=d) + self.assertRaises(TypeError, Metadata, + fileobj=fp, mapping=d) + self.assertRaises(TypeError, Metadata, + PKG_INFO, mapping=m, fileobj=fp) + + def test_metadata_read_write(self): + PKG_INFO = os.path.join(os.path.dirname(__file__), 'PKG-INFO') + metadata = Metadata(PKG_INFO) + out = StringIO() + metadata.write_file(out) + out.seek(0) + res = Metadata() + res.read_file(out) + for k in metadata: + self.assertEqual(metadata[k], res[k]) + + def test_metadata_markers(self): + # see if we can be platform-aware + PKG_INFO = os.path.join(os.path.dirname(__file__), 'PKG-INFO') + with open(PKG_INFO, 'r', encoding='utf-8') as f: + content = f.read() % sys.platform + metadata = Metadata(platform_dependent=True) + + metadata.read_file(StringIO(content)) + self.assertEqual(metadata['Requires-Dist'], ['bar']) + metadata['Name'] = "baz; sys.platform == 'blah'" + # FIXME is None or 'UNKNOWN' correct here? + # where is that documented? + self.assertEqual(metadata['Name'], None) + + # test with context + context = {'sys.platform': 'okook'} + metadata = Metadata(platform_dependent=True, + execution_context=context) + metadata.read_file(StringIO(content)) + self.assertEqual(metadata['Requires-Dist'], ['foo']) + + def test_description(self): + PKG_INFO = os.path.join(os.path.dirname(__file__), 'PKG-INFO') + with open(PKG_INFO, 'r', encoding='utf-8') as f: + content = f.read() % sys.platform + metadata = Metadata() + metadata.read_file(StringIO(content)) + + # see if we can read the description now + DESC = os.path.join(os.path.dirname(__file__), 'LONG_DESC.txt') + with open(DESC) as f: + wanted = f.read() + self.assertEqual(wanted, metadata['Description']) + + # save the file somewhere and make sure we can read it back + out = StringIO() + metadata.write_file(out) + out.seek(0) + metadata.read_file(out) + self.assertEqual(wanted, metadata['Description']) + + def test_mapping_api(self): + PKG_INFO = os.path.join(os.path.dirname(__file__), 'PKG-INFO') + with open(PKG_INFO, 'r', encoding='utf-8') as f: + content = f.read() % sys.platform + metadata = Metadata(fileobj=StringIO(content)) + self.assertIn('Version', metadata.keys()) + self.assertIn('0.5', metadata.values()) + self.assertIn(('Version', '0.5'), metadata.items()) + + metadata.update({'version': '0.6'}) + self.assertEqual(metadata['Version'], '0.6') + metadata.update([('version', '0.7')]) + self.assertEqual(metadata['Version'], '0.7') + + self.assertEqual(list(metadata), list(metadata.keys())) + + def test_versions(self): + metadata = Metadata() + metadata['Obsoletes'] = 'ok' + self.assertEqual(metadata['Metadata-Version'], '1.1') + + del metadata['Obsoletes'] + metadata['Obsoletes-Dist'] = 'ok' + self.assertEqual(metadata['Metadata-Version'], '1.2') + + self.assertRaises(MetadataConflictError, metadata.set, + 'Obsoletes', 'ok') + + del metadata['Obsoletes'] + del metadata['Obsoletes-Dist'] + metadata['Version'] = '1' + self.assertEqual(metadata['Metadata-Version'], '1.0') + + PKG_INFO = os.path.join(os.path.dirname(__file__), + 'SETUPTOOLS-PKG-INFO') + with open(PKG_INFO, 'r', encoding='utf-8') as f: + content = f.read() + metadata.read_file(StringIO(content)) + self.assertEqual(metadata['Metadata-Version'], '1.0') + + PKG_INFO = os.path.join(os.path.dirname(__file__), + 'SETUPTOOLS-PKG-INFO2') + with open(PKG_INFO, 'r', encoding='utf-8') as f: + content = f.read() + metadata.read_file(StringIO(content)) + self.assertEqual(metadata['Metadata-Version'], '1.1') + + # Update the _fields dict directly to prevent 'Metadata-Version' + # from being updated by the _set_best_version() method. + metadata._fields['Metadata-Version'] = '1.618' + self.assertRaises(MetadataUnrecognizedVersionError, metadata.keys) + + def test_warnings(self): + metadata = Metadata() + + # these should raise a warning + values = (('Requires-Dist', 'Funky (Groovie)'), + ('Requires-Python', '1-4')) + + for name, value in values: + metadata.set(name, value) + + # we should have a certain amount of warnings + self.assertEqual(len(self.get_logs()), 2) + + def test_multiple_predicates(self): + metadata = Metadata() + + # see for "3" instead of "3.0" ??? + # its seems like the MINOR VERSION can be omitted + metadata['Requires-Python'] = '>=2.6, <3.0' + metadata['Requires-Dist'] = ['Foo (>=2.6, <3.0)'] + + self.assertEqual([], self.get_logs(logging.WARNING)) + + def test_project_url(self): + metadata = Metadata() + metadata['Project-URL'] = [('one', 'http://ok')] + self.assertEqual(metadata['Project-URL'], + [('one', 'http://ok')]) + self.assertEqual(metadata['Metadata-Version'], '1.2') + + def test_check_version(self): + metadata = Metadata() + metadata['Name'] = 'vimpdb' + metadata['Home-page'] = 'http://pypi.python.org' + metadata['Author'] = 'Monty Python' + metadata.docutils_support = False + missing, warnings = metadata.check() + self.assertEqual(missing, ['Version']) + + def test_check_version_strict(self): + metadata = Metadata() + metadata['Name'] = 'vimpdb' + metadata['Home-page'] = 'http://pypi.python.org' + metadata['Author'] = 'Monty Python' + metadata.docutils_support = False + self.assertRaises(MetadataMissingError, metadata.check, strict=True) + + def test_check_name(self): + metadata = Metadata() + metadata['Version'] = '1.0' + metadata['Home-page'] = 'http://pypi.python.org' + metadata['Author'] = 'Monty Python' + metadata.docutils_support = False + missing, warnings = metadata.check() + self.assertEqual(missing, ['Name']) + + def test_check_name_strict(self): + metadata = Metadata() + metadata['Version'] = '1.0' + metadata['Home-page'] = 'http://pypi.python.org' + metadata['Author'] = 'Monty Python' + metadata.docutils_support = False + self.assertRaises(MetadataMissingError, metadata.check, strict=True) + + def test_check_author(self): + metadata = Metadata() + metadata['Version'] = '1.0' + metadata['Name'] = 'vimpdb' + metadata['Home-page'] = 'http://pypi.python.org' + metadata.docutils_support = False + missing, warnings = metadata.check() + self.assertEqual(missing, ['Author']) + + def test_check_homepage(self): + metadata = Metadata() + metadata['Version'] = '1.0' + metadata['Name'] = 'vimpdb' + metadata['Author'] = 'Monty Python' + metadata.docutils_support = False + missing, warnings = metadata.check() + self.assertEqual(missing, ['Home-page']) + + def test_check_predicates(self): + metadata = Metadata() + metadata['Version'] = 'rr' + metadata['Name'] = 'vimpdb' + metadata['Home-page'] = 'http://pypi.python.org' + metadata['Author'] = 'Monty Python' + metadata['Requires-dist'] = ['Foo (a)'] + metadata['Obsoletes-dist'] = ['Foo (a)'] + metadata['Provides-dist'] = ['Foo (a)'] + if metadata.docutils_support: + missing, warnings = metadata.check() + self.assertEqual(len(warnings), 4) + metadata.docutils_support = False + missing, warnings = metadata.check() + self.assertEqual(len(warnings), 4) + + def test_best_choice(self): + metadata = Metadata() + metadata['Version'] = '1.0' + self.assertEqual(metadata['Metadata-Version'], + PKG_INFO_PREFERRED_VERSION) + metadata['Classifier'] = ['ok'] + self.assertEqual(metadata['Metadata-Version'], '1.2') + + def test_project_urls(self): + # project-url is a bit specific, make sure we write it + # properly in PKG-INFO + metadata = Metadata() + metadata['Version'] = '1.0' + metadata['Project-Url'] = [('one', 'http://ok')] + self.assertEqual(metadata['Project-Url'], [('one', 'http://ok')]) + file_ = StringIO() + metadata.write_file(file_) + file_.seek(0) + res = file_.read().split('\n') + self.assertIn('Project-URL: one,http://ok', res) + + file_.seek(0) + metadata = Metadata() + metadata.read_file(file_) + self.assertEqual(metadata['Project-Url'], [('one', 'http://ok')]) + + +def test_suite(): + return unittest.makeSuite(MetadataTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_mixin2to3.py b/Lib/packaging/tests/test_mixin2to3.py new file mode 100644 index 0000000000..926f754d01 --- /dev/null +++ b/Lib/packaging/tests/test_mixin2to3.py @@ -0,0 +1,84 @@ +import sys +import textwrap + +from packaging.tests import unittest, support +from packaging.compat import Mixin2to3 + + +class Mixin2to3TestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') + def test_convert_code_only(self): + # used to check if code gets converted properly. + code = "print 'test'" + + with self.mktempfile() as fp: + fp.write(code) + + mixin2to3 = Mixin2to3() + mixin2to3._run_2to3([fp.name]) + expected = "print('test')" + + with open(fp.name) as fp: + converted = fp.read() + + self.assertEqual(expected, converted) + + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') + def test_doctests_only(self): + # used to check if doctests gets converted properly. + doctest = textwrap.dedent('''\ + """Example docstring. + + >>> print test + test + + It works. + """''') + + with self.mktempfile() as fp: + fp.write(doctest) + + mixin2to3 = Mixin2to3() + mixin2to3._run_2to3([fp.name]) + expected = textwrap.dedent('''\ + """Example docstring. + + >>> print(test) + test + + It works. + """\n''') + + with open(fp.name) as fp: + converted = fp.read() + + self.assertEqual(expected, converted) + + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') + def test_additional_fixers(self): + # used to check if use_2to3_fixers works + code = 'type(x) is not T' + + with self.mktempfile() as fp: + fp.write(code) + + mixin2to3 = Mixin2to3() + mixin2to3._run_2to3(files=[fp.name], doctests=[fp.name], + fixers=['packaging.tests.fixer']) + + expected = 'not isinstance(x, T)' + + with open(fp.name) as fp: + converted = fp.read() + + self.assertEqual(expected, converted) + + +def test_suite(): + return unittest.makeSuite(Mixin2to3TestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_msvc9compiler.py b/Lib/packaging/tests/test_msvc9compiler.py new file mode 100644 index 0000000000..dc3ae65a43 --- /dev/null +++ b/Lib/packaging/tests/test_msvc9compiler.py @@ -0,0 +1,140 @@ +"""Tests for packaging.compiler.msvc9compiler.""" +import os +import sys + +from packaging.errors import PackagingPlatformError + +from packaging.tests import unittest, support + +_MANIFEST = """\ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<assembly xmlns="urn:schemas-microsoft-com:asm.v1" + manifestVersion="1.0"> + <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3"> + <security> + <requestedPrivileges> + <requestedExecutionLevel level="asInvoker" uiAccess="false"> + </requestedExecutionLevel> + </requestedPrivileges> + </security> + </trustInfo> + <dependency> + <dependentAssembly> + <assemblyIdentity type="win32" name="Microsoft.VC90.CRT" + version="9.0.21022.8" processorArchitecture="x86" + publicKeyToken="XXXX"> + </assemblyIdentity> + </dependentAssembly> + </dependency> + <dependency> + <dependentAssembly> + <assemblyIdentity type="win32" name="Microsoft.VC90.MFC" + version="9.0.21022.8" processorArchitecture="x86" + publicKeyToken="XXXX"></assemblyIdentity> + </dependentAssembly> + </dependency> +</assembly> +""" + +_CLEANED_MANIFEST = """\ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<assembly xmlns="urn:schemas-microsoft-com:asm.v1" + manifestVersion="1.0"> + <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3"> + <security> + <requestedPrivileges> + <requestedExecutionLevel level="asInvoker" uiAccess="false"> + </requestedExecutionLevel> + </requestedPrivileges> + </security> + </trustInfo> + <dependency> + + </dependency> + <dependency> + <dependentAssembly> + <assemblyIdentity type="win32" name="Microsoft.VC90.MFC" + version="9.0.21022.8" processorArchitecture="x86" + publicKeyToken="XXXX"></assemblyIdentity> + </dependentAssembly> + </dependency> +</assembly>""" + + +class msvc9compilerTestCase(support.TempdirManager, + unittest.TestCase): + + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") + def test_no_compiler(self): + # make sure query_vcvarsall raises a PackagingPlatformError if + # the compiler is not found + from packaging.compiler.msvccompiler import get_build_version + if get_build_version() < 8.0: + raise unittest.SkipTest('only for MSVC8.0 or above') + + from packaging.compiler import msvc9compiler + from packaging.compiler.msvc9compiler import query_vcvarsall + + def _find_vcvarsall(version): + return None + + old_find_vcvarsall = msvc9compiler.find_vcvarsall + msvc9compiler.find_vcvarsall = _find_vcvarsall + try: + self.assertRaises(PackagingPlatformError, query_vcvarsall, + 'wont find this version') + finally: + msvc9compiler.find_vcvarsall = old_find_vcvarsall + + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") + def test_reg_class(self): + from packaging.compiler.msvccompiler import get_build_version + if get_build_version() < 8.0: + raise unittest.SkipTest("requires MSVC 8.0 or later") + + from packaging.compiler.msvc9compiler import Reg + self.assertRaises(KeyError, Reg.get_value, 'xxx', 'xxx') + + # looking for values that should exist on all + # windows registeries versions. + path = r'Control Panel\Desktop' + v = Reg.get_value(path, 'dragfullwindows') + self.assertIn(v, ('0', '1', '2')) + + import winreg + HKCU = winreg.HKEY_CURRENT_USER + keys = Reg.read_keys(HKCU, 'xxxx') + self.assertEqual(keys, None) + + keys = Reg.read_keys(HKCU, r'Control Panel') + self.assertIn('Desktop', keys) + + @unittest.skipUnless(sys.platform == "win32", "runs only on win32") + def test_remove_visual_c_ref(self): + from packaging.compiler.msvccompiler import get_build_version + if get_build_version() < 8.0: + raise unittest.SkipTest("requires MSVC 8.0 or later") + + from packaging.compiler.msvc9compiler import MSVCCompiler + tempdir = self.mkdtemp() + manifest = os.path.join(tempdir, 'manifest') + with open(manifest, 'w') as f: + f.write(_MANIFEST) + + compiler = MSVCCompiler() + compiler._remove_visual_c_ref(manifest) + + # see what we got + with open(manifest) as f: + # removing trailing spaces + content = '\n'.join(line.rstrip() for line in f.readlines()) + + # makes sure the manifest was properly cleaned + self.assertEqual(content, _CLEANED_MANIFEST) + + +def test_suite(): + return unittest.makeSuite(msvc9compilerTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_pypi_dist.py b/Lib/packaging/tests/test_pypi_dist.py new file mode 100644 index 0000000000..ff9a16e27b --- /dev/null +++ b/Lib/packaging/tests/test_pypi_dist.py @@ -0,0 +1,285 @@ +"""Tests for the packaging.pypi.dist module.""" + +import os +from packaging.version import VersionPredicate +from packaging.pypi.dist import (ReleaseInfo, ReleasesList, DistInfo, + split_archive_name, get_infos_from_url) +from packaging.pypi.errors import HashDoesNotMatch, UnsupportedHashName + +from packaging.tests import unittest +from packaging.tests.support import TempdirManager, requires_zlib, fake_dec +try: + import threading + from packaging.tests.pypi_server import use_pypi_server +except ImportError: + threading = None + use_pypi_server = fake_dec + + +def Dist(*args, **kwargs): + # DistInfo takes a release as a first parameter, avoid this in tests. + return DistInfo(None, *args, **kwargs) + + +class TestReleaseInfo(unittest.TestCase): + + def test_instantiation(self): + # Test the DistInfo class provides us the good attributes when + # given on construction + release = ReleaseInfo("FooBar", "1.1") + self.assertEqual("FooBar", release.name) + self.assertEqual("1.1", "%s" % release.version) + + def test_add_dist(self): + # empty distribution type should assume "sdist" + release = ReleaseInfo("FooBar", "1.1") + release.add_distribution(url="http://example.org/") + # should not fail + release['sdist'] + + def test_get_unknown_distribution(self): + # should raise a KeyError + pass + + def test_get_infos_from_url(self): + # Test that the the URLs are parsed the right way + url_list = { + 'FooBar-1.1.0.tar.gz': { + 'name': 'foobar', # lowercase the name + 'version': '1.1.0', + }, + 'Foo-Bar-1.1.0.zip': { + 'name': 'foo-bar', # keep the dash + 'version': '1.1.0', + }, + 'foobar-1.1b2.tar.gz#md5=123123123123123': { + 'name': 'foobar', + 'version': '1.1b2', + 'url': 'http://example.org/foobar-1.1b2.tar.gz', # no hash + 'hashval': '123123123123123', + 'hashname': 'md5', + }, + 'foobar-1.1-rc2.tar.gz': { # use suggested name + 'name': 'foobar', + 'version': '1.1c2', + 'url': 'http://example.org/foobar-1.1-rc2.tar.gz', + } + } + + for url, attributes in url_list.items(): + # for each url + infos = get_infos_from_url("http://example.org/" + url) + for attribute, expected in attributes.items(): + got = infos.get(attribute) + if attribute == "version": + self.assertEqual("%s" % got, expected) + else: + self.assertEqual(got, expected) + + def test_split_archive_name(self): + # Test we can split the archive names + names = { + 'foo-bar-baz-1.0-rc2': ('foo-bar-baz', '1.0c2'), + 'foo-bar-baz-1.0': ('foo-bar-baz', '1.0'), + 'foobarbaz-1.0': ('foobarbaz', '1.0'), + } + for name, results in names.items(): + self.assertEqual(results, split_archive_name(name)) + + +class TestDistInfo(TempdirManager, unittest.TestCase): + srcpath = "/packages/source/f/foobar/foobar-0.1.tar.gz" + + def test_get_url(self): + # Test that the url property works well + + d = Dist(url="test_url") + self.assertDictEqual(d.url, { + "url": "test_url", + "is_external": True, + "hashname": None, + "hashval": None, + }) + + # add a new url + d.add_url(url="internal_url", is_external=False) + self.assertEqual(d._url, None) + self.assertDictEqual(d.url, { + "url": "internal_url", + "is_external": False, + "hashname": None, + "hashval": None, + }) + self.assertEqual(2, len(d.urls)) + + def test_comparison(self): + # Test that we can compare DistInfoributionInfoList + foo1 = ReleaseInfo("foo", "1.0") + foo2 = ReleaseInfo("foo", "2.0") + bar = ReleaseInfo("bar", "2.0") + # assert we use the version to compare + self.assertTrue(foo1 < foo2) + self.assertFalse(foo1 > foo2) + self.assertFalse(foo1 == foo2) + + # assert we can't compare dists with different names + self.assertRaises(TypeError, foo1.__eq__, bar) + + @unittest.skipIf(threading is None, 'needs threading') + @use_pypi_server("downloads_with_md5") + def test_download(self, server): + # Download is possible, and the md5 is checked if given + + url = server.full_address + self.srcpath + + # check that a md5 if given + dist = Dist(url=url, hashname="md5", + hashval="fe18804c5b722ff024cabdf514924fc4") + dist.download(self.mkdtemp()) + + # a wrong md5 fails + dist2 = Dist(url=url, hashname="md5", hashval="wrongmd5") + + self.assertRaises(HashDoesNotMatch, dist2.download, self.mkdtemp()) + + # we can omit the md5 hash + dist3 = Dist(url=url) + dist3.download(self.mkdtemp()) + + # and specify a temporary location + # for an already downloaded dist + path1 = self.mkdtemp() + dist3.download(path=path1) + # and for a new one + path2_base = self.mkdtemp() + dist4 = Dist(url=url) + path2 = dist4.download(path=path2_base) + self.assertIn(path2_base, path2) + + def test_hashname(self): + # Invalid hashnames raises an exception on assignation + Dist(hashname="md5", hashval="value") + + self.assertRaises(UnsupportedHashName, Dist, + hashname="invalid_hashname", + hashval="value") + + @unittest.skipIf(threading is None, 'needs threading') + @requires_zlib + @use_pypi_server('downloads_with_md5') + def test_unpack(self, server): + url = server.full_address + self.srcpath + dist1 = Dist(url=url) + + # unpack the distribution in a specfied folder + dist1_here = self.mkdtemp() + dist1_there = dist1.unpack(path=dist1_here) + + # assert we unpack to the path provided + self.assertEqual(dist1_here, dist1_there) + dist1_result = os.listdir(dist1_there) + self.assertIn('paf', dist1_result) + os.remove(os.path.join(dist1_there, 'paf')) + + # Test unpack works without a path argument + dist2 = Dist(url=url) + # doing an unpack + dist2_there = dist2.unpack() + dist2_result = os.listdir(dist2_there) + self.assertIn('paf', dist2_result) + os.remove(os.path.join(dist2_there, 'paf')) + + +class TestReleasesList(unittest.TestCase): + + def test_filter(self): + # Test we filter the distributions the right way, using version + # predicate match method + releases = ReleasesList('FooBar', ( + ReleaseInfo("FooBar", "1.1"), + ReleaseInfo("FooBar", "1.1.1"), + ReleaseInfo("FooBar", "1.2"), + ReleaseInfo("FooBar", "1.2.1"), + )) + filtered = releases.filter(VersionPredicate("FooBar (<1.2)")) + self.assertNotIn(releases[2], filtered) + self.assertNotIn(releases[3], filtered) + self.assertIn(releases[0], filtered) + self.assertIn(releases[1], filtered) + + def test_append(self): + # When adding a new item to the list, the behavior is to test if + # a release with the same name and version number already exists, + # and if so, to add a new distribution for it. If the distribution type + # is already defined too, add url informations to the existing DistInfo + # object. + + releases = ReleasesList("FooBar", [ + ReleaseInfo("FooBar", "1.1", url="external_url", + dist_type="sdist"), + ]) + self.assertEqual(1, len(releases)) + releases.add_release(release=ReleaseInfo("FooBar", "1.1", + url="internal_url", + is_external=False, + dist_type="sdist")) + self.assertEqual(1, len(releases)) + self.assertEqual(2, len(releases[0]['sdist'].urls)) + + releases.add_release(release=ReleaseInfo("FooBar", "1.1.1", + dist_type="sdist")) + self.assertEqual(2, len(releases)) + + # when adding a distribution whith a different type, a new distribution + # has to be added. + releases.add_release(release=ReleaseInfo("FooBar", "1.1.1", + dist_type="bdist")) + self.assertEqual(2, len(releases)) + self.assertEqual(2, len(releases[1].dists)) + + def test_prefer_final(self): + # Can order the distributions using prefer_final + fb10 = ReleaseInfo("FooBar", "1.0") # final distribution + fb11a = ReleaseInfo("FooBar", "1.1a1") # alpha + fb12a = ReleaseInfo("FooBar", "1.2a1") # alpha + fb12b = ReleaseInfo("FooBar", "1.2b1") # beta + dists = ReleasesList("FooBar", [fb10, fb11a, fb12a, fb12b]) + + dists.sort_releases(prefer_final=True) + self.assertEqual(fb10, dists[0]) + + dists.sort_releases(prefer_final=False) + self.assertEqual(fb12b, dists[0]) + + @unittest.skip('method not implemented yet') + def test_prefer_source(self): + # Ordering supports prefer_source + fb_source = Dist("FooBar", "1.0", type="source") + fb_binary = Dist("FooBar", "1.0", type="binary") + fb2_binary = Dist("FooBar", "2.0", type="binary") + dists = ReleasesList([fb_binary, fb_source]) + + dists.sort_distributions(prefer_source=True) + self.assertEqual(fb_source, dists[0]) + + dists.sort_distributions(prefer_source=False) + self.assertEqual(fb_binary, dists[0]) + + dists.append(fb2_binary) + dists.sort_distributions(prefer_source=True) + self.assertEqual(fb2_binary, dists[0]) + + def test_get_last(self): + dists = ReleasesList('Foo') + self.assertEqual(dists.get_last('Foo 1.0'), None) + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestDistInfo)) + suite.addTest(unittest.makeSuite(TestReleaseInfo)) + suite.addTest(unittest.makeSuite(TestReleasesList)) + return suite + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_pypi_server.py b/Lib/packaging/tests/test_pypi_server.py new file mode 100644 index 0000000000..2c4ec0d34c --- /dev/null +++ b/Lib/packaging/tests/test_pypi_server.py @@ -0,0 +1,87 @@ +"""Tests for packaging.command.bdist.""" +import sys + +import urllib.request +import urllib.parse +import urllib.error + +try: + import threading + from packaging.tests.pypi_server import PyPIServer, PYPI_DEFAULT_STATIC_PATH +except ImportError: + threading = None + PyPIServer = None + PYPI_DEFAULT_STATIC_PATH = None + +from packaging.tests import unittest + + +@unittest.skipIf(threading is None, "Needs threading") +class PyPIServerTest(unittest.TestCase): + + def test_records_requests(self): + # We expect that PyPIServer can log our requests + server = PyPIServer() + server.default_response_status = 200 + + try: + server.start() + self.assertEqual(len(server.requests), 0) + + data = b'Rock Around The Bunker' + + headers = {"X-test-header": "Mister Iceberg"} + + request = urllib.request.Request(server.full_address, data, headers) + urllib.request.urlopen(request) + self.assertEqual(len(server.requests), 1) + handler, request_data = server.requests[-1] + self.assertIn(data, request_data) + self.assertIn("x-test-header", handler.headers) + self.assertEqual(handler.headers["x-test-header"], "Mister Iceberg") + + finally: + server.stop() + + + def test_serve_static_content(self): + # PYPI Mocked server can serve static content from disk. + + def uses_local_files_for(server, url_path): + """Test that files are served statically (eg. the output from the + server is the same than the one made by a simple file read. + """ + url = server.full_address + url_path + request = urllib.request.Request(url) + response = urllib.request.urlopen(request) + with open(PYPI_DEFAULT_STATIC_PATH + "/test_pypi_server" + + url_path) as file: + return response.read().decode() == file.read() + + server = PyPIServer(static_uri_paths=["simple", "external"], + static_filesystem_paths=["test_pypi_server"]) + server.start() + try: + # the file does not exists on the disc, so it might not be served + url = server.full_address + "/simple/unexisting_page" + request = urllib.request.Request(url) + try: + urllib.request.urlopen(request) + except urllib.error.HTTPError as e: + self.assertEqual(e.code, 404) + + # now try serving a content that do exists + self.assertTrue(uses_local_files_for(server, "/simple/index.html")) + + # and another one in another root path + self.assertTrue(uses_local_files_for(server, "/external/index.html")) + + finally: + server.stop() + + +def test_suite(): + return unittest.makeSuite(PyPIServerTest) + +if __name__ == '__main__': + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_pypi_simple.py b/Lib/packaging/tests/test_pypi_simple.py new file mode 100644 index 0000000000..bd50d01b39 --- /dev/null +++ b/Lib/packaging/tests/test_pypi_simple.py @@ -0,0 +1,353 @@ +"""Tests for the packaging.pypi.simple module.""" +import re +import os +import sys +import http.client +import urllib.error +import urllib.parse +import urllib.request + +from packaging.pypi.simple import Crawler + +from packaging.tests import unittest +from packaging.tests.support import (TempdirManager, LoggingCatcher, + fake_dec) + +try: + import _thread + from packaging.tests.pypi_server import (use_pypi_server, PyPIServer, + PYPI_DEFAULT_STATIC_PATH) +except ImportError: + _thread = None + use_pypi_server = fake_dec + PYPI_DEFAULT_STATIC_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'pypiserver') + + + +class SimpleCrawlerTestCase(TempdirManager, + LoggingCatcher, + unittest.TestCase): + + def _get_simple_crawler(self, server, base_url="/simple/", hosts=None, + *args, **kwargs): + """Build and return a SimpleIndex with the test server urls""" + if hosts is None: + hosts = (server.full_address.replace("http://", ""),) + kwargs['hosts'] = hosts + return Crawler(server.full_address + base_url, *args, + **kwargs) + + @unittest.skipIf(_thread is None, 'needs threads') + @use_pypi_server() + def test_bad_urls(self, server): + crawler = Crawler() + url = 'http://127.0.0.1:0/nonesuch/test_simple' + try: + v = crawler._open_url(url) + except Exception as v: + self.assertIn(url, str(v)) + else: + v.close() + self.assertIsInstance(v, urllib.error.HTTPError) + + # issue 16 + # easy_install inquant.contentmirror.plone breaks because of a typo + # in its home URL + crawler = Crawler(hosts=('example.org',)) + url = ('url:%20https://svn.plone.org/svn/collective/' + 'inquant.contentmirror.plone/trunk') + try: + v = crawler._open_url(url) + except Exception as v: + self.assertIn(url, str(v)) + else: + v.close() + self.assertIsInstance(v, urllib.error.HTTPError) + + def _urlopen(*args): + raise http.client.BadStatusLine('line') + + old_urlopen = urllib.request.urlopen + urllib.request.urlopen = _urlopen + url = 'http://example.org' + try: + v = crawler._open_url(url) + except Exception as v: + self.assertIn('line', str(v)) + else: + v.close() + # TODO use self.assertRaises + raise AssertionError('Should have raise here!') + finally: + urllib.request.urlopen = old_urlopen + + # issue 20 + url = 'http://http://svn.pythonpaste.org/Paste/wphp/trunk' + try: + crawler._open_url(url) + except Exception as v: + self.assertIn('nonnumeric port', str(v)) + + # issue #160 + url = server.full_address + page = ('<a href="http://www.famfamfam.com](' + 'http://www.famfamfam.com/">') + crawler._process_url(url, page) + + @unittest.skipIf(_thread is None, 'needs threads') + @use_pypi_server("test_found_links") + def test_found_links(self, server): + # Browse the index, asking for a specified release version + # The PyPI index contains links for version 1.0, 1.1, 2.0 and 2.0.1 + crawler = self._get_simple_crawler(server) + last_release = crawler.get_release("foobar") + + # we have scanned the index page + self.assertIn(server.full_address + "/simple/foobar/", + crawler._processed_urls) + + # we have found 4 releases in this page + self.assertEqual(len(crawler._projects["foobar"]), 4) + + # and returned the most recent one + self.assertEqual("%s" % last_release.version, '2.0.1') + + def test_is_browsable(self): + crawler = Crawler(follow_externals=False) + self.assertTrue(crawler._is_browsable(crawler.index_url + "test")) + + # Now, when following externals, we can have a list of hosts to trust. + # and don't follow other external links than the one described here. + crawler = Crawler(hosts=["pypi.python.org", "example.org"], + follow_externals=True) + good_urls = ( + "http://pypi.python.org/foo/bar", + "http://pypi.python.org/simple/foobar", + "http://example.org", + "http://example.org/", + "http://example.org/simple/", + ) + bad_urls = ( + "http://python.org", + "http://example.tld", + ) + + for url in good_urls: + self.assertTrue(crawler._is_browsable(url)) + + for url in bad_urls: + self.assertFalse(crawler._is_browsable(url)) + + # allow all hosts + crawler = Crawler(follow_externals=True, hosts=("*",)) + self.assertTrue(crawler._is_browsable("http://an-external.link/path")) + self.assertTrue(crawler._is_browsable("pypi.example.org/a/path")) + + # specify a list of hosts we want to allow + crawler = Crawler(follow_externals=True, + hosts=("*.example.org",)) + self.assertFalse(crawler._is_browsable("http://an-external.link/path")) + self.assertTrue( + crawler._is_browsable("http://pypi.example.org/a/path")) + + @unittest.skipIf(_thread is None, 'needs threads') + @use_pypi_server("with_externals") + def test_follow_externals(self, server): + # Include external pages + # Try to request the package index, wich contains links to "externals" + # resources. They have to be scanned too. + crawler = self._get_simple_crawler(server, follow_externals=True) + crawler.get_release("foobar") + self.assertIn(server.full_address + "/external/external.html", + crawler._processed_urls) + + @unittest.skipIf(_thread is None, 'needs threads') + @use_pypi_server("with_real_externals") + def test_restrict_hosts(self, server): + # Only use a list of allowed hosts is possible + # Test that telling the simple pyPI client to not retrieve external + # works + crawler = self._get_simple_crawler(server, follow_externals=False) + crawler.get_release("foobar") + self.assertNotIn(server.full_address + "/external/external.html", + crawler._processed_urls) + + @unittest.skipIf(_thread is None, 'needs threads') + @use_pypi_server(static_filesystem_paths=["with_externals"], + static_uri_paths=["simple", "external"]) + def test_links_priority(self, server): + # Download links from the pypi simple index should be used before + # external download links. + # http://bitbucket.org/tarek/distribute/issue/163/md5-validation-error + # + # Usecase : + # - someone uploads a package on pypi, a md5 is generated + # - someone manually coindexes this link (with the md5 in the url) onto + # an external page accessible from the package page. + # - someone reuploads the package (with a different md5) + # - while easy_installing, an MD5 error occurs because the external + # link is used + # -> The index should use the link from pypi, not the external one. + + # start an index server + index_url = server.full_address + '/simple/' + + # scan a test index + crawler = Crawler(index_url, follow_externals=True) + releases = crawler.get_releases("foobar") + server.stop() + + # we have only one link, because links are compared without md5 + self.assertEqual(1, len(releases)) + self.assertEqual(1, len(releases[0].dists)) + # the link should be from the index + self.assertEqual(2, len(releases[0].dists['sdist'].urls)) + self.assertEqual('12345678901234567', + releases[0].dists['sdist'].url['hashval']) + self.assertEqual('md5', releases[0].dists['sdist'].url['hashname']) + + @unittest.skipIf(_thread is None, 'needs threads') + @use_pypi_server(static_filesystem_paths=["with_norel_links"], + static_uri_paths=["simple", "external"]) + def test_not_scan_all_links(self, server): + # Do not follow all index page links. + # The links not tagged with rel="download" and rel="homepage" have + # to not be processed by the package index, while processing "pages". + + # process the pages + crawler = self._get_simple_crawler(server, follow_externals=True) + crawler.get_releases("foobar") + # now it should have processed only pages with links rel="download" + # and rel="homepage" + self.assertIn("%s/simple/foobar/" % server.full_address, + crawler._processed_urls) # it's the simple index page + self.assertIn("%s/external/homepage.html" % server.full_address, + crawler._processed_urls) # the external homepage is rel="homepage" + self.assertNotIn("%s/external/nonrel.html" % server.full_address, + crawler._processed_urls) # this link contains no rel=* + self.assertNotIn("%s/unrelated-0.2.tar.gz" % server.full_address, + crawler._processed_urls) # linked from simple index (no rel) + self.assertIn("%s/foobar-0.1.tar.gz" % server.full_address, + crawler._processed_urls) # linked from simple index (rel) + self.assertIn("%s/foobar-2.0.tar.gz" % server.full_address, + crawler._processed_urls) # linked from external homepage (rel) + + @unittest.skipIf(_thread is None, 'needs threads') + def test_uses_mirrors(self): + # When the main repository seems down, try using the given mirrors""" + server = PyPIServer("foo_bar_baz") + mirror = PyPIServer("foo_bar_baz") + mirror.start() # we dont start the server here + + try: + # create the index using both servers + crawler = Crawler(server.full_address + "/simple/", hosts=('*',), + # set the timeout to 1s for the tests + timeout=1, mirrors=[mirror.full_address]) + + # this should not raise a timeout + self.assertEqual(4, len(crawler.get_releases("foo"))) + finally: + mirror.stop() + server.stop() + + def test_simple_link_matcher(self): + # Test that the simple link matcher finds the right links""" + crawler = Crawler(follow_externals=False) + + # Here, we define: + # 1. one link that must be followed, cause it's a download one + # 2. one link that must *not* be followed, cause the is_browsable + # returns false for it. + # 3. one link that must be followed cause it's a homepage that is + # browsable + # 4. one link that must be followed, because it contain a md5 hash + self.assertTrue(crawler._is_browsable("%stest" % crawler.index_url)) + self.assertFalse(crawler._is_browsable("http://dl-link2")) + content = """ + <a href="http://dl-link1" rel="download">download_link1</a> + <a href="http://dl-link2" rel="homepage">homepage_link1</a> + <a href="%(index_url)stest" rel="homepage">homepage_link2</a> + <a href="%(index_url)stest/foobar-1.tar.gz#md5=abcdef>download_link2</a> + """ % {'index_url': crawler.index_url} + + # Test that the simple link matcher yield the good links. + generator = crawler._simple_link_matcher(content, crawler.index_url) + self.assertEqual(('%stest/foobar-1.tar.gz#md5=abcdef' % + crawler.index_url, True), next(generator)) + self.assertEqual(('http://dl-link1', True), next(generator)) + self.assertEqual(('%stest' % crawler.index_url, False), + next(generator)) + self.assertRaises(StopIteration, generator.__next__) + + # Follow the external links is possible (eg. homepages) + crawler.follow_externals = True + generator = crawler._simple_link_matcher(content, crawler.index_url) + self.assertEqual(('%stest/foobar-1.tar.gz#md5=abcdef' % + crawler.index_url, True), next(generator)) + self.assertEqual(('http://dl-link1', True), next(generator)) + self.assertEqual(('http://dl-link2', False), next(generator)) + self.assertEqual(('%stest' % crawler.index_url, False), + next(generator)) + self.assertRaises(StopIteration, generator.__next__) + + def test_browse_local_files(self): + # Test that we can browse local files""" + index_url = "file://" + PYPI_DEFAULT_STATIC_PATH + if sys.platform == 'win32': + # under windows the correct syntax is: + # file:///C|\the\path\here + # instead of + # file://C:\the\path\here + fix = re.compile(r'^(file://)([A-Za-z])(:)') + index_url = fix.sub('\\1/\\2|', index_url) + + index_path = os.sep.join([index_url, "test_found_links", "simple"]) + crawler = Crawler(index_path) + dists = crawler.get_releases("foobar") + self.assertEqual(4, len(dists)) + + def test_get_link_matcher(self): + crawler = Crawler("http://example.org") + self.assertEqual('_simple_link_matcher', crawler._get_link_matcher( + "http://example.org/some/file").__name__) + self.assertEqual('_default_link_matcher', crawler._get_link_matcher( + "http://other-url").__name__) + + def test_default_link_matcher(self): + crawler = Crawler("http://example.org", mirrors=[]) + crawler.follow_externals = True + crawler._is_browsable = lambda *args: True + base_url = "http://example.org/some/file/" + content = """ +<a href="../homepage" rel="homepage">link</a> +<a href="../download" rel="download">link2</a> +<a href="../simpleurl">link2</a> + """ + found_links = set(uri for uri, _ in + crawler._default_link_matcher(content, base_url)) + self.assertIn('http://example.org/some/homepage', found_links) + self.assertIn('http://example.org/some/simpleurl', found_links) + self.assertIn('http://example.org/some/download', found_links) + + @unittest.skipIf(_thread is None, 'needs threads') + @use_pypi_server("project_list") + def test_search_projects(self, server): + # we can search the index for some projects, on their names + # the case used no matters here + crawler = self._get_simple_crawler(server) + tests = (('Foobar', ['FooBar-bar', 'Foobar-baz', 'Baz-FooBar']), + ('foobar*', ['FooBar-bar', 'Foobar-baz']), + ('*foobar', ['Baz-FooBar'])) + + for search, expected in tests: + projects = [p.name for p in crawler.search_projects(search)] + self.assertListEqual(expected, projects) + + +def test_suite(): + return unittest.makeSuite(SimpleCrawlerTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_pypi_xmlrpc.py b/Lib/packaging/tests/test_pypi_xmlrpc.py new file mode 100644 index 0000000000..b7b382dba2 --- /dev/null +++ b/Lib/packaging/tests/test_pypi_xmlrpc.py @@ -0,0 +1,101 @@ +"""Tests for the packaging.pypi.xmlrpc module.""" + +from packaging.pypi.xmlrpc import Client, InvalidSearchField, ProjectNotFound + +from packaging.tests import unittest +from packaging.tests.support import fake_dec + +try: + import threading + from packaging.tests.pypi_server import use_xmlrpc_server +except ImportError: + threading = None + use_xmlrpc_server = fake_dec + + +@unittest.skipIf(threading is None, "Needs threading") +class TestXMLRPCClient(unittest.TestCase): + def _get_client(self, server, *args, **kwargs): + return Client(server.full_address, *args, **kwargs) + + @use_xmlrpc_server() + def test_search_projects(self, server): + client = self._get_client(server) + server.xmlrpc.set_search_result(['FooBar', 'Foo', 'FooFoo']) + results = [r.name for r in client.search_projects(name='Foo')] + self.assertEqual(3, len(results)) + self.assertIn('FooBar', results) + self.assertIn('Foo', results) + self.assertIn('FooFoo', results) + + def test_search_projects_bad_fields(self): + client = Client() + self.assertRaises(InvalidSearchField, client.search_projects, + invalid="test") + + @use_xmlrpc_server() + def test_get_releases(self, server): + client = self._get_client(server) + server.xmlrpc.set_distributions([ + {'name': 'FooBar', 'version': '1.1'}, + {'name': 'FooBar', 'version': '1.2', 'url': 'http://some/url/'}, + {'name': 'FooBar', 'version': '1.3', 'url': 'http://other/url/'}, + ]) + + # use a lambda here to avoid an useless mock call + server.xmlrpc.list_releases = lambda *a, **k: ['1.1', '1.2', '1.3'] + + releases = client.get_releases('FooBar (<=1.2)') + # dont call release_data and release_url; just return name and version. + self.assertEqual(2, len(releases)) + versions = releases.get_versions() + self.assertIn('1.1', versions) + self.assertIn('1.2', versions) + self.assertNotIn('1.3', versions) + + self.assertRaises(ProjectNotFound, client.get_releases, 'Foo') + + @use_xmlrpc_server() + def test_get_distributions(self, server): + client = self._get_client(server) + server.xmlrpc.set_distributions([ + {'name': 'FooBar', 'version': '1.1', + 'url': 'http://example.org/foobar-1.1-sdist.tar.gz', + 'digest': '1234567', + 'type': 'sdist', 'python_version': 'source'}, + {'name':'FooBar', 'version': '1.1', + 'url': 'http://example.org/foobar-1.1-bdist.tar.gz', + 'digest': '8912345', 'type': 'bdist'}, + ]) + + releases = client.get_releases('FooBar', '1.1') + client.get_distributions('FooBar', '1.1') + release = releases.get_release('1.1') + self.assertTrue('http://example.org/foobar-1.1-sdist.tar.gz', + release['sdist'].url['url']) + self.assertTrue('http://example.org/foobar-1.1-bdist.tar.gz', + release['bdist'].url['url']) + self.assertEqual(release['sdist'].python_version, 'source') + + @use_xmlrpc_server() + def test_get_metadata(self, server): + client = self._get_client(server) + server.xmlrpc.set_distributions([ + {'name': 'FooBar', + 'version': '1.1', + 'keywords': '', + 'obsoletes_dist': ['FooFoo'], + 'requires_external': ['Foo'], + }]) + release = client.get_metadata('FooBar', '1.1') + self.assertEqual(['Foo'], release.metadata['requires_external']) + self.assertEqual(['FooFoo'], release.metadata['obsoletes_dist']) + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestXMLRPCClient)) + return suite + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_run.py b/Lib/packaging/tests/test_run.py new file mode 100644 index 0000000000..cb576b7c1d --- /dev/null +++ b/Lib/packaging/tests/test_run.py @@ -0,0 +1,85 @@ +"""Tests for packaging.run.""" + +import os +import sys +import shutil +from tempfile import mkstemp +from io import StringIO + +from packaging import install +from packaging.tests import unittest, support, TESTFN +from packaging.run import main + +# setup script that uses __file__ +setup_using___file__ = """\ + +__file__ + +from packaging.run import setup +setup() +""" + +setup_prints_cwd = """\ + +import os +print os.getcwd() + +from packaging.run import setup +setup() +""" + + +class CoreTestCase(support.TempdirManager, support.LoggingCatcher, + unittest.TestCase): + + def setUp(self): + super(CoreTestCase, self).setUp() + self.old_stdout = sys.stdout + self.cleanup_testfn() + self.old_argv = sys.argv, sys.argv[:] + + def tearDown(self): + sys.stdout = self.old_stdout + self.cleanup_testfn() + sys.argv = self.old_argv[0] + sys.argv[:] = self.old_argv[1] + super(CoreTestCase, self).tearDown() + + def cleanup_testfn(self): + path = TESTFN + if os.path.isfile(path): + os.remove(path) + elif os.path.isdir(path): + shutil.rmtree(path) + + def write_setup(self, text, path=TESTFN): + with open(path, "w") as fp: + fp.write(text) + return path + + # TODO restore the tests removed six months ago and port them to pysetup + + def test_install(self): + # making sure install returns 0 or 1 exit codes + project = os.path.join(os.path.dirname(__file__), 'package.tgz') + install_path = self.mkdtemp() + old_get_path = install.get_path + install.get_path = lambda path: install_path + old_mod = os.stat(install_path).st_mode + os.chmod(install_path, 0) + old_stderr = sys.stderr + sys.stderr = StringIO() + try: + self.assertFalse(install.install(project)) + self.assertEqual(main(['install', 'blabla']), 1) + finally: + sys.stderr = old_stderr + os.chmod(install_path, old_mod) + install.get_path = old_get_path + + +def test_suite(): + return unittest.makeSuite(CoreTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_uninstall.py b/Lib/packaging/tests/test_uninstall.py new file mode 100644 index 0000000000..c7702dedc3 --- /dev/null +++ b/Lib/packaging/tests/test_uninstall.py @@ -0,0 +1,129 @@ +"""Tests for the uninstall command.""" +import os +import sys +from io import StringIO +import stat +import packaging.util + +from packaging.database import disable_cache, enable_cache +from packaging.run import main +from packaging.errors import PackagingError +from packaging.install import remove +from packaging.command.install_dist import install_dist + +from packaging.tests import unittest, support + +SETUP_CFG = """ +[metadata] +name = %(name)s +version = %(version)s + +[files] +packages = + %(pkg)s + %(pkg)s.sub +""" + + +class UninstallTestCase(support.TempdirManager, + support.LoggingCatcher, + support.EnvironRestorer, + unittest.TestCase): + + restore_environ = ['PLAT'] + + def setUp(self): + super(UninstallTestCase, self).setUp() + self.addCleanup(setattr, sys, 'stdout', sys.stdout) + self.addCleanup(setattr, sys, 'stderr', sys.stderr) + self.addCleanup(os.chdir, os.getcwd()) + self.addCleanup(enable_cache) + self.root_dir = self.mkdtemp() + self.cwd = os.getcwd() + disable_cache() + + def tearDown(self): + os.chdir(self.cwd) + packaging.util._path_created.clear() + super(UninstallTestCase, self).tearDown() + + def run_setup(self, *args): + # run setup with args + args = ['run'] + list(args) + dist = main(args) + return dist + + def get_path(self, dist, name): + cmd = install_dist(dist) + cmd.prefix = self.root_dir + cmd.finalize_options() + return getattr(cmd, 'install_' + name) + + def make_dist(self, name='Foo', **kw): + kw['name'] = name + pkg = name.lower() + if 'version' not in kw: + kw['version'] = '0.1' + project_dir, dist = self.create_dist(**kw) + kw['pkg'] = pkg + + pkg_dir = os.path.join(project_dir, pkg) + os.mkdir(pkg_dir) + os.mkdir(os.path.join(pkg_dir, 'sub')) + + self.write_file((project_dir, 'setup.cfg'), SETUP_CFG % kw) + self.write_file((pkg_dir, '__init__.py'), '#') + self.write_file((pkg_dir, pkg + '_utils.py'), '#') + self.write_file((pkg_dir, 'sub', '__init__.py'), '#') + self.write_file((pkg_dir, 'sub', pkg + '_utils.py'), '#') + + return project_dir + + def install_dist(self, name='Foo', dirname=None, **kw): + if not dirname: + dirname = self.make_dist(name, **kw) + os.chdir(dirname) + old_out = sys.stderr + sys.stderr = StringIO() + dist = self.run_setup('install_dist', '--prefix=' + self.root_dir) + install_lib = self.get_path(dist, 'purelib') + return dist, install_lib + + def test_uninstall_unknow_distribution(self): + self.assertRaises(PackagingError, remove, 'Foo', + paths=[self.root_dir]) + + def test_uninstall(self): + dist, install_lib = self.install_dist() + self.assertIsFile(install_lib, 'foo', '__init__.py') + self.assertIsFile(install_lib, 'foo', 'sub', '__init__.py') + self.assertIsFile(install_lib, 'Foo-0.1.dist-info', 'RECORD') + self.assertTrue(remove('Foo', paths=[install_lib])) + self.assertIsNotFile(install_lib, 'foo', 'sub', '__init__.py') + self.assertIsNotFile(install_lib, 'Foo-0.1.dist-info', 'RECORD') + + def test_remove_issue(self): + # makes sure if there are OSErrors (like permission denied) + # remove() stops and display a clean error + dist, install_lib = self.install_dist('Meh') + + # breaking os.rename + old = os.rename + + def _rename(source, target): + raise OSError + + os.rename = _rename + try: + self.assertFalse(remove('Meh', paths=[install_lib])) + finally: + os.rename = old + + self.assertTrue(remove('Meh', paths=[install_lib])) + + +def test_suite(): + return unittest.makeSuite(UninstallTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/packaging/tests/test_unixccompiler.py b/Lib/packaging/tests/test_unixccompiler.py new file mode 100644 index 0000000000..16a1af300e --- /dev/null +++ b/Lib/packaging/tests/test_unixccompiler.py @@ -0,0 +1,132 @@ +"""Tests for packaging.unixccompiler.""" +import sys + +import sysconfig +from packaging.compiler.unixccompiler import UnixCCompiler +from packaging.tests import unittest + + +class UnixCCompilerTestCase(unittest.TestCase): + + def setUp(self): + self._backup_platform = sys.platform + self._backup_get_config_var = sysconfig.get_config_var + + class CompilerWrapper(UnixCCompiler): + def rpath_foo(self): + return self.runtime_library_dir_option('/foo') + self.cc = CompilerWrapper() + + def tearDown(self): + sys.platform = self._backup_platform + sysconfig.get_config_var = self._backup_get_config_var + + @unittest.skipIf(sys.platform == 'win32', 'irrelevant on win32') + def test_runtime_libdir_option(self): + + # Issue #5900: Ensure RUNPATH is added to extension + # modules with RPATH if GNU ld is used + + # darwin + sys.platform = 'darwin' + self.assertEqual(self.cc.rpath_foo(), '-L/foo') + + # hp-ux + sys.platform = 'hp-ux' + old_gcv = sysconfig.get_config_var + + def gcv(v): + return 'xxx' + sysconfig.get_config_var = gcv + self.assertEqual(self.cc.rpath_foo(), ['+s', '-L/foo']) + + def gcv(v): + return 'gcc' + sysconfig.get_config_var = gcv + self.assertEqual(self.cc.rpath_foo(), ['-Wl,+s', '-L/foo']) + + def gcv(v): + return 'g++' + sysconfig.get_config_var = gcv + self.assertEqual(self.cc.rpath_foo(), ['-Wl,+s', '-L/foo']) + + sysconfig.get_config_var = old_gcv + + # irix646 + sys.platform = 'irix646' + self.assertEqual(self.cc.rpath_foo(), ['-rpath', '/foo']) + + # osf1V5 + sys.platform = 'osf1V5' + self.assertEqual(self.cc.rpath_foo(), ['-rpath', '/foo']) + + # GCC GNULD + sys.platform = 'bar' + + def gcv(v): + if v == 'CC': + return 'gcc' + elif v == 'GNULD': + return 'yes' + sysconfig.get_config_var = gcv + self.assertEqual(self.cc.rpath_foo(), '-Wl,--enable-new-dtags,-R/foo') + + # GCC non-GNULD + sys.platform = 'bar' + + def gcv(v): + if v == 'CC': + return 'gcc' + elif v == 'GNULD': + return 'no' + sysconfig.get_config_var = gcv + self.assertEqual(self.cc.rpath_foo(), '-Wl,-R/foo') + + # GCC GNULD with fully qualified configuration prefix + # see #7617 + sys.platform = 'bar' + + def gcv(v): + if v == 'CC': + return 'x86_64-pc-linux-gnu-gcc-4.4.2' + elif v == 'GNULD': + return 'yes' + sysconfig.get_config_var = gcv + self.assertEqual(self.cc.rpath_foo(), '-Wl,--enable-new-dtags,-R/foo') + + # non-GCC GNULD + sys.platform = 'bar' + + def gcv(v): + if v == 'CC': + return 'cc' + elif v == 'GNULD': + return 'yes' + sysconfig.get_config_var = gcv + self.assertEqual(self.cc.rpath_foo(), '-R/foo') + + # non-GCC non-GNULD + sys.platform = 'bar' + + def gcv(v): + if v == 'CC': + return 'cc' + elif v == 'GNULD': + return 'no' + sysconfig.get_config_var = gcv + self.assertEqual(self.cc.rpath_foo(), '-R/foo') + + # AIX C/C++ linker + sys.platform = 'aix' + + def gcv(v): + return 'xxx' + sysconfig.get_config_var = gcv + self.assertEqual(self.cc.rpath_foo(), '-blibpath:/foo') + + +def test_suite(): + return unittest.makeSuite(UnixCCompilerTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_util.py b/Lib/packaging/tests/test_util.py new file mode 100644 index 0000000000..5e804c27a6 --- /dev/null +++ b/Lib/packaging/tests/test_util.py @@ -0,0 +1,992 @@ +"""Tests for packaging.util.""" +import os +import sys +import time +import logging +import tempfile +import subprocess +from io import StringIO + +from packaging.tests import support, unittest +from packaging.tests.test_config import SETUP_CFG +from packaging.errors import ( + PackagingPlatformError, PackagingByteCompileError, PackagingFileError, + PackagingExecError, InstallationException) +from packaging import util +from packaging.dist import Distribution +from packaging.util import ( + convert_path, change_root, split_quoted, strtobool, + get_compiler_versions, _MAC_OS_X_LD_VERSION, byte_compile, find_packages, + spawn, get_pypirc_path, generate_pypirc, read_pypirc, resolve_name, iglob, + RICH_GLOB, egginfo_to_distinfo, is_setuptools, is_distutils, is_packaging, + get_install_method, cfg_to_args, encode_multipart) + + +PYPIRC = """\ +[distutils] +index-servers = + pypi + server1 + +[pypi] +username:me +password:xxxx + +[server1] +repository:http://example.com +username:tarek +password:secret +""" + +PYPIRC_OLD = """\ +[server-login] +username:tarek +password:secret +""" + +WANTED = """\ +[distutils] +index-servers = + pypi + +[pypi] +username:tarek +password:xxx +""" + +EXPECTED_MULTIPART_OUTPUT = [ + b'---x', + b'Content-Disposition: form-data; name="username"', + b'', + b'wok', + b'---x', + b'Content-Disposition: form-data; name="password"', + b'', + b'secret', + b'---x', + b'Content-Disposition: form-data; name="picture"; filename="wok.png"', + b'', + b'PNG89', + b'---x--', + b'', +] + + +class FakePopen: + test_class = None + + def __init__(self, args, bufsize=0, executable=None, + stdin=None, stdout=None, stderr=None, + preexec_fn=None, close_fds=False, + shell=False, cwd=None, env=None, universal_newlines=False, + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False, + pass_fds=()): + if isinstance(args, str): + args = args.split() + self.cmd = args[0] + exes = self.test_class._exes + if self.cmd not in exes: + # we don't want to call the system, returning an empty + # output so it doesn't match + self.stdout = StringIO() + self.stderr = StringIO() + else: + self.stdout = StringIO(exes[self.cmd]) + self.stderr = StringIO() + + def communicate(self, input=None, timeout=None): + return self.stdout.read(), self.stderr.read() + + def wait(self, timeout=None): + return 0 + + +class UtilTestCase(support.EnvironRestorer, + support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + restore_environ = ['HOME', 'PLAT'] + + def setUp(self): + super(UtilTestCase, self).setUp() + self.addCleanup(os.chdir, os.getcwd()) + tempdir = self.mkdtemp() + self.rc = os.path.join(tempdir, '.pypirc') + os.environ['HOME'] = tempdir + os.chdir(tempdir) + # saving the environment + self.name = os.name + self.platform = sys.platform + self.version = sys.version + self.sep = os.sep + self.join = os.path.join + self.isabs = os.path.isabs + self.splitdrive = os.path.splitdrive + + # patching os.uname + if hasattr(os, 'uname'): + self.uname = os.uname + self._uname = os.uname() + else: + self.uname = None + self._uname = None + os.uname = self._get_uname + + # patching POpen + self.old_find_executable = util.find_executable + util.find_executable = self._find_executable + self._exes = {} + self.old_popen = subprocess.Popen + self.old_stdout = sys.stdout + self.old_stderr = sys.stderr + FakePopen.test_class = self + subprocess.Popen = FakePopen + + def tearDown(self): + # getting back the environment + os.name = self.name + sys.platform = self.platform + sys.version = self.version + os.sep = self.sep + os.path.join = self.join + os.path.isabs = self.isabs + os.path.splitdrive = self.splitdrive + if self.uname is not None: + os.uname = self.uname + else: + del os.uname + util.find_executable = self.old_find_executable + subprocess.Popen = self.old_popen + sys.old_stdout = self.old_stdout + sys.old_stderr = self.old_stderr + super(UtilTestCase, self).tearDown() + + def _set_uname(self, uname): + self._uname = uname + + def _get_uname(self): + return self._uname + + def test_convert_path(self): + # linux/mac + os.sep = '/' + + def _join(path): + return '/'.join(path) + os.path.join = _join + + self.assertEqual(convert_path('/home/to/my/stuff'), + '/home/to/my/stuff') + + # win + os.sep = '\\' + + def _join(*path): + return '\\'.join(path) + os.path.join = _join + + self.assertRaises(ValueError, convert_path, '/home/to/my/stuff') + self.assertRaises(ValueError, convert_path, 'home/to/my/stuff/') + + self.assertEqual(convert_path('home/to/my/stuff'), + 'home\\to\\my\\stuff') + self.assertEqual(convert_path('.'), + os.curdir) + + def test_change_root(self): + # linux/mac + os.name = 'posix' + + def _isabs(path): + return path[0] == '/' + os.path.isabs = _isabs + + def _join(*path): + return '/'.join(path) + os.path.join = _join + + self.assertEqual(change_root('/root', '/old/its/here'), + '/root/old/its/here') + self.assertEqual(change_root('/root', 'its/here'), + '/root/its/here') + + # windows + os.name = 'nt' + + def _isabs(path): + return path.startswith('c:\\') + os.path.isabs = _isabs + + def _splitdrive(path): + if path.startswith('c:'): + return '', path.replace('c:', '') + return '', path + os.path.splitdrive = _splitdrive + + def _join(*path): + return '\\'.join(path) + os.path.join = _join + + self.assertEqual(change_root('c:\\root', 'c:\\old\\its\\here'), + 'c:\\root\\old\\its\\here') + self.assertEqual(change_root('c:\\root', 'its\\here'), + 'c:\\root\\its\\here') + + # BugsBunny os (it's a great os) + os.name = 'BugsBunny' + self.assertRaises(PackagingPlatformError, + change_root, 'c:\\root', 'its\\here') + + # XXX platforms to be covered: os2, mac + + def test_split_quoted(self): + self.assertEqual(split_quoted('""one"" "two" \'three\' \\four'), + ['one', 'two', 'three', 'four']) + + def test_strtobool(self): + yes = ('y', 'Y', 'yes', 'True', 't', 'true', 'True', 'On', 'on', '1') + no = ('n', 'no', 'f', 'false', 'off', '0', 'Off', 'No', 'N') + + for y in yes: + self.assertTrue(strtobool(y)) + + for n in no: + self.assertFalse(strtobool(n)) + + def test_find_exe_version(self): + # the ld version scheme under MAC OS is: + # ^@(#)PROGRAM:ld PROJECT:ld64-VERSION + # + # where VERSION is a 2-digit number for major + # revisions. For instance under Leopard, it's + # currently 77 + # + # Dots are used when branching is done. + # + # The SnowLeopard ld64 is currently 95.2.12 + + for output, version in (('@(#)PROGRAM:ld PROJECT:ld64-77', '77'), + ('@(#)PROGRAM:ld PROJECT:ld64-95.2.12', + '95.2.12')): + result = _MAC_OS_X_LD_VERSION.search(output) + self.assertEqual(result.group(1), version) + + def _find_executable(self, name): + if name in self._exes: + return name + return None + + def test_get_compiler_versions(self): + # get_versions calls distutils.spawn.find_executable on + # 'gcc', 'ld' and 'dllwrap' + self.assertEqual(get_compiler_versions(), (None, None, None)) + + # Let's fake we have 'gcc' and it returns '3.4.5' + self._exes['gcc'] = 'gcc (GCC) 3.4.5 (mingw special)\nFSF' + res = get_compiler_versions() + self.assertEqual(str(res[0]), '3.4.5') + + # and let's see what happens when the version + # doesn't match the regular expression + # (\d+\.\d+(\.\d+)*) + self._exes['gcc'] = 'very strange output' + res = get_compiler_versions() + self.assertEqual(res[0], None) + + # same thing for ld + if sys.platform != 'darwin': + self._exes['ld'] = 'GNU ld version 2.17.50 20060824' + res = get_compiler_versions() + self.assertEqual(str(res[1]), '2.17.50') + self._exes['ld'] = '@(#)PROGRAM:ld PROJECT:ld64-77' + res = get_compiler_versions() + self.assertEqual(res[1], None) + else: + self._exes['ld'] = 'GNU ld version 2.17.50 20060824' + res = get_compiler_versions() + self.assertEqual(res[1], None) + self._exes['ld'] = '@(#)PROGRAM:ld PROJECT:ld64-77' + res = get_compiler_versions() + self.assertEqual(str(res[1]), '77') + + # and dllwrap + self._exes['dllwrap'] = 'GNU dllwrap 2.17.50 20060824\nFSF' + res = get_compiler_versions() + self.assertEqual(str(res[2]), '2.17.50') + self._exes['dllwrap'] = 'Cheese Wrap' + res = get_compiler_versions() + self.assertEqual(res[2], None) + + @unittest.skipUnless(hasattr(sys, 'dont_write_bytecode'), + 'sys.dont_write_bytecode not supported') + def test_dont_write_bytecode(self): + # makes sure byte_compile raise a PackagingError + # if sys.dont_write_bytecode is True + old_dont_write_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + self.assertRaises(PackagingByteCompileError, byte_compile, []) + finally: + sys.dont_write_bytecode = old_dont_write_bytecode + + def test_newer(self): + self.assertRaises(PackagingFileError, util.newer, 'xxx', 'xxx') + self.newer_f1 = self.mktempfile() + time.sleep(1) + self.newer_f2 = self.mktempfile() + self.assertTrue(util.newer(self.newer_f2.name, self.newer_f1.name)) + + def test_find_packages(self): + # let's create a structure we want to scan: + # + # pkg1 + # __init__ + # pkg2 + # __init__ + # pkg3 + # __init__ + # pkg6 + # __init__ + # pkg4 <--- not a pkg + # pkg8 + # __init__ + # pkg5 + # __init__ + # + root = self.mkdtemp() + pkg1 = os.path.join(root, 'pkg1') + os.mkdir(pkg1) + self.write_file(os.path.join(pkg1, '__init__.py')) + os.mkdir(os.path.join(pkg1, 'pkg2')) + self.write_file(os.path.join(pkg1, 'pkg2', '__init__.py')) + os.mkdir(os.path.join(pkg1, 'pkg3')) + self.write_file(os.path.join(pkg1, 'pkg3', '__init__.py')) + os.mkdir(os.path.join(pkg1, 'pkg3', 'pkg6')) + self.write_file(os.path.join(pkg1, 'pkg3', 'pkg6', '__init__.py')) + os.mkdir(os.path.join(pkg1, 'pkg4')) + os.mkdir(os.path.join(pkg1, 'pkg4', 'pkg8')) + self.write_file(os.path.join(pkg1, 'pkg4', 'pkg8', '__init__.py')) + pkg5 = os.path.join(root, 'pkg5') + os.mkdir(pkg5) + self.write_file(os.path.join(pkg5, '__init__.py')) + + res = find_packages([root], ['pkg1.pkg2']) + self.assertEqual(set(res), set(['pkg1', 'pkg5', 'pkg1.pkg3', + 'pkg1.pkg3.pkg6'])) + + def test_resolve_name(self): + self.assertIs(str, resolve_name('builtins.str')) + self.assertEqual( + UtilTestCase.__name__, + resolve_name("packaging.tests.test_util.UtilTestCase").__name__) + self.assertEqual( + UtilTestCase.test_resolve_name.__name__, + resolve_name("packaging.tests.test_util.UtilTestCase." + "test_resolve_name").__name__) + + self.assertRaises(ImportError, resolve_name, + "packaging.tests.test_util.UtilTestCaseNot") + self.assertRaises(ImportError, resolve_name, + "packaging.tests.test_util.UtilTestCase." + "nonexistent_attribute") + + def test_import_nested_first_time(self): + tmp_dir = self.mkdtemp() + os.makedirs(os.path.join(tmp_dir, 'a', 'b')) + self.write_file(os.path.join(tmp_dir, 'a', '__init__.py'), '') + self.write_file(os.path.join(tmp_dir, 'a', 'b', '__init__.py'), '') + self.write_file(os.path.join(tmp_dir, 'a', 'b', 'c.py'), + 'class Foo: pass') + + try: + sys.path.append(tmp_dir) + resolve_name("a.b.c.Foo") + # assert nothing raised + finally: + sys.path.remove(tmp_dir) + + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') + def test_run_2to3_on_code(self): + content = "print 'test'" + converted_content = "print('test')" + file_handle = self.mktempfile() + file_name = file_handle.name + file_handle.write(content) + file_handle.flush() + file_handle.seek(0) + from packaging.util import run_2to3 + run_2to3([file_name]) + new_content = "".join(file_handle.read()) + file_handle.close() + self.assertEqual(new_content, converted_content) + + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') + def test_run_2to3_on_doctests(self): + # to check if text files containing doctests only get converted. + content = ">>> print 'test'\ntest\n" + converted_content = ">>> print('test')\ntest\n\n" + file_handle = self.mktempfile() + file_name = file_handle.name + file_handle.write(content) + file_handle.flush() + file_handle.seek(0) + from packaging.util import run_2to3 + run_2to3([file_name], doctests_only=True) + new_content = "".join(file_handle.readlines()) + file_handle.close() + self.assertEqual(new_content, converted_content) + + @unittest.skipUnless(os.name in ('nt', 'posix'), + 'runs only under posix or nt') + def test_spawn(self): + # no patching of Popen here + subprocess.Popen = self.old_popen + tmpdir = self.mkdtemp() + + # creating something executable + # through the shell that returns 1 + if os.name == 'posix': + exe = os.path.join(tmpdir, 'foo.sh') + self.write_file(exe, '#!/bin/sh\nexit 1') + os.chmod(exe, 0o777) + else: + exe = os.path.join(tmpdir, 'foo.bat') + self.write_file(exe, 'exit 1') + + os.chmod(exe, 0o777) + self.assertRaises(PackagingExecError, spawn, [exe]) + + # now something that works + if os.name == 'posix': + exe = os.path.join(tmpdir, 'foo.sh') + self.write_file(exe, '#!/bin/sh\nexit 0') + os.chmod(exe, 0o777) + else: + exe = os.path.join(tmpdir, 'foo.bat') + self.write_file(exe, 'exit 0') + + os.chmod(exe, 0o777) + spawn([exe]) # should work without any error + + def test_server_registration(self): + # This test makes sure we know how to: + # 1. handle several sections in .pypirc + # 2. handle the old format + + # new format + self.write_file(self.rc, PYPIRC) + config = read_pypirc() + + config = sorted(config.items()) + expected = [('password', 'xxxx'), ('realm', 'pypi'), + ('repository', 'http://pypi.python.org/pypi'), + ('server', 'pypi'), ('username', 'me')] + self.assertEqual(config, expected) + + # old format + self.write_file(self.rc, PYPIRC_OLD) + config = read_pypirc() + config = sorted(config.items()) + expected = [('password', 'secret'), ('realm', 'pypi'), + ('repository', 'http://pypi.python.org/pypi'), + ('server', 'server-login'), ('username', 'tarek')] + self.assertEqual(config, expected) + + def test_server_empty_registration(self): + rc = get_pypirc_path() + self.assertFalse(os.path.exists(rc)) + generate_pypirc('tarek', 'xxx') + self.assertTrue(os.path.exists(rc)) + with open(rc) as f: + content = f.read() + self.assertEqual(content, WANTED) + + def test_cfg_to_args(self): + opts = {'description-file': 'README', 'extra-files': '', + 'setup-hooks': 'packaging.tests.test_config.version_hook'} + self.write_file('setup.cfg', SETUP_CFG % opts, encoding='utf-8') + self.write_file('README', 'loooong description') + + args = cfg_to_args() + # use Distribution to get the contents of the setup.cfg file + dist = Distribution() + dist.parse_config_files() + metadata = dist.metadata + + self.assertEqual(args['name'], metadata['Name']) + # + .dev1 because the test SETUP_CFG also tests a hook function in + # test_config.py for appending to the version string + self.assertEqual(args['version'] + '.dev1', metadata['Version']) + self.assertEqual(args['author'], metadata['Author']) + self.assertEqual(args['author_email'], metadata['Author-Email']) + self.assertEqual(args['maintainer'], metadata['Maintainer']) + self.assertEqual(args['maintainer_email'], + metadata['Maintainer-Email']) + self.assertEqual(args['description'], metadata['Summary']) + self.assertEqual(args['long_description'], metadata['Description']) + self.assertEqual(args['classifiers'], metadata['Classifier']) + self.assertEqual(args['requires'], metadata['Requires-Dist']) + self.assertEqual(args['provides'], metadata['Provides-Dist']) + + self.assertEqual(args['package_dir'].get(''), dist.package_dir) + self.assertEqual(args['packages'], dist.packages) + self.assertEqual(args['scripts'], dist.scripts) + self.assertEqual(args['py_modules'], dist.py_modules) + + def test_encode_multipart(self): + fields = [('username', 'wok'), ('password', 'secret')] + files = [('picture', 'wok.png', b'PNG89')] + content_type, body = encode_multipart(fields, files, b'-x') + self.assertEqual(b'multipart/form-data; boundary=-x', content_type) + self.assertEqual(EXPECTED_MULTIPART_OUTPUT, body.split(b'\r\n')) + + +class GlobTestCaseBase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def build_files_tree(self, files): + tempdir = self.mkdtemp() + for filepath in files: + is_dir = filepath.endswith('/') + filepath = os.path.join(tempdir, *filepath.split('/')) + if is_dir: + dirname = filepath + else: + dirname = os.path.dirname(filepath) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname) + if not is_dir: + self.write_file(filepath, 'babar') + return tempdir + + @staticmethod + def os_dependent_path(path): + path = path.rstrip('/').split('/') + return os.path.join(*path) + + def clean_tree(self, spec): + files = [] + for path, includes in spec.items(): + if includes: + files.append(self.os_dependent_path(path)) + return files + + +class GlobTestCase(GlobTestCaseBase): + + def setUp(self): + super(GlobTestCase, self).setUp() + self.cwd = os.getcwd() + + def tearDown(self): + os.chdir(self.cwd) + super(GlobTestCase, self).tearDown() + + def assertGlobMatch(self, glob, spec): + """""" + tempdir = self.build_files_tree(spec) + expected = self.clean_tree(spec) + os.chdir(tempdir) + result = list(iglob(glob)) + self.assertCountEqual(expected, result) + + def test_regex_rich_glob(self): + matches = RICH_GLOB.findall( + r"babar aime les {fraises} est les {huitres}") + self.assertEqual(["fraises", "huitres"], matches) + + def test_simple_glob(self): + glob = '*.tp?' + spec = {'coucou.tpl': True, + 'coucou.tpj': True, + 'Donotwant': False} + self.assertGlobMatch(glob, spec) + + def test_simple_glob_in_dir(self): + glob = os.path.join('babar', '*.tp?') + spec = {'babar/coucou.tpl': True, + 'babar/coucou.tpj': True, + 'babar/toto.bin': False, + 'Donotwant': False} + self.assertGlobMatch(glob, spec) + + def test_recursive_glob_head(self): + glob = os.path.join('**', 'tip', '*.t?l') + spec = {'babar/zaza/zuzu/tip/coucou.tpl': True, + 'babar/z/tip/coucou.tpl': True, + 'babar/tip/coucou.tpl': True, + 'babar/zeop/tip/babar/babar.tpl': False, + 'babar/z/tip/coucou.bin': False, + 'babar/toto.bin': False, + 'zozo/zuzu/tip/babar.tpl': True, + 'zozo/tip/babar.tpl': True, + 'Donotwant': False} + self.assertGlobMatch(glob, spec) + + def test_recursive_glob_tail(self): + glob = os.path.join('babar', '**') + spec = {'babar/zaza/': True, + 'babar/zaza/zuzu/': True, + 'babar/zaza/zuzu/babar.xml': True, + 'babar/zaza/zuzu/toto.xml': True, + 'babar/zaza/zuzu/toto.csv': True, + 'babar/zaza/coucou.tpl': True, + 'babar/bubu.tpl': True, + 'zozo/zuzu/tip/babar.tpl': False, + 'zozo/tip/babar.tpl': False, + 'Donotwant': False} + self.assertGlobMatch(glob, spec) + + def test_recursive_glob_middle(self): + glob = os.path.join('babar', '**', 'tip', '*.t?l') + spec = {'babar/zaza/zuzu/tip/coucou.tpl': True, + 'babar/z/tip/coucou.tpl': True, + 'babar/tip/coucou.tpl': True, + 'babar/zeop/tip/babar/babar.tpl': False, + 'babar/z/tip/coucou.bin': False, + 'babar/toto.bin': False, + 'zozo/zuzu/tip/babar.tpl': False, + 'zozo/tip/babar.tpl': False, + 'Donotwant': False} + self.assertGlobMatch(glob, spec) + + def test_glob_set_tail(self): + glob = os.path.join('bin', '*.{bin,sh,exe}') + spec = {'bin/babar.bin': True, + 'bin/zephir.sh': True, + 'bin/celestine.exe': True, + 'bin/cornelius.bat': False, + 'bin/cornelius.xml': False, + 'toto/yurg': False, + 'Donotwant': False} + self.assertGlobMatch(glob, spec) + + def test_glob_set_middle(self): + glob = os.path.join('xml', '{babar,toto}.xml') + spec = {'xml/babar.xml': True, + 'xml/toto.xml': True, + 'xml/babar.xslt': False, + 'xml/cornelius.sgml': False, + 'xml/zephir.xml': False, + 'toto/yurg.xml': False, + 'Donotwant': False} + self.assertGlobMatch(glob, spec) + + def test_glob_set_head(self): + glob = os.path.join('{xml,xslt}', 'babar.*') + spec = {'xml/babar.xml': True, + 'xml/toto.xml': False, + 'xslt/babar.xslt': True, + 'xslt/toto.xslt': False, + 'toto/yurg.xml': False, + 'Donotwant': False} + self.assertGlobMatch(glob, spec) + + def test_glob_all(self): + dirs = '{%s,%s}' % (os.path.join('xml', '*'), + os.path.join('xslt', '**')) + glob = os.path.join(dirs, 'babar.xml') + spec = {'xml/a/babar.xml': True, + 'xml/b/babar.xml': True, + 'xml/a/c/babar.xml': False, + 'xslt/a/babar.xml': True, + 'xslt/b/babar.xml': True, + 'xslt/a/c/babar.xml': True, + 'toto/yurg.xml': False, + 'Donotwant': False} + self.assertGlobMatch(glob, spec) + + def test_invalid_glob_pattern(self): + invalids = [ + 'ppooa**', + 'azzaeaz4**/', + '/**ddsfs', + '**##1e"&e', + 'DSFb**c009', + '{', + '{aaQSDFa', + '}', + 'aQSDFSaa}', + '{**a,', + ',**a}', + '{a**,', + ',b**}', + '{a**a,babar}', + '{bob,b**z}', + ] + for pattern in invalids: + self.assertRaises(ValueError, iglob, pattern) + + +class EggInfoToDistInfoTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def get_metadata_file_paths(self, distinfo_path): + req_metadata_files = ['METADATA', 'RECORD', 'INSTALLER'] + metadata_file_paths = [] + for metadata_file in req_metadata_files: + path = os.path.join(distinfo_path, metadata_file) + metadata_file_paths.append(path) + return metadata_file_paths + + def test_egginfo_to_distinfo_setuptools(self): + distinfo = 'hello-0.1.1-py3.3.dist-info' + egginfo = 'hello-0.1.1-py3.3.egg-info' + dirs = [egginfo] + files = ['hello.py', 'hello.pyc'] + extra_metadata = ['dependency_links.txt', 'entry_points.txt', + 'not-zip-safe', 'PKG-INFO', 'top_level.txt', + 'SOURCES.txt'] + for f in extra_metadata: + files.append(os.path.join(egginfo, f)) + + tempdir, record_file = self.build_dist_tree(files, dirs) + distinfo_path = os.path.join(tempdir, distinfo) + egginfo_path = os.path.join(tempdir, egginfo) + metadata_file_paths = self.get_metadata_file_paths(distinfo_path) + + egginfo_to_distinfo(record_file) + # test that directories and files get created + self.assertTrue(os.path.isdir(distinfo_path)) + self.assertTrue(os.path.isdir(egginfo_path)) + + for mfile in metadata_file_paths: + self.assertTrue(os.path.isfile(mfile)) + + def test_egginfo_to_distinfo_distutils(self): + distinfo = 'hello-0.1.1-py3.3.dist-info' + egginfo = 'hello-0.1.1-py3.3.egg-info' + # egginfo is a file in distutils which contains the metadata + files = ['hello.py', 'hello.pyc', egginfo] + + tempdir, record_file = self.build_dist_tree(files, dirs=[]) + distinfo_path = os.path.join(tempdir, distinfo) + egginfo_path = os.path.join(tempdir, egginfo) + metadata_file_paths = self.get_metadata_file_paths(distinfo_path) + + egginfo_to_distinfo(record_file) + # test that directories and files get created + self.assertTrue(os.path.isdir(distinfo_path)) + self.assertTrue(os.path.isfile(egginfo_path)) + + for mfile in metadata_file_paths: + self.assertTrue(os.path.isfile(mfile)) + + def build_dist_tree(self, files, dirs): + tempdir = self.mkdtemp() + record_file_path = os.path.join(tempdir, 'RECORD') + file_paths, dir_paths = ([], []) + for d in dirs: + path = os.path.join(tempdir, d) + os.makedirs(path) + dir_paths.append(path) + for f in files: + path = os.path.join(tempdir, f) + with open(path, 'w') as _f: + _f.write(f) + file_paths.append(path) + + with open(record_file_path, 'w') as record_file: + for fpath in file_paths: + record_file.write(fpath + '\n') + for dpath in dir_paths: + record_file.write(dpath + '\n') + + return (tempdir, record_file_path) + + +class PackagingLibChecks(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def setUp(self): + super(PackagingLibChecks, self).setUp() + self._empty_dir = self.mkdtemp() + + def test_empty_package_is_not_based_on_anything(self): + self.assertFalse(is_setuptools(self._empty_dir)) + self.assertFalse(is_distutils(self._empty_dir)) + self.assertFalse(is_packaging(self._empty_dir)) + + def test_setup_py_importing_setuptools_is_setuptools_based(self): + self.assertTrue(is_setuptools(self._setuptools_setup_py_pkg())) + + def test_egg_info_dir_and_setup_py_is_setuptools_based(self): + self.assertTrue(is_setuptools(self._setuptools_egg_info_pkg())) + + def test_egg_info_and_non_setuptools_setup_py_is_setuptools_based(self): + self.assertTrue(is_setuptools(self._egg_info_with_no_setuptools())) + + def test_setup_py_not_importing_setuptools_is_not_setuptools_based(self): + self.assertFalse(is_setuptools(self._random_setup_py_pkg())) + + def test_setup_py_importing_distutils_is_distutils_based(self): + self.assertTrue(is_distutils(self._distutils_setup_py_pkg())) + + def test_pkg_info_file_and_setup_py_is_distutils_based(self): + self.assertTrue(is_distutils(self._distutils_pkg_info())) + + def test_pkg_info_and_non_distutils_setup_py_is_distutils_based(self): + self.assertTrue(is_distutils(self._pkg_info_with_no_distutils())) + + def test_setup_py_not_importing_distutils_is_not_distutils_based(self): + self.assertFalse(is_distutils(self._random_setup_py_pkg())) + + def test_setup_cfg_with_no_metadata_section_is_not_packaging_based(self): + self.assertFalse(is_packaging(self._setup_cfg_with_no_metadata_pkg())) + + def test_setup_cfg_with_valid_metadata_section_is_packaging_based(self): + self.assertTrue(is_packaging(self._valid_setup_cfg_pkg())) + + def test_setup_cfg_and_invalid_setup_cfg_is_not_packaging_based(self): + self.assertFalse(is_packaging(self._invalid_setup_cfg_pkg())) + + def test_get_install_method_with_setuptools_pkg(self): + path = self._setuptools_setup_py_pkg() + self.assertEqual("setuptools", get_install_method(path)) + + def test_get_install_method_with_distutils_pkg(self): + path = self._distutils_pkg_info() + self.assertEqual("distutils", get_install_method(path)) + + def test_get_install_method_with_packaging_pkg(self): + path = self._valid_setup_cfg_pkg() + self.assertEqual("packaging", get_install_method(path)) + + def test_get_install_method_with_unknown_pkg(self): + path = self._invalid_setup_cfg_pkg() + self.assertRaises(InstallationException, get_install_method, path) + + def test_is_setuptools_logs_setup_py_text_found(self): + is_setuptools(self._setuptools_setup_py_pkg()) + expected = ['setup.py file found.', + 'No egg-info directory found.', + 'Found setuptools text in setup.py.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) + + def test_is_setuptools_logs_setup_py_text_not_found(self): + directory = self._random_setup_py_pkg() + is_setuptools(directory) + expected = ['setup.py file found.', 'No egg-info directory found.', + 'No setuptools text found in setup.py.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) + + def test_is_setuptools_logs_egg_info_dir_found(self): + is_setuptools(self._setuptools_egg_info_pkg()) + expected = ['setup.py file found.', 'Found egg-info directory.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) + + def test_is_distutils_logs_setup_py_text_found(self): + is_distutils(self._distutils_setup_py_pkg()) + expected = ['setup.py file found.', + 'No PKG-INFO file found.', + 'Found distutils text in setup.py.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) + + def test_is_distutils_logs_setup_py_text_not_found(self): + directory = self._random_setup_py_pkg() + is_distutils(directory) + expected = ['setup.py file found.', 'No PKG-INFO file found.', + 'No distutils text found in setup.py.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) + + def test_is_distutils_logs_pkg_info_file_found(self): + is_distutils(self._distutils_pkg_info()) + expected = ['setup.py file found.', 'PKG-INFO file found.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) + + def test_is_packaging_logs_setup_cfg_found(self): + is_packaging(self._valid_setup_cfg_pkg()) + expected = ['setup.cfg file found.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) + + def test_is_packaging_logs_setup_cfg_not_found(self): + is_packaging(self._empty_dir) + expected = ['No setup.cfg file found.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) + + def _write_setuptools_setup_py(self, directory): + self.write_file((directory, 'setup.py'), + "from setuptools import setup") + + def _write_distutils_setup_py(self, directory): + self.write_file([directory, 'setup.py'], + "from distutils.core import setup") + + def _write_packaging_setup_cfg(self, directory): + self.write_file([directory, 'setup.cfg'], + ("[metadata]\n" + "name = mypackage\n" + "version = 0.1.0\n")) + + def _setuptools_setup_py_pkg(self): + tmp = self.mkdtemp() + self._write_setuptools_setup_py(tmp) + return tmp + + def _distutils_setup_py_pkg(self): + tmp = self.mkdtemp() + self._write_distutils_setup_py(tmp) + return tmp + + def _valid_setup_cfg_pkg(self): + tmp = self.mkdtemp() + self._write_packaging_setup_cfg(tmp) + return tmp + + def _setuptools_egg_info_pkg(self): + tmp = self.mkdtemp() + self._write_setuptools_setup_py(tmp) + tempfile.mkdtemp(suffix='.egg-info', dir=tmp) + return tmp + + def _distutils_pkg_info(self): + tmp = self._distutils_setup_py_pkg() + self.write_file([tmp, 'PKG-INFO'], '') + return tmp + + def _setup_cfg_with_no_metadata_pkg(self): + tmp = self.mkdtemp() + self.write_file([tmp, 'setup.cfg'], + ("[othersection]\n" + "foo = bar\n")) + return tmp + + def _invalid_setup_cfg_pkg(self): + tmp = self.mkdtemp() + self.write_file([tmp, 'setup.cfg'], + ("[metadata]\n" + "name = john\n" + "last_name = doe\n")) + return tmp + + def _egg_info_with_no_setuptools(self): + tmp = self._random_setup_py_pkg() + tempfile.mkdtemp(suffix='.egg-info', dir=tmp) + return tmp + + def _pkg_info_with_no_distutils(self): + tmp = self._random_setup_py_pkg() + self.write_file([tmp, 'PKG-INFO'], '') + return tmp + + def _random_setup_py_pkg(self): + tmp = self.mkdtemp() + self.write_file((tmp, 'setup.py'), "from mypackage import setup") + return tmp + + +def test_suite(): + suite = unittest.makeSuite(UtilTestCase) + suite.addTest(unittest.makeSuite(GlobTestCase)) + suite.addTest(unittest.makeSuite(EggInfoToDistInfoTestCase)) + suite.addTest(unittest.makeSuite(PackagingLibChecks)) + return suite + + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/tests/test_version.py b/Lib/packaging/tests/test_version.py new file mode 100644 index 0000000000..f94c800114 --- /dev/null +++ b/Lib/packaging/tests/test_version.py @@ -0,0 +1,252 @@ +"""Tests for packaging.version.""" +import doctest +import os + +from packaging.version import NormalizedVersion as V +from packaging.version import HugeMajorVersionNumError, IrrationalVersionError +from packaging.version import suggest_normalized_version as suggest +from packaging.version import VersionPredicate +from packaging.tests import unittest + + +class VersionTestCase(unittest.TestCase): + + versions = ((V('1.0'), '1.0'), + (V('1.1'), '1.1'), + (V('1.2.3'), '1.2.3'), + (V('1.2'), '1.2'), + (V('1.2.3a4'), '1.2.3a4'), + (V('1.2c4'), '1.2c4'), + (V('1.2.3.4'), '1.2.3.4'), + (V('1.2.3.4.0b3'), '1.2.3.4b3'), + (V('1.2.0.0.0'), '1.2'), + (V('1.0.dev345'), '1.0.dev345'), + (V('1.0.post456.dev623'), '1.0.post456.dev623')) + + def test_repr(self): + + self.assertEqual(repr(V('1.0')), "NormalizedVersion('1.0')") + + def test_basic_versions(self): + + for v, s in self.versions: + self.assertEqual(str(v), s) + + def test_hash(self): + + for v, s in self.versions: + self.assertEqual(hash(v), hash(V(s))) + + versions = set([v for v, s in self.versions]) + for v, s in self.versions: + self.assertIn(v, versions) + + self.assertEqual(set([V('1.0')]), set([V('1.0'), V('1.0')])) + + def test_from_parts(self): + + for v, s in self.versions: + parts = v.parts + v2 = V.from_parts(*v.parts) + self.assertEqual(v, v2) + self.assertEqual(str(v), str(v2)) + + def test_irrational_versions(self): + + irrational = ('1', '1.2a', '1.2.3b', '1.02', '1.2a03', + '1.2a3.04', '1.2.dev.2', '1.2dev', '1.2.dev', + '1.2.dev2.post2', '1.2.post2.dev3.post4') + + for s in irrational: + self.assertRaises(IrrationalVersionError, V, s) + + def test_huge_version(self): + + self.assertEqual(str(V('1980.0')), '1980.0') + self.assertRaises(HugeMajorVersionNumError, V, '1981.0') + self.assertEqual(str(V('1981.0', error_on_huge_major_num=False)), + '1981.0') + + def test_comparison(self): + comparison_doctest_string = r""" + >>> V('1.2.0') == '1.2' + Traceback (most recent call last): + ... + TypeError: cannot compare NormalizedVersion and str + + >>> V('1.2') < '1.3' + Traceback (most recent call last): + ... + TypeError: cannot compare NormalizedVersion and str + + >>> V('1.2.0') == V('1.2') + True + >>> V('1.2.0') == V('1.2.3') + False + >>> V('1.2.0') != V('1.2.3') + True + >>> V('1.2.0') < V('1.2.3') + True + >>> V('1.2.0') < V('1.2.0') + False + >>> V('1.2.0') <= V('1.2.0') + True + >>> V('1.2.0') <= V('1.2.3') + True + >>> V('1.2.3') <= V('1.2.0') + False + >>> V('1.2.0') >= V('1.2.0') + True + >>> V('1.2.3') >= V('1.2.0') + True + >>> V('1.2.0') >= V('1.2.3') + False + >>> (V('1.0') > V('1.0b2')) + True + >>> (V('1.0') > V('1.0c2') > V('1.0c1') > V('1.0b2') > V('1.0b1') + ... > V('1.0a2') > V('1.0a1')) + True + >>> (V('1.0.0') > V('1.0.0c2') > V('1.0.0c1') > V('1.0.0b2') > V('1.0.0b1') + ... > V('1.0.0a2') > V('1.0.0a1')) + True + + >>> V('1.0') < V('1.0.post456.dev623') + True + + >>> V('1.0.post456.dev623') < V('1.0.post456') < V('1.0.post1234') + True + + >>> (V('1.0a1') + ... < V('1.0a2.dev456') + ... < V('1.0a2') + ... < V('1.0a2.1.dev456') # e.g. need to do a quick post release on 1.0a2 + ... < V('1.0a2.1') + ... < V('1.0b1.dev456') + ... < V('1.0b2') + ... < V('1.0c1.dev456') + ... < V('1.0c1') + ... < V('1.0.dev7') + ... < V('1.0.dev18') + ... < V('1.0.dev456') + ... < V('1.0.dev1234') + ... < V('1.0') + ... < V('1.0.post456.dev623') # development version of a post release + ... < V('1.0.post456')) + True + """ + doctest.script_from_examples(comparison_doctest_string) + + def test_suggest_normalized_version(self): + + self.assertEqual(suggest('1.0'), '1.0') + self.assertEqual(suggest('1.0-alpha1'), '1.0a1') + self.assertEqual(suggest('1.0c2'), '1.0c2') + self.assertEqual(suggest('walla walla washington'), None) + self.assertEqual(suggest('2.4c1'), '2.4c1') + self.assertEqual(suggest('v1.0'), '1.0') + + # from setuptools + self.assertEqual(suggest('0.4a1.r10'), '0.4a1.post10') + self.assertEqual(suggest('0.7a1dev-r66608'), '0.7a1.dev66608') + self.assertEqual(suggest('0.6a9.dev-r41475'), '0.6a9.dev41475') + self.assertEqual(suggest('2.4preview1'), '2.4c1') + self.assertEqual(suggest('2.4pre1'), '2.4c1') + self.assertEqual(suggest('2.1-rc2'), '2.1c2') + + # from pypi + self.assertEqual(suggest('0.1dev'), '0.1.dev0') + self.assertEqual(suggest('0.1.dev'), '0.1.dev0') + + # we want to be able to parse Twisted + # development versions are like post releases in Twisted + self.assertEqual(suggest('9.0.0+r2363'), '9.0.0.post2363') + + # pre-releases are using markers like "pre1" + self.assertEqual(suggest('9.0.0pre1'), '9.0.0c1') + + # we want to be able to parse Tcl-TK + # they us "p1" "p2" for post releases + self.assertEqual(suggest('1.4p1'), '1.4.post1') + + def test_predicate(self): + # VersionPredicate knows how to parse stuff like: + # + # Project (>=version, ver2) + + predicates = ('zope.interface (>3.5.0)', + 'AnotherProject (3.4)', + 'OtherProject (<3.0)', + 'NoVersion', + 'Hey (>=2.5,<2.7)') + + for predicate in predicates: + v = VersionPredicate(predicate) + + self.assertTrue(VersionPredicate('Hey (>=2.5,<2.7)').match('2.6')) + self.assertTrue(VersionPredicate('Ho').match('2.6')) + self.assertFalse(VersionPredicate('Hey (>=2.5,!=2.6,<2.7)').match('2.6')) + self.assertTrue(VersionPredicate('Ho (<3.0)').match('2.6')) + self.assertTrue(VersionPredicate('Ho (<3.0,!=2.5)').match('2.6.0')) + self.assertFalse(VersionPredicate('Ho (<3.0,!=2.6)').match('2.6.0')) + self.assertTrue(VersionPredicate('Ho (2.5)').match('2.5.4')) + self.assertFalse(VersionPredicate('Ho (!=2.5)').match('2.5.2')) + self.assertTrue(VersionPredicate('Hey (<=2.5)').match('2.5.9')) + self.assertFalse(VersionPredicate('Hey (<=2.5)').match('2.6.0')) + self.assertTrue(VersionPredicate('Hey (>=2.5)').match('2.5.1')) + + self.assertRaises(ValueError, VersionPredicate, '') + + self.assertTrue(VersionPredicate('Hey 2.5').match('2.5.1')) + + # XXX need to silent the micro version in this case + self.assertFalse(VersionPredicate('Ho (<3.0,!=2.6)').match('2.6.3')) + + # Make sure a predicate that ends with a number works + self.assertTrue(VersionPredicate('virtualenv5 (1.0)').match('1.0')) + self.assertTrue(VersionPredicate('virtualenv5').match('1.0')) + self.assertTrue(VersionPredicate('vi5two').match('1.0')) + self.assertTrue(VersionPredicate('5two').match('1.0')) + self.assertTrue(VersionPredicate('vi5two 1.0').match('1.0')) + self.assertTrue(VersionPredicate('5two 1.0').match('1.0')) + + # test repr + for predicate in predicates: + self.assertEqual(str(VersionPredicate(predicate)), predicate) + + def test_predicate_name(self): + # Test that names are parsed the right way + + self.assertEqual('Hey', VersionPredicate('Hey (<1.1)').name) + self.assertEqual('Foo-Bar', VersionPredicate('Foo-Bar (1.1)').name) + self.assertEqual('Foo Bar', VersionPredicate('Foo Bar (1.1)').name) + + def test_is_final(self): + # VersionPredicate knows is a distribution is a final one or not. + final_versions = ('1.0', '1.0.post456') + other_versions = ('1.0.dev1', '1.0a2', '1.0c3') + + for version in final_versions: + self.assertTrue(V(version).is_final) + for version in other_versions: + self.assertFalse(V(version).is_final) + + +class VersionWhiteBoxTestCase(unittest.TestCase): + + def test_parse_numdots(self): + # For code coverage completeness, as pad_zeros_length can't be set or + # influenced from the public interface + self.assertEqual(V('1.0')._parse_numdots('1.0', '1.0', + pad_zeros_length=3), + [1, 0, 0]) + + +def test_suite(): + #README = os.path.join(os.path.dirname(__file__), 'README.txt') + #suite = [doctest.DocFileSuite(README), unittest.makeSuite(VersionTestCase)] + suite = [unittest.makeSuite(VersionTestCase), + unittest.makeSuite(VersionWhiteBoxTestCase)] + return unittest.TestSuite(suite) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff --git a/Lib/packaging/util.py b/Lib/packaging/util.py new file mode 100644 index 0000000000..49c5991ec0 --- /dev/null +++ b/Lib/packaging/util.py @@ -0,0 +1,1439 @@ +"""Miscellaneous utility functions.""" + +import os +import re +import csv +import sys +import errno +import shutil +import string +import hashlib +import posixpath +import subprocess +import sysconfig +from glob import iglob as std_iglob +from fnmatch import fnmatchcase +from inspect import getsource +from configparser import RawConfigParser + +from packaging import logger +from packaging.errors import (PackagingPlatformError, PackagingFileError, + PackagingByteCompileError, PackagingExecError, + InstallationException, PackagingInternalError) + +__all__ = [ + # file dependencies + 'newer', 'newer_group', + # helpers for commands (dry-run system) + 'execute', 'write_file', + # spawning programs + 'find_executable', 'spawn', + # path manipulation + 'convert_path', 'change_root', + # 2to3 conversion + 'Mixin2to3', 'run_2to3', + # packaging compatibility helpers + 'cfg_to_args', 'generate_setup_py', + 'egginfo_to_distinfo', + 'get_install_method', + # misc + 'ask', 'check_environ', 'encode_multipart', 'resolve_name', + # querying for information TODO move to sysconfig + 'get_compiler_versions', 'get_platform', 'set_platform', + # configuration TODO move to packaging.config + 'get_pypirc_path', 'read_pypirc', 'generate_pypirc', + 'strtobool', 'split_multiline', +] + +_PLATFORM = None +_DEFAULT_INSTALLER = 'packaging' + + +def newer(source, target): + """Tell if the target is newer than the source. + + Returns true if 'source' exists and is more recently modified than + 'target', or if 'source' exists and 'target' doesn't. + + Returns false if both exist and 'target' is the same age or younger + than 'source'. Raise PackagingFileError if 'source' does not exist. + + Note that this test is not very accurate: files created in the same second + will have the same "age". + """ + if not os.path.exists(source): + raise PackagingFileError("file '%s' does not exist" % + os.path.abspath(source)) + if not os.path.exists(target): + return True + + return os.stat(source).st_mtime > os.stat(target).st_mtime + + +def get_platform(): + """Return a string that identifies the current platform. + + By default, will return the value returned by sysconfig.get_platform(), + but it can be changed by calling set_platform(). + """ + global _PLATFORM + if _PLATFORM is None: + _PLATFORM = sysconfig.get_platform() + return _PLATFORM + + +def set_platform(identifier): + """Set the platform string identifier returned by get_platform(). + + Note that this change doesn't impact the value returned by + sysconfig.get_platform(); it is local to packaging. + """ + global _PLATFORM + _PLATFORM = identifier + + +def convert_path(pathname): + """Return 'pathname' as a name that will work on the native filesystem. + + The path is split on '/' and put back together again using the current + directory separator. Needed because filenames in the setup script are + always supplied in Unix style, and have to be converted to the local + convention before we can actually use them in the filesystem. Raises + ValueError on non-Unix-ish systems if 'pathname' either starts or + ends with a slash. + """ + if os.sep == '/': + return pathname + if not pathname: + return pathname + if pathname[0] == '/': + raise ValueError("path '%s' cannot be absolute" % pathname) + if pathname[-1] == '/': + raise ValueError("path '%s' cannot end with '/'" % pathname) + + paths = pathname.split('/') + while os.curdir in paths: + paths.remove(os.curdir) + if not paths: + return os.curdir + return os.path.join(*paths) + + +def change_root(new_root, pathname): + """Return 'pathname' with 'new_root' prepended. + + If 'pathname' is relative, this is equivalent to + os.path.join(new_root,pathname). Otherwise, it requires making 'pathname' + relative and then joining the two, which is tricky on DOS/Windows. + """ + if os.name == 'posix': + if not os.path.isabs(pathname): + return os.path.join(new_root, pathname) + else: + return os.path.join(new_root, pathname[1:]) + + elif os.name == 'nt': + drive, path = os.path.splitdrive(pathname) + if path[0] == '\\': + path = path[1:] + return os.path.join(new_root, path) + + elif os.name == 'os2': + drive, path = os.path.splitdrive(pathname) + if path[0] == os.sep: + path = path[1:] + return os.path.join(new_root, path) + + else: + raise PackagingPlatformError("nothing known about " + "platform '%s'" % os.name) + +_environ_checked = False + + +def check_environ(): + """Ensure that 'os.environ' has all the environment variables needed. + + We guarantee that users can use in config files, command-line options, + etc. Currently this includes: + HOME - user's home directory (Unix only) + PLAT - description of the current platform, including hardware + and OS (see 'get_platform()') + """ + global _environ_checked + if _environ_checked: + return + + if os.name == 'posix' and 'HOME' not in os.environ: + import pwd + os.environ['HOME'] = pwd.getpwuid(os.getuid())[5] + + if 'PLAT' not in os.environ: + os.environ['PLAT'] = sysconfig.get_platform() + + _environ_checked = True + + +# Needed by 'split_quoted()' +_wordchars_re = _squote_re = _dquote_re = None + + +def _init_regex(): + global _wordchars_re, _squote_re, _dquote_re + _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace) + _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'") + _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') + + +# TODO replace with shlex.split after testing + +def split_quoted(s): + """Split a string up according to Unix shell-like rules for quotes and + backslashes. + + In short: words are delimited by spaces, as long as those + spaces are not escaped by a backslash, or inside a quoted string. + Single and double quotes are equivalent, and the quote characters can + be backslash-escaped. The backslash is stripped from any two-character + escape sequence, leaving only the escaped character. The quote + characters are stripped from any quoted string. Returns a list of + words. + """ + # This is a nice algorithm for splitting up a single string, since it + # doesn't require character-by-character examination. It was a little + # bit of a brain-bender to get it working right, though... + if _wordchars_re is None: + _init_regex() + + s = s.strip() + words = [] + pos = 0 + + while s: + m = _wordchars_re.match(s, pos) + end = m.end() + if end == len(s): + words.append(s[:end]) + break + + if s[end] in string.whitespace: # unescaped, unquoted whitespace: now + words.append(s[:end]) # we definitely have a word delimiter + s = s[end:].lstrip() + pos = 0 + + elif s[end] == '\\': # preserve whatever is being escaped; + # will become part of the current word + s = s[:end] + s[end + 1:] + pos = end + 1 + + else: + if s[end] == "'": # slurp singly-quoted string + m = _squote_re.match(s, end) + elif s[end] == '"': # slurp doubly-quoted string + m = _dquote_re.match(s, end) + else: + raise RuntimeError("this can't happen " + "(bad char '%c')" % s[end]) + + if m is None: + raise ValueError("bad string (mismatched %s quotes?)" % s[end]) + + beg, end = m.span() + s = s[:beg] + s[beg + 1:end - 1] + s[end:] + pos = m.end() - 2 + + if pos >= len(s): + words.append(s) + break + + return words + + +def split_multiline(value): + """Split a multiline string into a list, excluding blank lines.""" + + return [element for element in + (line.strip() for line in value.split('\n')) + if element] + + +def execute(func, args, msg=None, verbose=0, dry_run=False): + """Perform some action that affects the outside world. + + Some actions (e.g. writing to the filesystem) are special because + they are disabled by the 'dry_run' flag. This method takes care of all + that bureaucracy for you; all you have to do is supply the + function to call and an argument tuple for it (to embody the + "external action" being performed), and an optional message to + print. + """ + if msg is None: + msg = "%s%r" % (func.__name__, args) + if msg[-2:] == ',)': # correct for singleton tuple + msg = msg[0:-2] + ')' + + logger.info(msg) + if not dry_run: + func(*args) + + +def strtobool(val): + """Convert a string representation of truth to a boolean. + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + """ + val = val.lower() + if val in ('y', 'yes', 't', 'true', 'on', '1'): + return True + elif val in ('n', 'no', 'f', 'false', 'off', '0'): + return False + else: + raise ValueError("invalid truth value %r" % (val,)) + + +def byte_compile(py_files, optimize=0, force=False, prefix=None, + base_dir=None, verbose=0, dry_run=False, direct=None): + """Byte-compile a collection of Python source files to either .pyc + or .pyo files in the same directory. + + 'py_files' is a list of files to compile; any files that don't end in + ".py" are silently skipped. 'optimize' must be one of the following: + 0 - don't optimize (generate .pyc) + 1 - normal optimization (like "python -O") + 2 - extra optimization (like "python -OO") + If 'force' is true, all files are recompiled regardless of + timestamps. + + The source filename encoded in each bytecode file defaults to the + filenames listed in 'py_files'; you can modify these with 'prefix' and + 'basedir'. 'prefix' is a string that will be stripped off of each + source filename, and 'base_dir' is a directory name that will be + prepended (after 'prefix' is stripped). You can supply either or both + (or neither) of 'prefix' and 'base_dir', as you wish. + + If 'dry_run' is true, doesn't actually do anything that would + affect the filesystem. + + Byte-compilation is either done directly in this interpreter process + with the standard py_compile module, or indirectly by writing a + temporary script and executing it. Normally, you should let + 'byte_compile()' figure out to use direct compilation or not (see + the source for details). The 'direct' flag is used by the script + generated in indirect mode; unless you know what you're doing, leave + it set to None. + """ + # nothing is done if sys.dont_write_bytecode is True + # FIXME this should not raise an error + if hasattr(sys, 'dont_write_bytecode') and sys.dont_write_bytecode: + raise PackagingByteCompileError('byte-compiling is disabled.') + + # First, if the caller didn't force us into direct or indirect mode, + # figure out which mode we should be in. We take a conservative + # approach: choose direct mode *only* if the current interpreter is + # in debug mode and optimize is 0. If we're not in debug mode (-O + # or -OO), we don't know which level of optimization this + # interpreter is running with, so we can't do direct + # byte-compilation and be certain that it's the right thing. Thus, + # always compile indirectly if the current interpreter is in either + # optimize mode, or if either optimization level was requested by + # the caller. + if direct is None: + direct = (__debug__ and optimize == 0) + + # "Indirect" byte-compilation: write a temporary script and then + # run it with the appropriate flags. + if not direct: + from tempfile import mkstemp + # XXX script_fd may leak, use something better than mkstemp + script_fd, script_name = mkstemp(".py") + logger.info("writing byte-compilation script '%s'", script_name) + if not dry_run: + if script_fd is not None: + script = os.fdopen(script_fd, "w", encoding='utf-8') + else: + script = open(script_name, "w", encoding='utf-8') + + with script: + script.write("""\ +from packaging.util import byte_compile +files = [ +""") + + # XXX would be nice to write absolute filenames, just for + # safety's sake (script should be more robust in the face of + # chdir'ing before running it). But this requires abspath'ing + # 'prefix' as well, and that breaks the hack in build_lib's + # 'byte_compile()' method that carefully tacks on a trailing + # slash (os.sep really) to make sure the prefix here is "just + # right". This whole prefix business is rather delicate -- the + # problem is that it's really a directory, but I'm treating it + # as a dumb string, so trailing slashes and so forth matter. + + #py_files = map(os.path.abspath, py_files) + #if prefix: + # prefix = os.path.abspath(prefix) + + script.write(",\n".join(map(repr, py_files)) + "]\n") + script.write(""" +byte_compile(files, optimize=%r, force=%r, + prefix=%r, base_dir=%r, + verbose=%r, dry_run=False, + direct=True) +""" % (optimize, force, prefix, base_dir, verbose)) + + cmd = [sys.executable, script_name] + if optimize == 1: + cmd.insert(1, "-O") + elif optimize == 2: + cmd.insert(1, "-OO") + + env = os.environ.copy() + env['PYTHONPATH'] = os.path.pathsep.join(sys.path) + try: + spawn(cmd, env=env) + finally: + execute(os.remove, (script_name,), "removing %s" % script_name, + dry_run=dry_run) + + # "Direct" byte-compilation: use the py_compile module to compile + # right here, right now. Note that the script generated in indirect + # mode simply calls 'byte_compile()' in direct mode, a weird sort of + # cross-process recursion. Hey, it works! + else: + from py_compile import compile + + for file in py_files: + if file[-3:] != ".py": + # This lets us be lazy and not filter filenames in + # the "install_lib" command. + continue + + # Terminology from the py_compile module: + # cfile - byte-compiled file + # dfile - purported source filename (same as 'file' by default) + cfile = file + (__debug__ and "c" or "o") + dfile = file + if prefix: + if file[:len(prefix)] != prefix: + raise ValueError("invalid prefix: filename %r doesn't " + "start with %r" % (file, prefix)) + dfile = dfile[len(prefix):] + if base_dir: + dfile = os.path.join(base_dir, dfile) + + cfile_base = os.path.basename(cfile) + if direct: + if force or newer(file, cfile): + logger.info("byte-compiling %s to %s", file, cfile_base) + if not dry_run: + compile(file, cfile, dfile) + else: + logger.debug("skipping byte-compilation of %s to %s", + file, cfile_base) + + +_RE_VERSION = re.compile('(\d+\.\d+(\.\d+)*)') +_MAC_OS_X_LD_VERSION = re.compile('^@\(#\)PROGRAM:ld ' + 'PROJECT:ld64-((\d+)(\.\d+)*)') + + +def _find_ld_version(): + """Find the ld version. The version scheme differs under Mac OS X.""" + if sys.platform == 'darwin': + return _find_exe_version('ld -v', _MAC_OS_X_LD_VERSION) + else: + return _find_exe_version('ld -v') + + +def _find_exe_version(cmd, pattern=_RE_VERSION): + """Find the version of an executable by running `cmd` in the shell. + + `pattern` is a compiled regular expression. If not provided, defaults + to _RE_VERSION. If the command is not found, or the output does not + match the mattern, returns None. + """ + from subprocess import Popen, PIPE + executable = cmd.split()[0] + if find_executable(executable) is None: + return None + pipe = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + try: + stdout, stderr = pipe.communicate() + finally: + pipe.stdout.close() + pipe.stderr.close() + # some commands like ld under MacOS X, will give the + # output in the stderr, rather than stdout. + if stdout != '': + out_string = stdout + else: + out_string = stderr + + result = pattern.search(out_string) + if result is None: + return None + return result.group(1) + + +def get_compiler_versions(): + """Return a tuple providing the versions of gcc, ld and dllwrap + + For each command, if a command is not found, None is returned. + Otherwise a string with the version is returned. + """ + gcc = _find_exe_version('gcc -dumpversion') + ld = _find_ld_version() + dllwrap = _find_exe_version('dllwrap --version') + return gcc, ld, dllwrap + + +def newer_group(sources, target, missing='error'): + """Return true if 'target' is out-of-date with respect to any file + listed in 'sources'. + + In other words, if 'target' exists and is newer + than every file in 'sources', return false; otherwise return true. + 'missing' controls what we do when a source file is missing; the + default ("error") is to blow up with an OSError from inside 'stat()'; + if it is "ignore", we silently drop any missing source files; if it is + "newer", any missing source files make us assume that 'target' is + out-of-date (this is handy in "dry-run" mode: it'll make you pretend to + carry out commands that wouldn't work because inputs are missing, but + that doesn't matter because you're not actually going to run the + commands). + """ + # If the target doesn't even exist, then it's definitely out-of-date. + if not os.path.exists(target): + return True + + # Otherwise we have to find out the hard way: if *any* source file + # is more recent than 'target', then 'target' is out-of-date and + # we can immediately return true. If we fall through to the end + # of the loop, then 'target' is up-to-date and we return false. + target_mtime = os.stat(target).st_mtime + + for source in sources: + if not os.path.exists(source): + if missing == 'error': # blow up when we stat() the file + pass + elif missing == 'ignore': # missing source dropped from + continue # target's dependency list + elif missing == 'newer': # missing source means target is + return True # out-of-date + + if os.stat(source).st_mtime > target_mtime: + return True + + return False + + +def write_file(filename, contents): + """Create *filename* and write *contents* to it. + + *contents* is a sequence of strings without line terminators. + + This functions is not intended to replace the usual with open + write + idiom in all cases, only with Command.execute, which runs depending on + the dry_run argument and also logs its arguments). + """ + with open(filename, "w") as f: + for line in contents: + f.write(line + "\n") + + +def _is_package(path): + return os.path.isdir(path) and os.path.isfile( + os.path.join(path, '__init__.py')) + + +# Code taken from the pip project +def _is_archive_file(name): + archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar') + ext = splitext(name)[1].lower() + return ext in archives + + +def _under(path, root): + # XXX use os.path + path = path.split(os.sep) + root = root.split(os.sep) + if len(root) > len(path): + return False + for pos, part in enumerate(root): + if path[pos] != part: + return False + return True + + +def _package_name(root_path, path): + # Return a dotted package name, given a subpath + if not _under(path, root_path): + raise ValueError('"%s" is not a subpath of "%s"' % (path, root_path)) + return path[len(root_path) + 1:].replace(os.sep, '.') + + +def find_packages(paths=(os.curdir,), exclude=()): + """Return a list all Python packages found recursively within + directories 'paths' + + 'paths' should be supplied as a sequence of "cross-platform" + (i.e. URL-style) path; it will be converted to the appropriate local + path syntax. + + 'exclude' is a sequence of package names to exclude; '*' can be used as + a wildcard in the names, such that 'foo.*' will exclude all subpackages + of 'foo' (but not 'foo' itself). + """ + packages = [] + discarded = [] + + def _discarded(path): + for discard in discarded: + if _under(path, discard): + return True + return False + + for path in paths: + path = convert_path(path) + for root, dirs, files in os.walk(path): + for dir_ in dirs: + fullpath = os.path.join(root, dir_) + if _discarded(fullpath): + continue + # we work only with Python packages + if not _is_package(fullpath): + discarded.append(fullpath) + continue + # see if it's excluded + excluded = False + package_name = _package_name(path, fullpath) + for pattern in exclude: + if fnmatchcase(package_name, pattern): + excluded = True + break + if excluded: + continue + + # adding it to the list + packages.append(package_name) + return packages + + +def resolve_name(name): + """Resolve a name like ``module.object`` to an object and return it. + + Raise ImportError if the module or name is not found. + """ + parts = name.split('.') + cursor = len(parts) + module_name = parts[:cursor] + + while cursor > 0: + try: + ret = __import__('.'.join(module_name)) + break + except ImportError: + if cursor == 0: + raise + cursor -= 1 + module_name = parts[:cursor] + ret = '' + + for part in parts[1:]: + try: + ret = getattr(ret, part) + except AttributeError as exc: + raise ImportError(exc) + + return ret + + +def splitext(path): + """Like os.path.splitext, but take off .tar too""" + base, ext = posixpath.splitext(path) + if base.lower().endswith('.tar'): + ext = base[-4:] + ext + base = base[:-4] + return base, ext + + +if sys.platform == 'darwin': + _cfg_target = None + _cfg_target_split = None + + +def spawn(cmd, search_path=True, verbose=0, dry_run=False, env=None): + """Run another program specified as a command list 'cmd' in a new process. + + 'cmd' is just the argument list for the new process, ie. + cmd[0] is the program to run and cmd[1:] are the rest of its arguments. + There is no way to run a program with a name different from that of its + executable. + + If 'search_path' is true (the default), the system's executable + search path will be used to find the program; otherwise, cmd[0] + must be the exact path to the executable. If 'dry_run' is true, + the command will not actually be run. + + If 'env' is given, it's a environment dictionary used for the execution + environment. + + Raise PackagingExecError if running the program fails in any way; just + return on success. + """ + logger.debug('spawn: running %r', cmd) + if dry_run: + logger.debug('dry run, no process actually spawned') + return + if sys.platform == 'darwin': + global _cfg_target, _cfg_target_split + if _cfg_target is None: + _cfg_target = sysconfig.get_config_var( + 'MACOSX_DEPLOYMENT_TARGET') or '' + if _cfg_target: + _cfg_target_split = [int(x) for x in _cfg_target.split('.')] + if _cfg_target: + # ensure that the deployment target of build process is not less + # than that used when the interpreter was built. This ensures + # extension modules are built with correct compatibility values + env = env or os.environ + cur_target = env.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) + if _cfg_target_split > [int(x) for x in cur_target.split('.')]: + my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: ' + 'now "%s" but "%s" during configure' + % (cur_target, _cfg_target)) + raise PackagingPlatformError(my_msg) + env = dict(env, MACOSX_DEPLOYMENT_TARGET=cur_target) + + exit_status = subprocess.call(cmd, env=env) + if exit_status != 0: + msg = "command %r failed with exit status %d" + raise PackagingExecError(msg % (cmd, exit_status)) + + +def find_executable(executable, path=None): + """Try to find 'executable' in the directories listed in 'path'. + + *path* is a string listing directories separated by 'os.pathsep' and + defaults to os.environ['PATH']. Returns the complete filename or None + if not found. + """ + if path is None: + path = os.environ['PATH'] + paths = path.split(os.pathsep) + base, ext = os.path.splitext(executable) + + if (sys.platform == 'win32' or os.name == 'os2') and (ext != '.exe'): + executable = executable + '.exe' + + if not os.path.isfile(executable): + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None + else: + return executable + + +DEFAULT_REPOSITORY = 'http://pypi.python.org/pypi' +DEFAULT_REALM = 'pypi' +DEFAULT_PYPIRC = """\ +[distutils] +index-servers = + pypi + +[pypi] +username:%s +password:%s +""" + + +def get_pypirc_path(): + """Return path to pypirc config file.""" + return os.path.join(os.path.expanduser('~'), '.pypirc') + + +def generate_pypirc(username, password): + """Create a default .pypirc file.""" + rc = get_pypirc_path() + with open(rc, 'w') as f: + f.write(DEFAULT_PYPIRC % (username, password)) + try: + os.chmod(rc, 0o600) + except OSError: + # should do something better here + pass + + +def read_pypirc(repository=DEFAULT_REPOSITORY, realm=DEFAULT_REALM): + """Read the .pypirc file.""" + rc = get_pypirc_path() + if os.path.exists(rc): + config = RawConfigParser() + config.read(rc) + sections = config.sections() + if 'distutils' in sections: + # let's get the list of servers + index_servers = config.get('distutils', 'index-servers') + _servers = [server.strip() for server in + index_servers.split('\n') + if server.strip() != ''] + if _servers == []: + # nothing set, let's try to get the default pypi + if 'pypi' in sections: + _servers = ['pypi'] + else: + # the file is not properly defined, returning + # an empty dict + return {} + for server in _servers: + current = {'server': server} + current['username'] = config.get(server, 'username') + + # optional params + for key, default in (('repository', DEFAULT_REPOSITORY), + ('realm', DEFAULT_REALM), + ('password', None)): + if config.has_option(server, key): + current[key] = config.get(server, key) + else: + current[key] = default + if (current['server'] == repository or + current['repository'] == repository): + return current + elif 'server-login' in sections: + # old format + server = 'server-login' + if config.has_option(server, 'repository'): + repository = config.get(server, 'repository') + else: + repository = DEFAULT_REPOSITORY + + return {'username': config.get(server, 'username'), + 'password': config.get(server, 'password'), + 'repository': repository, + 'server': server, + 'realm': DEFAULT_REALM} + + return {} + + +# utility functions for 2to3 support + +def run_2to3(files, doctests_only=False, fixer_names=None, + options=None, explicit=None): + """ Wrapper function around the refactor() class which + performs the conversions on a list of python files. + Invoke 2to3 on a list of Python files. The files should all come + from the build area, as the modification is done in-place.""" + + #if not files: + # return + + # Make this class local, to delay import of 2to3 + from lib2to3.refactor import get_fixers_from_package, RefactoringTool + fixers = [] + fixers = get_fixers_from_package('lib2to3.fixes') + + if fixer_names: + for fixername in fixer_names: + fixers.extend(fixer for fixer in + get_fixers_from_package(fixername)) + r = RefactoringTool(fixers, options=options) + r.refactor(files, write=True, doctests_only=doctests_only) + + +class Mixin2to3: + """ Wrapper class for commands that run 2to3. + To configure 2to3, setup scripts may either change + the class variables, or inherit from this class + to override how 2to3 is invoked. + """ + # provide list of fixers to run. + # defaults to all from lib2to3.fixers + fixer_names = None + + # options dictionary + options = None + + # list of fixers to invoke even though they are marked as explicit + explicit = None + + def run_2to3(self, files, doctests_only=False): + """ Issues a call to util.run_2to3. """ + return run_2to3(files, doctests_only, self.fixer_names, + self.options, self.explicit) + +RICH_GLOB = re.compile(r'\{([^}]*)\}') +_CHECK_RECURSIVE_GLOB = re.compile(r'[^/\\,{]\*\*|\*\*[^/\\,}]') +_CHECK_MISMATCH_SET = re.compile(r'^[^{]*\}|\{[^}]*$') + + +def iglob(path_glob): + """Extended globbing function that supports ** and {opt1,opt2,opt3}.""" + if _CHECK_RECURSIVE_GLOB.search(path_glob): + msg = """invalid glob %r: recursive glob "**" must be used alone""" + raise ValueError(msg % path_glob) + if _CHECK_MISMATCH_SET.search(path_glob): + msg = """invalid glob %r: mismatching set marker '{' or '}'""" + raise ValueError(msg % path_glob) + return _iglob(path_glob) + + +def _iglob(path_glob): + rich_path_glob = RICH_GLOB.split(path_glob, 1) + if len(rich_path_glob) > 1: + assert len(rich_path_glob) == 3, rich_path_glob + prefix, set, suffix = rich_path_glob + for item in set.split(','): + for path in _iglob(''.join((prefix, item, suffix))): + yield path + else: + if '**' not in path_glob: + for item in std_iglob(path_glob): + yield item + else: + prefix, radical = path_glob.split('**', 1) + if prefix == '': + prefix = '.' + if radical == '': + radical = '*' + else: + # we support both + radical = radical.lstrip('/') + radical = radical.lstrip('\\') + for path, dir, files in os.walk(prefix): + path = os.path.normpath(path) + for file in _iglob(os.path.join(path, radical)): + yield file + + +def cfg_to_args(path='setup.cfg'): + """Compatibility helper to use setup.cfg in setup.py. + + This functions uses an existing setup.cfg to generate a dictionnary of + keywords that can be used by distutils.core.setup(**kwargs). It is used + by generate_setup_py. + + *file* is the path to the setup.cfg file. If it doesn't exist, + PackagingFileError is raised. + """ + # We need to declare the following constants here so that it's easier to + # generate the setup.py afterwards, using inspect.getsource. + + # XXX ** == needs testing + D1_D2_SETUP_ARGS = {"name": ("metadata",), + "version": ("metadata",), + "author": ("metadata",), + "author_email": ("metadata",), + "maintainer": ("metadata",), + "maintainer_email": ("metadata",), + "url": ("metadata", "home_page"), + "description": ("metadata", "summary"), + "long_description": ("metadata", "description"), + "download-url": ("metadata",), + "classifiers": ("metadata", "classifier"), + "platforms": ("metadata", "platform"), # ** + "license": ("metadata",), + "requires": ("metadata", "requires_dist"), + "provides": ("metadata", "provides_dist"), # ** + "obsoletes": ("metadata", "obsoletes_dist"), # ** + "package_dir": ("files", 'packages_root'), + "packages": ("files",), + "scripts": ("files",), + "py_modules": ("files", "modules"), # ** + } + + MULTI_FIELDS = ("classifiers", + "platforms", + "requires", + "provides", + "obsoletes", + "packages", + "scripts", + "py_modules") + + def has_get_option(config, section, option): + if config.has_option(section, option): + return config.get(section, option) + elif config.has_option(section, option.replace('_', '-')): + return config.get(section, option.replace('_', '-')) + else: + return False + + # The real code starts here + config = RawConfigParser() + if not os.path.exists(path): + raise PackagingFileError("file '%s' does not exist" % + os.path.abspath(path)) + config.read(path, encoding='utf-8') + + kwargs = {} + for arg in D1_D2_SETUP_ARGS: + if len(D1_D2_SETUP_ARGS[arg]) == 2: + # The distutils field name is different than packaging's + section, option = D1_D2_SETUP_ARGS[arg] + + else: + # The distutils field name is the same thant packaging's + section = D1_D2_SETUP_ARGS[arg][0] + option = arg + + in_cfg_value = has_get_option(config, section, option) + if not in_cfg_value: + # There is no such option in the setup.cfg + if arg == 'long_description': + filenames = has_get_option(config, section, 'description-file') + if filenames: + filenames = split_multiline(filenames) + in_cfg_value = [] + for filename in filenames: + with open(filename) as fp: + in_cfg_value.append(fp.read()) + in_cfg_value = '\n\n'.join(in_cfg_value) + else: + continue + + if arg == 'package_dir' and in_cfg_value: + in_cfg_value = {'': in_cfg_value} + + if arg in MULTI_FIELDS: + # support multiline options + in_cfg_value = split_multiline(in_cfg_value) + + kwargs[arg] = in_cfg_value + + return kwargs + + +_SETUP_TMPL = """\ +# This script was automatically generated by packaging +import os +from distutils.core import setup +from ConfigParser import RawConfigParser + +%(func)s + +setup(**cfg_to_args()) +""" + + +def generate_setup_py(): + """Generate a distutils compatible setup.py using an existing setup.cfg. + + Raises a PackagingFileError when a setup.py already exists. + """ + if os.path.exists("setup.py"): + raise PackagingFileError("a setup.py file already exists") + + with open("setup.py", "w", encoding='utf-8') as fp: + fp.write(_SETUP_TMPL % {'func': getsource(cfg_to_args)}) + + +# Taken from the pip project +# https://github.com/pypa/pip/blob/master/pip/util.py +def ask(message, options): + """Prompt the user with *message*; *options* contains allowed responses.""" + while True: + response = input(message) + response = response.strip().lower() + if response not in options: + print('invalid response:', repr(response)) + print('choose one of', ', '.join(repr(o) for o in options)) + else: + return response + + +def _parse_record_file(record_file): + distinfo, extra_metadata, installed = ({}, [], []) + with open(record_file, 'r') as rfile: + for path in rfile: + path = path.strip() + if path.endswith('egg-info') and os.path.isfile(path): + distinfo_dir = path.replace('egg-info', 'dist-info') + metadata = path + egginfo = path + elif path.endswith('egg-info') and os.path.isdir(path): + distinfo_dir = path.replace('egg-info', 'dist-info') + egginfo = path + for metadata_file in os.listdir(path): + metadata_fpath = os.path.join(path, metadata_file) + if metadata_file == 'PKG-INFO': + metadata = metadata_fpath + else: + extra_metadata.append(metadata_fpath) + elif 'egg-info' in path and os.path.isfile(path): + # skip extra metadata files + continue + else: + installed.append(path) + + distinfo['egginfo'] = egginfo + distinfo['metadata'] = metadata + distinfo['distinfo_dir'] = distinfo_dir + distinfo['installer_path'] = os.path.join(distinfo_dir, 'INSTALLER') + distinfo['metadata_path'] = os.path.join(distinfo_dir, 'METADATA') + distinfo['record_path'] = os.path.join(distinfo_dir, 'RECORD') + distinfo['requested_path'] = os.path.join(distinfo_dir, 'REQUESTED') + installed.extend([distinfo['installer_path'], distinfo['metadata_path']]) + distinfo['installed'] = installed + distinfo['extra_metadata'] = extra_metadata + return distinfo + + +def _write_record_file(record_path, installed_files): + with open(record_path, 'w', encoding='utf-8') as f: + writer = csv.writer(f, delimiter=',', lineterminator=os.linesep, + quotechar='"') + + for fpath in installed_files: + if fpath.endswith('.pyc') or fpath.endswith('.pyo'): + # do not put size and md5 hash, as in PEP-376 + writer.writerow((fpath, '', '')) + else: + hash = hashlib.md5() + with open(fpath, 'rb') as fp: + hash.update(fp.read()) + md5sum = hash.hexdigest() + size = os.path.getsize(fpath) + writer.writerow((fpath, md5sum, size)) + + # add the RECORD file itself + writer.writerow((record_path, '', '')) + return record_path + + +def egginfo_to_distinfo(record_file, installer=_DEFAULT_INSTALLER, + requested=False, remove_egginfo=False): + """Create files and directories required for PEP 376 + + :param record_file: path to RECORD file as produced by setup.py --record + :param installer: installer name + :param requested: True if not installed as a dependency + :param remove_egginfo: delete egginfo dir? + """ + distinfo = _parse_record_file(record_file) + distinfo_dir = distinfo['distinfo_dir'] + if os.path.isdir(distinfo_dir) and not os.path.islink(distinfo_dir): + shutil.rmtree(distinfo_dir) + elif os.path.exists(distinfo_dir): + os.unlink(distinfo_dir) + + os.makedirs(distinfo_dir) + + # copy setuptools extra metadata files + if distinfo['extra_metadata']: + for path in distinfo['extra_metadata']: + shutil.copy2(path, distinfo_dir) + new_path = path.replace('egg-info', 'dist-info') + distinfo['installed'].append(new_path) + + metadata_path = distinfo['metadata_path'] + logger.info('creating %s', metadata_path) + shutil.copy2(distinfo['metadata'], metadata_path) + + installer_path = distinfo['installer_path'] + logger.info('creating %s', installer_path) + with open(installer_path, 'w') as f: + f.write(installer) + + if requested: + requested_path = distinfo['requested_path'] + logger.info('creating %s', requested_path) + open(requested_path, 'wb').close() + distinfo['installed'].append(requested_path) + + record_path = distinfo['record_path'] + logger.info('creating %s', record_path) + _write_record_file(record_path, distinfo['installed']) + + if remove_egginfo: + egginfo = distinfo['egginfo'] + logger.info('removing %s', egginfo) + if os.path.isfile(egginfo): + os.remove(egginfo) + else: + shutil.rmtree(egginfo) + + +def _has_egg_info(srcdir): + if os.path.isdir(srcdir): + for item in os.listdir(srcdir): + full_path = os.path.join(srcdir, item) + if item.endswith('.egg-info') and os.path.isdir(full_path): + logger.debug("Found egg-info directory.") + return True + logger.debug("No egg-info directory found.") + return False + + +def _has_setuptools_text(setup_py): + return _has_text(setup_py, 'setuptools') + + +def _has_distutils_text(setup_py): + return _has_text(setup_py, 'distutils') + + +def _has_text(setup_py, installer): + installer_pattern = re.compile('import {0}|from {0}'.format(installer)) + with open(setup_py, 'r', encoding='utf-8') as setup: + for line in setup: + if re.search(installer_pattern, line): + logger.debug("Found %s text in setup.py.", installer) + return True + logger.debug("No %s text found in setup.py.", installer) + return False + + +def _has_required_metadata(setup_cfg): + config = RawConfigParser() + config.read([setup_cfg], encoding='utf8') + return (config.has_section('metadata') and + 'name' in config.options('metadata') and + 'version' in config.options('metadata')) + + +def _has_pkg_info(srcdir): + pkg_info = os.path.join(srcdir, 'PKG-INFO') + has_pkg_info = os.path.isfile(pkg_info) + if has_pkg_info: + logger.debug("PKG-INFO file found.") + else: + logger.debug("No PKG-INFO file found.") + return has_pkg_info + + +def _has_setup_py(srcdir): + setup_py = os.path.join(srcdir, 'setup.py') + if os.path.isfile(setup_py): + logger.debug('setup.py file found.') + return True + return False + + +def _has_setup_cfg(srcdir): + setup_cfg = os.path.join(srcdir, 'setup.cfg') + if os.path.isfile(setup_cfg): + logger.debug('setup.cfg file found.') + return True + logger.debug("No setup.cfg file found.") + return False + + +def is_setuptools(path): + """Check if the project is based on setuptools. + + :param path: path to source directory containing a setup.py script. + + Return True if the project requires setuptools to install, else False. + """ + srcdir = os.path.abspath(path) + setup_py = os.path.join(srcdir, 'setup.py') + + return _has_setup_py(srcdir) and (_has_egg_info(srcdir) or + _has_setuptools_text(setup_py)) + + +def is_distutils(path): + """Check if the project is based on distutils. + + :param path: path to source directory containing a setup.py script. + + Return True if the project requires distutils to install, else False. + """ + srcdir = os.path.abspath(path) + setup_py = os.path.join(srcdir, 'setup.py') + + return _has_setup_py(srcdir) and (_has_pkg_info(srcdir) or + _has_distutils_text(setup_py)) + + +def is_packaging(path): + """Check if the project is based on packaging + + :param path: path to source directory containing a setup.cfg file. + + Return True if the project has a valid setup.cfg, else False. + """ + srcdir = os.path.abspath(path) + setup_cfg = os.path.join(srcdir, 'setup.cfg') + + return _has_setup_cfg(srcdir) and _has_required_metadata(setup_cfg) + + +def get_install_method(path): + """Check if the project is based on packaging, setuptools, or distutils + + :param path: path to source directory containing a setup.cfg file, + or setup.py. + + Returns a string representing the best install method to use. + """ + if is_packaging(path): + return "packaging" + elif is_setuptools(path): + return "setuptools" + elif is_distutils(path): + return "distutils" + else: + raise InstallationException('Cannot detect install method') + + +# XXX to be replaced by shutil.copytree +def copy_tree(src, dst, preserve_mode=True, preserve_times=True, + preserve_symlinks=False, update=False, verbose=True, + dry_run=False): + from distutils.file_util import copy_file + + if not dry_run and not os.path.isdir(src): + raise PackagingFileError( + "cannot copy tree '%s': not a directory" % src) + try: + names = os.listdir(src) + except os.error as e: + errstr = e[1] + if dry_run: + names = [] + else: + raise PackagingFileError( + "error listing files in '%s': %s" % (src, errstr)) + + if not dry_run: + _mkpath(dst, verbose=verbose) + + outputs = [] + + for n in names: + src_name = os.path.join(src, n) + dst_name = os.path.join(dst, n) + + if preserve_symlinks and os.path.islink(src_name): + link_dest = os.readlink(src_name) + if verbose >= 1: + logger.info("linking %s -> %s", dst_name, link_dest) + if not dry_run: + os.symlink(link_dest, dst_name) + outputs.append(dst_name) + + elif os.path.isdir(src_name): + outputs.extend( + copy_tree(src_name, dst_name, preserve_mode, + preserve_times, preserve_symlinks, update, + verbose=verbose, dry_run=dry_run)) + else: + copy_file(src_name, dst_name, preserve_mode, + preserve_times, update, verbose=verbose, + dry_run=dry_run) + outputs.append(dst_name) + + return outputs + +# cache for by mkpath() -- in addition to cheapening redundant calls, +# eliminates redundant "creating /foo/bar/baz" messages in dry-run mode +_path_created = set() + + +# I don't use os.makedirs because a) it's new to Python 1.5.2, and +# b) it blows up if the directory already exists (I want to silently +# succeed in that case). +def _mkpath(name, mode=0o777, verbose=True, dry_run=False): + # Detect a common bug -- name is None + if not isinstance(name, str): + raise PackagingInternalError( + "mkpath: 'name' must be a string (got %r)" % (name,)) + + # XXX what's the better way to handle verbosity? print as we create + # each directory in the path (the current behaviour), or only announce + # the creation of the whole path? (quite easy to do the latter since + # we're not using a recursive algorithm) + + name = os.path.normpath(name) + created_dirs = [] + if os.path.isdir(name) or name == '': + return created_dirs + if os.path.abspath(name) in _path_created: + return created_dirs + + head, tail = os.path.split(name) + tails = [tail] # stack of lone dirs to create + + while head and tail and not os.path.isdir(head): + head, tail = os.path.split(head) + tails.insert(0, tail) # push next higher dir onto stack + + # now 'head' contains the deepest directory that already exists + # (that is, the child of 'head' in 'name' is the highest directory + # that does *not* exist) + for d in tails: + head = os.path.join(head, d) + abs_head = os.path.abspath(head) + + if abs_head in _path_created: + continue + + if verbose >= 1: + logger.info("creating %s", head) + + if not dry_run: + try: + os.mkdir(head, mode) + except OSError as exc: + if not (exc.errno == errno.EEXIST and os.path.isdir(head)): + raise PackagingFileError( + "could not create '%s': %s" % (head, exc.args[-1])) + created_dirs.append(head) + + _path_created.add(abs_head) + return created_dirs + + +def encode_multipart(fields, files, boundary=None): + """Prepare a multipart HTTP request. + + *fields* is a sequence of (name: str, value: str) elements for regular + form fields, *files* is a sequence of (name: str, filename: str, value: + bytes) elements for data to be uploaded as files. + + Returns (content_type: bytes, body: bytes) ready for http.client.HTTP. + """ + # Taken from + # http://code.activestate.com/recipes/146306-http-client-to-post-using-multipartform-data/ + + if boundary is None: + boundary = b'--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' + elif not isinstance(boundary, bytes): + raise TypeError('boundary must be bytes, not %r' % type(boundary)) + + l = [] + for key, values in fields: + # handle multiple entries for the same name + if not isinstance(values, (tuple, list)): + values = [values] + + for value in values: + l.extend(( + b'--' + boundary, + ('Content-Disposition: form-data; name="%s"' % + key).encode('utf-8'), + b'', + value.encode('utf-8'))) + + for key, filename, value in files: + l.extend(( + b'--' + boundary, + ('Content-Disposition: form-data; name="%s"; filename="%s"' % + (key, filename)).encode('utf-8'), + b'', + value)) + + l.append(b'--' + boundary + b'--') + l.append(b'') + + body = b'\r\n'.join(l) + content_type = b'multipart/form-data; boundary=' + boundary + return content_type, body diff --git a/Lib/packaging/version.py b/Lib/packaging/version.py new file mode 100644 index 0000000000..0eaf80b829 --- /dev/null +++ b/Lib/packaging/version.py @@ -0,0 +1,449 @@ +"""Implementation of the versioning scheme defined in PEP 386.""" + +import re + +from packaging.errors import IrrationalVersionError, HugeMajorVersionNumError + +__all__ = ['NormalizedVersion', 'suggest_normalized_version', + 'VersionPredicate', 'is_valid_version', 'is_valid_versions', + 'is_valid_predicate'] + +# A marker used in the second and third parts of the `parts` tuple, for +# versions that don't have those segments, to sort properly. An example +# of versions in sort order ('highest' last): +# 1.0b1 ((1,0), ('b',1), ('f',)) +# 1.0.dev345 ((1,0), ('f',), ('dev', 345)) +# 1.0 ((1,0), ('f',), ('f',)) +# 1.0.post256.dev345 ((1,0), ('f',), ('f', 'post', 256, 'dev', 345)) +# 1.0.post345 ((1,0), ('f',), ('f', 'post', 345, 'f')) +# ^ ^ ^ +# 'b' < 'f' ---------------------/ | | +# | | +# 'dev' < 'f' < 'post' -------------------/ | +# | +# 'dev' < 'f' ----------------------------------------------/ +# Other letters would do, but 'f' for 'final' is kind of nice. +_FINAL_MARKER = ('f',) + +_VERSION_RE = re.compile(r''' + ^ + (?P<version>\d+\.\d+) # minimum 'N.N' + (?P<extraversion>(?:\.\d+)*) # any number of extra '.N' segments + (?: + (?P<prerel>[abc]|rc) # 'a'=alpha, 'b'=beta, 'c'=release candidate + # 'rc'= alias for release candidate + (?P<prerelversion>\d+(?:\.\d+)*) + )? + (?P<postdev>(\.post(?P<post>\d+))?(\.dev(?P<dev>\d+))?)? + $''', re.VERBOSE) + + +class NormalizedVersion: + """A rational version. + + Good: + 1.2 # equivalent to "1.2.0" + 1.2.0 + 1.2a1 + 1.2.3a2 + 1.2.3b1 + 1.2.3c1 + 1.2.3.4 + TODO: fill this out + + Bad: + 1 # mininum two numbers + 1.2a # release level must have a release serial + 1.2.3b + """ + def __init__(self, s, error_on_huge_major_num=True): + """Create a NormalizedVersion instance from a version string. + + @param s {str} The version string. + @param error_on_huge_major_num {bool} Whether to consider an + apparent use of a year or full date as the major version number + an error. Default True. One of the observed patterns on PyPI before + the introduction of `NormalizedVersion` was version numbers like + this: + 2009.01.03 + 20040603 + 2005.01 + This guard is here to strongly encourage the package author to + use an alternate version, because a release deployed into PyPI + and, e.g. downstream Linux package managers, will forever remove + the possibility of using a version number like "1.0" (i.e. + where the major number is less than that huge major number). + """ + self.is_final = True # by default, consider a version as final. + self._parse(s, error_on_huge_major_num) + + @classmethod + def from_parts(cls, version, prerelease=_FINAL_MARKER, + devpost=_FINAL_MARKER): + return cls(cls.parts_to_str((version, prerelease, devpost))) + + def _parse(self, s, error_on_huge_major_num=True): + """Parses a string version into parts.""" + match = _VERSION_RE.search(s) + if not match: + raise IrrationalVersionError(s) + + groups = match.groupdict() + parts = [] + + # main version + block = self._parse_numdots(groups['version'], s, False, 2) + extraversion = groups.get('extraversion') + if extraversion not in ('', None): + block += self._parse_numdots(extraversion[1:], s) + parts.append(tuple(block)) + + # prerelease + prerel = groups.get('prerel') + if prerel is not None: + block = [prerel] + block += self._parse_numdots(groups.get('prerelversion'), s, + pad_zeros_length=1) + parts.append(tuple(block)) + self.is_final = False + else: + parts.append(_FINAL_MARKER) + + # postdev + if groups.get('postdev'): + post = groups.get('post') + dev = groups.get('dev') + postdev = [] + if post is not None: + postdev.extend((_FINAL_MARKER[0], 'post', int(post))) + if dev is None: + postdev.append(_FINAL_MARKER[0]) + if dev is not None: + postdev.extend(('dev', int(dev))) + self.is_final = False + parts.append(tuple(postdev)) + else: + parts.append(_FINAL_MARKER) + self.parts = tuple(parts) + if error_on_huge_major_num and self.parts[0][0] > 1980: + raise HugeMajorVersionNumError("huge major version number, %r, " + "which might cause future problems: %r" % (self.parts[0][0], s)) + + def _parse_numdots(self, s, full_ver_str, drop_trailing_zeros=True, + pad_zeros_length=0): + """Parse 'N.N.N' sequences, return a list of ints. + + @param s {str} 'N.N.N...' sequence to be parsed + @param full_ver_str {str} The full version string from which this + comes. Used for error strings. + @param drop_trailing_zeros {bool} Whether to drop trailing zeros + from the returned list. Default True. + @param pad_zeros_length {int} The length to which to pad the + returned list with zeros, if necessary. Default 0. + """ + nums = [] + for n in s.split("."): + if len(n) > 1 and n[0] == '0': + raise IrrationalVersionError("cannot have leading zero in " + "version number segment: '%s' in %r" % (n, full_ver_str)) + nums.append(int(n)) + if drop_trailing_zeros: + while nums and nums[-1] == 0: + nums.pop() + while len(nums) < pad_zeros_length: + nums.append(0) + return nums + + def __str__(self): + return self.parts_to_str(self.parts) + + @classmethod + def parts_to_str(cls, parts): + """Transforms a version expressed in tuple into its string + representation.""" + # XXX This doesn't check for invalid tuples + main, prerel, postdev = parts + s = '.'.join(str(v) for v in main) + if prerel is not _FINAL_MARKER: + s += prerel[0] + s += '.'.join(str(v) for v in prerel[1:]) + if postdev and postdev is not _FINAL_MARKER: + if postdev[0] == 'f': + postdev = postdev[1:] + i = 0 + while i < len(postdev): + if i % 2 == 0: + s += '.' + s += str(postdev[i]) + i += 1 + return s + + def __repr__(self): + return "%s('%s')" % (self.__class__.__name__, self) + + def _cannot_compare(self, other): + raise TypeError("cannot compare %s and %s" + % (type(self).__name__, type(other).__name__)) + + def __eq__(self, other): + if not isinstance(other, NormalizedVersion): + self._cannot_compare(other) + return self.parts == other.parts + + def __lt__(self, other): + if not isinstance(other, NormalizedVersion): + self._cannot_compare(other) + return self.parts < other.parts + + def __ne__(self, other): + return not self.__eq__(other) + + def __gt__(self, other): + return not (self.__lt__(other) or self.__eq__(other)) + + def __le__(self, other): + return self.__eq__(other) or self.__lt__(other) + + def __ge__(self, other): + return self.__eq__(other) or self.__gt__(other) + + # See http://docs.python.org/reference/datamodel#object.__hash__ + def __hash__(self): + return hash(self.parts) + + +def suggest_normalized_version(s): + """Suggest a normalized version close to the given version string. + + If you have a version string that isn't rational (i.e. NormalizedVersion + doesn't like it) then you might be able to get an equivalent (or close) + rational version from this function. + + This does a number of simple normalizations to the given string, based + on observation of versions currently in use on PyPI. Given a dump of + those version during PyCon 2009, 4287 of them: + - 2312 (53.93%) match NormalizedVersion without change + with the automatic suggestion + - 3474 (81.04%) match when using this suggestion method + + @param s {str} An irrational version string. + @returns A rational version string, or None, if couldn't determine one. + """ + try: + NormalizedVersion(s) + return s # already rational + except IrrationalVersionError: + pass + + rs = s.lower() + + # part of this could use maketrans + for orig, repl in (('-alpha', 'a'), ('-beta', 'b'), ('alpha', 'a'), + ('beta', 'b'), ('rc', 'c'), ('-final', ''), + ('-pre', 'c'), + ('-release', ''), ('.release', ''), ('-stable', ''), + ('+', '.'), ('_', '.'), (' ', ''), ('.final', ''), + ('final', '')): + rs = rs.replace(orig, repl) + + # if something ends with dev or pre, we add a 0 + rs = re.sub(r"pre$", r"pre0", rs) + rs = re.sub(r"dev$", r"dev0", rs) + + # if we have something like "b-2" or "a.2" at the end of the + # version, that is pobably beta, alpha, etc + # let's remove the dash or dot + rs = re.sub(r"([abc|rc])[\-\.](\d+)$", r"\1\2", rs) + + # 1.0-dev-r371 -> 1.0.dev371 + # 0.1-dev-r79 -> 0.1.dev79 + rs = re.sub(r"[\-\.](dev)[\-\.]?r?(\d+)$", r".\1\2", rs) + + # Clean: 2.0.a.3, 2.0.b1, 0.9.0~c1 + rs = re.sub(r"[.~]?([abc])\.?", r"\1", rs) + + # Clean: v0.3, v1.0 + if rs.startswith('v'): + rs = rs[1:] + + # Clean leading '0's on numbers. + #TODO: unintended side-effect on, e.g., "2003.05.09" + # PyPI stats: 77 (~2%) better + rs = re.sub(r"\b0+(\d+)(?!\d)", r"\1", rs) + + # Clean a/b/c with no version. E.g. "1.0a" -> "1.0a0". Setuptools infers + # zero. + # PyPI stats: 245 (7.56%) better + rs = re.sub(r"(\d+[abc])$", r"\g<1>0", rs) + + # the 'dev-rNNN' tag is a dev tag + rs = re.sub(r"\.?(dev-r|dev\.r)\.?(\d+)$", r".dev\2", rs) + + # clean the - when used as a pre delimiter + rs = re.sub(r"-(a|b|c)(\d+)$", r"\1\2", rs) + + # a terminal "dev" or "devel" can be changed into ".dev0" + rs = re.sub(r"[\.\-](dev|devel)$", r".dev0", rs) + + # a terminal "dev" can be changed into ".dev0" + rs = re.sub(r"(?![\.\-])dev$", r".dev0", rs) + + # a terminal "final" or "stable" can be removed + rs = re.sub(r"(final|stable)$", "", rs) + + # The 'r' and the '-' tags are post release tags + # 0.4a1.r10 -> 0.4a1.post10 + # 0.9.33-17222 -> 0.9.3.post17222 + # 0.9.33-r17222 -> 0.9.3.post17222 + rs = re.sub(r"\.?(r|-|-r)\.?(\d+)$", r".post\2", rs) + + # Clean 'r' instead of 'dev' usage: + # 0.9.33+r17222 -> 0.9.3.dev17222 + # 1.0dev123 -> 1.0.dev123 + # 1.0.git123 -> 1.0.dev123 + # 1.0.bzr123 -> 1.0.dev123 + # 0.1a0dev.123 -> 0.1a0.dev123 + # PyPI stats: ~150 (~4%) better + rs = re.sub(r"\.?(dev|git|bzr)\.?(\d+)$", r".dev\2", rs) + + # Clean '.pre' (normalized from '-pre' above) instead of 'c' usage: + # 0.2.pre1 -> 0.2c1 + # 0.2-c1 -> 0.2c1 + # 1.0preview123 -> 1.0c123 + # PyPI stats: ~21 (0.62%) better + rs = re.sub(r"\.?(pre|preview|-c)(\d+)$", r"c\g<2>", rs) + + # Tcl/Tk uses "px" for their post release markers + rs = re.sub(r"p(\d+)$", r".post\1", rs) + + try: + NormalizedVersion(rs) + return rs # already rational + except IrrationalVersionError: + pass + return None + + +# A predicate is: "ProjectName (VERSION1, VERSION2, ..) +_PREDICATE = re.compile(r"(?i)^\s*(\w[\s\w-]*(?:\.\w*)*)(.*)") +_VERSIONS = re.compile(r"^\s*\((?P<versions>.*)\)\s*$|^\s*" + "(?P<versions2>.*)\s*$") +_PLAIN_VERSIONS = re.compile(r"^\s*(.*)\s*$") +_SPLIT_CMP = re.compile(r"^\s*(<=|>=|<|>|!=|==)\s*([^\s,]+)\s*$") + + +def _split_predicate(predicate): + match = _SPLIT_CMP.match(predicate) + if match is None: + # probably no op, we'll use "==" + comp, version = '==', predicate + else: + comp, version = match.groups() + return comp, NormalizedVersion(version) + + +class VersionPredicate: + """Defines a predicate: ProjectName (>ver1,ver2, ..)""" + + _operators = {"<": lambda x, y: x < y, + ">": lambda x, y: x > y, + "<=": lambda x, y: str(x).startswith(str(y)) or x < y, + ">=": lambda x, y: str(x).startswith(str(y)) or x > y, + "==": lambda x, y: str(x).startswith(str(y)), + "!=": lambda x, y: not str(x).startswith(str(y)), + } + + def __init__(self, predicate): + self._string = predicate + predicate = predicate.strip() + match = _PREDICATE.match(predicate) + if match is None: + raise ValueError('Bad predicate "%s"' % predicate) + + name, predicates = match.groups() + self.name = name.strip() + self.predicates = [] + if predicates is None: + return + + predicates = _VERSIONS.match(predicates.strip()) + if predicates is None: + return + + predicates = predicates.groupdict() + if predicates['versions'] is not None: + versions = predicates['versions'] + else: + versions = predicates.get('versions2') + + if versions is not None: + for version in versions.split(','): + if version.strip() == '': + continue + self.predicates.append(_split_predicate(version)) + + def match(self, version): + """Check if the provided version matches the predicates.""" + if isinstance(version, str): + version = NormalizedVersion(version) + for operator, predicate in self.predicates: + if not self._operators[operator](version, predicate): + return False + return True + + def __repr__(self): + return self._string + + +class _Versions(VersionPredicate): + def __init__(self, predicate): + predicate = predicate.strip() + match = _PLAIN_VERSIONS.match(predicate) + self.name = None + predicates = match.groups()[0] + self.predicates = [_split_predicate(pred.strip()) + for pred in predicates.split(',')] + + +class _Version(VersionPredicate): + def __init__(self, predicate): + predicate = predicate.strip() + match = _PLAIN_VERSIONS.match(predicate) + self.name = None + self.predicates = _split_predicate(match.groups()[0]) + + +def is_valid_predicate(predicate): + try: + VersionPredicate(predicate) + except (ValueError, IrrationalVersionError): + return False + else: + return True + + +def is_valid_versions(predicate): + try: + _Versions(predicate) + except (ValueError, IrrationalVersionError): + return False + else: + return True + + +def is_valid_version(predicate): + try: + _Version(predicate) + except (ValueError, IrrationalVersionError): + return False + else: + return True + + +def get_version_predicate(requirements): + """Return a VersionPredicate object, from a string or an already + existing object. + """ + if isinstance(requirements, str): + requirements = VersionPredicate(requirements) + return requirements diff --git a/Lib/pickle.py b/Lib/pickle.py index 32ae02bc7d..0aee77af77 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -23,8 +23,6 @@ Misc variables: """ -__version__ = "$Revision$" # Code version - from types import FunctionType, BuiltinFunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache diff --git a/Lib/pipes.py b/Lib/pipes.py index 4297053816..f1a16f63de 100644 --- a/Lib/pipes.py +++ b/Lib/pipes.py @@ -60,7 +60,9 @@ To create a new template object initialized to a given one: import re import os import tempfile -import string +# we import the quote function rather than the module for backward compat +# (quote used to be an undocumented but used function in pipes) +from shlex import quote __all__ = ["Template"] @@ -243,22 +245,3 @@ def makepipeline(infile, steps, outfile): cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd # return cmdlist - - -# Reliably quote a string as a single argument for /bin/sh - -# Safe unquoted -_safechars = frozenset(string.ascii_letters + string.digits + '@%_-+=:,./') - -def quote(file): - """Return a shell-escaped version of the file string.""" - for c in file: - if c not in _safechars: - break - else: - if not file: - return "''" - return file - # use single quotes, and put single quotes into double quotes - # the string $'b is then quoted as '$'"'"'b' - return "'" + file.replace("'", "'\"'\"'") + "'" diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 2d885683e0..2dd20cd8e6 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -248,7 +248,7 @@ class ImpLoader: if self.file and self.file.closed: mod_type = self.etc[2] if mod_type==imp.PY_SOURCE: - self.file = open(self.filename, 'rU') + self.file = open(self.filename, 'r') elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): self.file = open(self.filename, 'rb') @@ -293,7 +293,7 @@ class ImpLoader: self.file.close() elif mod_type==imp.PY_COMPILED: if os.path.exists(self.filename[:-1]): - f = open(self.filename[:-1], 'rU') + f = open(self.filename[:-1], 'r') self.source = f.read() f.close() elif mod_type==imp.PKG_DIRECTORY: diff --git a/Lib/plat-linux2/CDROM.py b/Lib/plat-linux/CDROM.py index 434093684c..434093684c 100644 --- a/Lib/plat-linux2/CDROM.py +++ b/Lib/plat-linux/CDROM.py diff --git a/Lib/plat-linux2/DLFCN.py b/Lib/plat-linux/DLFCN.py index dd10ac4ead..dd10ac4ead 100644 --- a/Lib/plat-linux2/DLFCN.py +++ b/Lib/plat-linux/DLFCN.py diff --git a/Lib/plat-linux2/IN.py b/Lib/plat-linux/IN.py index d7d30024c2..d7d30024c2 100644 --- a/Lib/plat-linux2/IN.py +++ b/Lib/plat-linux/IN.py diff --git a/Lib/plat-linux2/TYPES.py b/Lib/plat-linux/TYPES.py index e7a324b25a..e7a324b25a 100644 --- a/Lib/plat-linux2/TYPES.py +++ b/Lib/plat-linux/TYPES.py diff --git a/Lib/plat-linux2/regen b/Lib/plat-linux/regen index c76950e232..c76950e232 100755 --- a/Lib/plat-linux2/regen +++ b/Lib/plat-linux/regen diff --git a/Lib/plat-linux3/CDROM.py b/Lib/plat-linux3/CDROM.py deleted file mode 100644 index 434093684c..0000000000 --- a/Lib/plat-linux3/CDROM.py +++ /dev/null @@ -1,207 +0,0 @@ -# Generated by h2py from /usr/include/linux/cdrom.h - -CDROMPAUSE = 0x5301 -CDROMRESUME = 0x5302 -CDROMPLAYMSF = 0x5303 -CDROMPLAYTRKIND = 0x5304 -CDROMREADTOCHDR = 0x5305 -CDROMREADTOCENTRY = 0x5306 -CDROMSTOP = 0x5307 -CDROMSTART = 0x5308 -CDROMEJECT = 0x5309 -CDROMVOLCTRL = 0x530a -CDROMSUBCHNL = 0x530b -CDROMREADMODE2 = 0x530c -CDROMREADMODE1 = 0x530d -CDROMREADAUDIO = 0x530e -CDROMEJECT_SW = 0x530f -CDROMMULTISESSION = 0x5310 -CDROM_GET_MCN = 0x5311 -CDROM_GET_UPC = CDROM_GET_MCN -CDROMRESET = 0x5312 -CDROMVOLREAD = 0x5313 -CDROMREADRAW = 0x5314 -CDROMREADCOOKED = 0x5315 -CDROMSEEK = 0x5316 -CDROMPLAYBLK = 0x5317 -CDROMREADALL = 0x5318 -CDROMGETSPINDOWN = 0x531d -CDROMSETSPINDOWN = 0x531e -CDROMCLOSETRAY = 0x5319 -CDROM_SET_OPTIONS = 0x5320 -CDROM_CLEAR_OPTIONS = 0x5321 -CDROM_SELECT_SPEED = 0x5322 -CDROM_SELECT_DISC = 0x5323 -CDROM_MEDIA_CHANGED = 0x5325 -CDROM_DRIVE_STATUS = 0x5326 -CDROM_DISC_STATUS = 0x5327 -CDROM_CHANGER_NSLOTS = 0x5328 -CDROM_LOCKDOOR = 0x5329 -CDROM_DEBUG = 0x5330 -CDROM_GET_CAPABILITY = 0x5331 -CDROMAUDIOBUFSIZ = 0x5382 -DVD_READ_STRUCT = 0x5390 -DVD_WRITE_STRUCT = 0x5391 -DVD_AUTH = 0x5392 -CDROM_SEND_PACKET = 0x5393 -CDROM_NEXT_WRITABLE = 0x5394 -CDROM_LAST_WRITTEN = 0x5395 -CDROM_PACKET_SIZE = 12 -CGC_DATA_UNKNOWN = 0 -CGC_DATA_WRITE = 1 -CGC_DATA_READ = 2 -CGC_DATA_NONE = 3 -CD_MINS = 74 -CD_SECS = 60 -CD_FRAMES = 75 -CD_SYNC_SIZE = 12 -CD_MSF_OFFSET = 150 -CD_CHUNK_SIZE = 24 -CD_NUM_OF_CHUNKS = 98 -CD_FRAMESIZE_SUB = 96 -CD_HEAD_SIZE = 4 -CD_SUBHEAD_SIZE = 8 -CD_EDC_SIZE = 4 -CD_ZERO_SIZE = 8 -CD_ECC_SIZE = 276 -CD_FRAMESIZE = 2048 -CD_FRAMESIZE_RAW = 2352 -CD_FRAMESIZE_RAWER = 2646 -CD_FRAMESIZE_RAW1 = (CD_FRAMESIZE_RAW-CD_SYNC_SIZE) -CD_FRAMESIZE_RAW0 = (CD_FRAMESIZE_RAW-CD_SYNC_SIZE-CD_HEAD_SIZE) -CD_XA_HEAD = (CD_HEAD_SIZE+CD_SUBHEAD_SIZE) -CD_XA_TAIL = (CD_EDC_SIZE+CD_ECC_SIZE) -CD_XA_SYNC_HEAD = (CD_SYNC_SIZE+CD_XA_HEAD) -CDROM_LBA = 0x01 -CDROM_MSF = 0x02 -CDROM_DATA_TRACK = 0x04 -CDROM_LEADOUT = 0xAA -CDROM_AUDIO_INVALID = 0x00 -CDROM_AUDIO_PLAY = 0x11 -CDROM_AUDIO_PAUSED = 0x12 -CDROM_AUDIO_COMPLETED = 0x13 -CDROM_AUDIO_ERROR = 0x14 -CDROM_AUDIO_NO_STATUS = 0x15 -CDC_CLOSE_TRAY = 0x1 -CDC_OPEN_TRAY = 0x2 -CDC_LOCK = 0x4 -CDC_SELECT_SPEED = 0x8 -CDC_SELECT_DISC = 0x10 -CDC_MULTI_SESSION = 0x20 -CDC_MCN = 0x40 -CDC_MEDIA_CHANGED = 0x80 -CDC_PLAY_AUDIO = 0x100 -CDC_RESET = 0x200 -CDC_IOCTLS = 0x400 -CDC_DRIVE_STATUS = 0x800 -CDC_GENERIC_PACKET = 0x1000 -CDC_CD_R = 0x2000 -CDC_CD_RW = 0x4000 -CDC_DVD = 0x8000 -CDC_DVD_R = 0x10000 -CDC_DVD_RAM = 0x20000 -CDS_NO_INFO = 0 -CDS_NO_DISC = 1 -CDS_TRAY_OPEN = 2 -CDS_DRIVE_NOT_READY = 3 -CDS_DISC_OK = 4 -CDS_AUDIO = 100 -CDS_DATA_1 = 101 -CDS_DATA_2 = 102 -CDS_XA_2_1 = 103 -CDS_XA_2_2 = 104 -CDS_MIXED = 105 -CDO_AUTO_CLOSE = 0x1 -CDO_AUTO_EJECT = 0x2 -CDO_USE_FFLAGS = 0x4 -CDO_LOCK = 0x8 -CDO_CHECK_TYPE = 0x10 -CD_PART_MAX = 64 -CD_PART_MASK = (CD_PART_MAX - 1) -GPCMD_BLANK = 0xa1 -GPCMD_CLOSE_TRACK = 0x5b -GPCMD_FLUSH_CACHE = 0x35 -GPCMD_FORMAT_UNIT = 0x04 -GPCMD_GET_CONFIGURATION = 0x46 -GPCMD_GET_EVENT_STATUS_NOTIFICATION = 0x4a -GPCMD_GET_PERFORMANCE = 0xac -GPCMD_INQUIRY = 0x12 -GPCMD_LOAD_UNLOAD = 0xa6 -GPCMD_MECHANISM_STATUS = 0xbd -GPCMD_MODE_SELECT_10 = 0x55 -GPCMD_MODE_SENSE_10 = 0x5a -GPCMD_PAUSE_RESUME = 0x4b -GPCMD_PLAY_AUDIO_10 = 0x45 -GPCMD_PLAY_AUDIO_MSF = 0x47 -GPCMD_PLAY_AUDIO_TI = 0x48 -GPCMD_PLAY_CD = 0xbc -GPCMD_PREVENT_ALLOW_MEDIUM_REMOVAL = 0x1e -GPCMD_READ_10 = 0x28 -GPCMD_READ_12 = 0xa8 -GPCMD_READ_CDVD_CAPACITY = 0x25 -GPCMD_READ_CD = 0xbe -GPCMD_READ_CD_MSF = 0xb9 -GPCMD_READ_DISC_INFO = 0x51 -GPCMD_READ_DVD_STRUCTURE = 0xad -GPCMD_READ_FORMAT_CAPACITIES = 0x23 -GPCMD_READ_HEADER = 0x44 -GPCMD_READ_TRACK_RZONE_INFO = 0x52 -GPCMD_READ_SUBCHANNEL = 0x42 -GPCMD_READ_TOC_PMA_ATIP = 0x43 -GPCMD_REPAIR_RZONE_TRACK = 0x58 -GPCMD_REPORT_KEY = 0xa4 -GPCMD_REQUEST_SENSE = 0x03 -GPCMD_RESERVE_RZONE_TRACK = 0x53 -GPCMD_SCAN = 0xba -GPCMD_SEEK = 0x2b -GPCMD_SEND_DVD_STRUCTURE = 0xad -GPCMD_SEND_EVENT = 0xa2 -GPCMD_SEND_KEY = 0xa3 -GPCMD_SEND_OPC = 0x54 -GPCMD_SET_READ_AHEAD = 0xa7 -GPCMD_SET_STREAMING = 0xb6 -GPCMD_START_STOP_UNIT = 0x1b -GPCMD_STOP_PLAY_SCAN = 0x4e -GPCMD_TEST_UNIT_READY = 0x00 -GPCMD_VERIFY_10 = 0x2f -GPCMD_WRITE_10 = 0x2a -GPCMD_WRITE_AND_VERIFY_10 = 0x2e -GPCMD_SET_SPEED = 0xbb -GPCMD_PLAYAUDIO_TI = 0x48 -GPCMD_GET_MEDIA_STATUS = 0xda -GPMODE_R_W_ERROR_PAGE = 0x01 -GPMODE_WRITE_PARMS_PAGE = 0x05 -GPMODE_AUDIO_CTL_PAGE = 0x0e -GPMODE_POWER_PAGE = 0x1a -GPMODE_FAULT_FAIL_PAGE = 0x1c -GPMODE_TO_PROTECT_PAGE = 0x1d -GPMODE_CAPABILITIES_PAGE = 0x2a -GPMODE_ALL_PAGES = 0x3f -GPMODE_CDROM_PAGE = 0x0d -DVD_STRUCT_PHYSICAL = 0x00 -DVD_STRUCT_COPYRIGHT = 0x01 -DVD_STRUCT_DISCKEY = 0x02 -DVD_STRUCT_BCA = 0x03 -DVD_STRUCT_MANUFACT = 0x04 -DVD_LAYERS = 4 -DVD_LU_SEND_AGID = 0 -DVD_HOST_SEND_CHALLENGE = 1 -DVD_LU_SEND_KEY1 = 2 -DVD_LU_SEND_CHALLENGE = 3 -DVD_HOST_SEND_KEY2 = 4 -DVD_AUTH_ESTABLISHED = 5 -DVD_AUTH_FAILURE = 6 -DVD_LU_SEND_TITLE_KEY = 7 -DVD_LU_SEND_ASF = 8 -DVD_INVALIDATE_AGID = 9 -DVD_LU_SEND_RPC_STATE = 10 -DVD_HOST_SEND_RPC_STATE = 11 -DVD_CPM_NO_COPYRIGHT = 0 -DVD_CPM_COPYRIGHTED = 1 -DVD_CP_SEC_NONE = 0 -DVD_CP_SEC_EXIST = 1 -DVD_CGMS_UNRESTRICTED = 0 -DVD_CGMS_SINGLE = 2 -DVD_CGMS_RESTRICTED = 3 - -CDROM_MAX_SLOTS = 256 diff --git a/Lib/plat-linux3/DLFCN.py b/Lib/plat-linux3/DLFCN.py deleted file mode 100644 index dd10ac4ead..0000000000 --- a/Lib/plat-linux3/DLFCN.py +++ /dev/null @@ -1,83 +0,0 @@ -# Generated by h2py from /usr/include/dlfcn.h -_DLFCN_H = 1 - -# Included from features.h -_FEATURES_H = 1 -__USE_ANSI = 1 -__FAVOR_BSD = 1 -_ISOC99_SOURCE = 1 -_POSIX_SOURCE = 1 -_POSIX_C_SOURCE = 199506 -_XOPEN_SOURCE = 600 -_XOPEN_SOURCE_EXTENDED = 1 -_LARGEFILE64_SOURCE = 1 -_BSD_SOURCE = 1 -_SVID_SOURCE = 1 -_BSD_SOURCE = 1 -_SVID_SOURCE = 1 -__USE_ISOC99 = 1 -_POSIX_SOURCE = 1 -_POSIX_C_SOURCE = 2 -_POSIX_C_SOURCE = 199506 -__USE_POSIX = 1 -__USE_POSIX2 = 1 -__USE_POSIX199309 = 1 -__USE_POSIX199506 = 1 -__USE_XOPEN = 1 -__USE_XOPEN_EXTENDED = 1 -__USE_UNIX98 = 1 -_LARGEFILE_SOURCE = 1 -__USE_XOPEN2K = 1 -__USE_ISOC99 = 1 -__USE_XOPEN_EXTENDED = 1 -__USE_LARGEFILE = 1 -__USE_LARGEFILE64 = 1 -__USE_FILE_OFFSET64 = 1 -__USE_MISC = 1 -__USE_BSD = 1 -__USE_SVID = 1 -__USE_GNU = 1 -__USE_REENTRANT = 1 -__STDC_IEC_559__ = 1 -__STDC_IEC_559_COMPLEX__ = 1 -__STDC_ISO_10646__ = 200009 -__GNU_LIBRARY__ = 6 -__GLIBC__ = 2 -__GLIBC_MINOR__ = 2 - -# Included from sys/cdefs.h -_SYS_CDEFS_H = 1 -def __PMT(args): return args - -def __P(args): return args - -def __PMT(args): return args - -def __STRING(x): return #x - -__flexarr = [] -__flexarr = [0] -__flexarr = [] -__flexarr = [1] -def __ASMNAME(cname): return __ASMNAME2 (__USER_LABEL_PREFIX__, cname) - -def __attribute__(xyz): return - -def __attribute_format_arg__(x): return __attribute__ ((__format_arg__ (x))) - -def __attribute_format_arg__(x): return - -__USE_LARGEFILE = 1 -__USE_LARGEFILE64 = 1 -__USE_EXTERN_INLINES = 1 - -# Included from gnu/stubs.h - -# Included from bits/dlfcn.h -RTLD_LAZY = 0x00001 -RTLD_NOW = 0x00002 -RTLD_BINDING_MASK = 0x3 -RTLD_NOLOAD = 0x00004 -RTLD_GLOBAL = 0x00100 -RTLD_LOCAL = 0 -RTLD_NODELETE = 0x01000 diff --git a/Lib/plat-linux3/IN.py b/Lib/plat-linux3/IN.py deleted file mode 100644 index d7d30024c2..0000000000 --- a/Lib/plat-linux3/IN.py +++ /dev/null @@ -1,615 +0,0 @@ -# Generated by h2py from /usr/include/netinet/in.h -_NETINET_IN_H = 1 - -# Included from features.h -_FEATURES_H = 1 -__USE_ANSI = 1 -__FAVOR_BSD = 1 -_ISOC99_SOURCE = 1 -_POSIX_SOURCE = 1 -_POSIX_C_SOURCE = 199506 -_XOPEN_SOURCE = 600 -_XOPEN_SOURCE_EXTENDED = 1 -_LARGEFILE64_SOURCE = 1 -_BSD_SOURCE = 1 -_SVID_SOURCE = 1 -_BSD_SOURCE = 1 -_SVID_SOURCE = 1 -__USE_ISOC99 = 1 -_POSIX_SOURCE = 1 -_POSIX_C_SOURCE = 2 -_POSIX_C_SOURCE = 199506 -__USE_POSIX = 1 -__USE_POSIX2 = 1 -__USE_POSIX199309 = 1 -__USE_POSIX199506 = 1 -__USE_XOPEN = 1 -__USE_XOPEN_EXTENDED = 1 -__USE_UNIX98 = 1 -_LARGEFILE_SOURCE = 1 -__USE_XOPEN2K = 1 -__USE_ISOC99 = 1 -__USE_XOPEN_EXTENDED = 1 -__USE_LARGEFILE = 1 -__USE_LARGEFILE64 = 1 -__USE_FILE_OFFSET64 = 1 -__USE_MISC = 1 -__USE_BSD = 1 -__USE_SVID = 1 -__USE_GNU = 1 -__USE_REENTRANT = 1 -__STDC_IEC_559__ = 1 -__STDC_IEC_559_COMPLEX__ = 1 -__STDC_ISO_10646__ = 200009 -__GNU_LIBRARY__ = 6 -__GLIBC__ = 2 -__GLIBC_MINOR__ = 2 - -# Included from sys/cdefs.h -_SYS_CDEFS_H = 1 -def __PMT(args): return args - -def __P(args): return args - -def __PMT(args): return args - -def __STRING(x): return #x - -__flexarr = [] -__flexarr = [0] -__flexarr = [] -__flexarr = [1] -def __ASMNAME(cname): return __ASMNAME2 (__USER_LABEL_PREFIX__, cname) - -def __attribute__(xyz): return - -def __attribute_format_arg__(x): return __attribute__ ((__format_arg__ (x))) - -def __attribute_format_arg__(x): return - -__USE_LARGEFILE = 1 -__USE_LARGEFILE64 = 1 -__USE_EXTERN_INLINES = 1 - -# Included from gnu/stubs.h - -# Included from stdint.h -_STDINT_H = 1 - -# Included from bits/wchar.h -_BITS_WCHAR_H = 1 -__WCHAR_MIN = (-2147483647 - 1) -__WCHAR_MAX = (2147483647) - -# Included from bits/wordsize.h -__WORDSIZE = 32 -def __INT64_C(c): return c ## L - -def __UINT64_C(c): return c ## UL - -def __INT64_C(c): return c ## LL - -def __UINT64_C(c): return c ## ULL - -INT8_MIN = (-128) -INT16_MIN = (-32767-1) -INT32_MIN = (-2147483647-1) -INT64_MIN = (-__INT64_C(9223372036854775807)-1) -INT8_MAX = (127) -INT16_MAX = (32767) -INT32_MAX = (2147483647) -INT64_MAX = (__INT64_C(9223372036854775807)) -UINT8_MAX = (255) -UINT16_MAX = (65535) -UINT64_MAX = (__UINT64_C(18446744073709551615)) -INT_LEAST8_MIN = (-128) -INT_LEAST16_MIN = (-32767-1) -INT_LEAST32_MIN = (-2147483647-1) -INT_LEAST64_MIN = (-__INT64_C(9223372036854775807)-1) -INT_LEAST8_MAX = (127) -INT_LEAST16_MAX = (32767) -INT_LEAST32_MAX = (2147483647) -INT_LEAST64_MAX = (__INT64_C(9223372036854775807)) -UINT_LEAST8_MAX = (255) -UINT_LEAST16_MAX = (65535) -UINT_LEAST64_MAX = (__UINT64_C(18446744073709551615)) -INT_FAST8_MIN = (-128) -INT_FAST16_MIN = (-9223372036854775807-1) -INT_FAST32_MIN = (-9223372036854775807-1) -INT_FAST16_MIN = (-2147483647-1) -INT_FAST32_MIN = (-2147483647-1) -INT_FAST64_MIN = (-__INT64_C(9223372036854775807)-1) -INT_FAST8_MAX = (127) -INT_FAST16_MAX = (9223372036854775807) -INT_FAST32_MAX = (9223372036854775807) -INT_FAST16_MAX = (2147483647) -INT_FAST32_MAX = (2147483647) -INT_FAST64_MAX = (__INT64_C(9223372036854775807)) -UINT_FAST8_MAX = (255) -UINT_FAST64_MAX = (__UINT64_C(18446744073709551615)) -INTPTR_MIN = (-9223372036854775807-1) -INTPTR_MAX = (9223372036854775807) -INTPTR_MIN = (-2147483647-1) -INTPTR_MAX = (2147483647) -INTMAX_MIN = (-__INT64_C(9223372036854775807)-1) -INTMAX_MAX = (__INT64_C(9223372036854775807)) -UINTMAX_MAX = (__UINT64_C(18446744073709551615)) -PTRDIFF_MIN = (-9223372036854775807-1) -PTRDIFF_MAX = (9223372036854775807) -PTRDIFF_MIN = (-2147483647-1) -PTRDIFF_MAX = (2147483647) -SIG_ATOMIC_MIN = (-2147483647-1) -SIG_ATOMIC_MAX = (2147483647) -WCHAR_MIN = __WCHAR_MIN -WCHAR_MAX = __WCHAR_MAX -def INT8_C(c): return c - -def INT16_C(c): return c - -def INT32_C(c): return c - -def INT64_C(c): return c ## L - -def INT64_C(c): return c ## LL - -def UINT8_C(c): return c ## U - -def UINT16_C(c): return c ## U - -def UINT32_C(c): return c ## U - -def UINT64_C(c): return c ## UL - -def UINT64_C(c): return c ## ULL - -def INTMAX_C(c): return c ## L - -def UINTMAX_C(c): return c ## UL - -def INTMAX_C(c): return c ## LL - -def UINTMAX_C(c): return c ## ULL - - -# Included from bits/types.h -_BITS_TYPES_H = 1 -__FD_SETSIZE = 1024 - -# Included from bits/pthreadtypes.h -_BITS_PTHREADTYPES_H = 1 - -# Included from bits/sched.h -SCHED_OTHER = 0 -SCHED_FIFO = 1 -SCHED_RR = 2 -CSIGNAL = 0x000000ff -CLONE_VM = 0x00000100 -CLONE_FS = 0x00000200 -CLONE_FILES = 0x00000400 -CLONE_SIGHAND = 0x00000800 -CLONE_PID = 0x00001000 -CLONE_PTRACE = 0x00002000 -CLONE_VFORK = 0x00004000 -__defined_schedparam = 1 -def IN_CLASSA(a): return ((((in_addr_t)(a)) & (-2147483648)) == 0) - -IN_CLASSA_NET = (-16777216) -IN_CLASSA_NSHIFT = 24 -IN_CLASSA_HOST = ((-1) & ~IN_CLASSA_NET) -IN_CLASSA_MAX = 128 -def IN_CLASSB(a): return ((((in_addr_t)(a)) & (-1073741824)) == (-2147483648)) - -IN_CLASSB_NET = (-65536) -IN_CLASSB_NSHIFT = 16 -IN_CLASSB_HOST = ((-1) & ~IN_CLASSB_NET) -IN_CLASSB_MAX = 65536 -def IN_CLASSC(a): return ((((in_addr_t)(a)) & (-536870912)) == (-1073741824)) - -IN_CLASSC_NET = (-256) -IN_CLASSC_NSHIFT = 8 -IN_CLASSC_HOST = ((-1) & ~IN_CLASSC_NET) -def IN_CLASSD(a): return ((((in_addr_t)(a)) & (-268435456)) == (-536870912)) - -def IN_MULTICAST(a): return IN_CLASSD(a) - -def IN_EXPERIMENTAL(a): return ((((in_addr_t)(a)) & (-536870912)) == (-536870912)) - -def IN_BADCLASS(a): return ((((in_addr_t)(a)) & (-268435456)) == (-268435456)) - -IN_LOOPBACKNET = 127 -INET_ADDRSTRLEN = 16 -INET6_ADDRSTRLEN = 46 - -# Included from bits/socket.h - -# Included from limits.h -_LIBC_LIMITS_H_ = 1 -MB_LEN_MAX = 16 -_LIMITS_H = 1 -CHAR_BIT = 8 -SCHAR_MIN = (-128) -SCHAR_MAX = 127 -UCHAR_MAX = 255 -CHAR_MIN = 0 -CHAR_MAX = UCHAR_MAX -CHAR_MIN = SCHAR_MIN -CHAR_MAX = SCHAR_MAX -SHRT_MIN = (-32768) -SHRT_MAX = 32767 -USHRT_MAX = 65535 -INT_MAX = 2147483647 -LONG_MAX = 9223372036854775807 -LONG_MAX = 2147483647 -LONG_MIN = (-LONG_MAX - 1) - -# Included from bits/posix1_lim.h -_BITS_POSIX1_LIM_H = 1 -_POSIX_AIO_LISTIO_MAX = 2 -_POSIX_AIO_MAX = 1 -_POSIX_ARG_MAX = 4096 -_POSIX_CHILD_MAX = 6 -_POSIX_DELAYTIMER_MAX = 32 -_POSIX_LINK_MAX = 8 -_POSIX_MAX_CANON = 255 -_POSIX_MAX_INPUT = 255 -_POSIX_MQ_OPEN_MAX = 8 -_POSIX_MQ_PRIO_MAX = 32 -_POSIX_NGROUPS_MAX = 0 -_POSIX_OPEN_MAX = 16 -_POSIX_FD_SETSIZE = _POSIX_OPEN_MAX -_POSIX_NAME_MAX = 14 -_POSIX_PATH_MAX = 256 -_POSIX_PIPE_BUF = 512 -_POSIX_RTSIG_MAX = 8 -_POSIX_SEM_NSEMS_MAX = 256 -_POSIX_SEM_VALUE_MAX = 32767 -_POSIX_SIGQUEUE_MAX = 32 -_POSIX_SSIZE_MAX = 32767 -_POSIX_STREAM_MAX = 8 -_POSIX_TZNAME_MAX = 6 -_POSIX_QLIMIT = 1 -_POSIX_HIWAT = _POSIX_PIPE_BUF -_POSIX_UIO_MAXIOV = 16 -_POSIX_TTY_NAME_MAX = 9 -_POSIX_TIMER_MAX = 32 -_POSIX_LOGIN_NAME_MAX = 9 -_POSIX_CLOCKRES_MIN = 20000000 - -# Included from bits/local_lim.h - -# Included from linux/limits.h -NR_OPEN = 1024 -NGROUPS_MAX = 32 -ARG_MAX = 131072 -CHILD_MAX = 999 -OPEN_MAX = 256 -LINK_MAX = 127 -MAX_CANON = 255 -MAX_INPUT = 255 -NAME_MAX = 255 -PATH_MAX = 4096 -PIPE_BUF = 4096 -RTSIG_MAX = 32 -_POSIX_THREAD_KEYS_MAX = 128 -PTHREAD_KEYS_MAX = 1024 -_POSIX_THREAD_DESTRUCTOR_ITERATIONS = 4 -PTHREAD_DESTRUCTOR_ITERATIONS = _POSIX_THREAD_DESTRUCTOR_ITERATIONS -_POSIX_THREAD_THREADS_MAX = 64 -PTHREAD_THREADS_MAX = 1024 -AIO_PRIO_DELTA_MAX = 20 -PTHREAD_STACK_MIN = 16384 -TIMER_MAX = 256 -SSIZE_MAX = LONG_MAX -NGROUPS_MAX = _POSIX_NGROUPS_MAX - -# Included from bits/posix2_lim.h -_BITS_POSIX2_LIM_H = 1 -_POSIX2_BC_BASE_MAX = 99 -_POSIX2_BC_DIM_MAX = 2048 -_POSIX2_BC_SCALE_MAX = 99 -_POSIX2_BC_STRING_MAX = 1000 -_POSIX2_COLL_WEIGHTS_MAX = 2 -_POSIX2_EXPR_NEST_MAX = 32 -_POSIX2_LINE_MAX = 2048 -_POSIX2_RE_DUP_MAX = 255 -_POSIX2_CHARCLASS_NAME_MAX = 14 -BC_BASE_MAX = _POSIX2_BC_BASE_MAX -BC_DIM_MAX = _POSIX2_BC_DIM_MAX -BC_SCALE_MAX = _POSIX2_BC_SCALE_MAX -BC_STRING_MAX = _POSIX2_BC_STRING_MAX -COLL_WEIGHTS_MAX = 255 -EXPR_NEST_MAX = _POSIX2_EXPR_NEST_MAX -LINE_MAX = _POSIX2_LINE_MAX -CHARCLASS_NAME_MAX = 2048 -RE_DUP_MAX = (0x7fff) - -# Included from bits/xopen_lim.h -_XOPEN_LIM_H = 1 - -# Included from bits/stdio_lim.h -L_tmpnam = 20 -TMP_MAX = 238328 -FILENAME_MAX = 4096 -L_ctermid = 9 -L_cuserid = 9 -FOPEN_MAX = 16 -IOV_MAX = 1024 -_XOPEN_IOV_MAX = _POSIX_UIO_MAXIOV -NL_ARGMAX = _POSIX_ARG_MAX -NL_LANGMAX = _POSIX2_LINE_MAX -NL_MSGMAX = INT_MAX -NL_NMAX = INT_MAX -NL_SETMAX = INT_MAX -NL_TEXTMAX = INT_MAX -NZERO = 20 -WORD_BIT = 16 -WORD_BIT = 32 -WORD_BIT = 64 -WORD_BIT = 16 -WORD_BIT = 32 -WORD_BIT = 64 -WORD_BIT = 32 -LONG_BIT = 32 -LONG_BIT = 64 -LONG_BIT = 32 -LONG_BIT = 64 -LONG_BIT = 64 -LONG_BIT = 32 -from TYPES import * -PF_UNSPEC = 0 -PF_LOCAL = 1 -PF_UNIX = PF_LOCAL -PF_FILE = PF_LOCAL -PF_INET = 2 -PF_AX25 = 3 -PF_IPX = 4 -PF_APPLETALK = 5 -PF_NETROM = 6 -PF_BRIDGE = 7 -PF_ATMPVC = 8 -PF_X25 = 9 -PF_INET6 = 10 -PF_ROSE = 11 -PF_DECnet = 12 -PF_NETBEUI = 13 -PF_SECURITY = 14 -PF_KEY = 15 -PF_NETLINK = 16 -PF_ROUTE = PF_NETLINK -PF_PACKET = 17 -PF_ASH = 18 -PF_ECONET = 19 -PF_ATMSVC = 20 -PF_SNA = 22 -PF_IRDA = 23 -PF_PPPOX = 24 -PF_WANPIPE = 25 -PF_BLUETOOTH = 31 -PF_MAX = 32 -AF_UNSPEC = PF_UNSPEC -AF_LOCAL = PF_LOCAL -AF_UNIX = PF_UNIX -AF_FILE = PF_FILE -AF_INET = PF_INET -AF_AX25 = PF_AX25 -AF_IPX = PF_IPX -AF_APPLETALK = PF_APPLETALK -AF_NETROM = PF_NETROM -AF_BRIDGE = PF_BRIDGE -AF_ATMPVC = PF_ATMPVC -AF_X25 = PF_X25 -AF_INET6 = PF_INET6 -AF_ROSE = PF_ROSE -AF_DECnet = PF_DECnet -AF_NETBEUI = PF_NETBEUI -AF_SECURITY = PF_SECURITY -AF_KEY = PF_KEY -AF_NETLINK = PF_NETLINK -AF_ROUTE = PF_ROUTE -AF_PACKET = PF_PACKET -AF_ASH = PF_ASH -AF_ECONET = PF_ECONET -AF_ATMSVC = PF_ATMSVC -AF_SNA = PF_SNA -AF_IRDA = PF_IRDA -AF_PPPOX = PF_PPPOX -AF_WANPIPE = PF_WANPIPE -AF_BLUETOOTH = PF_BLUETOOTH -AF_MAX = PF_MAX -SOL_RAW = 255 -SOL_DECNET = 261 -SOL_X25 = 262 -SOL_PACKET = 263 -SOL_ATM = 264 -SOL_AAL = 265 -SOL_IRDA = 266 -SOMAXCONN = 128 - -# Included from bits/sockaddr.h -_BITS_SOCKADDR_H = 1 -def __SOCKADDR_COMMON(sa_prefix): return \ - -_SS_SIZE = 128 -def CMSG_FIRSTHDR(mhdr): return \ - - -# Included from asm/socket.h - -# Included from asm/sockios.h -FIOSETOWN = 0x8901 -SIOCSPGRP = 0x8902 -FIOGETOWN = 0x8903 -SIOCGPGRP = 0x8904 -SIOCATMARK = 0x8905 -SIOCGSTAMP = 0x8906 -SOL_SOCKET = 1 -SO_DEBUG = 1 -SO_REUSEADDR = 2 -SO_TYPE = 3 -SO_ERROR = 4 -SO_DONTROUTE = 5 -SO_BROADCAST = 6 -SO_SNDBUF = 7 -SO_RCVBUF = 8 -SO_KEEPALIVE = 9 -SO_OOBINLINE = 10 -SO_NO_CHECK = 11 -SO_PRIORITY = 12 -SO_LINGER = 13 -SO_BSDCOMPAT = 14 -SO_PASSCRED = 16 -SO_PEERCRED = 17 -SO_RCVLOWAT = 18 -SO_SNDLOWAT = 19 -SO_RCVTIMEO = 20 -SO_SNDTIMEO = 21 -SO_SECURITY_AUTHENTICATION = 22 -SO_SECURITY_ENCRYPTION_TRANSPORT = 23 -SO_SECURITY_ENCRYPTION_NETWORK = 24 -SO_BINDTODEVICE = 25 -SO_ATTACH_FILTER = 26 -SO_DETACH_FILTER = 27 -SO_PEERNAME = 28 -SO_TIMESTAMP = 29 -SCM_TIMESTAMP = SO_TIMESTAMP -SO_ACCEPTCONN = 30 -SOCK_STREAM = 1 -SOCK_DGRAM = 2 -SOCK_RAW = 3 -SOCK_RDM = 4 -SOCK_SEQPACKET = 5 -SOCK_PACKET = 10 -SOCK_MAX = (SOCK_PACKET+1) - -# Included from bits/in.h -IP_TOS = 1 -IP_TTL = 2 -IP_HDRINCL = 3 -IP_OPTIONS = 4 -IP_ROUTER_ALERT = 5 -IP_RECVOPTS = 6 -IP_RETOPTS = 7 -IP_PKTINFO = 8 -IP_PKTOPTIONS = 9 -IP_PMTUDISC = 10 -IP_MTU_DISCOVER = 10 -IP_RECVERR = 11 -IP_RECVTTL = 12 -IP_RECVTOS = 13 -IP_MULTICAST_IF = 32 -IP_MULTICAST_TTL = 33 -IP_MULTICAST_LOOP = 34 -IP_ADD_MEMBERSHIP = 35 -IP_DROP_MEMBERSHIP = 36 -IP_RECVRETOPTS = IP_RETOPTS -IP_PMTUDISC_DONT = 0 -IP_PMTUDISC_WANT = 1 -IP_PMTUDISC_DO = 2 -SOL_IP = 0 -IP_DEFAULT_MULTICAST_TTL = 1 -IP_DEFAULT_MULTICAST_LOOP = 1 -IP_MAX_MEMBERSHIPS = 20 -IPV6_ADDRFORM = 1 -IPV6_PKTINFO = 2 -IPV6_HOPOPTS = 3 -IPV6_DSTOPTS = 4 -IPV6_RTHDR = 5 -IPV6_PKTOPTIONS = 6 -IPV6_CHECKSUM = 7 -IPV6_HOPLIMIT = 8 -IPV6_NEXTHOP = 9 -IPV6_AUTHHDR = 10 -IPV6_UNICAST_HOPS = 16 -IPV6_MULTICAST_IF = 17 -IPV6_MULTICAST_HOPS = 18 -IPV6_MULTICAST_LOOP = 19 -IPV6_JOIN_GROUP = 20 -IPV6_LEAVE_GROUP = 21 -IPV6_ROUTER_ALERT = 22 -IPV6_MTU_DISCOVER = 23 -IPV6_MTU = 24 -IPV6_RECVERR = 25 -IPV6_RXHOPOPTS = IPV6_HOPOPTS -IPV6_RXDSTOPTS = IPV6_DSTOPTS -IPV6_ADD_MEMBERSHIP = IPV6_JOIN_GROUP -IPV6_DROP_MEMBERSHIP = IPV6_LEAVE_GROUP -IPV6_PMTUDISC_DONT = 0 -IPV6_PMTUDISC_WANT = 1 -IPV6_PMTUDISC_DO = 2 -SOL_IPV6 = 41 -SOL_ICMPV6 = 58 -IPV6_RTHDR_LOOSE = 0 -IPV6_RTHDR_STRICT = 1 -IPV6_RTHDR_TYPE_0 = 0 - -# Included from endian.h -_ENDIAN_H = 1 -__LITTLE_ENDIAN = 1234 -__BIG_ENDIAN = 4321 -__PDP_ENDIAN = 3412 - -# Included from bits/endian.h -__BYTE_ORDER = __LITTLE_ENDIAN -__FLOAT_WORD_ORDER = __BYTE_ORDER -LITTLE_ENDIAN = __LITTLE_ENDIAN -BIG_ENDIAN = __BIG_ENDIAN -PDP_ENDIAN = __PDP_ENDIAN -BYTE_ORDER = __BYTE_ORDER - -# Included from bits/byteswap.h -_BITS_BYTESWAP_H = 1 -def __bswap_constant_16(x): return \ - -def __bswap_16(x): return \ - -def __bswap_16(x): return __bswap_constant_16 (x) - -def __bswap_constant_32(x): return \ - -def __bswap_32(x): return \ - -def __bswap_32(x): return \ - -def __bswap_32(x): return __bswap_constant_32 (x) - -def __bswap_constant_64(x): return \ - -def __bswap_64(x): return \ - -def ntohl(x): return (x) - -def ntohs(x): return (x) - -def htonl(x): return (x) - -def htons(x): return (x) - -def ntohl(x): return __bswap_32 (x) - -def ntohs(x): return __bswap_16 (x) - -def htonl(x): return __bswap_32 (x) - -def htons(x): return __bswap_16 (x) - -def IN6_IS_ADDR_UNSPECIFIED(a): return \ - -def IN6_IS_ADDR_LOOPBACK(a): return \ - -def IN6_IS_ADDR_LINKLOCAL(a): return \ - -def IN6_IS_ADDR_SITELOCAL(a): return \ - -def IN6_IS_ADDR_V4MAPPED(a): return \ - -def IN6_IS_ADDR_V4COMPAT(a): return \ - -def IN6_IS_ADDR_MC_NODELOCAL(a): return \ - -def IN6_IS_ADDR_MC_LINKLOCAL(a): return \ - -def IN6_IS_ADDR_MC_SITELOCAL(a): return \ - -def IN6_IS_ADDR_MC_ORGLOCAL(a): return \ - -def IN6_IS_ADDR_MC_GLOBAL(a): return diff --git a/Lib/plat-linux3/TYPES.py b/Lib/plat-linux3/TYPES.py deleted file mode 100644 index e7a324b25a..0000000000 --- a/Lib/plat-linux3/TYPES.py +++ /dev/null @@ -1,170 +0,0 @@ -# Generated by h2py from /usr/include/sys/types.h -_SYS_TYPES_H = 1 - -# Included from features.h -_FEATURES_H = 1 -__USE_ANSI = 1 -__FAVOR_BSD = 1 -_ISOC99_SOURCE = 1 -_POSIX_SOURCE = 1 -_POSIX_C_SOURCE = 199506 -_XOPEN_SOURCE = 600 -_XOPEN_SOURCE_EXTENDED = 1 -_LARGEFILE64_SOURCE = 1 -_BSD_SOURCE = 1 -_SVID_SOURCE = 1 -_BSD_SOURCE = 1 -_SVID_SOURCE = 1 -__USE_ISOC99 = 1 -_POSIX_SOURCE = 1 -_POSIX_C_SOURCE = 2 -_POSIX_C_SOURCE = 199506 -__USE_POSIX = 1 -__USE_POSIX2 = 1 -__USE_POSIX199309 = 1 -__USE_POSIX199506 = 1 -__USE_XOPEN = 1 -__USE_XOPEN_EXTENDED = 1 -__USE_UNIX98 = 1 -_LARGEFILE_SOURCE = 1 -__USE_XOPEN2K = 1 -__USE_ISOC99 = 1 -__USE_XOPEN_EXTENDED = 1 -__USE_LARGEFILE = 1 -__USE_LARGEFILE64 = 1 -__USE_FILE_OFFSET64 = 1 -__USE_MISC = 1 -__USE_BSD = 1 -__USE_SVID = 1 -__USE_GNU = 1 -__USE_REENTRANT = 1 -__STDC_IEC_559__ = 1 -__STDC_IEC_559_COMPLEX__ = 1 -__STDC_ISO_10646__ = 200009 -__GNU_LIBRARY__ = 6 -__GLIBC__ = 2 -__GLIBC_MINOR__ = 2 - -# Included from sys/cdefs.h -_SYS_CDEFS_H = 1 -def __PMT(args): return args - -def __P(args): return args - -def __PMT(args): return args - -def __STRING(x): return #x - -__flexarr = [] -__flexarr = [0] -__flexarr = [] -__flexarr = [1] -def __ASMNAME(cname): return __ASMNAME2 (__USER_LABEL_PREFIX__, cname) - -def __attribute__(xyz): return - -def __attribute_format_arg__(x): return __attribute__ ((__format_arg__ (x))) - -def __attribute_format_arg__(x): return - -__USE_LARGEFILE = 1 -__USE_LARGEFILE64 = 1 -__USE_EXTERN_INLINES = 1 - -# Included from gnu/stubs.h - -# Included from bits/types.h -_BITS_TYPES_H = 1 -__FD_SETSIZE = 1024 - -# Included from bits/pthreadtypes.h -_BITS_PTHREADTYPES_H = 1 - -# Included from bits/sched.h -SCHED_OTHER = 0 -SCHED_FIFO = 1 -SCHED_RR = 2 -CSIGNAL = 0x000000ff -CLONE_VM = 0x00000100 -CLONE_FS = 0x00000200 -CLONE_FILES = 0x00000400 -CLONE_SIGHAND = 0x00000800 -CLONE_PID = 0x00001000 -CLONE_PTRACE = 0x00002000 -CLONE_VFORK = 0x00004000 -__defined_schedparam = 1 - -# Included from time.h -_TIME_H = 1 - -# Included from bits/time.h -_BITS_TIME_H = 1 -CLOCKS_PER_SEC = 1000000 -CLOCK_REALTIME = 0 -CLOCK_PROCESS_CPUTIME_ID = 2 -CLOCK_THREAD_CPUTIME_ID = 3 -TIMER_ABSTIME = 1 -_STRUCT_TIMEVAL = 1 -CLK_TCK = CLOCKS_PER_SEC -__clock_t_defined = 1 -__time_t_defined = 1 -__clockid_t_defined = 1 -__timer_t_defined = 1 -__timespec_defined = 1 -def __isleap(year): return \ - -__BIT_TYPES_DEFINED__ = 1 - -# Included from endian.h -_ENDIAN_H = 1 -__LITTLE_ENDIAN = 1234 -__BIG_ENDIAN = 4321 -__PDP_ENDIAN = 3412 - -# Included from bits/endian.h -__BYTE_ORDER = __LITTLE_ENDIAN -__FLOAT_WORD_ORDER = __BYTE_ORDER -LITTLE_ENDIAN = __LITTLE_ENDIAN -BIG_ENDIAN = __BIG_ENDIAN -PDP_ENDIAN = __PDP_ENDIAN -BYTE_ORDER = __BYTE_ORDER - -# Included from sys/select.h -_SYS_SELECT_H = 1 - -# Included from bits/select.h -def __FD_ZERO(fdsp): return \ - -def __FD_ZERO(set): return \ - - -# Included from bits/sigset.h -_SIGSET_H_types = 1 -_SIGSET_H_fns = 1 -def __sigmask(sig): return \ - -def __sigemptyset(set): return \ - -def __sigfillset(set): return \ - -def __sigisemptyset(set): return \ - -def __FDELT(d): return ((d) / __NFDBITS) - -FD_SETSIZE = __FD_SETSIZE -def FD_ZERO(fdsetp): return __FD_ZERO (fdsetp) - - -# Included from sys/sysmacros.h -_SYS_SYSMACROS_H = 1 -def major(dev): return ((int)(((dev) >> 8) & 0xff)) - -def minor(dev): return ((int)((dev) & 0xff)) - -def major(dev): return (((dev).__val[1] >> 8) & 0xff) - -def minor(dev): return ((dev).__val[1] & 0xff) - -def major(dev): return (((dev).__val[0] >> 8) & 0xff) - -def minor(dev): return ((dev).__val[0] & 0xff) diff --git a/Lib/plat-linux3/regen b/Lib/plat-linux3/regen deleted file mode 100755 index c76950e232..0000000000 --- a/Lib/plat-linux3/regen +++ /dev/null @@ -1,8 +0,0 @@ -#! /bin/sh -case `uname` in -Linux*) ;; -*) echo Probably not on a Linux system 1>&2 - exit 1;; -esac -set -v -h2py -i '(u_long)' /usr/include/sys/types.h /usr/include/netinet/in.h /usr/include/dlfcn.h diff --git a/Lib/platform.py b/Lib/platform.py index 6e301da0e1..0cb0a4fc43 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -357,92 +357,13 @@ def dist(distname='',version='',id='', supported_dists=supported_dists, full_distribution_name=0) -class _popen: - - """ Fairly portable (alternative) popen implementation. - - This is mostly needed in case os.popen() is not available, or - doesn't work as advertised, e.g. in Win9X GUI programs like - PythonWin or IDLE. - - Writing to the pipe is currently not supported. - - """ - tmpfile = '' - pipe = None - bufsize = None - mode = 'r' - - def __init__(self,cmd,mode='r',bufsize=None): - - if mode != 'r': - raise ValueError('popen()-emulation only supports read mode') - import tempfile - self.tmpfile = tmpfile = tempfile.mktemp() - os.system(cmd + ' > %s' % tmpfile) - self.pipe = open(tmpfile,'rb') - self.bufsize = bufsize - self.mode = mode - - def read(self): - - return self.pipe.read() - - def readlines(self): - - if self.bufsize is not None: - return self.pipe.readlines() - - def close(self, - - remove=os.unlink,error=os.error): - - if self.pipe: - rc = self.pipe.close() - else: - rc = 255 - if self.tmpfile: - try: - remove(self.tmpfile) - except error: - pass - return rc - - # Alias - __del__ = close - def popen(cmd, mode='r', bufsize=-1): """ Portable popen() interface. """ - # Find a working popen implementation preferring win32pipe.popen - # over os.popen over _popen - popen = None - if os.environ.get('OS','') == 'Windows_NT': - # On NT win32pipe should work; on Win9x it hangs due to bugs - # in the MS C lib (see MS KnowledgeBase article Q150956) - try: - import win32pipe - except ImportError: - pass - else: - popen = win32pipe.popen - if popen is None: - if hasattr(os,'popen'): - popen = os.popen - # Check whether it works... it doesn't in GUI programs - # on Windows platforms - if sys.platform == 'win32': # XXX Others too ? - try: - popen('') - except os.error: - popen = _popen - else: - popen = _popen - if bufsize is None: - return popen(cmd,mode) - else: - return popen(cmd,mode,bufsize) + import warnings + warnings.warn('use os.popen instead', DeprecationWarning, stacklevel=2) + return os.popen(cmd, mode, bufsize) def _norm_version(version, build=''): diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 2e7e5126d7..21076db9a5 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -266,13 +266,13 @@ class _InternalDict(dict): raise AttributeError(attr) from warnings import warn warn("Attribute access from plist dicts is deprecated, use d[key] " - "notation instead", PendingDeprecationWarning, 2) + "notation instead", DeprecationWarning, 2) return value def __setattr__(self, attr, value): from warnings import warn warn("Attribute access from plist dicts is deprecated, use d[key] " - "notation instead", PendingDeprecationWarning, 2) + "notation instead", DeprecationWarning, 2) self[attr] = value def __delattr__(self, attr): @@ -282,14 +282,14 @@ class _InternalDict(dict): raise AttributeError(attr) from warnings import warn warn("Attribute access from plist dicts is deprecated, use d[key] " - "notation instead", PendingDeprecationWarning, 2) + "notation instead", DeprecationWarning, 2) class Dict(_InternalDict): def __init__(self, **kwargs): from warnings import warn warn("The plistlib.Dict class is deprecated, use builtin dict instead", - PendingDeprecationWarning, 2) + DeprecationWarning, 2) super().__init__(**kwargs) @@ -302,7 +302,7 @@ class Plist(_InternalDict): def __init__(self, **kwargs): from warnings import warn warn("The Plist class is deprecated, use the readPlist() and " - "writePlist() functions instead", PendingDeprecationWarning, 2) + "writePlist() functions instead", DeprecationWarning, 2) super().__init__(**kwargs) def fromFile(cls, pathOrFile): diff --git a/Lib/poplib.py b/Lib/poplib.py index 84ea88de46..d42d9dd320 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -250,15 +250,18 @@ class POP3: def quit(self): """Signoff: commit changes on server, unlock mailbox, close connection.""" - try: - resp = self._shortcmd('QUIT') - except error_proto as val: - resp = val - self.file.close() - self.sock.close() - del self.file, self.sock + resp = self._shortcmd('QUIT') + self.close() return resp + def close(self): + """Close the connection without assuming anything about it.""" + if self.file is not None: + self.file.close() + if self.sock is not None: + self.sock.close() + self.file = self.sock = None + #__del__ = quit diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 8e2e47bd15..f0b020012e 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -22,11 +22,6 @@ Run "pydoc -b" to start an HTTP server on an arbitrary unused port and open a Web browser to interactively browse documentation. The -p option can be used with the -b option to explicitly specify the server port. -For platforms without a command line, "pydoc -g" starts the HTTP server -and also pops up a little window for controlling it. This option is -deprecated, since the server can now be controlled directly from HTTP -clients. - Run "pydoc -w <name>" to write out the HTML documentation for a module to a file named "<name>.html". @@ -42,7 +37,6 @@ __all__ = ['help'] __author__ = "Ka-Ping Yee <ping@lfw.org>" __date__ = "26 February 2001" -__version__ = "$Revision$" __credits__ = """Guido van Rossum, for an excellent programming language. Tommy Burnette, the original creator of manpy. Paul Prescod, for all his work on onlinehelp. @@ -169,11 +163,11 @@ def _split_list(s, predicate): def visiblename(name, all=None, obj=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant. - _hidden_names = ('__builtins__', '__doc__', '__file__', '__path__', + if name in {'__builtins__', '__doc__', '__file__', '__path__', '__module__', '__name__', '__slots__', '__package__', '__cached__', '__author__', '__credits__', '__date__', - '__version__') - if name in _hidden_names: return 0 + '__version__'}: + return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 # Namedtuples have public fields and methods with a single leading underscore @@ -951,6 +945,9 @@ class HTMLDoc(Doc): modpkgs = [] if shadowed is None: shadowed = {} for importer, name, ispkg in pkgutil.iter_modules([dir]): + if any((0xD800 <= ord(ch) <= 0xDFFF) for ch in name): + # ignore a module if its name contains a surrogate character + continue modpkgs.append((name, '', ispkg, name in shadowed)) shadowed[name] = 1 @@ -2053,272 +2050,6 @@ def apropos(key): warnings.filterwarnings('ignore') # ignore problems during import ModuleScanner().run(callback, key, onerror=onerror) -# --------------------------------------------------- Web browser interface - -def serve(port, callback=None, completer=None): - import http.server, email.message, select - - msg = 'the pydoc.serve() function is deprecated' - warnings.warn(msg, DeprecationWarning, stacklevel=2) - - class DocHandler(http.server.BaseHTTPRequestHandler): - def send_document(self, title, contents): - try: - self.send_response(200) - self.send_header('Content-Type', 'text/html; charset=UTF-8') - self.end_headers() - self.wfile.write(html.page(title, contents).encode('utf-8')) - except IOError: pass - - def do_GET(self): - path = self.path - if path[-5:] == '.html': path = path[:-5] - if path[:1] == '/': path = path[1:] - if path and path != '.': - try: - obj = locate(path, forceload=1) - except ErrorDuringImport as value: - self.send_document(path, html.escape(str(value))) - return - if obj: - self.send_document(describe(obj), html.document(obj, path)) - else: - self.send_document(path, -'no Python documentation found for %s' % repr(path)) - else: - heading = html.heading( -'<big><big><strong>Python: Index of Modules</strong></big></big>', -'#ffffff', '#7799ee') - def bltinlink(name): - return '<a href="%s.html">%s</a>' % (name, name) - names = [x for x in sys.builtin_module_names if x != '__main__'] - contents = html.multicolumn(names, bltinlink) - indices = ['<p>' + html.bigsection( - 'Built-in Modules', '#ffffff', '#ee77aa', contents)] - - seen = {} - for dir in sys.path: - indices.append(html.index(dir, seen)) - contents = heading + ' '.join(indices) + '''<p align=right> -<font color="#909090" face="helvetica, arial"><strong> -pydoc</strong> by Ka-Ping Yee <ping@lfw.org></font>''' - self.send_document('Index of Modules', contents) - - def log_message(self, *args): pass - - class DocServer(http.server.HTTPServer): - def __init__(self, port, callback): - host = 'localhost' - self.address = (host, port) - self.url = 'http://%s:%d/' % (host, port) - self.callback = callback - self.base.__init__(self, self.address, self.handler) - - def serve_until_quit(self): - import select - self.quit = False - while not self.quit: - rd, wr, ex = select.select([self.socket.fileno()], [], [], 1) - if rd: self.handle_request() - self.server_close() - - def server_activate(self): - self.base.server_activate(self) - if self.callback: self.callback(self) - - DocServer.base = http.server.HTTPServer - DocServer.handler = DocHandler - DocHandler.MessageClass = email.message.Message - try: - try: - DocServer(port, callback).serve_until_quit() - except (KeyboardInterrupt, select.error): - pass - finally: - if completer: completer() - -# ----------------------------------------------------- graphical interface - -def gui(): - """Graphical interface (starts Web server and pops up a control window).""" - - msg = ('the pydoc.gui() function and "pydoc -g" option are deprecated\n', - 'use "pydoc.browse() function and "pydoc -b" option instead.') - warnings.warn(msg, DeprecationWarning, stacklevel=2) - - class GUI: - def __init__(self, window, port=7464): - self.window = window - self.server = None - self.scanner = None - - import tkinter - self.server_frm = tkinter.Frame(window) - self.title_lbl = tkinter.Label(self.server_frm, - text='Starting server...\n ') - self.open_btn = tkinter.Button(self.server_frm, - text='open browser', command=self.open, state='disabled') - self.quit_btn = tkinter.Button(self.server_frm, - text='quit serving', command=self.quit, state='disabled') - - self.search_frm = tkinter.Frame(window) - self.search_lbl = tkinter.Label(self.search_frm, text='Search for') - self.search_ent = tkinter.Entry(self.search_frm) - self.search_ent.bind('<Return>', self.search) - self.stop_btn = tkinter.Button(self.search_frm, - text='stop', pady=0, command=self.stop, state='disabled') - if sys.platform == 'win32': - # Trying to hide and show this button crashes under Windows. - self.stop_btn.pack(side='right') - - self.window.title('pydoc') - self.window.protocol('WM_DELETE_WINDOW', self.quit) - self.title_lbl.pack(side='top', fill='x') - self.open_btn.pack(side='left', fill='x', expand=1) - self.quit_btn.pack(side='right', fill='x', expand=1) - self.server_frm.pack(side='top', fill='x') - - self.search_lbl.pack(side='left') - self.search_ent.pack(side='right', fill='x', expand=1) - self.search_frm.pack(side='top', fill='x') - self.search_ent.focus_set() - - font = ('helvetica', sys.platform == 'win32' and 8 or 10) - self.result_lst = tkinter.Listbox(window, font=font, height=6) - self.result_lst.bind('<Button-1>', self.select) - self.result_lst.bind('<Double-Button-1>', self.goto) - self.result_scr = tkinter.Scrollbar(window, - orient='vertical', command=self.result_lst.yview) - self.result_lst.config(yscrollcommand=self.result_scr.set) - - self.result_frm = tkinter.Frame(window) - self.goto_btn = tkinter.Button(self.result_frm, - text='go to selected', command=self.goto) - self.hide_btn = tkinter.Button(self.result_frm, - text='hide results', command=self.hide) - self.goto_btn.pack(side='left', fill='x', expand=1) - self.hide_btn.pack(side='right', fill='x', expand=1) - - self.window.update() - self.minwidth = self.window.winfo_width() - self.minheight = self.window.winfo_height() - self.bigminheight = (self.server_frm.winfo_reqheight() + - self.search_frm.winfo_reqheight() + - self.result_lst.winfo_reqheight() + - self.result_frm.winfo_reqheight()) - self.bigwidth, self.bigheight = self.minwidth, self.bigminheight - self.expanded = 0 - self.window.wm_geometry('%dx%d' % (self.minwidth, self.minheight)) - self.window.wm_minsize(self.minwidth, self.minheight) - self.window.tk.willdispatch() - - import threading - threading.Thread( - target=serve, args=(port, self.ready, self.quit)).start() - - def ready(self, server): - self.server = server - self.title_lbl.config( - text='Python documentation server at\n' + server.url) - self.open_btn.config(state='normal') - self.quit_btn.config(state='normal') - - def open(self, event=None, url=None): - url = url or self.server.url - import webbrowser - webbrowser.open(url) - - def quit(self, event=None): - if self.server: - self.server.quit = 1 - self.window.quit() - - def search(self, event=None): - key = self.search_ent.get() - self.stop_btn.pack(side='right') - self.stop_btn.config(state='normal') - self.search_lbl.config(text='Searching for "%s"...' % key) - self.search_ent.forget() - self.search_lbl.pack(side='left') - self.result_lst.delete(0, 'end') - self.goto_btn.config(state='disabled') - self.expand() - - import threading - if self.scanner: - self.scanner.quit = 1 - self.scanner = ModuleScanner() - threading.Thread(target=self.scanner.run, - args=(self.update, key, self.done)).start() - - def update(self, path, modname, desc): - if modname[-9:] == '.__init__': - modname = modname[:-9] + ' (package)' - self.result_lst.insert('end', - modname + ' - ' + (desc or '(no description)')) - - def stop(self, event=None): - if self.scanner: - self.scanner.quit = 1 - self.scanner = None - - def done(self): - self.scanner = None - self.search_lbl.config(text='Search for') - self.search_lbl.pack(side='left') - self.search_ent.pack(side='right', fill='x', expand=1) - if sys.platform != 'win32': self.stop_btn.forget() - self.stop_btn.config(state='disabled') - - def select(self, event=None): - self.goto_btn.config(state='normal') - - def goto(self, event=None): - selection = self.result_lst.curselection() - if selection: - modname = self.result_lst.get(selection[0]).split()[0] - self.open(url=self.server.url + modname + '.html') - - def collapse(self): - if not self.expanded: return - self.result_frm.forget() - self.result_scr.forget() - self.result_lst.forget() - self.bigwidth = self.window.winfo_width() - self.bigheight = self.window.winfo_height() - self.window.wm_geometry('%dx%d' % (self.minwidth, self.minheight)) - self.window.wm_minsize(self.minwidth, self.minheight) - self.expanded = 0 - - def expand(self): - if self.expanded: return - self.result_frm.pack(side='bottom', fill='x') - self.result_scr.pack(side='right', fill='y') - self.result_lst.pack(side='top', fill='both', expand=1) - self.window.wm_geometry('%dx%d' % (self.bigwidth, self.bigheight)) - self.window.wm_minsize(self.minwidth, self.bigminheight) - self.expanded = 1 - - def hide(self, event=None): - self.stop() - self.collapse() - - import tkinter - try: - root = tkinter.Tk() - # Tk will crash if pythonw.exe has an XP .manifest - # file and the root has is not destroyed explicitly. - # If the problem is ever fixed in Tk, the explicit - # destroy can go. - try: - gui = GUI(root) - root.mainloop() - finally: - root.destroy() - except KeyboardInterrupt: - pass - - # --------------------------------------- enhanced Web browser interface def _start_server(urlhandler, port): @@ -2775,15 +2506,12 @@ def cli(): sys.path.insert(0, '.') try: - opts, args = getopt.getopt(sys.argv[1:], 'bgk:p:w') + opts, args = getopt.getopt(sys.argv[1:], 'bk:p:w') writing = False start_server = False open_browser = False port = None for opt, val in opts: - if opt == '-g': - gui() - return if opt == '-b': start_server = True open_browser = True @@ -2844,9 +2572,6 @@ def cli(): to interactively browse documentation. The -p option can be used with the -b option to explicitly specify the server port. -{cmd} -g - Deprecated. - {cmd} -w <name> ... Write out the HTML documentation for a module to a file in the current directory. If <name> contains a '{sep}', it is treated as a filename; if diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 55fa26dbfd..7edc551761 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Sun Jul 3 09:27:35 2011 +# Autogenerated by Sphinx on Thu Apr 28 07:53:12 2011 topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names. In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O). The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime. Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n', 'assignment': '\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n | "*" target\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list, optionally enclosed in\nparentheses or square brackets, is recursively defined as follows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets.\n\n * If the target list contains one target prefixed with an asterisk,\n called a "starred" target: The object must be a sequence with at\n least as many items as there are targets in the target list, minus\n one. The first items of the sequence are assigned, from left to\n right, to the targets before the starred target. The final items\n of the sequence are assigned to the targets after the starred\n target. A list of the remaining items in the sequence is then\n assigned to the starred target (the list can be empty).\n\n * Else: The object must be a sequence with the same number of items\n as there are targets in the target list, and the items are\n assigned, from left to right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` or ``nonlocal``\n statement in the current code block: the name is bound to the\n object in the current local namespace.\n\n * Otherwise: the name is bound to the object in the global namespace\n or the outer namespace determined by ``nonlocal``, respectively.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield an integer. If it is negative, the sequence\'s\n length is added to it. The resulting value must be a nonnegative\n integer less than the sequence\'s length, and the sequence is asked\n to assign the assigned object to its item with that index. If the\n index is out of range, ``IndexError`` is raised (assignment to a\n subscripted sequence cannot add new items to a list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n For user-defined objects, the ``__setitem__()`` method is called\n with appropriate arguments.\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to integers. If either bound is\n negative, the sequence\'s length is added to it. The resulting\n bounds are clipped to lie between zero and the sequence\'s length,\n inclusive. Finally, the sequence object is asked to replace the\n slice with the items of the assigned sequence. The length of the\n slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print(x)\n\nSee also:\n\n **PEP 3132** - Extended Iterable Unpacking\n The specification for the ``*target`` feature.\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', 'atom-identifiers': '\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name in front of the name, with leading underscores removed, and\na single underscore inserted in front of the class name. For example,\nthe identifier ``__spam`` occurring in a class named ``Ham`` will be\ntransformed to ``_Ham__spam``. This transformation is independent of\nthe syntactical context in which the identifier is used. If the\ntransformed name is extremely long (longer than 255 characters),\nimplementation defined truncation may happen. If the class name\nconsists only of underscores, no transformation is done.\n', @@ -16,9 +16,9 @@ topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAss 'break': '\nThe ``break`` statement\n***********************\n\n break_stmt ::= "break"\n\n``break`` may only occur syntactically nested in a ``for`` or\n``while`` loop, but not nested in a function or class definition\nwithin that loop.\n\nIt terminates the nearest enclosing loop, skipping the optional\n``else`` clause if the loop has one.\n\nIf a ``for`` loop is terminated by ``break``, the loop control target\nkeeps its current value.\n\nWhen ``break`` passes control out of a ``try`` statement with a\n``finally`` clause, that ``finally`` clause is executed before really\nleaving the loop.\n', 'callable-types': '\nEmulating callable objects\n**************************\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n ``x.__call__(arg1, arg2, ...)``.\n', 'calls': '\nCalls\n*****\n\nA call calls a callable object (e.g., a function) with a possibly\nempty series of arguments:\n\n call ::= primary "(" [argument_list [","] | comprehension] ")"\n argument_list ::= positional_arguments ["," keyword_arguments]\n ["," "*" expression] ["," keyword_arguments]\n ["," "**" expression]\n | keyword_arguments ["," "*" expression]\n ["," keyword_arguments] ["," "**" expression]\n | "*" expression ["," keyword_arguments] ["," "**" expression]\n | "**" expression\n positional_arguments ::= expression ("," expression)*\n keyword_arguments ::= keyword_item ("," keyword_item)*\n keyword_item ::= identifier "=" expression\n\nA trailing comma may be present after the positional and keyword\narguments but does not affect the semantics.\n\nThe primary must evaluate to a callable object (user-defined\nfunctions, built-in functions, methods of built-in objects, class\nobjects, methods of class instances, and all objects having a\n``__call__()`` method are callable). All argument expressions are\nevaluated before the call is attempted. Please refer to section\n*Function definitions* for the syntax of formal parameter lists.\n\nIf keyword arguments are present, they are first converted to\npositional arguments, as follows. First, a list of unfilled slots is\ncreated for the formal parameters. If there are N positional\narguments, they are placed in the first N slots. Next, for each\nkeyword argument, the identifier is used to determine the\ncorresponding slot (if the identifier is the same as the first formal\nparameter name, the first slot is used, and so on). If the slot is\nalready filled, a ``TypeError`` exception is raised. Otherwise, the\nvalue of the argument is placed in the slot, filling it (even if the\nexpression is ``None``, it fills the slot). When all arguments have\nbeen processed, the slots that are still unfilled are filled with the\ncorresponding default value from the function definition. (Default\nvalues are calculated, once, when the function is defined; thus, a\nmutable object such as a list or dictionary used as default value will\nbe shared by all calls that don\'t specify an argument value for the\ncorresponding slot; this should usually be avoided.) If there are any\nunfilled slots for which no default value is specified, a\n``TypeError`` exception is raised. Otherwise, the list of filled\nslots is used as the argument list for the call.\n\n**CPython implementation detail:** An implementation may provide\nbuilt-in functions whose positional parameters do not have names, even\nif they are \'named\' for the purpose of documentation, and which\ntherefore cannot be supplied by keyword. In CPython, this is the case\nfor functions implemented in C that use ``PyArg_ParseTuple()`` to\nparse their arguments.\n\nIf there are more positional arguments than there are formal parameter\nslots, a ``TypeError`` exception is raised, unless a formal parameter\nusing the syntax ``*identifier`` is present; in this case, that formal\nparameter receives a tuple containing the excess positional arguments\n(or an empty tuple if there were no excess positional arguments).\n\nIf any keyword argument does not correspond to a formal parameter\nname, a ``TypeError`` exception is raised, unless a formal parameter\nusing the syntax ``**identifier`` is present; in this case, that\nformal parameter receives a dictionary containing the excess keyword\narguments (using the keywords as keys and the argument values as\ncorresponding values), or a (new) empty dictionary if there were no\nexcess keyword arguments.\n\nIf the syntax ``*expression`` appears in the function call,\n``expression`` must evaluate to a sequence. Elements from this\nsequence are treated as if they were additional positional arguments;\nif there are positional arguments *x1*,..., *xN*, and ``expression``\nevaluates to a sequence *y1*, ..., *yM*, this is equivalent to a call\nwith M+N positional arguments *x1*, ..., *xN*, *y1*, ..., *yM*.\n\nA consequence of this is that although the ``*expression`` syntax may\nappear *after* some keyword arguments, it is processed *before* the\nkeyword arguments (and the ``**expression`` argument, if any -- see\nbelow). So:\n\n >>> def f(a, b):\n ... print(a, b)\n ...\n >>> f(b=1, *(2,))\n 2 1\n >>> f(a=1, *(2,))\n Traceback (most recent call last):\n File "<stdin>", line 1, in ?\n TypeError: f() got multiple values for keyword argument \'a\'\n >>> f(1, *(2,))\n 1 2\n\nIt is unusual for both keyword arguments and the ``*expression``\nsyntax to be used in the same call, so in practice this confusion does\nnot arise.\n\nIf the syntax ``**expression`` appears in the function call,\n``expression`` must evaluate to a mapping, the contents of which are\ntreated as additional keyword arguments. In the case of a keyword\nappearing in both ``expression`` and as an explicit keyword argument,\na ``TypeError`` exception is raised.\n\nFormal parameters using the syntax ``*identifier`` or ``**identifier``\ncannot be used as positional argument slots or as keyword argument\nnames.\n\nA call always returns some value, possibly ``None``, unless it raises\nan exception. How this value is computed depends on the type of the\ncallable object.\n\nIf it is---\n\na user-defined function:\n The code block for the function is executed, passing it the\n argument list. The first thing the code block will do is bind the\n formal parameters to the arguments; this is described in section\n *Function definitions*. When the code block executes a ``return``\n statement, this specifies the return value of the function call.\n\na built-in function or method:\n The result is up to the interpreter; see *Built-in Functions* for\n the descriptions of built-in functions and methods.\n\na class object:\n A new instance of that class is returned.\n\na class instance method:\n The corresponding user-defined function is called, with an argument\n list that is one longer than the argument list of the call: the\n instance becomes the first argument.\n\na class instance:\n The class must define a ``__call__()`` method; the effect is then\n the same as if that method was called.\n', - 'class': '\nClass definitions\n*****************\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= [decorators] "class" classname [inheritance] ":" suite\n inheritance ::= "(" [argument_list [","] | comprehension] ")"\n classname ::= identifier\n\nA class definition is an executable statement. The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing. Classes without an inheritance\nlist inherit, by default, from the base class ``object``; hence,\n\n class Foo:\n pass\n\nis equivalent to\n\n class Foo(object):\n pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.) When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n @f1(arg)\n @f2\n class Foo: pass\n\nis equivalent to\n\n class Foo: pass\n Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators. The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances. Instance attributes\ncan be set in a method with ``self.name = value``. Both class and\ninstance attributes are accessible through the notation\n"``self.name``", and an instance attribute hides a class attribute\nwith the same name when accessed in this way. Class attributes can be\nused as defaults for instance attributes, but using mutable values\nthere can lead to unexpected results. *Descriptors* can be used to\ncreate instance variables with different implementation details.\n\nSee also:\n\n **PEP 3115** - Metaclasses in Python 3 **PEP 3129** - Class\n Decorators\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack unless there\n is a ``finally`` clause which happens to raise another exception.\n That new exception causes the old one to be lost.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', + 'class': '\nClass definitions\n*****************\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= [decorators] "class" classname [inheritance] ":" suite\n inheritance ::= "(" [argument_list [","] | comprehension] ")"\n classname ::= identifier\n\nA class definition is an executable statement. The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing. Classes without an inheritance\nlist inherit, by default, from the base class ``object``; hence,\n\n class Foo:\n pass\n\nis equivalent to\n\n class Foo(object):\n pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.) When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n @f1(arg)\n @f2\n class Foo: pass\n\nis equivalent to\n\n class Foo: pass\n Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators. The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances. Instance attributes\ncan be set in a method with ``self.name = value``. Both class and\ninstance attributes are accessible through the notation\n"``self.name``", and an instance attribute hides a class attribute\nwith the same name when accessed in this way. Class attributes can be\nused as defaults for instance attributes, but using mutable values\nthere can lead to unexpected results. *Descriptors* can be used to\ncreate instance variables with different implementation details.\n\nSee also:\n\n **PEP 3115** - Metaclasses in Python 3 **PEP 3129** - Class\n Decorators\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', 'comparisons': '\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects. The objects need not have the same type.\nIf both are numbers, they are converted to a common type. Otherwise,\nthe ``==`` and ``!=`` operators *always* consider objects of different\ntypes to be unequal, while the ``<``, ``>``, ``>=`` and ``<=``\noperators raise a ``TypeError`` when comparing objects of different\ntypes that do not implement these operators for the given pair of\ntypes. You can control comparison behavior of objects of non-built-in\ntypes by defining rich comparison methods like ``__gt__()``, described\nin section *Basic customization*.\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* The values ``float(\'NaN\')`` and ``Decimal(\'NaN\')`` are special. The\n are identical to themselves, ``x is x`` but are not equal to\n themselves, ``x != x``. Additionally, comparing any value to a\n not-a-number value will return ``False``. For example, both ``3 <\n float(\'NaN\')`` and ``float(\'NaN\') < 3`` will return ``False``.\n\n* Bytes objects are compared lexicographically using the numeric\n values of their elements.\n\n* Strings are compared lexicographically using the numeric equivalents\n (the result of the built-in function ``ord()``) of their characters.\n [3] String and bytes object can\'t be compared!\n\n* Tuples and lists are compared lexicographically using comparison of\n corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, ``[1,2,x] <= [1,2,y]`` has the\n same value as ``x <= y``. If the corresponding element does not\n exist, the shorter sequence is ordered first (for example, ``[1,2] <\n [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if they have the\n same ``(key, value)`` pairs. Order comparisons ``(\'<\', \'<=\', \'>=\',\n \'>\')`` raise ``TypeError``.\n\n* Sets and frozensets define comparison operators to mean subset and\n superset tests. Those relations do not define total orderings (the\n two sets ``{1,2}`` and {2,3} are not equal, nor subsets of one\n another, nor supersets of one another). Accordingly, sets are not\n appropriate arguments for functions which depend on total ordering.\n For example, ``min()``, ``max()``, and ``sorted()`` produce\n undefined results given a list of sets as inputs.\n\n* Most other objects of built-in types compare unequal unless they are\n the same object; the choice whether one object is considered smaller\n or larger than another one is made arbitrarily but consistently\n within one execution of a program.\n\nComparison of objects of the differing types depends on whether either\nof the types provide explicit support for the comparison. Most\nnumeric types can be compared with one another, but comparisons of\n``float`` and ``Decimal`` are not supported to avoid the inevitable\nconfusion arising from representation issues such as ``float(\'1.1\')``\nbeing inexactly represented and therefore not exactly equal to\n``Decimal(\'1.1\')`` which is. When cross-type comparison is not\nsupported, the comparison method returns ``NotImplemented``. This can\ncreate the illusion of non-transitivity between supported cross-type\ncomparisons and unsupported comparisons. For example, ``Decimal(2) ==\n2`` and ``2 == float(2)`` but ``Decimal(2) != float(2)``.\n\nThe operators ``in`` and ``not in`` test for membership. ``x in s``\nevaluates to true if *x* is a member of *s*, and false otherwise. ``x\nnot in s`` returns the negation of ``x in s``. All built-in sequences\nand set types support this as well as dictionary, for which ``in``\ntests whether a the dictionary has a given key. For container types\nsuch as list, tuple, set, frozenset, dict, or collections.deque, the\nexpression ``x in y`` is equivalent to ``any(x is e or x == e for e in\ny)``.\n\nFor the string and bytes types, ``x in y`` is true if and only if *x*\nis a substring of *y*. An equivalent test is ``y.find(x) != -1``.\nEmpty strings are always considered to be a substring of any other\nstring, so ``"" in "abc"`` will return ``True``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``. If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception. (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object. ``x is\nnot y`` yields the inverse truth value. [4]\n', - 'compound': '\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way. In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs. ``try`` specifies exception handlers and/or\ncleanup code for a group of statements, while the ``with`` statement\nallows the execution of initialization and finalization code around a\nblock of code. Function and class definitions are also syntactically\ncompound statements.\n\nCompound statements consist of one or more \'clauses.\' A clause\nconsists of a header and a \'suite.\' The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon. A suite is a group of statements controlled by a\nclause. A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines. Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n if test1: if test2: print(x)\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print()`` calls are executed:\n\n if x < y < z: print(x); print(y); print(z)\n\nSummarizing:\n\n compound_stmt ::= if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n statement ::= stmt_list NEWLINE | compound_stmt\n stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``. Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted. When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a ``StopIteration``\nexception), the suite in the ``else`` clause, if present, is executed,\nand the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, it will not have been assigned to at all\nby the loop. Hint: the built-in function ``range()`` returns an\niterator of integers suitable to emulate the effect of Pascal\'s ``for\ni := a to b do``; e.g., ``list(range(3))`` returns the list ``[0, 1,\n2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An\n internal counter is used to keep track of which item is used next,\n and this is incremented on each iteration. When this counter has\n reached the length of the sequence the loop terminates. This means\n that if the suite deletes the current (or a previous) item from the\n sequence, the next item will be skipped (since it gets the index of\n the current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression ["as" target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object or a tuple containing an item compatible with the\nexception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the ``as`` keyword in that except clause,\nif present, and the except clause\'s suite is executed. All except\nclauses must have an executable block. When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using ``as target``, it is cleared\nat the end of the except clause. This is as if\n\n except E as N:\n foo\n\nwas translated to\n\n except E as N:\n try:\n foo\n finally:\n del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause. Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the ``sys`` module and can be access via\n``sys.exc_info()``. ``sys.exc_info()`` returns a 3-tuple consisting of\nthe exception class, the exception instance and a traceback object\n(see section *The standard type hierarchy*) identifying the point in\nthe program where the exception occurred. ``sys.exc_info()`` values\nare restored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost. The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the ``with_item``)\n is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n funcdef ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" [parameter] ("," defparameter)*\n [, "**" parameter]\n | "**" parameter\n | defparameter [","] )\n parameter ::= identifier [":" expression]\n defparameter ::= parameter ["=" expression]\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters up until the "``*``" must also have a default value ---\nthis is a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after "``*``" or "``*identifier``" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "``: expression``"\nfollowing the parameter name. Any parameter may have an annotation\neven those of the form ``*identifier`` or ``**identifier``. Functions\nmay have "return" annotation of the form "``-> expression``" after the\nparameter list. These annotations can be any valid Python expression\nand are evaluated when the function definition is executed.\nAnnotations may be evaluated in a different order than they appear in\nthe source code. The presence of annotations does not change the\nsemantics of a function. The annotation values are available as\nvalues of a dictionary keyed by the parameters\' names in the\n``__annotations__`` attribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= [decorators] "class" classname [inheritance] ":" suite\n inheritance ::= "(" [argument_list [","] | comprehension] ")"\n classname ::= identifier\n\nA class definition is an executable statement. The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing. Classes without an inheritance\nlist inherit, by default, from the base class ``object``; hence,\n\n class Foo:\n pass\n\nis equivalent to\n\n class Foo(object):\n pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.) When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n @f1(arg)\n @f2\n class Foo: pass\n\nis equivalent to\n\n class Foo: pass\n Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators. The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances. Instance attributes\ncan be set in a method with ``self.name = value``. Both class and\ninstance attributes are accessible through the notation\n"``self.name``", and an instance attribute hides a class attribute\nwith the same name when accessed in this way. Class attributes can be\nused as defaults for instance attributes, but using mutable values\nthere can lead to unexpected results. *Descriptors* can be used to\ncreate instance variables with different implementation details.\n\nSee also:\n\n **PEP 3115** - Metaclasses in Python 3 **PEP 3129** - Class\n Decorators\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack unless there\n is a ``finally`` clause which happens to raise another exception.\n That new exception causes the old one to be lost.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', + 'compound': '\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way. In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe ``if``, ``while`` and ``for`` statements implement traditional\ncontrol flow constructs. ``try`` specifies exception handlers and/or\ncleanup code for a group of statements, while the ``with`` statement\nallows the execution of initialization and finalization code around a\nblock of code. Function and class definitions are also syntactically\ncompound statements.\n\nCompound statements consist of one or more \'clauses.\' A clause\nconsists of a header and a \'suite.\' The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon. A suite is a group of statements controlled by a\nclause. A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines. Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which ``if`` clause a following ``else`` clause would belong:\n\n if test1: if test2: print(x)\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n``print()`` calls are executed:\n\n if x < y < z: print(x); print(y); print(z)\n\nSummarizing:\n\n compound_stmt ::= if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n statement ::= stmt_list NEWLINE | compound_stmt\n stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a ``NEWLINE`` possibly followed by\na ``DEDENT``. Also note that optional continuation clauses always\nbegin with a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling ``else``\' problem is solved in Python by\nrequiring nested ``if`` statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe ``if`` statement\n====================\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n\n\nThe ``while`` statement\n=======================\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n\n\nThe ``for`` statement\n=====================\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted. When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a ``StopIteration``\nexception), the suite in the ``else`` clause, if present, is executed,\nand the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, it will not have been assigned to at all\nby the loop. Hint: the built-in function ``range()`` returns an\niterator of integers suitable to emulate the effect of Pascal\'s ``for\ni := a to b do``; e.g., ``list(range(3))`` returns the list ``[0, 1,\n2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An\n internal counter is used to keep track of which item is used next,\n and this is incremented on each iteration. When this counter has\n reached the length of the sequence the loop terminates. This means\n that if the suite deletes the current (or a previous) item from the\n sequence, the next item will be skipped (since it gets the index of\n the current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n\n\nThe ``try`` statement\n=====================\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression ["as" target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object or a tuple containing an item compatible with the\nexception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the ``as`` keyword in that except clause,\nif present, and the except clause\'s suite is executed. All except\nclauses must have an executable block. When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using ``as target``, it is cleared\nat the end of the except clause. This is as if\n\n except E as N:\n foo\n\nwas translated to\n\n except E as N:\n try:\n foo\n finally:\n del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause. Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the ``sys`` module and can be access via\n``sys.exc_info()``. ``sys.exc_info()`` returns a 3-tuple consisting of\nthe exception class, the exception instance and a traceback object\n(see section *The standard type hierarchy*) identifying the point in\nthe program where the exception occurred. ``sys.exc_info()`` values\nare restored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is lost. The exception information is not available to the\nprogram during execution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe ``with`` statement\n======================\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the ``with_item``)\n is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n funcdef ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" [parameter] ("," defparameter)*\n [, "**" parameter]\n | "**" parameter\n | defparameter [","] )\n parameter ::= identifier [":" expression]\n defparameter ::= parameter ["=" expression]\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more parameters have the form *parameter* ``=``\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding argument may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters up until the "``*``" must also have a default value ---\nthis is a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that that same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after "``*``" or "``*identifier``" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "``: expression``"\nfollowing the parameter name. Any parameter may have an annotation\neven those of the form ``*identifier`` or ``**identifier``. Functions\nmay have "return" annotation of the form "``-> expression``" after the\nparameter list. These annotations can be any valid Python expression\nand are evaluated when the function definition is executed.\nAnnotations may be evaluated in a different order than they appear in\nthe source code. The presence of annotations does not change the\nsemantics of a function. The annotation values are available as\nvalues of a dictionary keyed by the parameters\' names in the\n``__annotations__`` attribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= [decorators] "class" classname [inheritance] ":" suite\n inheritance ::= "(" [argument_list [","] | comprehension] ")"\n classname ::= identifier\n\nA class definition is an executable statement. The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing. Classes without an inheritance\nlist inherit, by default, from the base class ``object``; hence,\n\n class Foo:\n pass\n\nis equivalent to\n\n class Foo(object):\n pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.) When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n @f1(arg)\n @f2\n class Foo: pass\n\nis equivalent to\n\n class Foo: pass\n Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators. The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances. Instance attributes\ncan be set in a method with ``self.name = value``. Both class and\ninstance attributes are accessible through the notation\n"``self.name``", and an instance attribute hides a class attribute\nwith the same name when accessed in this way. Class attributes can be\nused as defaults for instance attributes, but using mutable values\nthere can lead to unexpected results. *Descriptors* can be used to\ncreate instance variables with different implementation details.\n\nSee also:\n\n **PEP 3115** - Metaclasses in Python 3 **PEP 3129** - Class\n Decorators\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack only if there\n is no ``finally`` clause that negates the exception.\n\n[2] Currently, control "flows off the end" except in the case of an\n exception or the execution of a ``return``, ``continue``, or\n ``break`` statement.\n\n[3] A string literal appearing as the first statement in the function\n body is transformed into the function\'s ``__doc__`` attribute and\n therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s ``__doc__`` item and\n therefore the class\'s *docstring*.\n', 'context-managers': '\nWith Statement Context Managers\n*******************************\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', 'continue': '\nThe ``continue`` statement\n**************************\n\n continue_stmt ::= "continue"\n\n``continue`` may only occur syntactically nested in a ``for`` or\n``while`` loop, but not nested in a function or class definition or\n``finally`` clause within that loop. It continues with the next cycle\nof the nearest enclosing loop.\n\nWhen ``continue`` passes control out of a ``try`` statement with a\n``finally`` clause, that ``finally`` clause is executed before really\nstarting the next loop cycle.\n', 'conversions': '\nArithmetic conversions\n**********************\n\nWhen a description of an arithmetic operator below uses the phrase\n"the numeric arguments are converted to a common type," this means\nthat the operator implementation for built-in types works that way:\n\n* If either argument is a complex number, the other is converted to\n complex;\n\n* otherwise, if either argument is a floating point number, the other\n is converted to floating point;\n\n* otherwise, both must be integers and no conversion is necessary.\n\nSome additional rules apply for certain operators (e.g., a string left\nargument to the \'%\' operator). Extensions must define their own\nconversion behavior.\n', @@ -60,7 +60,7 @@ topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAss 'slicings': '\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list). Slicings may be used as expressions or as\ntargets in assignment or ``del`` statements. The syntax for a\nslicing:\n\n slicing ::= primary "[" slice_list "]"\n slice_list ::= slice_item ("," slice_item)* [","]\n slice_item ::= expression | proper_slice\n proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" [stride] ]\n lower_bound ::= expression\n upper_bound ::= expression\n stride ::= expression\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing. Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice).\n\nThe semantics for a slicing are as follows. The primary must evaluate\nto a mapping object, and it is indexed (using the same\n``__getitem__()`` method as normal subscription) with a key that is\nconstructed from the slice list, as follows. If the slice list\ncontains at least one comma, the key is a tuple containing the\nconversion of the slice items; otherwise, the conversion of the lone\nslice item is the key. The conversion of a slice item that is an\nexpression is that expression. The conversion of a proper slice is a\nslice object (see section *The standard type hierarchy*) whose\n``start``, ``stop`` and ``step`` attributes are the values of the\nexpressions given as lower bound, upper bound and stride,\nrespectively, substituting ``None`` for missing expressions.\n', 'specialattrs': "\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant. Some of these are not reported\nby the ``dir()`` built-in function.\n\nobject.__dict__\n\n A dictionary or other mapping object used to store an object's\n (writable) attributes.\n\ninstance.__class__\n\n The class to which a class instance belongs.\n\nclass.__bases__\n\n The tuple of base classes of a class object.\n\nclass.__name__\n\n The name of the class or type.\n\nThe following attributes are only supported by *new-style class*es.\n\nclass.__mro__\n\n This attribute is a tuple of classes that are considered when\n looking for base classes during method resolution.\n\nclass.mro()\n\n This method can be overridden by a metaclass to customize the\n method resolution order for its instances. It is called at class\n instantiation, and its result is stored in ``__mro__``.\n\nclass.__subclasses__()\n\n Each new-style class keeps a list of weak references to its\n immediate subclasses. This method returns a list of all those\n references still alive. Example:\n\n >>> int.__subclasses__()\n [<type 'bool'>]\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found in\n the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list ``[1, 2]`` is considered equal to\n ``[1.0, 2.0]``, and similarly for tuples.\n\n[3] They must have since the parser can't tell the type of the\n operands.\n\n[4] To format only a tuple you should therefore provide a singleton\n tuple whose only element is the tuple to be formatted.\n", 'specialnames': '\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``type(x).__getitem__(x,\ni)``. Except where mentioned, attempts to execute an operation raise\nan exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_info()[2]`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.last_traceback``. Circular references which are garbage are\n detected when the option cycle detector is enabled (it\'s on by\n default), but can only be cleaned up if there are no Python-\n level ``__del__()`` methods involved. Refer to the documentation\n for the ``gc`` module for more information about how\n ``__del__()`` methods are handled by the cycle detector,\n particularly the description of the ``garbage`` value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function to compute the\n "official" string representation of an object. If at all possible,\n this should look like a valid Python expression that could be used\n to recreate an object with the same value (given an appropriate\n environment). If this is not possible, a string of the form\n ``<...some useful description...>`` should be returned. The return\n value must be a string object. If a class defines ``__repr__()``\n but not ``__str__()``, then ``__repr__()`` is also used when an\n "informal" string representation of instances of that class is\n required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print()``\n function to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__format__(self, format_spec)\n\n Called by the ``format()`` built-in function (and by extension, the\n ``format()`` method of class ``str``) to produce a "formatted"\n string representation of an object. The ``format_spec`` argument is\n a string that contains a description of the formatting options\n desired. The interpretation of the ``format_spec`` argument is up\n to the type implementing ``__format__()``, however most classes\n will either delegate formatting to one of the built-in types, or\n use a similar formatting option syntax.\n\n See *Format Specification Mini-Language* for a description of the\n standard formatting syntax.\n\n The return value must be a string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n These are the so-called "rich comparison" methods. The\n correspondence between operator symbols and method names is as\n follows: ``x<y`` calls ``x.__lt__(y)``, ``x<=y`` calls\n ``x.__le__(y)``, ``x==y`` calls ``x.__eq__(y)``, ``x!=y`` calls\n ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and ``x>=y`` calls\n ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define an ``__eq__()`` method it should not\n define a ``__hash__()`` operation either; if it defines\n ``__eq__()`` but not ``__hash__()``, its instances will not be\n usable as items in hashable collections. If a class defines\n mutable objects and implements an ``__eq__()`` method, it should\n not implement ``__hash__()``, since the implementation of hashable\n collections requires that a key\'s hash value is immutable (if the\n object\'s hash value changes, it will be in the wrong hash bucket).\n\n User-defined classes have ``__eq__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__eq__()`` such that the hash value\n returned is no longer appropriate (e.g. by switching to a value-\n based concept of equality instead of the default identity based\n equality) can explicitly flag themselves as being unhashable by\n setting ``__hash__ = None`` in the class definition. Doing so means\n that not only will instances of the class raise an appropriate\n ``TypeError`` when a program attempts to retrieve their hash value,\n but they will also be correctly identified as unhashable when\n checking ``isinstance(obj, collections.Hashable)`` (unlike classes\n which define their own ``__hash__()`` to explicitly raise\n ``TypeError``).\n\n If a class that overrides ``__eq__()`` needs to retain the\n implementation of ``__hash__()`` from a parent class, the\n interpreter must be told this explicitly by setting ``__hash__ =\n <ParentClass>.__hash__``. Otherwise the inheritance of\n ``__hash__()`` will be blocked, just as if ``__hash__`` had been\n explicitly set to ``None``.\n\nobject.__bool__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``. When this method\n is not defined, ``__len__()`` is called, if it is defined, and the\n object is considered true if its result is nonzero. If a class\n defines neither ``__len__()`` nor ``__bool__()``, all its instances\n are considered true.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control over attribute access.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should call the base class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\nobject.__dir__(self)\n\n Called when ``dir()`` is called on the object. A list must be\n returned.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to an object instance, ``a.x`` is transformed into the\n call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a class, ``A.x`` is transformed into the call:\n ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of classes have a dictionary for attribute\nstorage. This wastes space for objects having very few instance\nvariables. The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable. Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n class, *__slots__* reserves space for the declared variables and\n prevents the automatic creation of *__dict__* and *__weakref__* for\n each instance.\n\n\nNotes on using *__slots__*\n~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``int``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, classes are constructed using ``type()``. A class\ndefinition is read into a separate namespace and the value of class\nname is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if a callable ``metaclass`` keyword\nargument is passed after the bases in the class definition, the\ncallable given will be called instead of ``type()``. If other keyword\narguments are passed, they will also be passed to the metaclass. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties. This example adds a new\nelement to the class dictionary before creating the class:\n\n class metacls(type):\n def __new__(mcs, name, bases, dict):\n dict[\'foo\'] = \'metacls was here\'\n return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\nIf the metaclass has a ``__prepare__()`` attribute (usually\nimplemented as a class or static method), it is called before the\nclass body is evaluated with the name of the class and a tuple of its\nbases for arguments. It should return an object that supports the\nmapping interface that will be used to store the namespace of the\nclass. The default is a plain dictionary. This could be used, for\nexample, to keep track of the order that class attributes are declared\nin by returning an ordered dictionary.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If the ``metaclass`` keyword argument is passed with the bases, it\n is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n used.\n\n* Otherwise, the default metaclass (``type``) is used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\nHere is an example of a metaclass that uses an\n``collections.OrderedDict`` to remember the order that class members\nwere defined:\n\n class OrderedClass(type):\n\n @classmethod\n def __prepare__(metacls, name, bases, **kwds):\n return collections.OrderedDict()\n\n def __new__(cls, name, bases, classdict):\n result = type.__new__(cls, name, bases, dict(classdict))\n result.members = tuple(classdict)\n return result\n\n class A(metaclass=OrderedClass):\n def one(self): pass\n def two(self): pass\n def three(self): pass\n def four(self): pass\n\n >>> A.members\n (\'__module__\', \'one\', \'two\', \'three\', \'four\')\n\nWhen the class definition for *A* gets executed, the process begins\nwith calling the metaclass\'s ``__prepare__()`` method which returns an\nempty ``collections.OrderedDict``. That mapping records the methods\nand attributes of *A* as they are defined within the body of the class\nstatement. Once those definitions are executed, the ordered dictionary\nis fully populated and the metaclass\'s ``__new__()`` method gets\ninvoked. That method builds the new type and it saves the ordered\ndictionary keys in an attribute called ``members``.\n\n\nCustomizing instance and subclass checks\n========================================\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n **PEP 3119** - Introducing Abstract Base Classes\n Includes the specification for customizing ``isinstance()`` and\n ``issubclass()`` behavior through ``__instancecheck__()`` and\n ``__subclasscheck__()``, with motivation for this functionality\n in the context of adding Abstract Base Classes (see the ``abc``\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``get()``,\n``clear()``, ``setdefault()``, ``pop()``, ``popitem()``, ``copy()``,\nand ``update()`` behaving similar to those for Python\'s standard\ndictionary objects. The ``collections`` module provides a\n``MutableMapping`` abstract base class to help create those methods\nfrom a base set of ``__getitem__()``, ``__setitem__()``,\n``__delitem__()``, and ``keys()``. Mutable sequences should provide\nmethods ``append()``, ``count()``, ``index()``, ``extend()``,\n``insert()``, ``pop()``, ``remove()``, ``reverse()`` and ``sort()``,\nlike Python standard list objects. Finally, sequence types should\nimplement addition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods ``__add__()``, ``__radd__()``,\n``__iadd__()``, ``__mul__()``, ``__rmul__()`` and ``__imul__()``\ndescribed below; they should not define other numerical operators. It\nis recommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should search the mapping\'s keys; for\nsequences, it should search through the values. It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``keys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function ``len()``. Should return\n the length of the object, an integer ``>=`` 0. Also, an object\n that doesn\'t define a ``__bool__()`` method and whose ``__len__()``\n method returns zero is considered to be false in a Boolean context.\n\nNote: Slicing is done exclusively with the following three methods. A\n call like\n\n a[1:2] = b\n\n is translated to\n\n a[slice(1, 2, None)] = b\n\n and so forth. Missing slice items are always filled in with\n ``None``.\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of ``self[key]``. For sequence\n types, the accepted keys should be integers and slice objects.\n Note that the special interpretation of negative indexes (if the\n class wishes to emulate a sequence type) is up to the\n ``__getitem__()`` method. If *key* is of an inappropriate type,\n ``TypeError`` may be raised; if of a value outside the set of\n indexes for the sequence (after any special interpretation of\n negative values), ``IndexError`` should be raised. For mapping\n types, if *key* is missing (not in the container), ``KeyError``\n should be raised.\n\n Note: ``for`` loops expect that an ``IndexError`` will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the ``__getitem__()``\n method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method ``keys()``.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the ``reversed()`` built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the ``__reversed__()`` method is not provided, the\n ``reversed()`` built-in will fall back to using the sequence\n protocol (``__len__()`` and ``__getitem__()``). Objects that\n support the sequence protocol should only provide\n ``__reversed__()`` if they can provide an implementation that is\n more efficient than the one provided by ``reversed()``.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define ``__contains__()``, the membership\n test first tries iteration via ``__iter__()``, then the old\n sequence iteration protocol via ``__getitem__()``, see *this\n section in the language reference*.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``,\n ``divmod()``, ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``,\n ``|``). For instance, to evaluate the expression ``x + y``, where\n *x* is an instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()``. Note that\n ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``,\n ``divmod()``, ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``,\n ``|``) with reflected (swapped) operands. These functions are only\n called if the left operand does not support the corresponding\n operation and the operands are of different types. [2] For\n instance, to evaluate the expression ``x - y``, where *y* is an\n instance of a class that has an ``__rsub__()`` method,\n ``y.__rsub__(x)`` is called if ``x.__sub__(y)`` returns\n *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``float()`` and ``round()``. Should return a value of\n the appropriate type.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing, or in the\n built-in ``bin()``, ``hex()`` and ``oct()`` functions). Must return\n an integer.\n\n\nWith Statement Context Managers\n===============================\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nSpecial method lookup\n=====================\n\nFor custom classes, implicit invocations of special methods are only\nguaranteed to work correctly if defined on an object\'s type, not in\nthe object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception:\n\n >>> class C:\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print("Metaclass getattribute invoked")\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object, metaclass=Meta):\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print("Class getattribute invoked")\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n certain controlled conditions. It generally isn\'t a good idea\n though, since it can lead to some very strange behaviour if it is\n handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as ``__add__()``) fails the operation is\n not supported, which is why the reflected method is not called.\n', - 'string-methods': '\nString Methods\n**************\n\nString objects support the methods listed below.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, bytes, bytearray, list,\ntuple, range* section. To output formatted strings, see the *String\nFormatting* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.encode(encoding="utf-8", errors="strict")\n\n Return an encoded version of the string as a bytes object. Default\n encoding is ``\'utf-8\'``. *errors* may be given to set a different\n error handling scheme. The default for *errors* is ``\'strict\'``,\n meaning that encoding errors raise a ``UnicodeError``. Other\n possible values are ``\'ignore\'``, ``\'replace\'``,\n ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and any other name\n registered via ``codecs.register_error()``, see section *Codec Base\n Classes*. For a list of possible encodings, see section *Standard\n Encodings*.\n\n Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\nstr.format_map(mapping)\n\n Similar to ``str.format(**mapping)``, except that ``mapping`` is\n used directly and not copied to a ``dict`` . This is useful if for\n example ``mapping`` is a dict subclass:\n\n >>> class Default(dict):\n ... def __missing__(self, key):\n ... return key\n ...\n >>> \'{name} was born in {country}\'.format_map(Default(name=\'Guido\'))\n \'Guido was born in country\'\n\n New in version 3.2.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise. A character\n ``c`` is alphanumeric if one of the following returns ``True``:\n ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or\n ``c.isnumeric()``.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise. Alphabetic\n characters are those characters defined in the Unicode character\n database as "Letter", i.e., those with general category property\n being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is\n different from the "Alphabetic" property defined in the Unicode\n Standard.\n\nstr.isdecimal()\n\n Return true if all characters in the string are decimal characters\n and there is at least one character, false otherwise. Decimal\n characters are those from general category "Nd". This category\n includes digit characters, and all characters that that can be used\n to form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT\n ZERO.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise. Digits include decimal\n characters and digits that need special handling, such as the\n compatibility superscript digits. Formally, a digit is a character\n that has the property value Numeric_Type=Digit or\n Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n Return true if the string is a valid identifier according to the\n language definition, section *Identifiers and keywords*.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and lowercase characters are those with general\n category property "Ll".\n\nstr.isnumeric()\n\n Return true if all characters in the string are numeric characters,\n and there is at least one character, false otherwise. Numeric\n characters include digit characters, and all characters that have\n the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n ONE FIFTH. Formally, numeric characters are those with the\n property value Numeric_Type=Digit, Numeric_Type=Decimal or\n Numeric_Type=Numeric.\n\nstr.isprintable()\n\n Return true if all characters in the string are printable or the\n string is empty, false otherwise. Nonprintable characters are\n those characters defined in the Unicode character database as\n "Other" or "Separator", excepting the ASCII space (0x20) which is\n considered printable. (Note that printable characters in this\n context are those which should not be escaped when ``repr()`` is\n invoked on a string. It has no bearing on the handling of strings\n written to ``sys.stdout`` or ``sys.stderr``.)\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise. Whitespace\n characters are those characters defined in the Unicode character\n database as "Other" or "Separator" and those with bidirectional\n property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and uppercase characters are those with general\n category property "Lu".\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. A ``TypeError`` will be raised if there are\n any non-string values in *seq*, including ``bytes`` objects. The\n separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n This static method returns a translation table usable for\n ``str.translate()``.\n\n If there is only one argument, it must be a dictionary mapping\n Unicode ordinals (integers) or characters (strings of length 1) to\n Unicode ordinals, strings (of arbitrary lengths) or None.\n Character keys will then be converted to ordinals.\n\n If there are two arguments, they must be strings of equal length,\n and in the resulting dictionary, each character in x will be mapped\n to the character at the same position in y. If there is a third\n argument, it must be a string, whose characters will be mapped to\n None in the result.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n Return a copy of the *s* where all characters have been mapped\n through the *map* which must be a dictionary of Unicode ordinals\n (integers) to Unicode ordinals, strings or ``None``. Unmapped\n characters are left untouched. Characters mapped to ``None`` are\n deleted.\n\n You can use ``str.maketrans()`` to create a translation map from\n character-to-character mappings in different formats.\n\n Note: An even more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see\n ``encodings.cp1251`` for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n', + 'string-methods': '\nString Methods\n**************\n\nString objects support the methods listed below.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, bytes, bytearray, list,\ntuple, range* section. To output formatted strings, see the *String\nFormatting* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.encode(encoding="utf-8", errors="strict")\n\n Return an encoded version of the string as a bytes object. Default\n encoding is ``\'utf-8\'``. *errors* may be given to set a different\n error handling scheme. The default for *errors* is ``\'strict\'``,\n meaning that encoding errors raise a ``UnicodeError``. Other\n possible values are ``\'ignore\'``, ``\'replace\'``,\n ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and any other name\n registered via ``codecs.register_error()``, see section *Codec Base\n Classes*. For a list of possible encodings, see section *Standard\n Encodings*.\n\n Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\nstr.format_map(mapping)\n\n Similar to ``str.format(**mapping)``, except that ``mapping`` is\n used directly and not copied to a ``dict`` . This is useful if for\n example ``mapping`` is a dict subclass:\n\n >>> class Default(dict):\n ... def __missing__(self, key):\n ... return key\n ...\n >>> \'{name} was born in {country}\'.format_map(Default(name=\'Guido\'))\n \'Guido was born in country\'\n\n New in version 3.2.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise. A character\n ``c`` is alphanumeric if one of the following returns ``True``:\n ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or\n ``c.isnumeric()``.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise. Alphabetic\n characters are those characters defined in the Unicode character\n database as "Letter", i.e., those with general category property\n being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is\n different from the "Alphabetic" property defined in the Unicode\n Standard.\n\nstr.isdecimal()\n\n Return true if all characters in the string are decimal characters\n and there is at least one character, false otherwise. Decimal\n characters are those from general category "Nd". This category\n includes digit characters, and all characters that that can be used\n to form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT\n ZERO.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise. Digits include decimal\n characters and digits that need special handling, such as the\n compatibility superscript digits. Formally, a digit is a character\n that has the property value Numeric_Type=Digit or\n Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n Return true if the string is a valid identifier according to the\n language definition, section *Identifiers and keywords*.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and lowercase characters are those with general\n category property "Ll".\n\nstr.isnumeric()\n\n Return true if all characters in the string are numeric characters,\n and there is at least one character, false otherwise. Numeric\n characters include digit characters, and all characters that have\n the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n ONE FIFTH. Formally, numeric characters are those with the\n property value Numeric_Type=Digit, Numeric_Type=Decimal or\n Numeric_Type=Numeric.\n\nstr.isprintable()\n\n Return true if all characters in the string are printable or the\n string is empty, false otherwise. Nonprintable characters are\n those characters defined in the Unicode character database as\n "Other" or "Separator", excepting the ASCII space (0x20) which is\n considered printable. (Note that printable characters in this\n context are those which should not be escaped when ``repr()`` is\n invoked on a string. It has no bearing on the handling of strings\n written to ``sys.stdout`` or ``sys.stderr``.)\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise. Whitespace\n characters are those characters defined in the Unicode character\n database as "Other" or "Separator" and those with bidirectional\n property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and uppercase characters are those with general\n category property "Lu".\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. A ``TypeError`` will be raised if there are\n any non-string values in *seq*, including ``bytes`` objects. The\n separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n This static method returns a translation table usable for\n ``str.translate()``.\n\n If there is only one argument, it must be a dictionary mapping\n Unicode ordinals (integers) or characters (strings of length 1) to\n Unicode ordinals, strings (of arbitrary lengths) or None.\n Character keys will then be converted to ordinals.\n\n If there are two arguments, they must be strings of equal length,\n and in the resulting dictionary, each character in x will be mapped\n to the character at the same position in y. If there is a third\n argument, it must be a string, whose characters will be mapped to\n None in the result.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n Return a copy of the *s* where all characters have been mapped\n through the *map* which must be a dictionary of Unicode ordinals\n (integers) to Unicode ordinals, strings or ``None``. Unmapped\n characters are left untouched. Characters mapped to ``None`` are\n deleted.\n\n You can use ``str.maketrans()`` to create a translation map from\n character-to-character mappings in different formats.\n\n Note: An even more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see\n ``encodings.cp1251`` for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n', 'strings': '\nString and Bytes literals\n*************************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "R"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'" | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | stringescapeseq\n longstringitem ::= longstringchar | stringescapeseq\n shortstringchar ::= <any source character except "\\" or newline or the quote>\n longstringchar ::= <any source character except "\\">\n stringescapeseq ::= "\\" <any source character>\n\n bytesliteral ::= bytesprefix(shortbytes | longbytes)\n bytesprefix ::= "b" | "B" | "br" | "Br" | "bR" | "BR"\n shortbytes ::= "\'" shortbytesitem* "\'" | \'"\' shortbytesitem* \'"\'\n longbytes ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' longbytesitem* \'"""\'\n shortbytesitem ::= shortbyteschar | bytesescapeseq\n longbytesitem ::= longbyteschar | bytesescapeseq\n shortbyteschar ::= <any ASCII character except "\\" or newline or the quote>\n longbyteschar ::= <any ASCII character except "\\">\n bytesescapeseq ::= "\\" <any ASCII character>\n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the ``stringprefix`` or\n``bytesprefix`` and the rest of the literal. The source character set\nis defined by the encoding declaration; it is UTF-8 if no encoding\ndeclaration is given in the source file; see section *Encoding\ndeclarations*.\n\nIn plain English: Both types of literals can be enclosed in matching\nsingle quotes (``\'``) or double quotes (``"``). They can also be\nenclosed in matching groups of three single or double quotes (these\nare generally referred to as *triple-quoted strings*). The backslash\n(``\\``) character is used to escape characters that otherwise have a\nspecial meaning, such as newline, backslash itself, or the quote\ncharacter.\n\nBytes literals are always prefixed with ``\'b\'`` or ``\'B\'``; they\nproduce an instance of the ``bytes`` type instead of the ``str`` type.\nThey may only contain ASCII characters; bytes with a numeric value of\n128 or greater must be expressed with escapes.\n\nBoth string and bytes literals may optionally be prefixed with a\nletter ``\'r\'`` or ``\'R\'``; such strings are called *raw strings* and\ntreat backslashes as literal characters. As a result, in string\nliterals, ``\'\\U\'`` and ``\'\\u\'`` escapes in raw strings are not treated\nspecially.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string. (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\newline`` | Backslash and newline ignored | |\n+-------------------+-----------------------------------+---------+\n| ``\\\\`` | Backslash (``\\``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\\'`` | Single quote (``\'``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\"`` | Double quote (``"``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\a`` | ASCII Bell (BEL) | |\n+-------------------+-----------------------------------+---------+\n| ``\\b`` | ASCII Backspace (BS) | |\n+-------------------+-----------------------------------+---------+\n| ``\\f`` | ASCII Formfeed (FF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\n`` | ASCII Linefeed (LF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\r`` | ASCII Carriage Return (CR) | |\n+-------------------+-----------------------------------+---------+\n| ``\\t`` | ASCII Horizontal Tab (TAB) | |\n+-------------------+-----------------------------------+---------+\n| ``\\v`` | ASCII Vertical Tab (VT) | |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo`` | Character with octal value *ooo* | (1,3) |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh`` | Character with hex value *hh* | (2,3) |\n+-------------------+-----------------------------------+---------+\n\nEscape sequences only recognized in string literals are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\N{name}`` | Character named *name* in the | |\n| | Unicode database | |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx`` | Character with 16-bit hex value | (4) |\n| | *xxxx* | |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx`` | Character with 32-bit hex value | (5) |\n| | *xxxxxxxx* | |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. As in Standard C, up to three octal digits are accepted.\n\n2. Unlike in Standard C, exactly two hex digits are required.\n\n3. In a bytes literal, hexadecimal and octal escapes denote the byte\n with the given value. In a string literal, these escapes denote a\n Unicode character with the given value.\n\n4. Individual code units which form parts of a surrogate pair can be\n encoded using this escape sequence. Exactly four hex digits are\n required.\n\n5. Any Unicode character can be encoded this way, but characters\n outside the Basic Multilingual Plane (BMP) will be encoded using a\n surrogate pair if Python is compiled to use 16-bit code units (the\n default). Exactly eight hex digits are required.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences only recognized in string\nliterals fall into the category of unrecognized escapes for bytes\nliterals.\n\nEven in a raw string, string quotes can be escaped with a backslash,\nbut the backslash remains in the string; for example, ``r"\\""`` is a\nvalid string literal consisting of two characters: a backslash and a\ndouble quote; ``r"\\"`` is not a valid string literal (even a raw\nstring cannot end in an odd number of backslashes). Specifically, *a\nraw string cannot end in a single backslash* (since the backslash\nwould escape the following quote character). Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n', 'subscriptions': '\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object that supports subscription,\ne.g. a list or dictionary. User-defined objects can support\nsubscription by defining a ``__getitem__()`` method.\n\nFor built-in objects, there are two types of objects that support\nsubscription:\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey. (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to\nan integer or a slice (as discussed in the following section).\n\nThe formal syntax makes no special provision for negative indices in\nsequences; however, built-in sequences all provide a ``__getitem__()``\nmethod that interprets negative indices by adding the length of the\nsequence to the index (so that ``x[-1]`` selects the last item of\n``x``). The resulting value must be a nonnegative integer less than\nthe number of items in the sequence, and the subscription selects the\nitem whose index is that value (counting from zero). Since the support\nfor negative indices and slicing occurs in the object\'s\n``__getitem__()`` method, subclasses overriding this method will need\nto explicitly add that support.\n\nA string\'s items are characters. A character is not a separate data\ntype but a string of exactly one character.\n', 'truth': "\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an ``if`` or\n``while`` condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* ``None``\n\n* ``False``\n\n* zero of any numeric type, for example, ``0``, ``0.0``, ``0j``.\n\n* any empty sequence, for example, ``''``, ``()``, ``[]``.\n\n* any empty mapping, for example, ``{}``.\n\n* instances of user-defined classes, if the class defines a\n ``__bool__()`` or ``__len__()`` method, when that method returns the\n integer zero or ``bool`` value ``False``. [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn ``0`` or ``False`` for false and ``1`` or ``True`` for true,\nunless otherwise stated. (Important exception: the Boolean operations\n``or`` and ``and`` always return one of their operands.)\n", @@ -70,8 +70,8 @@ topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAss 'typesmapping': '\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry. (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n Return a new dictionary initialized from an optional positional\n argument or from a set of keyword arguments. If no arguments are\n given, return a new empty dictionary. If the positional argument\n *arg* is a mapping object, return a dictionary mapping the same\n keys to the same values as does the mapping object. Otherwise the\n positional argument must be a sequence, a container that supports\n iteration, or an iterator object. The elements of the argument\n must each also be of one of those kinds, and each must in turn\n contain exactly two objects. The first is used as a key in the new\n dictionary, and the second as the key\'s value. If a given key is\n seen more than once, the last value associated with it is retained\n in the new dictionary.\n\n If keyword arguments are given, the keywords themselves with their\n associated values are added as items to the dictionary. If a key\n is specified both in the positional argument and as a keyword\n argument, the value associated with the keyword is retained in the\n dictionary. For example, these all return a dictionary equal to\n ``{"one": 1, "two": 2}``:\n\n * ``dict(one=1, two=2)``\n\n * ``dict({\'one\': 1, \'two\': 2})``\n\n * ``dict(zip((\'one\', \'two\'), (1, 2)))``\n\n * ``dict([[\'two\', 2], [\'one\', 1]])``\n\n The first example only works for keys that are valid Python\n identifiers; the others work with any valid keys.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a ``KeyError`` if\n *key* is not in the map.\n\n If a subclass of dict defines a method ``__missing__()``, if the\n key *key* is not present, the ``d[key]`` operation calls that\n method with the key *key* as argument. The ``d[key]`` operation\n then returns or raises whatever is returned or raised by the\n ``__missing__(key)`` call if the key is not present. No other\n operations or methods invoke ``__missing__()``. If\n ``__missing__()`` is not defined, ``KeyError`` is raised.\n ``__missing__()`` must be a method; it cannot be an instance\n variable:\n\n >>> class Counter(dict):\n ... def __missing__(self, key):\n ... return 0\n >>> c = Counter()\n >>> c[\'red\']\n 0\n >>> c[\'red\'] += 1\n >>> c[\'red\']\n 1\n\n See ``collections.Counter`` for a complete implementation\n including other methods helpful for accumulating and managing\n tallies.\n\n d[key] = value\n\n Set ``d[key]`` to *value*.\n\n del d[key]\n\n Remove ``d[key]`` from *d*. Raises a ``KeyError`` if *key* is\n not in the map.\n\n key in d\n\n Return ``True`` if *d* has a key *key*, else ``False``.\n\n key not in d\n\n Equivalent to ``not key in d``.\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for ``iter(d.keys())``.\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n classmethod fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n ``fromkeys()`` is a class method that returns a new dictionary.\n *value* defaults to ``None``.\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to ``None``,\n so that this method never raises a ``KeyError``.\n\n items()\n\n Return a new view of the dictionary\'s items (``(key, value)``\n pairs). See below for documentation of view objects.\n\n keys()\n\n Return a new view of the dictionary\'s keys. See below for\n documentation of view objects.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a ``KeyError`` is raised.\n\n popitem()\n\n Remove and return an arbitrary ``(key, value)`` pair from the\n dictionary.\n\n ``popitem()`` is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling ``popitem()`` raises a ``KeyError``.\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to ``None``.\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return ``None``.\n\n ``update()`` accepts either another dictionary object or an\n iterable of key/value pairs (as tuples or other iterables of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: ``d.update(red=1,\n blue=2)``.\n\n values()\n\n Return a new view of the dictionary\'s values. See below for\n documentation of view objects.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.keys()``, ``dict.values()`` and\n``dict.items()`` are *view objects*. They provide a dynamic view on\nthe dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of ``(key, value)``) in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of ``(value, key)`` pairs\n using ``zip()``: ``pairs = zip(d.values(), d.keys())``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.items()]``.\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n Return ``True`` if *x* is in the underlying dictionary\'s keys,\n values or items (in the latter case, *x* should be a ``(key,\n value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that ``(key, value)`` pairs are unique\nand hashable, then the items view is also set-like. (Values views are\nnot treated as set-like since the entries are generally not unique.)\nFor set-like views, all of the operations defined for the abstract\nbase class ``collections.Set`` are available (for example, ``==``,\n``<``, or ``^``).\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.keys()\n >>> values = dishes.values()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n >>> keys ^ {\'sausage\', \'juice\'}\n {\'juice\', \'eggs\', \'bacon\', \'spam\'}\n', 'typesmethods': "\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as ``append()`` on\nlists) and class instance methods. Built-in methods are described\nwith the types that support them.\n\nIf you access a method (a function defined in a class namespace)\nthrough an instance, you get a special object: a *bound method* (also\ncalled *instance method*) object. When called, it will add the\n``self`` argument to the argument list. Bound methods have two\nspecial read-only attributes: ``m.__self__`` is the object on which\nthe method operates, and ``m.__func__`` is the function implementing\nthe method. Calling ``m(arg-1, arg-2, ..., arg-n)`` is completely\nequivalent to calling ``m.__func__(m.__self__, arg-1, arg-2, ...,\narg-n)``.\n\nLike function objects, bound method objects support getting arbitrary\nattributes. However, since method attributes are actually stored on\nthe underlying function object (``meth.__func__``), setting method\nattributes on bound methods is disallowed. Attempting to set a method\nattribute results in a ``TypeError`` being raised. In order to set a\nmethod attribute, you need to explicitly set it on the underlying\nfunction object:\n\n class C:\n def method(self):\n pass\n\n c = C()\n c.method.__func__.whoami = 'my name is c'\n\nSee *The standard type hierarchy* for more information.\n", 'typesmodules': "\nModules\n*******\n\nThe only special operation on a module is attribute access:\n``m.name``, where *m* is a module and *name* accesses a name defined\nin *m*'s symbol table. Module attributes can be assigned to. (Note\nthat the ``import`` statement is not, strictly speaking, an operation\non a module object; ``import foo`` does not require a module object\nnamed *foo* to exist, rather it requires an (external) *definition*\nfor a module named *foo* somewhere.)\n\nA special member of every module is ``__dict__``. This is the\ndictionary containing the module's symbol table. Modifying this\ndictionary will actually change the module's symbol table, but direct\nassignment to the ``__dict__`` attribute is not possible (you can\nwrite ``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but\nyou can't write ``m.__dict__ = {}``). Modifying ``__dict__`` directly\nis not recommended.\n\nModules built into the interpreter are written like this: ``<module\n'sys' (built-in)>``. If loaded from a file, they are written as\n``<module 'os' from '/usr/local/lib/pythonX.Y/os.pyc'>``.\n", - 'typesseq': '\nSequence Types --- ``str``, ``bytes``, ``bytearray``, ``list``, ``tuple``, ``range``\n************************************************************************************\n\nThere are six sequence types: strings, byte sequences (``bytes``\nobjects), byte arrays (``bytearray`` objects), lists, tuples, and\nrange objects. For other containers see the built in ``dict`` and\n``set`` classes, and the ``collections`` module.\n\nStrings contain Unicode characters. Their literals are written in\nsingle or double quotes: ``\'xyzzy\'``, ``"frobozz"``. See *String and\nBytes literals* for more about string literals. In addition to the\nfunctionality described here, there are also string-specific methods\ndescribed in the *String Methods* section.\n\nBytes and bytearray objects contain single bytes -- the former is\nimmutable while the latter is a mutable sequence. Bytes objects can\nbe constructed the constructor, ``bytes()``, and from literals; use a\n``b`` prefix with normal string syntax: ``b\'xyzzy\'``. To construct\nbyte arrays, use the ``bytearray()`` function.\n\nWhile string objects are sequences of characters (represented by\nstrings of length 1), bytes and bytearray objects are sequences of\n*integers* (between 0 and 255), representing the ASCII value of single\nbytes. That means that for a bytes or bytearray object *b*, ``b[0]``\nwill be an integer, while ``b[0:1]`` will be a bytes or bytearray\nobject of length 1. The representation of bytes objects uses the\nliteral format (``b\'...\'``) since it is generally more useful than\ne.g. ``bytes([50, 19, 100])``. You can always convert a bytes object\ninto a list of integers using ``list(b)``.\n\nAlso, while in previous Python versions, byte strings and Unicode\nstrings could be exchanged for each other rather freely (barring\nencoding issues), strings and bytes are now completely separate\nconcepts. There\'s no implicit en-/decoding if you pass an object of\nthe wrong type. A string always compares unequal to a bytes or\nbytearray object.\n\nLists are constructed with square brackets, separating items with\ncommas: ``[a, b, c]``. Tuples are constructed by the comma operator\n(not within square brackets), with or without enclosing parentheses,\nbut an empty tuple must have the enclosing parentheses, such as ``a,\nb, c`` or ``()``. A single item tuple must have a trailing comma,\nsuch as ``(d,)``.\n\nObjects of type range are created using the ``range()`` function.\nThey don\'t support concatenation or repetition, and using ``min()`` or\n``max()`` on them is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i*, *j* and *k* are\nintegers.\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*\'th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)`` | index of the first occurence of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)`` | total number of occurences of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must\ncompare equal and the two sequences must be of the same type and have\nthe same length. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string object, the ``in`` and ``not in`` operations\n act like a substring test.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n\nString Methods\n==============\n\nString objects support the methods listed below.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, bytes, bytearray, list,\ntuple, range* section. To output formatted strings, see the *String\nFormatting* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.encode(encoding="utf-8", errors="strict")\n\n Return an encoded version of the string as a bytes object. Default\n encoding is ``\'utf-8\'``. *errors* may be given to set a different\n error handling scheme. The default for *errors* is ``\'strict\'``,\n meaning that encoding errors raise a ``UnicodeError``. Other\n possible values are ``\'ignore\'``, ``\'replace\'``,\n ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and any other name\n registered via ``codecs.register_error()``, see section *Codec Base\n Classes*. For a list of possible encodings, see section *Standard\n Encodings*.\n\n Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\nstr.format_map(mapping)\n\n Similar to ``str.format(**mapping)``, except that ``mapping`` is\n used directly and not copied to a ``dict`` . This is useful if for\n example ``mapping`` is a dict subclass:\n\n >>> class Default(dict):\n ... def __missing__(self, key):\n ... return key\n ...\n >>> \'{name} was born in {country}\'.format_map(Default(name=\'Guido\'))\n \'Guido was born in country\'\n\n New in version 3.2.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise. A character\n ``c`` is alphanumeric if one of the following returns ``True``:\n ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or\n ``c.isnumeric()``.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise. Alphabetic\n characters are those characters defined in the Unicode character\n database as "Letter", i.e., those with general category property\n being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is\n different from the "Alphabetic" property defined in the Unicode\n Standard.\n\nstr.isdecimal()\n\n Return true if all characters in the string are decimal characters\n and there is at least one character, false otherwise. Decimal\n characters are those from general category "Nd". This category\n includes digit characters, and all characters that that can be used\n to form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT\n ZERO.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise. Digits include decimal\n characters and digits that need special handling, such as the\n compatibility superscript digits. Formally, a digit is a character\n that has the property value Numeric_Type=Digit or\n Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n Return true if the string is a valid identifier according to the\n language definition, section *Identifiers and keywords*.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and lowercase characters are those with general\n category property "Ll".\n\nstr.isnumeric()\n\n Return true if all characters in the string are numeric characters,\n and there is at least one character, false otherwise. Numeric\n characters include digit characters, and all characters that have\n the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n ONE FIFTH. Formally, numeric characters are those with the\n property value Numeric_Type=Digit, Numeric_Type=Decimal or\n Numeric_Type=Numeric.\n\nstr.isprintable()\n\n Return true if all characters in the string are printable or the\n string is empty, false otherwise. Nonprintable characters are\n those characters defined in the Unicode character database as\n "Other" or "Separator", excepting the ASCII space (0x20) which is\n considered printable. (Note that printable characters in this\n context are those which should not be escaped when ``repr()`` is\n invoked on a string. It has no bearing on the handling of strings\n written to ``sys.stdout`` or ``sys.stderr``.)\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise. Whitespace\n characters are those characters defined in the Unicode character\n database as "Other" or "Separator" and those with bidirectional\n property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and uppercase characters are those with general\n category property "Lu".\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. A ``TypeError`` will be raised if there are\n any non-string values in *seq*, including ``bytes`` objects. The\n separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n This static method returns a translation table usable for\n ``str.translate()``.\n\n If there is only one argument, it must be a dictionary mapping\n Unicode ordinals (integers) or characters (strings of length 1) to\n Unicode ordinals, strings (of arbitrary lengths) or None.\n Character keys will then be converted to ordinals.\n\n If there are two arguments, they must be strings of equal length,\n and in the resulting dictionary, each character in x will be mapped\n to the character at the same position in y. If there is a third\n argument, it must be a string, whose characters will be mapped to\n None in the result.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n Return a copy of the *s* where all characters have been mapped\n through the *map* which must be a dictionary of Unicode ordinals\n (integers) to Unicode ordinals, strings or ``None``. Unmapped\n characters are left untouched. Characters mapped to ``None`` are\n deleted.\n\n You can use ``str.maketrans()`` to create a translation map from\n character-to-character mappings in different formats.\n\n Note: An even more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see\n ``encodings.cp1251`` for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n\nOld String Formatting Operations\n================================\n\nNote: The formatting operations described here are obsolete and may go\n away in future versions of Python. Use the new *String Formatting*\n in new code.\n\nString objects have one unique built-in operation: the ``%`` operator\n(modulo). This is also known as the string *formatting* or\n*interpolation* operator. Given ``format % values`` (where *format* is\na string), ``%`` conversion specifications in *format* are replaced\nwith zero or more elements of *values*. The effect is similar to the\nusing ``sprintf()`` in the C language.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print(\'%(language)s has %(number)03d quote types.\' %\n... {\'language\': "Python", "number": 2})\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The precision determines the maximal number of characters used.\n\n1. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 3.1: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nRange Type\n==========\n\nThe ``range`` type is an immutable sequence which is commonly used for\nlooping. The advantage of the ``range`` type is that an ``range``\nobject will always take the same amount of memory, no matter the size\nof the range it represents.\n\nRange objects have relatively little behavior: they support indexing,\ncontains, iteration, the ``len()`` function, and the following\nmethods:\n\nrange.count(x)\n\n Return the number of *i*\'s for which ``s[i] == x``.\n\n New in version 3.2.\n\nrange.index(x)\n\n Return the smallest *i* such that ``s[i] == x``. Raises\n ``ValueError`` when *x* is not in the range.\n\n New in version 3.2.\n\n\nMutable Sequence Types\n======================\n\nList and bytearray objects support additional operations that allow\nin-place modification of the object. Other mutable sequence types\n(when added to the language) should also support these operations.\nStrings and tuples are immutable sequence types: such objects cannot\nbe modified once created. The following operations are defined on\nmutable sequence types (where *x* is an arbitrary object).\n\nNote that while lists allow their items to be of any type, bytearray\nobject "items" are all integers in the range 0 <= x < 256.\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (2) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (3) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (5) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (6) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([key[, reverse]])`` | sort the items of *s* in place | (6), (7), (8) |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. *x* can be any iterable object.\n\n3. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the sequence length is added, as for slice indices. If it\n is still negative, it is truncated to zero, as for slice indices.\n\n4. When a negative index is passed as the first parameter to the\n ``insert()`` method, the sequence length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n5. The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n6. The ``sort()`` and ``reverse()`` methods modify the sequence in\n place for economy of space when sorting or reversing a large\n sequence. To remind you that they operate by side effect, they\n don\'t return the sorted or reversed sequence.\n\n7. The ``sort()`` method takes optional arguments for controlling the\n comparisons. Each must be specified as a keyword argument.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``. Use ``functools.cmp_to_key()`` to\n convert an old-style *cmp* function to a *key* function.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n The ``sort()`` method is guaranteed to be stable. A sort is stable\n if it guarantees not to change the relative order of elements that\n compare equal --- this is helpful for sorting in multiple passes\n (for example, sort by department, then by salary grade).\n\n **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python makes the list appear\n empty for the duration, and raises ``ValueError`` if it can detect\n that the list has been mutated during a sort.\n\n8. ``sort()`` is not supported by ``bytearray`` objects.\n\n\nBytes and Byte Array Methods\n============================\n\nBytes and bytearray objects, being "strings of bytes", have all\nmethods found on strings, with the exception of ``encode()``,\n``format()`` and ``isidentifier()``, which do not make sense with\nthese types. For converting the objects to strings, they have a\n``decode()`` method.\n\nWherever one of these methods needs to interpret the bytes as\ncharacters (e.g. the ``is...()`` methods), the ASCII character set is\nassumed.\n\nNote: The methods on bytes and bytearray objects don\'t accept strings as\n their arguments, just as the methods on strings don\'t accept bytes\n as their arguments. For example, you have to write\n\n a = "abc"\n b = a.replace("a", "f")\n\n and\n\n a = b"abc"\n b = a.replace(b"a", b"f")\n\nbytes.decode(encoding="utf-8", errors="strict")\nbytearray.decode(encoding="utf-8", errors="strict")\n\n Return a string decoded from the given bytes. Default encoding is\n ``\'utf-8\'``. *errors* may be given to set a different error\n handling scheme. The default for *errors* is ``\'strict\'``, meaning\n that encoding errors raise a ``UnicodeError``. Other possible\n values are ``\'ignore\'``, ``\'replace\'`` and any other name\n registered via ``codecs.register_error()``, see section *Codec Base\n Classes*. For a list of possible encodings, see section *Standard\n Encodings*.\n\n Changed in version 3.1: Added support for keyword arguments.\n\nThe bytes and bytearray types have an additional class method:\n\nclassmethod bytes.fromhex(string)\nclassmethod bytearray.fromhex(string)\n\n This ``bytes`` class method returns a bytes or bytearray object,\n decoding the given string object. The string must contain two\n hexadecimal digits per byte, spaces are ignored.\n\n >>> bytes.fromhex(\'f0 f1f2 \')\n b\'\\xf0\\xf1\\xf2\'\n\nThe maketrans and translate methods differ in semantics from the\nversions available on strings:\n\nbytes.translate(table[, delete])\nbytearray.translate(table[, delete])\n\n Return a copy of the bytes or bytearray object where all bytes\n occurring in the optional argument *delete* are removed, and the\n remaining bytes have been mapped through the given translation\n table, which must be a bytes object of length 256.\n\n You can use the ``bytes.maketrans()`` method to create a\n translation table.\n\n Set the *table* argument to ``None`` for translations that only\n delete characters:\n\n >>> b\'read this short text\'.translate(None, b\'aeiou\')\n b\'rd ths shrt txt\'\n\nstatic bytes.maketrans(from, to)\nstatic bytearray.maketrans(from, to)\n\n This static method returns a translation table usable for\n ``bytes.translate()`` that will map each character in *from* into\n the character at the same position in *to*; *from* and *to* must be\n bytes objects and have the same length.\n\n New in version 3.1.\n', - 'typesseq-mutable': '\nMutable Sequence Types\n**********************\n\nList and bytearray objects support additional operations that allow\nin-place modification of the object. Other mutable sequence types\n(when added to the language) should also support these operations.\nStrings and tuples are immutable sequence types: such objects cannot\nbe modified once created. The following operations are defined on\nmutable sequence types (where *x* is an arbitrary object).\n\nNote that while lists allow their items to be of any type, bytearray\nobject "items" are all integers in the range 0 <= x < 256.\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (2) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (3) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (5) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (6) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([key[, reverse]])`` | sort the items of *s* in place | (6), (7), (8) |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. *x* can be any iterable object.\n\n3. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the sequence length is added, as for slice indices. If it\n is still negative, it is truncated to zero, as for slice indices.\n\n4. When a negative index is passed as the first parameter to the\n ``insert()`` method, the sequence length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n5. The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n6. The ``sort()`` and ``reverse()`` methods modify the sequence in\n place for economy of space when sorting or reversing a large\n sequence. To remind you that they operate by side effect, they\n don\'t return the sorted or reversed sequence.\n\n7. The ``sort()`` method takes optional arguments for controlling the\n comparisons. Each must be specified as a keyword argument.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``. Use ``functools.cmp_to_key()`` to\n convert an old-style *cmp* function to a *key* function.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n The ``sort()`` method is guaranteed to be stable. A sort is stable\n if it guarantees not to change the relative order of elements that\n compare equal --- this is helpful for sorting in multiple passes\n (for example, sort by department, then by salary grade).\n\n **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python makes the list appear\n empty for the duration, and raises ``ValueError`` if it can detect\n that the list has been mutated during a sort.\n\n8. ``sort()`` is not supported by ``bytearray`` objects.\n', + 'typesseq': '\nSequence Types --- ``str``, ``bytes``, ``bytearray``, ``list``, ``tuple``, ``range``\n************************************************************************************\n\nThere are six sequence types: strings, byte sequences (``bytes``\nobjects), byte arrays (``bytearray`` objects), lists, tuples, and\nrange objects. For other containers see the built in ``dict`` and\n``set`` classes, and the ``collections`` module.\n\nStrings contain Unicode characters. Their literals are written in\nsingle or double quotes: ``\'xyzzy\'``, ``"frobozz"``. See *String and\nBytes literals* for more about string literals. In addition to the\nfunctionality described here, there are also string-specific methods\ndescribed in the *String Methods* section.\n\nBytes and bytearray objects contain single bytes -- the former is\nimmutable while the latter is a mutable sequence. Bytes objects can\nbe constructed the constructor, ``bytes()``, and from literals; use a\n``b`` prefix with normal string syntax: ``b\'xyzzy\'``. To construct\nbyte arrays, use the ``bytearray()`` function.\n\nWhile string objects are sequences of characters (represented by\nstrings of length 1), bytes and bytearray objects are sequences of\n*integers* (between 0 and 255), representing the ASCII value of single\nbytes. That means that for a bytes or bytearray object *b*, ``b[0]``\nwill be an integer, while ``b[0:1]`` will be a bytes or bytearray\nobject of length 1. The representation of bytes objects uses the\nliteral format (``b\'...\'``) since it is generally more useful than\ne.g. ``bytes([50, 19, 100])``. You can always convert a bytes object\ninto a list of integers using ``list(b)``.\n\nAlso, while in previous Python versions, byte strings and Unicode\nstrings could be exchanged for each other rather freely (barring\nencoding issues), strings and bytes are now completely separate\nconcepts. There\'s no implicit en-/decoding if you pass an object of\nthe wrong type. A string always compares unequal to a bytes or\nbytearray object.\n\nLists are constructed with square brackets, separating items with\ncommas: ``[a, b, c]``. Tuples are constructed by the comma operator\n(not within square brackets), with or without enclosing parentheses,\nbut an empty tuple must have the enclosing parentheses, such as ``a,\nb, c`` or ``()``. A single item tuple must have a trailing comma,\nsuch as ``(d,)``.\n\nObjects of type range are created using the ``range()`` function.\nThey don\'t support concatenation or repetition, and using ``min()`` or\n``max()`` on them is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i*, *j* and *k* are\nintegers.\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*\'th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)`` | index of the first occurence of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)`` | total number of occurences of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must\ncompare equal and the two sequences must be of the same type and have\nthe same length. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string object, the ``in`` and ``not in`` operations\n act like a substring test.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n\nString Methods\n==============\n\nString objects support the methods listed below.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, bytes, bytearray, list,\ntuple, range* section. To output formatted strings, see the *String\nFormatting* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.encode(encoding="utf-8", errors="strict")\n\n Return an encoded version of the string as a bytes object. Default\n encoding is ``\'utf-8\'``. *errors* may be given to set a different\n error handling scheme. The default for *errors* is ``\'strict\'``,\n meaning that encoding errors raise a ``UnicodeError``. Other\n possible values are ``\'ignore\'``, ``\'replace\'``,\n ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and any other name\n registered via ``codecs.register_error()``, see section *Codec Base\n Classes*. For a list of possible encodings, see section *Standard\n Encodings*.\n\n Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\nstr.format_map(mapping)\n\n Similar to ``str.format(**mapping)``, except that ``mapping`` is\n used directly and not copied to a ``dict`` . This is useful if for\n example ``mapping`` is a dict subclass:\n\n >>> class Default(dict):\n ... def __missing__(self, key):\n ... return key\n ...\n >>> \'{name} was born in {country}\'.format_map(Default(name=\'Guido\'))\n \'Guido was born in country\'\n\n New in version 3.2.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise. A character\n ``c`` is alphanumeric if one of the following returns ``True``:\n ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or\n ``c.isnumeric()``.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise. Alphabetic\n characters are those characters defined in the Unicode character\n database as "Letter", i.e., those with general category property\n being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is\n different from the "Alphabetic" property defined in the Unicode\n Standard.\n\nstr.isdecimal()\n\n Return true if all characters in the string are decimal characters\n and there is at least one character, false otherwise. Decimal\n characters are those from general category "Nd". This category\n includes digit characters, and all characters that that can be used\n to form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT\n ZERO.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise. Digits include decimal\n characters and digits that need special handling, such as the\n compatibility superscript digits. Formally, a digit is a character\n that has the property value Numeric_Type=Digit or\n Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n Return true if the string is a valid identifier according to the\n language definition, section *Identifiers and keywords*.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and lowercase characters are those with general\n category property "Ll".\n\nstr.isnumeric()\n\n Return true if all characters in the string are numeric characters,\n and there is at least one character, false otherwise. Numeric\n characters include digit characters, and all characters that have\n the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n ONE FIFTH. Formally, numeric characters are those with the\n property value Numeric_Type=Digit, Numeric_Type=Decimal or\n Numeric_Type=Numeric.\n\nstr.isprintable()\n\n Return true if all characters in the string are printable or the\n string is empty, false otherwise. Nonprintable characters are\n those characters defined in the Unicode character database as\n "Other" or "Separator", excepting the ASCII space (0x20) which is\n considered printable. (Note that printable characters in this\n context are those which should not be escaped when ``repr()`` is\n invoked on a string. It has no bearing on the handling of strings\n written to ``sys.stdout`` or ``sys.stderr``.)\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise. Whitespace\n characters are those characters defined in the Unicode character\n database as "Other" or "Separator" and those with bidirectional\n property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and uppercase characters are those with general\n category property "Lu".\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. A ``TypeError`` will be raised if there are\n any non-string values in *seq*, including ``bytes`` objects. The\n separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n This static method returns a translation table usable for\n ``str.translate()``.\n\n If there is only one argument, it must be a dictionary mapping\n Unicode ordinals (integers) or characters (strings of length 1) to\n Unicode ordinals, strings (of arbitrary lengths) or None.\n Character keys will then be converted to ordinals.\n\n If there are two arguments, they must be strings of equal length,\n and in the resulting dictionary, each character in x will be mapped\n to the character at the same position in y. If there is a third\n argument, it must be a string, whose characters will be mapped to\n None in the result.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n Return a copy of the *s* where all characters have been mapped\n through the *map* which must be a dictionary of Unicode ordinals\n (integers) to Unicode ordinals, strings or ``None``. Unmapped\n characters are left untouched. Characters mapped to ``None`` are\n deleted.\n\n You can use ``str.maketrans()`` to create a translation map from\n character-to-character mappings in different formats.\n\n Note: An even more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see\n ``encodings.cp1251`` for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n\nOld String Formatting Operations\n================================\n\nNote: The formatting operations described here are obsolete and may go\n away in future versions of Python. Use the new *String Formatting*\n in new code.\n\nString objects have one unique built-in operation: the ``%`` operator\n(modulo). This is also known as the string *formatting* or\n*interpolation* operator. Given ``format % values`` (where *format* is\na string), ``%`` conversion specifications in *format* are replaced\nwith zero or more elements of *values*. The effect is similar to the\nusing ``sprintf()`` in the C language.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print(\'%(language)s has %(number)03d quote types.\' %\n... {\'language\': "Python", "number": 2})\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The precision determines the maximal number of characters used.\n\n1. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 3.1: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nRange Type\n==========\n\nThe ``range`` type is an immutable sequence which is commonly used for\nlooping. The advantage of the ``range`` type is that an ``range``\nobject will always take the same amount of memory, no matter the size\nof the range it represents.\n\nRange objects have relatively little behavior: they support indexing,\ncontains, iteration, the ``len()`` function, and the following\nmethods:\n\nrange.count(x)\n\n Return the number of *i*\'s for which ``s[i] == x``.\n\n New in version 3.2.\n\nrange.index(x)\n\n Return the smallest *i* such that ``s[i] == x``. Raises\n ``ValueError`` when *x* is not in the range.\n\n New in version 3.2.\n\n\nMutable Sequence Types\n======================\n\nList and bytearray objects support additional operations that allow\nin-place modification of the object. Other mutable sequence types\n(when added to the language) should also support these operations.\nStrings and tuples are immutable sequence types: such objects cannot\nbe modified once created. The following operations are defined on\nmutable sequence types (where *x* is an arbitrary object).\n\nNote that while lists allow their items to be of any type, bytearray\nobject "items" are all integers in the range 0 <= x < 256.\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (2) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.clear()`` | remove all items from ``s`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.copy()`` | return a shallow copy of ``s`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (3) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (5) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (6) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([key[, reverse]])`` | sort the items of *s* in place | (6), (7), (8) |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. *x* can be any iterable object.\n\n3. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the sequence length is added, as for slice indices. If it\n is still negative, it is truncated to zero, as for slice indices.\n\n4. When a negative index is passed as the first parameter to the\n ``insert()`` method, the sequence length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n5. The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n6. The ``sort()`` and ``reverse()`` methods modify the sequence in\n place for economy of space when sorting or reversing a large\n sequence. To remind you that they operate by side effect, they\n don\'t return the sorted or reversed sequence.\n\n7. The ``sort()`` method takes optional arguments for controlling the\n comparisons. Each must be specified as a keyword argument.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``. Use ``functools.cmp_to_key()`` to\n convert an old-style *cmp* function to a *key* function.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n The ``sort()`` method is guaranteed to be stable. A sort is stable\n if it guarantees not to change the relative order of elements that\n compare equal --- this is helpful for sorting in multiple passes\n (for example, sort by department, then by salary grade).\n\n **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python makes the list appear\n empty for the duration, and raises ``ValueError`` if it can detect\n that the list has been mutated during a sort.\n\n8. ``sort()`` is not supported by ``bytearray`` objects.\n\n New in version 3.3: ``clear()`` and ``copy()`` methods.\n\n\nBytes and Byte Array Methods\n============================\n\nBytes and bytearray objects, being "strings of bytes", have all\nmethods found on strings, with the exception of ``encode()``,\n``format()`` and ``isidentifier()``, which do not make sense with\nthese types. For converting the objects to strings, they have a\n``decode()`` method.\n\nWherever one of these methods needs to interpret the bytes as\ncharacters (e.g. the ``is...()`` methods), the ASCII character set is\nassumed.\n\nNote: The methods on bytes and bytearray objects don\'t accept strings as\n their arguments, just as the methods on strings don\'t accept bytes\n as their arguments. For example, you have to write\n\n a = "abc"\n b = a.replace("a", "f")\n\n and\n\n a = b"abc"\n b = a.replace(b"a", b"f")\n\nbytes.decode(encoding="utf-8", errors="strict")\nbytearray.decode(encoding="utf-8", errors="strict")\n\n Return a string decoded from the given bytes. Default encoding is\n ``\'utf-8\'``. *errors* may be given to set a different error\n handling scheme. The default for *errors* is ``\'strict\'``, meaning\n that encoding errors raise a ``UnicodeError``. Other possible\n values are ``\'ignore\'``, ``\'replace\'`` and any other name\n registered via ``codecs.register_error()``, see section *Codec Base\n Classes*. For a list of possible encodings, see section *Standard\n Encodings*.\n\n Changed in version 3.1: Added support for keyword arguments.\n\nThe bytes and bytearray types have an additional class method:\n\nclassmethod bytes.fromhex(string)\nclassmethod bytearray.fromhex(string)\n\n This ``bytes`` class method returns a bytes or bytearray object,\n decoding the given string object. The string must contain two\n hexadecimal digits per byte, spaces are ignored.\n\n >>> bytes.fromhex(\'f0 f1f2 \')\n b\'\\xf0\\xf1\\xf2\'\n\nThe maketrans and translate methods differ in semantics from the\nversions available on strings:\n\nbytes.translate(table[, delete])\nbytearray.translate(table[, delete])\n\n Return a copy of the bytes or bytearray object where all bytes\n occurring in the optional argument *delete* are removed, and the\n remaining bytes have been mapped through the given translation\n table, which must be a bytes object of length 256.\n\n You can use the ``bytes.maketrans()`` method to create a\n translation table.\n\n Set the *table* argument to ``None`` for translations that only\n delete characters:\n\n >>> b\'read this short text\'.translate(None, b\'aeiou\')\n b\'rd ths shrt txt\'\n\nstatic bytes.maketrans(from, to)\nstatic bytearray.maketrans(from, to)\n\n This static method returns a translation table usable for\n ``bytes.translate()`` that will map each character in *from* into\n the character at the same position in *to*; *from* and *to* must be\n bytes objects and have the same length.\n\n New in version 3.1.\n', + 'typesseq-mutable': '\nMutable Sequence Types\n**********************\n\nList and bytearray objects support additional operations that allow\nin-place modification of the object. Other mutable sequence types\n(when added to the language) should also support these operations.\nStrings and tuples are immutable sequence types: such objects cannot\nbe modified once created. The following operations are defined on\nmutable sequence types (where *x* is an arbitrary object).\n\nNote that while lists allow their items to be of any type, bytearray\nobject "items" are all integers in the range 0 <= x < 256.\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (2) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.clear()`` | remove all items from ``s`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.copy()`` | return a shallow copy of ``s`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (3) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (5) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (6) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([key[, reverse]])`` | sort the items of *s* in place | (6), (7), (8) |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. *x* can be any iterable object.\n\n3. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the sequence length is added, as for slice indices. If it\n is still negative, it is truncated to zero, as for slice indices.\n\n4. When a negative index is passed as the first parameter to the\n ``insert()`` method, the sequence length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n5. The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n6. The ``sort()`` and ``reverse()`` methods modify the sequence in\n place for economy of space when sorting or reversing a large\n sequence. To remind you that they operate by side effect, they\n don\'t return the sorted or reversed sequence.\n\n7. The ``sort()`` method takes optional arguments for controlling the\n comparisons. Each must be specified as a keyword argument.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``. Use ``functools.cmp_to_key()`` to\n convert an old-style *cmp* function to a *key* function.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n The ``sort()`` method is guaranteed to be stable. A sort is stable\n if it guarantees not to change the relative order of elements that\n compare equal --- this is helpful for sorting in multiple passes\n (for example, sort by department, then by salary grade).\n\n **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python makes the list appear\n empty for the duration, and raises ``ValueError`` if it can detect\n that the list has been mutated during a sort.\n\n8. ``sort()`` is not supported by ``bytearray`` objects.\n\n New in version 3.3: ``clear()`` and ``copy()`` methods.\n', 'unary': '\nUnary arithmetic and bitwise operations\n***************************************\n\nAll unary arithmetic and bitwise operations have the same priority:\n\n u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n\nThe unary ``-`` (minus) operator yields the negation of its numeric\nargument.\n\nThe unary ``+`` (plus) operator yields its numeric argument unchanged.\n\nThe unary ``~`` (invert) operator yields the bitwise inversion of its\ninteger argument. The bitwise inversion of ``x`` is defined as\n``-(x+1)``. It only applies to integral numbers.\n\nIn all three cases, if the argument does not have the proper type, a\n``TypeError`` exception is raised.\n', 'while': '\nThe ``while`` statement\n***********************\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n', 'with': '\nThe ``with`` statement\n**********************\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the ``with_item``)\n is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', diff --git a/Lib/random.py b/Lib/random.py index ccc304dd66..a8f1a43418 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -42,7 +42,7 @@ from types import MethodType as _MethodType, BuiltinMethodType as _BuiltinMethod from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin from os import urandom as _urandom -from collections import Set as _Set, Sequence as _Sequence +from collections.abc import Set as _Set, Sequence as _Sequence from hashlib import sha512 as _sha512 __all__ = ["Random","seed","random","uniform","randint","choice","sample", @@ -215,12 +215,14 @@ def template(pattern, flags=0): return _compile(pattern, flags|T) _alphanum_str = frozenset( - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890") + "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890") _alphanum_bytes = frozenset( - b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890") + b"_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890") def escape(pattern): - "Escape all non-alphanumeric characters in pattern." + """ + Escape all the characters in pattern except ASCII letters, numbers and '_'. + """ if isinstance(pattern, str): alphanum = _alphanum_str s = list(pattern) diff --git a/Lib/reprlib.py b/Lib/reprlib.py index f8033604da..092874a18f 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -5,7 +5,7 @@ __all__ = ["Repr", "repr", "recursive_repr"] import builtins from itertools import islice try: - from _thread import get_ident + from threading import get_ident except ImportError: from _dummy_thread import get_ident diff --git a/Lib/shlex.py b/Lib/shlex.py index 3edd3db1ed..69f3b456af 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -6,13 +6,14 @@ # Posix compliance, split(), string arguments, and # iterator interface by Gustavo Niemeyer, April 2003. -import os.path +import os +import re import sys from collections import deque from io import StringIO -__all__ = ["shlex", "split"] +__all__ = ["shlex", "split", "quote"] class shlex: "A lexical analyzer class for simple shell-like syntaxes." @@ -274,6 +275,21 @@ def split(s, comments=False, posix=True): lex.commenters = '' return list(lex) + +_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search + +def quote(s): + """Return a shell-escaped version of the string *s*.""" + if not s: + return "''" + if _find_unsafe(s) is None: + return s + + # use single quotes, and put single quotes into double quotes + # the string $'b is then quoted as '$'"'"'b' + return "'" + s.replace("'", "'\"'\"'") + "'" + + if __name__ == '__main__': if len(sys.argv) == 1: lexer = shlex() diff --git a/Lib/shutil.py b/Lib/shutil.py index a0d981b0c7..0c096cba55 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -34,7 +34,9 @@ __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", "ExecError", "make_archive", "get_archive_formats", "register_archive_format", "unregister_archive_format", "get_unpack_formats", "register_unpack_format", - "unregister_unpack_format", "unpack_archive", "ignore_patterns"] + "unregister_unpack_format", "unpack_archive", + "ignore_patterns", "chown"] + # disk_usage is added later, if available on the platform class Error(EnvironmentError): pass @@ -266,7 +268,7 @@ def rmtree(path, ignore_errors=False, onerror=None): names = [] try: names = os.listdir(path) - except os.error as err: + except os.error: onerror(os.listdir, path, sys.exc_info()) for name in names: fullname = os.path.join(path, name) @@ -279,7 +281,7 @@ def rmtree(path, ignore_errors=False, onerror=None): else: try: os.remove(fullname) - except os.error as err: + except os.error: onerror(os.remove, fullname, sys.exc_info()) try: os.rmdir(path) @@ -322,7 +324,7 @@ def move(src, dst): raise Error("Destination path '%s' already exists" % real_dst) try: os.rename(src, real_dst) - except OSError as exc: + except OSError: if os.path.isdir(src): if _destinsrc(src, dst): raise Error("Cannot move a directory '%s' into itself '%s'." % (src, dst)) @@ -754,3 +756,69 @@ def unpack_archive(filename, extract_dir=None, format=None): func = _UNPACK_FORMATS[format][1] kwargs = dict(_UNPACK_FORMATS[format][2]) func(filename, extract_dir, **kwargs) + + +if hasattr(os, 'statvfs'): + + __all__.append('disk_usage') + _ntuple_diskusage = collections.namedtuple('usage', 'total used free') + + def disk_usage(path): + """Return disk usage statistics about the given path. + + Returned valus is a named tuple with attributes 'total', 'used' and + 'free', which are the amount of total, used and free space, in bytes. + """ + st = os.statvfs(path) + free = st.f_bavail * st.f_frsize + total = st.f_blocks * st.f_frsize + used = (st.f_blocks - st.f_bfree) * st.f_frsize + return _ntuple_diskusage(total, used, free) + +elif os.name == 'nt': + + import nt + __all__.append('disk_usage') + _ntuple_diskusage = collections.namedtuple('usage', 'total used free') + + def disk_usage(path): + """Return disk usage statistics about the given path. + + Returned valus is a named tuple with attributes 'total', 'used' and + 'free', which are the amount of total, used and free space, in bytes. + """ + total, free = nt._getdiskusage(path) + used = total - free + return _ntuple_diskusage(total, used, free) + + +def chown(path, user=None, group=None): + """Change owner user and group of the given path. + + user and group can be the uid/gid or the user/group names, and in that case, + they are converted to their respective uid/gid. + """ + + if user is None and group is None: + raise ValueError("user and/or group must be set") + + _user = user + _group = group + + # -1 means don't change it + if user is None: + _user = -1 + # user can either be an int (the uid) or a string (the system username) + elif isinstance(user, str): + _user = _get_uid(user) + if _user is None: + raise LookupError("no such user: {!r}".format(user)) + + if group is None: + _group = -1 + elif not isinstance(group, int): + _group = _get_gid(group) + if _group is None: + raise LookupError("no such group: {!r}".format(group)) + + os.chown(path, _user, _group) diff --git a/Lib/site.py b/Lib/site.py index a2c0becbc1..46fa53ada9 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -138,7 +138,7 @@ def addpackage(sitedir, name, known_paths): reset = 0 fullname = os.path.join(sitedir, name) try: - f = open(fullname, "rU") + f = open(fullname, "r") except IOError: return with f: @@ -385,7 +385,7 @@ class _Printer(object): for filename in self.__files: filename = os.path.join(dir, filename) try: - fp = open(filename, "rU") + fp = open(filename, "r") data = fp.read() fp.close() break @@ -508,6 +508,11 @@ def execusercustomize(): def main(): + """Add standard site-specific directories to the module search path. + + This function is called automatically when this module is imported, + unless the python interpreter was started with the -S flag. + """ global ENABLE_USER_SITE abs_paths() @@ -526,7 +531,10 @@ def main(): if ENABLE_USER_SITE: execusercustomize() -main() +# Prevent edition of sys.path when python was started with -S and +# site is imported later. +if not sys.flags.no_site: + main() def _script(): help = """\ diff --git a/Lib/smtpd.py b/Lib/smtpd.py index 599e79b7ed..32f45ae768 100755 --- a/Lib/smtpd.py +++ b/Lib/smtpd.py @@ -275,7 +275,7 @@ class SMTPChannel(asynchat.async_chat): return elif limit: self.num_bytes += len(data) - self.received_lines.append(str(data, "utf8")) + self.received_lines.append(str(data, "utf-8")) # Implementation of base class abstract method def found_terminator(self): diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 13c56e196f..68fc646f27 100644 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -133,24 +133,18 @@ class SMTPAuthenticationError(SMTPResponseException): combination provided. """ -def quoteaddr(addr): +def quoteaddr(addrstring): """Quote a subset of the email addresses defined by RFC 821. Should be able to handle anything email.utils.parseaddr can handle. """ - m = (None, None) - try: - m = email.utils.parseaddr(addr)[1] - except AttributeError: - pass - if m == (None, None): # Indicates parse failure or AttributeError - # something weird here.. punt -ddm - return "<%s>" % addr - elif m is None: - # the sender wants an empty return address - return "<>" - else: - return "<%s>" % m + displayname, addr = email.utils.parseaddr(addrstring) + if (displayname, addr) == ('', ''): + # parseaddr couldn't parse it, use it as is and hope for the best. + if addrstring.strip().startswith('<'): + return addrstring + return "<%s>" % addrstring + return "<%s>" % addr def _addr_only(addrstring): displayname, addr = email.utils.parseaddr(addrstring) @@ -180,27 +174,6 @@ try: except ImportError: _have_ssl = False else: - class SSLFakeFile: - """A fake file like object that really wraps a SSLObject. - - It only supports what is needed in smtplib. - """ - def __init__(self, sslobj): - self.sslobj = sslobj - - def readline(self): - str = b"" - chr = None - while chr != b"\n": - chr = self.sslobj.read(1) - if not chr: - break - str += chr - return str - - def close(self): - pass - _have_ssl = True @@ -242,7 +215,8 @@ class SMTP: default_port = SMTP_PORT def __init__(self, host='', port=0, local_hostname=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None): """Initialize a new instance. If specified, `host' is the name of the remote host to which to @@ -250,11 +224,16 @@ class SMTP: By default, smtplib.SMTP_PORT is used. An SMTPConnectError is raised if the specified `host' doesn't respond correctly. If specified, `local_hostname` is used as the FQDN of the local host. By default, - the local hostname is found using socket.getfqdn(). + the local hostname is found using socket.getfqdn(). The + `source_address` parameter takes a 2-tuple (host, port) for the socket + to bind to as its source address before connecting. If the host is '' + and port is 0, the OS default behavior will be used. """ self.timeout = timeout self.esmtp_features = {} + self.source_address = source_address + if host: (code, msg) = self.connect(host, port) if code != 220: @@ -277,6 +256,19 @@ class SMTP: pass self.local_hostname = '[%s]' % addr + def __enter__(self): + return self + + def __exit__(self, *args): + try: + code, message = self.docmd("QUIT") + if code != 221: + raise SMTPResponseException(code, message) + except SMTPServerDisconnected: + pass + finally: + self.close() + def set_debuglevel(self, debuglevel): """Set the debug output level. @@ -290,10 +282,12 @@ class SMTP: # This makes it simpler for SMTP_SSL to use the SMTP connect code # and just alter the socket connection bit. if self.debuglevel > 0: - print('connect:', (host, port), file=stderr) - return socket.create_connection((host, port), timeout) + print('connect: to', (host, port), self.source_address, + file=stderr) + return socket.create_connection((host, port), timeout, + self.source_address) - def connect(self, host='localhost', port=0): + def connect(self, host='localhost', port=0, source_address=None): """Connect to a host on a given port. If the hostname ends with a colon (`:') followed by a number, and @@ -304,6 +298,10 @@ class SMTP: specified during instantiation. """ + + if source_address: + self.source_address = source_address + if not port and (host.find(':') == host.rfind(':')): i = host.rfind(':') if i >= 0: @@ -317,6 +315,7 @@ class SMTP: if self.debuglevel > 0: print('connect:', (host, port), file=stderr) self.sock = self._get_socket(host, port, self.timeout) + self.file = None (code, msg) = self.getreply() if self.debuglevel > 0: print("connect:", msg, file=stderr) @@ -388,7 +387,8 @@ class SMTP: errmsg = b"\n".join(resp) if self.debuglevel > 0: - print('reply: retcode (%s); Msg: %s' % (errcode, errmsg), file=stderr) + print('reply: retcode (%s); Msg: %s' % (errcode, errmsg), + file=stderr) return errcode, errmsg def docmd(self, cmd, args=""): @@ -632,7 +632,7 @@ class SMTP: # We could not login sucessfully. Return result of last attempt. raise SMTPAuthenticationError(code, resp) - def starttls(self, keyfile=None, certfile=None): + def starttls(self, keyfile=None, certfile=None, context=None): """Puts the connection to the SMTP server into TLS mode. If there has been no previous EHLO or HELO command this session, this @@ -656,8 +656,17 @@ class SMTP: if resp == 220: if not _have_ssl: raise RuntimeError("No SSL support included in this Python") - self.sock = ssl.wrap_socket(self.sock, keyfile, certfile) - self.file = SSLFakeFile(self.sock) + if context is not None and keyfile is not None: + raise ValueError("context and keyfile arguments are mutually " + "exclusive") + if context is not None and certfile is not None: + raise ValueError("context and certfile arguments are mutually " + "exclusive") + if context is not None: + self.sock = context.wrap_socket(self.sock) + else: + self.sock = ssl.wrap_socket(self.sock, keyfile, certfile) + self.file = None # RFC 3207: # The client MUST discard any knowledge obtained from # the server, such as the list of SMTP service extensions, @@ -786,7 +795,8 @@ class SMTP: # TODO implement heuristics to guess the correct Resent-* block with an # option allowing the user to enable the heuristics. (It should be # possible to guess correctly almost all of the time.) - resent =msg.get_all('Resent-Date') + + resent = msg.get_all('Resent-Date') if resent is None: header_prefix = '' elif len(resent) == 1: @@ -795,13 +805,13 @@ class SMTP: raise ValueError("message has more than one 'Resent-' header block") if from_addr is None: # Prefer the sender field per RFC 2822:3.6.2. - from_addr = (msg[header_prefix+'Sender'] - if (header_prefix+'Sender') in msg - else msg[header_prefix+'From']) + from_addr = (msg[header_prefix + 'Sender'] + if (header_prefix + 'Sender') in msg + else msg[header_prefix + 'From']) if to_addrs is None: - addr_fields = [f for f in (msg[header_prefix+'To'], - msg[header_prefix+'Bcc'], - msg[header_prefix+'Cc']) if f is not None] + addr_fields = [f for f in (msg[header_prefix + 'To'], + msg[header_prefix + 'Bcc'], + msg[header_prefix + 'Cc']) if f is not None] to_addrs = [a[1] for a in email.utils.getaddresses(addr_fields)] # Make a local copy so we can delete the bcc headers. msg_copy = copy.copy(msg) @@ -835,26 +845,41 @@ if _have_ssl: """ This is a subclass derived from SMTP that connects over an SSL encrypted socket (to use this class you need a socket module that was compiled with SSL support). If host is not specified, '' (the local host) is used. If port is - omitted, the standard SMTP-over-SSL port (465) is used. keyfile and certfile + omitted, the standard SMTP-over-SSL port (465) is used. The optional + source_address takes a two-tuple (host,port) for socket to bind to. keyfile and certfile are also optional - they can contain a PEM formatted private key and - certificate chain file for the SSL connection. + certificate chain file for the SSL connection. context also optional, can contain + a SSLContext, and is an alternative to keyfile and certfile; If it is specified both + keyfile and certfile must be None. """ default_port = SMTP_SSL_PORT def __init__(self, host='', port=0, local_hostname=None, keyfile=None, certfile=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, context=None): + if context is not None and keyfile is not None: + raise ValueError("context and keyfile arguments are mutually " + "exclusive") + if context is not None and certfile is not None: + raise ValueError("context and certfile arguments are mutually " + "exclusive") self.keyfile = keyfile self.certfile = certfile - SMTP.__init__(self, host, port, local_hostname, timeout) + self.context = context + SMTP.__init__(self, host, port, local_hostname, timeout, + source_address) def _get_socket(self, host, port, timeout): if self.debuglevel > 0: print('connect:', (host, port), file=stderr) - new_socket = socket.create_connection((host, port), timeout) - new_socket = ssl.wrap_socket(new_socket, self.keyfile, self.certfile) - self.file = SSLFakeFile(new_socket) + new_socket = socket.create_connection((host, port), timeout, + self.source_address) + if self.context is not None: + new_socket = self.context.wrap_socket(new_socket) + else: + new_socket = ssl.wrap_socket(new_socket, self.keyfile, self.certfile) return new_socket __all__.append("SMTP_SSL") @@ -879,18 +904,21 @@ class LMTP(SMTP): ehlo_msg = "lhlo" - def __init__(self, host='', port=LMTP_PORT, local_hostname=None): + def __init__(self, host='', port=LMTP_PORT, local_hostname=None, + source_address=None): """Initialize a new instance.""" - SMTP.__init__(self, host, port, local_hostname) + SMTP.__init__(self, host, port, local_hostname=local_hostname, + source_address=source_address) - def connect(self, host='localhost', port=0): + def connect(self, host='localhost', port=0, source_address=None): """Connect to the LMTP daemon, on either a Unix or a TCP socket.""" if host[0] != '/': - return SMTP.connect(self, host, port) + return SMTP.connect(self, host, port, source_address=source_address) # Handle Unix-domain sockets. try: self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.file = None self.sock.connect(host) except socket.error as msg: if self.debuglevel > 0: diff --git a/Lib/socket.py b/Lib/socket.py index 1e285493c4..57150349a5 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -112,6 +112,9 @@ class socket(_socket.socket): s[7:]) return s + def __getstate__(self): + raise TypeError("Cannot serialize socket object") + def dup(self): """dup() -> socket object diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 73896081d1..a2f0f3925d 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -82,7 +82,7 @@ On the other hand, if you are building e.g. an HTTP server, where all data is stored externally (e.g. in the file system), a synchronous class will essentially render the service "deaf" while one request is being handled -- which may be for a very long time if a client is slow -to reqd all the data it has requested. Here a threading or forking +to recv all the data it has requested. Here a threading or forking server is appropriate. In some cases, it may be appropriate to process part of a request @@ -170,6 +170,7 @@ class BaseServer: - process_request(request, client_address) - shutdown_request(request) - close_request(request) + - service_actions() - handle_error() Methods for derived classes: @@ -225,6 +226,8 @@ class BaseServer: r, w, e = select.select([self], [], [], poll_interval) if self in r: self._handle_request_noblock() + + self.service_actions() finally: self.__shutdown_request = False self.__is_shut_down.set() @@ -239,6 +242,14 @@ class BaseServer: self.__shutdown_request = True self.__is_shut_down.wait() + def service_actions(self): + """Called by the serve_forever() loop. + + May be overridden by a subclass / Mixin to implement any code that + needs to be run during the loop. + """ + pass + # The distinction between handling, getting, processing and # finishing a request is fairly arbitrary. Remember: # @@ -539,9 +550,15 @@ class ForkingMixIn: """ self.collect_children() + def service_actions(self): + """Collect the zombie child processes regularly in the ForkingMixin. + + service_actions is called in the BaseServer's serve_forver loop. + """ + self.collect_children() + def process_request(self, request, client_address): """Fork a new subprocess to process the request.""" - self.collect_children() pid = os.fork() if pid: # Parent process @@ -549,6 +566,7 @@ class ForkingMixIn: self.active_children = [] self.active_children.append(pid) self.close_request(request) + return else: # Child process. # This must never return, hence os._exit()! diff --git a/Lib/sqlite3/test/hooks.py b/Lib/sqlite3/test/hooks.py index a6161fac85..dad35d9674 100644 --- a/Lib/sqlite3/test/hooks.py +++ b/Lib/sqlite3/test/hooks.py @@ -175,10 +175,60 @@ class ProgressTests(unittest.TestCase): con.execute("select 1 union select 2 union select 3").fetchall() self.assertEqual(action, 0, "progress handler was not cleared") +class TraceCallbackTests(unittest.TestCase): + def CheckTraceCallbackUsed(self): + """ + Test that the trace callback is invoked once it is set. + """ + con = sqlite.connect(":memory:") + traced_statements = [] + def trace(statement): + traced_statements.append(statement) + con.set_trace_callback(trace) + con.execute("create table foo(a, b)") + self.assertTrue(traced_statements) + self.assertTrue(any("create table foo" in stmt for stmt in traced_statements)) + + def CheckClearTraceCallback(self): + """ + Test that setting the trace callback to None clears the previously set callback. + """ + con = sqlite.connect(":memory:") + traced_statements = [] + def trace(statement): + traced_statements.append(statement) + con.set_trace_callback(trace) + con.set_trace_callback(None) + con.execute("create table foo(a, b)") + self.assertFalse(traced_statements, "trace callback was not cleared") + + def CheckUnicodeContent(self): + """ + Test that the statement can contain unicode literals. + """ + unicode_value = '\xf6\xe4\xfc\xd6\xc4\xdc\xdf\u20ac' + con = sqlite.connect(":memory:") + traced_statements = [] + def trace(statement): + traced_statements.append(statement) + con.set_trace_callback(trace) + con.execute("create table foo(x)") + # Can't execute bound parameters as their values don't appear + # in traced statements before SQLite 3.6.21 + # (cf. http://www.sqlite.org/draft/releaselog/3_6_21.html) + con.execute('insert into foo(x) values ("%s")' % unicode_value) + con.commit() + self.assertTrue(any(unicode_value in stmt for stmt in traced_statements), + "Unicode data %s garbled in trace callback: %s" + % (ascii(unicode_value), ', '.join(map(ascii, traced_statements)))) + + + def suite(): collation_suite = unittest.makeSuite(CollationTests, "Check") progress_suite = unittest.makeSuite(ProgressTests, "Check") - return unittest.TestSuite((collation_suite, progress_suite)) + trace_suite = unittest.makeSuite(TraceCallbackTests, "Check") + return unittest.TestSuite((collation_suite, progress_suite, trace_suite)) def test(): runner = unittest.TextTestRunner() diff --git a/Lib/sqlite3/test/regression.py b/Lib/sqlite3/test/regression.py index 7d0553d8f1..c7551e35a1 100644 --- a/Lib/sqlite3/test/regression.py +++ b/Lib/sqlite3/test/regression.py @@ -281,6 +281,28 @@ class RegressionTests(unittest.TestCase): # Lone surrogate cannot be encoded to the default encoding (utf8) "\uDC80", collation_cb) + def CheckRecursiveCursorUse(self): + """ + http://bugs.python.org/issue10811 + + Recursively using a cursor, such as when reusing it from a generator led to segfaults. + Now we catch recursive cursor usage and raise a ProgrammingError. + """ + con = sqlite.connect(":memory:") + + cur = con.cursor() + cur.execute("create table a (bar)") + cur.execute("create table b (baz)") + + def foo(): + cur.execute("insert into a (bar) values (?)", (1,)) + yield 1 + + with self.assertRaises(sqlite.ProgrammingError): + cur.executemany("insert into b (baz) values (?)", + ((i,) for i in foo())) + + def suite(): regression_suite = unittest.makeSuite(RegressionTests, "Check") return unittest.TestSuite((regression_suite,)) diff --git a/Lib/sqlite3/test/types.py b/Lib/sqlite3/test/types.py index 29413e14ec..d214f3d630 100644 --- a/Lib/sqlite3/test/types.py +++ b/Lib/sqlite3/test/types.py @@ -85,7 +85,7 @@ class DeclTypesTests(unittest.TestCase): if isinstance(_val, bytes): # sqlite3 always calls __init__ with a bytes created from a # UTF-8 string when __conform__ was used to store the object. - _val = _val.decode('utf8') + _val = _val.decode('utf-8') self.val = _val def __cmp__(self, other): diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index 13737ca12f..ae63c31ebc 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -791,7 +791,7 @@ def parse_template(source, pattern): else: # The tokenizer implicitly decodes bytes objects as latin-1, we must # therefore re-encode the final representation. - encode = lambda x: x.encode('latin1') + encode = lambda x: x.encode('latin-1') for c, s in p: if c is MARK: groupsappend((i, s)) diff --git a/Lib/ssl.py b/Lib/ssl.py index ce9ebdf30a..39cef2c699 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -63,7 +63,7 @@ from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION from _ssl import _SSLContext, SSLError from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED from _ssl import OP_ALL, OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_TLSv1 -from _ssl import RAND_status, RAND_egd, RAND_add +from _ssl import RAND_status, RAND_egd, RAND_add, RAND_bytes, RAND_pseudo_bytes from _ssl import ( SSL_ERROR_ZERO_RETURN, SSL_ERROR_WANT_READ, @@ -76,7 +76,8 @@ from _ssl import ( SSL_ERROR_INVALID_ERROR_CODE, ) from _ssl import HAS_SNI -from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1 +from _ssl import (PROTOCOL_SSLv3, PROTOCOL_SSLv23, + PROTOCOL_TLSv1) from _ssl import _OPENSSL_API_VERSION _PROTOCOL_NAMES = { @@ -93,11 +94,15 @@ else: from socket import getnameinfo as _getnameinfo from socket import error as socket_error -from socket import socket, AF_INET, SOCK_STREAM +from socket import socket, AF_INET, SOCK_STREAM, create_connection import base64 # for DER-to-PEM translation import traceback import errno +if _ssl.HAS_TLS_UNIQUE: + CHANNEL_BINDING_TYPES = ['tls-unique'] +else: + CHANNEL_BINDING_TYPES = [] class CertificateError(ValueError): pass @@ -350,6 +355,12 @@ class SSLSocket(socket): else: return socket.sendto(self, data, flags_or_addr, addr) + def sendmsg(self, *args, **kwargs): + # Ensure programs don't send data unencrypted if they try to + # use this method. + raise NotImplementedError("sendmsg not allowed on instances of %s" % + self.__class__) + def sendall(self, data, flags=0): self._checkClosed() if self._sslobj: @@ -408,6 +419,14 @@ class SSLSocket(socket): else: return socket.recvfrom_into(self, buffer, nbytes, flags) + def recvmsg(self, *args, **kwargs): + raise NotImplementedError("recvmsg not allowed on instances of %s" % + self.__class__) + + def recvmsg_into(self, *args, **kwargs): + raise NotImplementedError("recvmsg_into not allowed on instances of " + "%s" % self.__class__) + def pending(self): self._checkClosed() if self._sslobj: @@ -494,6 +513,21 @@ class SSLSocket(socket): self.do_handshake_on_connect), addr) + def get_channel_binding(self, cb_type="tls-unique"): + """Get channel binding data for current connection. Raise ValueError + if the requested `cb_type` is not supported. Return bytes of the data + or None if the data is not available (e.g. before the handshake). + """ + if cb_type not in CHANNEL_BINDING_TYPES: + raise ValueError("Unsupported channel binding type") + if cb_type != "tls-unique": + raise NotImplementedError( + "{0} channel binding type not implemented" + .format(cb_type)) + if self._sslobj is None: + return None + return self._sslobj.tls_unique_cb() + def __del__(self): # sys.stderr.write("__del__ on %s\n" % repr(self)) self._real_close() @@ -558,9 +592,9 @@ def get_server_certificate(addr, ssl_version=PROTOCOL_SSLv3, ca_certs=None): cert_reqs = CERT_REQUIRED else: cert_reqs = CERT_NONE - s = wrap_socket(socket(), ssl_version=ssl_version, + s = create_connection(addr) + s = wrap_socket(s, ssl_version=ssl_version, cert_reqs=cert_reqs, ca_certs=ca_certs) - s.connect(addr) dercert = s.getpeercert(True) s.close() return DER_cert_to_PEM_cert(dercert) diff --git a/Lib/string.py b/Lib/string.py index ef0334c472..d4f9cd9355 100644 --- a/Lib/string.py +++ b/Lib/string.py @@ -46,23 +46,7 @@ def capwords(s, sep=None): #################################################################### import re as _re - -class _multimap: - """Helper class for combining multiple mappings. - - Used by .{safe_,}substitute() to combine the mapping and keyword - arguments. - """ - def __init__(self, primary, secondary): - self._primary = primary - self._secondary = secondary - - def __getitem__(self, key): - try: - return self._primary[key] - except KeyError: - return self._secondary[key] - +from collections import ChainMap class _TemplateMetaclass(type): pattern = r""" @@ -116,7 +100,7 @@ class Template(metaclass=_TemplateMetaclass): if not args: mapping = kws elif kws: - mapping = _multimap(kws, args[0]) + mapping = ChainMap(kws, args[0]) else: mapping = args[0] # Helper function for .sub() @@ -142,7 +126,7 @@ class Template(metaclass=_TemplateMetaclass): if not args: mapping = kws elif kws: - mapping = _multimap(kws, args[0]) + mapping = ChainMap(kws, args[0]) else: mapping = args[0] # Helper function for .sub() diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 017f58d717..2c5c888910 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -191,8 +191,10 @@ should prepare for OSErrors. A ValueError will be raised if Popen is called with invalid arguments. -check_call() and check_output() will raise CalledProcessError, if the -called process returns a non-zero return code. +Exceptions defined within this module inherit from SubprocessError. +check_call() and check_output() will raise CalledProcessError if the +called process returns a non-zero return code. TimeoutExpired +be raised if a timeout was specified and expired. Security @@ -340,6 +342,7 @@ mswindows = (sys.platform == "win32") import io import os +import time import traceback import gc import signal @@ -348,7 +351,10 @@ import warnings import errno # Exception classes used by this module. -class CalledProcessError(Exception): +class SubprocessError(Exception): pass + + +class CalledProcessError(SubprocessError): """This exception is raised when a process run by check_call() or check_output() returns a non-zero exit status. The exit status will be stored in the returncode attribute; @@ -362,6 +368,20 @@ class CalledProcessError(Exception): return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) +class TimeoutExpired(SubprocessError): + """This exception is raised when the timeout expires while waiting for a + child process. + """ + def __init__(self, cmd, timeout, output=None): + self.cmd = cmd + self.timeout = timeout + self.output = output + + def __str__(self): + return ("Command '%s' timed out after %s seconds" % + (self.cmd, self.timeout)) + + if mswindows: import threading import msvcrt @@ -377,42 +397,17 @@ if mswindows: else: import select _has_poll = hasattr(select, 'poll') - import fcntl - import pickle - - try: - import _posixsubprocess - except ImportError: - _posixsubprocess = None - warnings.warn("The _posixsubprocess module is not being used. " - "Child process reliability may suffer if your " - "program uses threads.", RuntimeWarning) + import _posixsubprocess + _create_pipe = _posixsubprocess.cloexec_pipe # When select or poll has indicated that the file is writable, # we can write up to _PIPE_BUF bytes without risk of blocking. # POSIX defines PIPE_BUF as >= 512. _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) - _FD_CLOEXEC = getattr(fcntl, 'FD_CLOEXEC', 1) - - def _set_cloexec(fd, cloexec): - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if cloexec: - fcntl.fcntl(fd, fcntl.F_SETFD, old | _FD_CLOEXEC) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~_FD_CLOEXEC) - - if _posixsubprocess: - _create_pipe = _posixsubprocess.cloexec_pipe - else: - def _create_pipe(): - fds = os.pipe() - _set_cloexec(fds[0], True) - _set_cloexec(fds[1], True) - return fds __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", - "getoutput", "check_output", "CalledProcessError"] + "getoutput", "check_output", "CalledProcessError", "DEVNULL"] if mswindows: from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, @@ -448,6 +443,7 @@ def _cleanup(): PIPE = -1 STDOUT = -2 +DEVNULL = -3 def _eintr_retry_call(func, *args): @@ -460,15 +456,21 @@ def _eintr_retry_call(func, *args): raise -def call(*popenargs, **kwargs): - """Run command with arguments. Wait for command to complete, then - return the returncode attribute. +def call(*popenargs, timeout=None, **kwargs): + """Run command with arguments. Wait for command to complete or + timeout, then return the returncode attribute. The arguments are the same as for the Popen constructor. Example: retcode = call(["ls", "-l"]) """ - return Popen(*popenargs, **kwargs).wait() + with Popen(*popenargs, **kwargs) as p: + try: + return p.wait(timeout=timeout) + except: + p.kill() + p.wait() + raise def check_call(*popenargs, **kwargs): @@ -477,7 +479,7 @@ def check_call(*popenargs, **kwargs): CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute. - The arguments are the same as for the Popen constructor. Example: + The arguments are the same as for the call function. Example: check_call(["ls", "-l"]) """ @@ -490,7 +492,7 @@ def check_call(*popenargs, **kwargs): return 0 -def check_output(*popenargs, **kwargs): +def check_output(*popenargs, timeout=None, **kwargs): r"""Run command with arguments and return its output as a byte string. If the exit code was non-zero it raises a CalledProcessError. The @@ -512,14 +514,20 @@ def check_output(*popenargs, **kwargs): """ if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') - process = Popen(*popenargs, stdout=PIPE, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd, output=output) + with Popen(*popenargs, stdout=PIPE, **kwargs) as process: + try: + output, unused_err = process.communicate(timeout=timeout) + except TimeoutExpired: + process.kill() + output, unused_err = process.communicate() + raise TimeoutExpired(process.args, timeout, output=output) + except: + process.kill() + process.wait() + raise + retcode = process.poll() + if retcode: + raise CalledProcessError(retcode, process.args, output=output) return output @@ -614,11 +622,19 @@ def getstatusoutput(cmd): >>> subprocess.getstatusoutput('/bin/junk') (256, 'sh: /bin/junk: not found') """ - pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r') - text = pipe.read() - sts = pipe.close() - if sts is None: sts = 0 - if text[-1:] == '\n': text = text[:-1] + with os.popen('{ ' + cmd + '; } 2>&1', 'r') as pipe: + try: + text = pipe.read() + sts = pipe.close() + except: + process = pipe._proc + process.kill() + process.wait() + raise + if sts is None: + sts = 0 + if text[-1:] == '\n': + text = text[:-1] return sts, text @@ -650,6 +666,8 @@ class Popen(object): _cleanup() self._child_created = False + self._input = None + self._communication_started = False if bufsize is None: bufsize = 0 # Restore default if not isinstance(bufsize, int): @@ -684,6 +702,7 @@ class Popen(object): raise ValueError("creationflags is only supported on Windows " "platforms") + self.args = args self.stdin = None self.stdout = None self.stderr = None @@ -784,19 +803,28 @@ class Popen(object): # Child is still running, keep us alive until we can wait on it. _active.append(self) + def _get_devnull(self): + if not hasattr(self, '_devnull'): + self._devnull = os.open(os.devnull, os.O_RDWR) + return self._devnull - def communicate(self, input=None): + def communicate(self, input=None, timeout=None): """Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached. Wait for - process to terminate. The optional input argument should be a - string to be sent to the child process, or None, if no data + process to terminate. The optional input argument should be + bytes to be sent to the child process, or None, if no data should be sent to the child. communicate() returns a tuple (stdout, stderr).""" - # Optimization: If we are only using one pipe, or no pipe at - # all, using select() or threads is unnecessary. - if [self.stdin, self.stdout, self.stderr].count(None) >= 2: + if self._communication_started and input: + raise ValueError("Cannot send input after starting communication") + + # Optimization: If we are not worried about timeouts, we haven't + # started communicating, and we have one or zero pipes, using select() + # or threads is unnecessary. + if (timeout is None and not self._communication_started and + [self.stdin, self.stdout, self.stderr].count(None) >= 2): stdout = None stderr = None if self.stdin: @@ -814,15 +842,42 @@ class Popen(object): stderr = _eintr_retry_call(self.stderr.read) self.stderr.close() self.wait() - return (stdout, stderr) + else: + if timeout is not None: + endtime = time.time() + timeout + else: + endtime = None + + try: + stdout, stderr = self._communicate(input, endtime, timeout) + finally: + self._communication_started = True + + sts = self.wait(timeout=self._remaining_time(endtime)) - return self._communicate(input) + return (stdout, stderr) def poll(self): return self._internal_poll() + def _remaining_time(self, endtime): + """Convenience for _communicate when computing timeouts.""" + if endtime is None: + return None + else: + return endtime - time.time() + + + def _check_timeout(self, endtime, orig_timeout): + """Convenience for checking if a timeout has expired.""" + if endtime is None: + return + if time.time() > endtime: + raise TimeoutExpired(self.args, orig_timeout) + + if mswindows: # # Windows methods @@ -844,6 +899,8 @@ class Popen(object): p2cread, _ = _subprocess.CreatePipe(None, 0) elif stdin == PIPE: p2cread, p2cwrite = _subprocess.CreatePipe(None, 0) + elif stdin == DEVNULL: + p2cread = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: @@ -857,6 +914,8 @@ class Popen(object): _, c2pwrite = _subprocess.CreatePipe(None, 0) elif stdout == PIPE: c2pread, c2pwrite = _subprocess.CreatePipe(None, 0) + elif stdout == DEVNULL: + c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: @@ -872,6 +931,8 @@ class Popen(object): errread, errwrite = _subprocess.CreatePipe(None, 0) elif stderr == STDOUT: errwrite = c2pwrite + elif stderr == DEVNULL: + errwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: @@ -981,6 +1042,8 @@ class Popen(object): c2pwrite.Close() if errwrite != -1: errwrite.Close() + if hasattr(self, '_devnull'): + os.close(self._devnull) # Retain the process handle, but close the thread handle self._child_created = True @@ -1005,12 +1068,20 @@ class Popen(object): return self.returncode - def wait(self): + def wait(self, timeout=None, endtime=None): """Wait for child process to terminate. Returns returncode attribute.""" + if endtime is not None: + timeout = self._remaining_time(endtime) + if timeout is None: + timeout_millis = _subprocess.INFINITE + else: + timeout_millis = int(timeout * 1000) if self.returncode is None: - _subprocess.WaitForSingleObject(self._handle, - _subprocess.INFINITE) + result = _subprocess.WaitForSingleObject(self._handle, + timeout_millis) + if result == _subprocess.WAIT_TIMEOUT: + raise TimeoutExpired(self.args, timeout) self.returncode = _subprocess.GetExitCodeProcess(self._handle) return self.returncode @@ -1020,22 +1091,23 @@ class Popen(object): fh.close() - def _communicate(self, input): - stdout = None # Return - stderr = None # Return - - if self.stdout: - stdout = [] - stdout_thread = threading.Thread(target=self._readerthread, - args=(self.stdout, stdout)) - stdout_thread.daemon = True - stdout_thread.start() - if self.stderr: - stderr = [] - stderr_thread = threading.Thread(target=self._readerthread, - args=(self.stderr, stderr)) - stderr_thread.daemon = True - stderr_thread.start() + def _communicate(self, input, endtime, orig_timeout): + # Start reader threads feeding into a list hanging off of this + # object, unless they've already been started. + if self.stdout and not hasattr(self, "_stdout_buff"): + self._stdout_buff = [] + self.stdout_thread = \ + threading.Thread(target=self._readerthread, + args=(self.stdout, self._stdout_buff)) + self.stdout_thread.daemon = True + self.stdout_thread.start() + if self.stderr and not hasattr(self, "_stderr_buff"): + self._stderr_buff = [] + self.stderr_thread = \ + threading.Thread(target=self._readerthread, + args=(self.stderr, self._stderr_buff)) + self.stderr_thread.daemon = True + self.stderr_thread.start() if self.stdin: if input is not None: @@ -1046,10 +1118,28 @@ class Popen(object): raise self.stdin.close() + # Wait for the reader threads, or time out. If we time out, the + # threads remain reading and the fds left open in case the user + # calls communicate again. + if self.stdout is not None: + self.stdout_thread.join(self._remaining_time(endtime)) + if self.stdout_thread.isAlive(): + raise TimeoutExpired(self.args, orig_timeout) + if self.stderr is not None: + self.stderr_thread.join(self._remaining_time(endtime)) + if self.stderr_thread.isAlive(): + raise TimeoutExpired(self.args, orig_timeout) + + # Collect the output from and close both pipes, now that we know + # both have been read successfully. + stdout = None + stderr = None if self.stdout: - stdout_thread.join() + stdout = self._stdout_buff + self.stdout.close() if self.stderr: - stderr_thread.join() + stderr = self._stderr_buff + self.stderr.close() # All data exchanged. Translate lists into strings. if stdout is not None: @@ -1057,7 +1147,6 @@ class Popen(object): if stderr is not None: stderr = stderr[0] - self.wait() return (stdout, stderr) def send_signal(self, sig): @@ -1095,6 +1184,8 @@ class Popen(object): pass elif stdin == PIPE: p2cread, p2cwrite = _create_pipe() + elif stdin == DEVNULL: + p2cread = self._get_devnull() elif isinstance(stdin, int): p2cread = stdin else: @@ -1105,6 +1196,8 @@ class Popen(object): pass elif stdout == PIPE: c2pread, c2pwrite = _create_pipe() + elif stdout == DEVNULL: + c2pwrite = self._get_devnull() elif isinstance(stdout, int): c2pwrite = stdout else: @@ -1117,6 +1210,8 @@ class Popen(object): errread, errwrite = _create_pipe() elif stderr == STDOUT: errwrite = c2pwrite + elif stderr == DEVNULL: + errwrite = self._get_devnull() elif isinstance(stderr, int): errwrite = stderr else: @@ -1147,7 +1242,7 @@ class Popen(object): restore_signals, start_new_session): """Execute program (POSIX version)""" - if isinstance(args, str): + if isinstance(args, (str, bytes)): args = [args] else: args = list(args) @@ -1166,149 +1261,34 @@ class Popen(object): errpipe_read, errpipe_write = _create_pipe() try: try: - - if _posixsubprocess: - # We must avoid complex work that could involve - # malloc or free in the child process to avoid - # potential deadlocks, thus we do all this here. - # and pass it to fork_exec() - - if env is not None: - env_list = [os.fsencode(k) + b'=' + os.fsencode(v) - for k, v in env.items()] - else: - env_list = None # Use execv instead of execve. - executable = os.fsencode(executable) - if os.path.dirname(executable): - executable_list = (executable,) - else: - # This matches the behavior of os._execvpe(). - executable_list = tuple( - os.path.join(os.fsencode(dir), executable) - for dir in os.get_exec_path(env)) - fds_to_keep = set(pass_fds) - fds_to_keep.add(errpipe_write) - self.pid = _posixsubprocess.fork_exec( - args, executable_list, - close_fds, sorted(fds_to_keep), cwd, env_list, - p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite, - errpipe_read, errpipe_write, - restore_signals, start_new_session, preexec_fn) - self._child_created = True + # We must avoid complex work that could involve + # malloc or free in the child process to avoid + # potential deadlocks, thus we do all this here. + # and pass it to fork_exec() + + if env is not None: + env_list = [os.fsencode(k) + b'=' + os.fsencode(v) + for k, v in env.items()] else: - # Pure Python implementation: It is not thread safe. - # This implementation may deadlock in the child if your - # parent process has any other threads running. - - gc_was_enabled = gc.isenabled() - # Disable gc to avoid bug where gc -> file_dealloc -> - # write to stderr -> hang. See issue1336 - gc.disable() - try: - self.pid = os.fork() - except: - if gc_was_enabled: - gc.enable() - raise - self._child_created = True - if self.pid == 0: - # Child - try: - # Close parent's pipe ends - if p2cwrite != -1: - os.close(p2cwrite) - if c2pread != -1: - os.close(c2pread) - if errread != -1: - os.close(errread) - os.close(errpipe_read) - - # When duping fds, if there arises a situation - # where one of the fds is either 0, 1 or 2, it - # is possible that it is overwritten (#12607). - if c2pwrite == 0: - c2pwrite = os.dup(c2pwrite) - if errwrite == 0 or errwrite == 1: - errwrite = os.dup(errwrite) - - # Dup fds for child - def _dup2(a, b): - # dup2() removes the CLOEXEC flag but - # we must do it ourselves if dup2() - # would be a no-op (issue #10806). - if a == b: - _set_cloexec(a, False) - elif a != -1: - os.dup2(a, b) - _dup2(p2cread, 0) - _dup2(c2pwrite, 1) - _dup2(errwrite, 2) - - # Close pipe fds. Make sure we don't close the - # same fd more than once, or standard fds. - closed = set() - for fd in [p2cread, c2pwrite, errwrite]: - if fd > 2 and fd not in closed: - os.close(fd) - closed.add(fd) - - # Close all other fds, if asked for - if close_fds: - fds_to_keep = set(pass_fds) - fds_to_keep.add(errpipe_write) - self._close_fds(fds_to_keep) - - - if cwd is not None: - os.chdir(cwd) - - # This is a copy of Python/pythonrun.c - # _Py_RestoreSignals(). If that were exposed - # as a sys._py_restoresignals func it would be - # better.. but this pure python implementation - # isn't likely to be used much anymore. - if restore_signals: - signals = ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ') - for sig in signals: - if hasattr(signal, sig): - signal.signal(getattr(signal, sig), - signal.SIG_DFL) - - if start_new_session and hasattr(os, 'setsid'): - os.setsid() - - if preexec_fn: - preexec_fn() - - if env is None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) - - except: - try: - exc_type, exc_value = sys.exc_info()[:2] - if isinstance(exc_value, OSError): - errno_num = exc_value.errno - else: - errno_num = 0 - message = '%s:%x:%s' % (exc_type.__name__, - errno_num, exc_value) - message = message.encode(errors="surrogatepass") - os.write(errpipe_write, message) - except Exception: - # We MUST not allow anything odd happening - # above to prevent us from exiting below. - pass - - # This exitcode won't be reported to applications - # so it really doesn't matter what we return. - os._exit(255) - - # Parent - if gc_was_enabled: - gc.enable() + env_list = None # Use execv instead of execve. + executable = os.fsencode(executable) + if os.path.dirname(executable): + executable_list = (executable,) + else: + # This matches the behavior of os._execvpe(). + executable_list = tuple( + os.path.join(os.fsencode(dir), executable) + for dir in os.get_exec_path(env)) + fds_to_keep = set(pass_fds) + fds_to_keep.add(errpipe_write) + self.pid = _posixsubprocess.fork_exec( + args, executable_list, + close_fds, sorted(fds_to_keep), cwd, env_list, + p2cread, p2cwrite, c2pread, c2pwrite, + errread, errwrite, + errpipe_read, errpipe_write, + restore_signals, start_new_session, preexec_fn) + self._child_created = True finally: # be sure the FD is closed no matter what os.close(errpipe_write) @@ -1319,6 +1299,8 @@ class Popen(object): os.close(c2pwrite) if errwrite != -1 and errread != -1: os.close(errwrite) + if hasattr(self, '_devnull'): + os.close(self._devnull) # Wait for exec to fail or succeed; possibly raising an # exception (limited in size) @@ -1396,25 +1378,57 @@ class Popen(object): return self.returncode - def wait(self): + def _try_wait(self, wait_flags): + try: + (pid, sts) = _eintr_retry_call(os.waitpid, self.pid, wait_flags) + except OSError as e: + if e.errno != errno.ECHILD: + raise + # This happens if SIGCLD is set to be ignored or waiting + # for child processes has otherwise been disabled for our + # process. This child is dead, we can't get the status. + pid = self.pid + sts = 0 + return (pid, sts) + + + def wait(self, timeout=None, endtime=None): """Wait for child process to terminate. Returns returncode attribute.""" - if self.returncode is None: - try: - pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError as e: - if e.errno != errno.ECHILD: - raise - # This happens if SIGCLD is set to be ignored or waiting - # for child processes has otherwise been disabled for our - # process. This child is dead, we can't get the status. - sts = 0 + if self.returncode is not None: + return self.returncode + + # endtime is preferred to timeout. timeout is only used for + # printing. + if endtime is not None or timeout is not None: + if endtime is None: + endtime = time.time() + timeout + elif timeout is None: + timeout = self._remaining_time(endtime) + + if endtime is not None: + # Enter a busy loop if we have a timeout. This busy loop was + # cribbed from Lib/threading.py in Thread.wait() at r71065. + delay = 0.0005 # 500 us -> initial delay of 1 ms + while True: + (pid, sts) = self._try_wait(os.WNOHANG) + assert pid == self.pid or pid == 0 + if pid == self.pid: + self._handle_exitstatus(sts) + break + remaining = self._remaining_time(endtime) + if remaining <= 0: + raise TimeoutExpired(self.args, timeout) + delay = min(delay * 2, remaining, .05) + time.sleep(delay) + elif self.returncode is None: + (pid, sts) = self._try_wait(0) self._handle_exitstatus(sts) return self.returncode - def _communicate(self, input): - if self.stdin: + def _communicate(self, input, endtime, orig_timeout): + if self.stdin and not self._communication_started: # Flush stdio buffer. This might block, if the user has # been writing to .stdin in an uncontrolled fashion. self.stdin.flush() @@ -1422,9 +1436,13 @@ class Popen(object): self.stdin.close() if _has_poll: - stdout, stderr = self._communicate_with_poll(input) + stdout, stderr = self._communicate_with_poll(input, endtime, + orig_timeout) else: - stdout, stderr = self._communicate_with_select(input) + stdout, stderr = self._communicate_with_select(input, endtime, + orig_timeout) + + self.wait(timeout=self._remaining_time(endtime)) # All data exchanged. Translate lists into strings. if stdout is not None: @@ -1442,67 +1460,87 @@ class Popen(object): stderr = self._translate_newlines(stderr, self.stderr.encoding) - self.wait() return (stdout, stderr) - def _communicate_with_poll(self, input): + def _communicate_with_poll(self, input, endtime, orig_timeout): stdout = None # Return stderr = None # Return - fd2file = {} - fd2output = {} + + if not self._communication_started: + self._fd2file = {} poller = select.poll() def register_and_append(file_obj, eventmask): poller.register(file_obj.fileno(), eventmask) - fd2file[file_obj.fileno()] = file_obj + self._fd2file[file_obj.fileno()] = file_obj def close_unregister_and_remove(fd): poller.unregister(fd) - fd2file[fd].close() - fd2file.pop(fd) + self._fd2file[fd].close() + self._fd2file.pop(fd) if self.stdin and input: register_and_append(self.stdin, select.POLLOUT) + # Only create this mapping if we haven't already. + if not self._communication_started: + self._fd2output = {} + if self.stdout: + self._fd2output[self.stdout.fileno()] = [] + if self.stderr: + self._fd2output[self.stderr.fileno()] = [] + select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI if self.stdout: register_and_append(self.stdout, select_POLLIN_POLLPRI) - fd2output[self.stdout.fileno()] = stdout = [] + stdout = self._fd2output[self.stdout.fileno()] if self.stderr: register_and_append(self.stderr, select_POLLIN_POLLPRI) - fd2output[self.stderr.fileno()] = stderr = [] - - input_offset = 0 - while fd2file: + stderr = self._fd2output[self.stderr.fileno()] + + # Save the input here so that if we time out while communicating, + # we can continue sending input if we retry. + if self.stdin and self._input is None: + self._input_offset = 0 + self._input = input + if self.universal_newlines: + self._input = self._input.encode(self.stdin.encoding) + + while self._fd2file: + timeout = self._remaining_time(endtime) + if timeout is not None and timeout < 0: + raise TimeoutExpired(self.args, orig_timeout) try: - ready = poller.poll() + ready = poller.poll(timeout) except select.error as e: if e.args[0] == errno.EINTR: continue raise + self._check_timeout(endtime, orig_timeout) # XXX Rewrite these to use non-blocking I/O on the # file objects; they are no longer using C stdio! for fd, mode in ready: if mode & select.POLLOUT: - chunk = input[input_offset : input_offset + _PIPE_BUF] + chunk = self._input[self._input_offset : + self._input_offset + _PIPE_BUF] try: - input_offset += os.write(fd, chunk) + self._input_offset += os.write(fd, chunk) except OSError as e: if e.errno == errno.EPIPE: close_unregister_and_remove(fd) else: raise else: - if input_offset >= len(input): + if self._input_offset >= len(self._input): close_unregister_and_remove(fd) elif mode & select_POLLIN_POLLPRI: data = os.read(fd, 4096) if not data: close_unregister_and_remove(fd) - fd2output[fd].append(data) + self._fd2output[fd].append(data) else: # Ignore hang up or errors. close_unregister_and_remove(fd) @@ -1510,61 +1548,87 @@ class Popen(object): return (stdout, stderr) - def _communicate_with_select(self, input): - read_set = [] - write_set = [] + def _communicate_with_select(self, input, endtime, orig_timeout): + if not self._communication_started: + self._read_set = [] + self._write_set = [] + if self.stdin and input: + self._write_set.append(self.stdin) + if self.stdout: + self._read_set.append(self.stdout) + if self.stderr: + self._read_set.append(self.stderr) + + if self.stdin and self._input is None: + self._input_offset = 0 + self._input = input + if self.universal_newlines: + self._input = self._input.encode(self.stdin.encoding) + stdout = None # Return stderr = None # Return - if self.stdin and input: - write_set.append(self.stdin) if self.stdout: - read_set.append(self.stdout) - stdout = [] + if not self._communication_started: + self._stdout_buff = [] + stdout = self._stdout_buff if self.stderr: - read_set.append(self.stderr) - stderr = [] - - input_offset = 0 - while read_set or write_set: + if not self._communication_started: + self._stderr_buff = [] + stderr = self._stderr_buff + + while self._read_set or self._write_set: + timeout = self._remaining_time(endtime) + if timeout is not None and timeout < 0: + raise TimeoutExpired(self.args, orig_timeout) try: - rlist, wlist, xlist = select.select(read_set, write_set, []) + (rlist, wlist, xlist) = \ + select.select(self._read_set, self._write_set, [], + timeout) except select.error as e: if e.args[0] == errno.EINTR: continue raise + # According to the docs, returning three empty lists indicates + # that the timeout expired. + if not (rlist or wlist or xlist): + raise TimeoutExpired(self.args, orig_timeout) + # We also check what time it is ourselves for good measure. + self._check_timeout(endtime, orig_timeout) + # XXX Rewrite these to use non-blocking I/O on the # file objects; they are no longer using C stdio! if self.stdin in wlist: - chunk = input[input_offset : input_offset + _PIPE_BUF] + chunk = self._input[self._input_offset : + self._input_offset + _PIPE_BUF] try: bytes_written = os.write(self.stdin.fileno(), chunk) except OSError as e: if e.errno == errno.EPIPE: self.stdin.close() - write_set.remove(self.stdin) + self._write_set.remove(self.stdin) else: raise else: - input_offset += bytes_written - if input_offset >= len(input): + self._input_offset += bytes_written + if self._input_offset >= len(self._input): self.stdin.close() - write_set.remove(self.stdin) + self._write_set.remove(self.stdin) if self.stdout in rlist: data = os.read(self.stdout.fileno(), 1024) if not data: self.stdout.close() - read_set.remove(self.stdout) + self._read_set.remove(self.stdout) stdout.append(data) if self.stderr in rlist: data = os.read(self.stderr.fileno(), 1024) if not data: self.stderr.close() - read_set.remove(self.stderr) + self._read_set.remove(self.stderr) stderr.append(data) return (stdout, stderr) @@ -1584,68 +1648,3 @@ class Popen(object): """Kill the process with SIGKILL """ self.send_signal(signal.SIGKILL) - - -def _demo_posix(): - # - # Example 1: Simple redirection: Get process list - # - plist = Popen(["ps"], stdout=PIPE).communicate()[0] - print("Process list:") - print(plist) - - # - # Example 2: Change uid before executing child - # - if os.getuid() == 0: - p = Popen(["id"], preexec_fn=lambda: os.setuid(100)) - p.wait() - - # - # Example 3: Connecting several subprocesses - # - print("Looking for 'hda'...") - p1 = Popen(["dmesg"], stdout=PIPE) - p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) - print(repr(p2.communicate()[0])) - - # - # Example 4: Catch execution error - # - print() - print("Trying a weird file...") - try: - print(Popen(["/this/path/does/not/exist"]).communicate()) - except OSError as e: - if e.errno == errno.ENOENT: - print("The file didn't exist. I thought so...") - print("Child traceback:") - print(e.child_traceback) - else: - print("Error", e.errno) - else: - print("Gosh. No error.", file=sys.stderr) - - -def _demo_windows(): - # - # Example 1: Connecting several subprocesses - # - print("Looking for 'PROMPT' in set output...") - p1 = Popen("set", stdout=PIPE, shell=True) - p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE) - print(repr(p2.communicate()[0])) - - # - # Example 2: Simple execution of program - # - print("Executing calc...") - p = Popen("calc") - p.wait() - - -if __name__ == "__main__": - if mswindows: - _demo_windows() - else: - _demo_posix() diff --git a/Lib/sysconfig.cfg b/Lib/sysconfig.cfg new file mode 100644 index 0000000000..573b12e8da --- /dev/null +++ b/Lib/sysconfig.cfg @@ -0,0 +1,111 @@ +[globals] +# These are useful categories that can be referenced at run time, +# using packaging.database.get_file. +# Configuration files +config = {confdir}/{distribution.name} +# Non-writable data that is independent of architecture (images, many xml/text files) +appdata = {datadir}/{distribution.name} +# Non-writable data that is architecture-dependent (some binary data formats) +appdata.arch = {libdir}/{distribution.name} +# Data, written by the app/lib, that must be preserved (databases) +appdata.persistent = {statedir}/lib/{distribution.name} +# Data, written by the app/lib, that can be safely discarded (cache) +appdata.disposable = {statedir}/cache/{distribution.name} +# Help or documentation files +help = {datadir}/{distribution.name} +icon = {datadir}/pixmaps +scripts = {base}/bin + +# Non-runtime files. These are valid categories for marking files for +# install, but they should not be referenced by the app/lib at run time. +# Help or documentation files +doc = {datadir}/doc/{distribution.name} +# GNU info documentation files +info = {datadir}/info +# man pages +man = {datadir}/man + +[posix_prefix] +# Configuration directories. Some of these come straight out of the +# configure script. They are for implementing the other variables, not to +# be used directly in [resource_locations]. +confdir = /etc +datadir = /usr/share +libdir = /usr/lib ; or /usr/lib64 on a multilib system +statedir = /var +# User resource directory +local = ~/.local/{distribution.name} + +stdlib = {base}/lib/python{py_version_short} +platstdlib = {platbase}/lib/python{py_version_short} +purelib = {base}/lib/python{py_version_short}/site-packages +platlib = {platbase}/lib/python{py_version_short}/site-packages +include = {base}/include/python{py_version_short}{abiflags} +platinclude = {platbase}/include/python{py_version_short}{abiflags} +data = {base} + +[posix_home] +stdlib = {base}/lib/python +platstdlib = {base}/lib/python +purelib = {base}/lib/python +platlib = {base}/lib/python +include = {base}/include/python +platinclude = {base}/include/python +scripts = {base}/bin +data = {base} + +[nt] +stdlib = {base}/Lib +platstdlib = {base}/Lib +purelib = {base}/Lib/site-packages +platlib = {base}/Lib/site-packages +include = {base}/Include +platinclude = {base}/Include +scripts = {base}/Scripts +data = {base} + +[os2] +stdlib = {base}/Lib +platstdlib = {base}/Lib +purelib = {base}/Lib/site-packages +platlib = {base}/Lib/site-packages +include = {base}/Include +platinclude = {base}/Include +scripts = {base}/Scripts +data = {base} + +[os2_home] +stdlib = {userbase}/lib/python{py_version_short} +platstdlib = {userbase}/lib/python{py_version_short} +purelib = {userbase}/lib/python{py_version_short}/site-packages +platlib = {userbase}/lib/python{py_version_short}/site-packages +include = {userbase}/include/python{py_version_short} +scripts = {userbase}/bin +data = {userbase} + +[nt_user] +stdlib = {userbase}/Python{py_version_nodot} +platstdlib = {userbase}/Python{py_version_nodot} +purelib = {userbase}/Python{py_version_nodot}/site-packages +platlib = {userbase}/Python{py_version_nodot}/site-packages +include = {userbase}/Python{py_version_nodot}/Include +scripts = {userbase}/Scripts +data = {userbase} + +[posix_user] +stdlib = {userbase}/lib/python{py_version_short} +platstdlib = {userbase}/lib/python{py_version_short} +purelib = {userbase}/lib/python{py_version_short}/site-packages +platlib = {userbase}/lib/python{py_version_short}/site-packages +include = {userbase}/include/python{py_version_short} +scripts = {userbase}/bin +data = {userbase} + +[osx_framework_user] +stdlib = {userbase}/lib/python +platstdlib = {userbase}/lib/python +purelib = {userbase}/lib/python/site-packages +platlib = {userbase}/lib/python/site-packages +include = {userbase}/include +scripts = {userbase}/bin +data = {userbase} diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 2279a51d5e..d07784d967 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -1,9 +1,10 @@ -"""Provide access to Python's configuration information. +"""Access to Python's configuration information.""" -""" -import sys import os +import re +import sys from os.path import pardir, realpath +from configparser import RawConfigParser __all__ = [ 'get_config_h_filename', @@ -17,91 +18,51 @@ __all__ = [ 'get_python_version', 'get_scheme_names', 'parse_config_h', - ] - -_INSTALL_SCHEMES = { - 'posix_prefix': { - 'stdlib': '{base}/lib/python{py_version_short}', - 'platstdlib': '{platbase}/lib/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/lib/python{py_version_short}/site-packages', - 'include': - '{base}/include/python{py_version_short}{abiflags}', - 'platinclude': - '{platbase}/include/python{py_version_short}{abiflags}', - 'scripts': '{base}/bin', - 'data': '{base}', - }, - 'posix_home': { - 'stdlib': '{base}/lib/python', - 'platstdlib': '{base}/lib/python', - 'purelib': '{base}/lib/python', - 'platlib': '{base}/lib/python', - 'include': '{base}/include/python', - 'platinclude': '{base}/include/python', - 'scripts': '{base}/bin', - 'data' : '{base}', - }, - 'nt': { - 'stdlib': '{base}/Lib', - 'platstdlib': '{base}/Lib', - 'purelib': '{base}/Lib/site-packages', - 'platlib': '{base}/Lib/site-packages', - 'include': '{base}/Include', - 'platinclude': '{base}/Include', - 'scripts': '{base}/Scripts', - 'data' : '{base}', - }, - 'os2': { - 'stdlib': '{base}/Lib', - 'platstdlib': '{base}/Lib', - 'purelib': '{base}/Lib/site-packages', - 'platlib': '{base}/Lib/site-packages', - 'include': '{base}/Include', - 'platinclude': '{base}/Include', - 'scripts': '{base}/Scripts', - 'data' : '{base}', - }, - 'os2_home': { - 'stdlib': '{userbase}/lib/python{py_version_short}', - 'platstdlib': '{userbase}/lib/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', - 'scripts': '{userbase}/bin', - 'data' : '{userbase}', - }, - 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot}', - 'platstdlib': '{userbase}/Python{py_version_nodot}', - 'purelib': '{userbase}/Python{py_version_nodot}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot}/site-packages', - 'include': '{userbase}/Python{py_version_nodot}/Include', - 'scripts': '{userbase}/Scripts', - 'data' : '{userbase}', - }, - 'posix_user': { - 'stdlib': '{userbase}/lib/python{py_version_short}', - 'platstdlib': '{userbase}/lib/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', - 'scripts': '{userbase}/bin', - 'data' : '{userbase}', - }, - 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', - 'include': '{userbase}/include', - 'scripts': '{userbase}/bin', - 'data' : '{userbase}', - }, - } - -_SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', - 'scripts', 'data') +] + +# let's read the configuration file +# XXX _CONFIG_DIR will be set by the Makefile later +_CONFIG_DIR = os.path.normpath(os.path.dirname(__file__)) +_CONFIG_FILE = os.path.join(_CONFIG_DIR, 'sysconfig.cfg') +_SCHEMES = RawConfigParser() +_SCHEMES.read(_CONFIG_FILE) +_VAR_REPL = re.compile(r'\{([^{]*?)\}') + + +def _expand_globals(config): + if config.has_section('globals'): + globals = config.items('globals') + else: + globals = tuple() + + sections = config.sections() + for section in sections: + if section == 'globals': + continue + for option, value in globals: + if config.has_option(section, option): + continue + config.set(section, option, value) + config.remove_section('globals') + + # now expanding local variables defined in the cfg file + # + for section in config.sections(): + variables = dict(config.items(section)) + + def _replacer(matchobj): + name = matchobj.group(1) + if name in variables: + return variables[name] + return matchobj.group(0) + + for option, value in config.items(section): + config.set(section, option, _VAR_REPL.sub(_replacer, value)) + +_expand_globals(_SCHEMES) + + # FIXME don't rely on sys.version here, its format is an implementatin detail + # of CPython, use sys.version_info or sys.hexversion _PY_VERSION = sys.version.split()[0] _PY_VERSION_SHORT = sys.version[:3] _PY_VERSION_SHORT_NO_DOT = _PY_VERSION[0] + _PY_VERSION[2] @@ -110,6 +71,7 @@ _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _CONFIG_VARS = None _USER_BASE = None + def _safe_realpath(path): try: return realpath(path) @@ -132,6 +94,7 @@ if os.name == "nt" and "\\pc\\v" in _PROJECT_BASE[-10:].lower(): if os.name == "nt" and "\\pcbuild\\amd64" in _PROJECT_BASE[-14:].lower(): _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir)) + def is_python_build(): for fn in ("Setup.dist", "Setup.local"): if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)): @@ -142,17 +105,25 @@ _PYTHON_BUILD = is_python_build() if _PYTHON_BUILD: for scheme in ('posix_prefix', 'posix_home'): - _INSTALL_SCHEMES[scheme]['include'] = '{srcdir}/Include' - _INSTALL_SCHEMES[scheme]['platinclude'] = '{projectbase}/.' + _SCHEMES.set(scheme, 'include', '{srcdir}/Include') + _SCHEMES.set(scheme, 'platinclude', '{projectbase}/.') + + +def _subst_vars(path, local_vars): + """In the string `path`, replace tokens like {some.thing} with the + corresponding value from the map `local_vars`. + + If there is no corresponding value, leave the token unchanged. + """ + def _replacer(matchobj): + name = matchobj.group(1) + if name in local_vars: + return local_vars[name] + elif name in os.environ: + return os.environ[name] + return matchobj.group(0) + return _VAR_REPL.sub(_replacer, path) -def _subst_vars(s, local_vars): - try: - return s.format(**local_vars) - except KeyError: - try: - return s.format(**os.environ) - except KeyError as var: - raise AttributeError('{%s}' % var) def _extend_dict(target_dict, other_dict): target_keys = target_dict.keys() @@ -161,41 +132,63 @@ def _extend_dict(target_dict, other_dict): continue target_dict[key] = value + def _expand_vars(scheme, vars): res = {} if vars is None: vars = {} _extend_dict(vars, get_config_vars()) - for key, value in _INSTALL_SCHEMES[scheme].items(): + for key, value in _SCHEMES.items(scheme): if os.name in ('posix', 'nt'): value = os.path.expanduser(value) res[key] = os.path.normpath(_subst_vars(value, vars)) return res + +def format_value(value, vars): + def _replacer(matchobj): + name = matchobj.group(1) + if name in vars: + return vars[name] + return matchobj.group(0) + return _VAR_REPL.sub(_replacer, value) + + def _get_default_scheme(): if os.name == 'posix': # the default scheme for posix is posix_prefix return 'posix_prefix' return os.name + def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) + def joinuser(*args): return os.path.expanduser(os.path.join(*args)) # what about 'os2emx', 'riscos' ? if os.name == "nt": base = os.environ.get("APPDATA") or "~" - return env_base if env_base else joinuser(base, "Python") + if env_base: + return env_base + else: + return joinuser(base, "Python") if sys.platform == "darwin": framework = get_config_var("PYTHONFRAMEWORK") if framework: - return env_base if env_base else joinuser("~", "Library", framework, "%d.%d"%( - sys.version_info[:2])) + if env_base: + return env_base + else: + return joinuser("~", "Library", framework, "%d.%d" % + sys.version_info[:2]) - return env_base if env_base else joinuser("~", ".local") + if env_base: + return env_base + else: + return joinuser("~", ".local") def _parse_makefile(filename, vars=None): @@ -205,7 +198,6 @@ def _parse_makefile(filename, vars=None): optional dictionary is passed in as the second argument, it is used instead of a new dictionary. """ - import re # Regexes needed for parsing Makefile (and similar syntaxes, # like old-style Setup files). _variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)") @@ -267,7 +259,8 @@ def _parse_makefile(filename, vars=None): item = os.environ[n] elif n in renamed_variables: - if name.startswith('PY_') and name[3:] in renamed_variables: + if (name.startswith('PY_') and + name[3:] in renamed_variables): item = "" elif 'PY_' + n in notdone: @@ -300,7 +293,6 @@ def _parse_makefile(filename, vars=None): if name not in done: done[name] = value - else: # bogus variable reference (e.g. "prefix=$/opt/python"); # just drop it since we can't deal @@ -321,9 +313,11 @@ def get_makefile_filename(): """Return the path of the Makefile.""" if _PYTHON_BUILD: return os.path.join(_PROJECT_BASE, "Makefile") - return os.path.join(get_path('stdlib'), - 'config-{}{}'.format(_PY_VERSION_SHORT, sys.abiflags), - 'Makefile') + if hasattr(sys, 'abiflags'): + config_dir_name = 'config-%s%s' % (_PY_VERSION_SHORT, sys.abiflags) + else: + config_dir_name = 'config' + return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') def _init_posix(vars): @@ -353,6 +347,7 @@ def _init_posix(vars): if _PYTHON_BUILD: vars['LDSHARED'] = vars['BLDSHARED'] + def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories @@ -376,7 +371,6 @@ def parse_config_h(fp, vars=None): optional dictionary is passed in as the second argument, it is used instead of a new dictionary. """ - import re if vars is None: vars = {} define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") @@ -389,8 +383,10 @@ def parse_config_h(fp, vars=None): m = define_rx.match(line) if m: n, v = m.group(1, 2) - try: v = int(v) - except ValueError: pass + try: + v = int(v) + except ValueError: + pass vars[n] = v else: m = undef_rx.match(line) @@ -398,6 +394,7 @@ def parse_config_h(fp, vars=None): vars[m.group(1)] = 0 return vars + def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: @@ -409,15 +406,17 @@ def get_config_h_filename(): inc_dir = get_path('platinclude') return os.path.join(inc_dir, 'pyconfig.h') + def get_scheme_names(): """Return a tuple containing the schemes names.""" - schemes = list(_INSTALL_SCHEMES.keys()) - schemes.sort() - return tuple(schemes) + return tuple(sorted(_SCHEMES.sections())) + def get_path_names(): """Return a tuple containing the paths names.""" - return _SCHEME_KEYS + # xxx see if we want a static list + return _SCHEMES.options('posix_prefix') + def get_paths(scheme=_get_default_scheme(), vars=None, expand=True): """Return a mapping containing an install scheme. @@ -428,7 +427,8 @@ def get_paths(scheme=_get_default_scheme(), vars=None, expand=True): if expand: return _expand_vars(scheme, vars) else: - return _INSTALL_SCHEMES[scheme] + return dict(_SCHEMES.items(scheme)) + def get_path(name, scheme=_get_default_scheme(), vars=None, expand=True): """Return a path corresponding to the scheme. @@ -437,6 +437,7 @@ def get_path(name, scheme=_get_default_scheme(), vars=None, expand=True): """ return get_paths(scheme, vars, expand)[name] + def get_config_vars(*args): """With no arguments, return a dictionary of all configuration variables relevant for the current platform. @@ -447,13 +448,12 @@ def get_config_vars(*args): With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. """ - import re global _CONFIG_VARS if _CONFIG_VARS is None: _CONFIG_VARS = {} # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the - # Distutils. + # packaging module. _CONFIG_VARS['prefix'] = _PREFIX _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX _CONFIG_VARS['py_version'] = _PY_VERSION @@ -475,14 +475,14 @@ def get_config_vars(*args): # Setting 'userbase' is done below the call to the # init function to enable using 'get_config_var' in # the init-function. - _CONFIG_VARS['userbase'] = _getuserbase() + if sys.version >= '2.6': + _CONFIG_VARS['userbase'] = _getuserbase() if 'srcdir' not in _CONFIG_VARS: _CONFIG_VARS['srcdir'] = _PROJECT_BASE else: _CONFIG_VARS['srcdir'] = _safe_realpath(_CONFIG_VARS['srcdir']) - # Convert srcdir into an absolute path if it appears necessary. # Normally it is relative to the build directory. However, during # testing, for example, we might be running a non-installed python @@ -502,7 +502,7 @@ def get_config_vars(*args): _CONFIG_VARS['srcdir'] = os.path.normpath(srcdir) if sys.platform == 'darwin': - kernel_version = os.uname()[2] # Kernel version (8.4.3) + kernel_version = os.uname()[2] # Kernel version (8.4.3) major_version = int(kernel_version.split('.')[0]) if major_version < 8: @@ -568,6 +568,7 @@ def get_config_vars(*args): else: return _CONFIG_VARS + def get_config_var(name): """Return the value of a single variable using the dictionary returned by 'get_config_vars()'. @@ -576,6 +577,7 @@ def get_config_var(name): """ return get_config_vars().get(name) + def get_platform(): """Return a string that identifies the current platform. @@ -601,7 +603,6 @@ def get_platform(): For other non-POSIX platforms, currently just returns 'sys.platform'. """ - import re if os.name == 'nt': # sniff sys.version for architecture. prefix = " bit (" @@ -646,7 +647,7 @@ def get_platform(): return "%s-%s.%s" % (osname, version, release) elif osname[:6] == "cygwin": osname = "cygwin" - rel_re = re.compile (r'[\d.]+') + rel_re = re.compile(r'[\d.]+') m = rel_re.match(release) if m: release = m.group() @@ -657,11 +658,10 @@ def get_platform(): # to. This makes the compatibility story a bit more sane because the # machine is going to compile and link as if it were # MACOSX_DEPLOYMENT_TARGET. - # cfgvars = get_config_vars() macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') - if 1: + if True: # Always calculate the release of the running machine, # needed to determine if we can build fat binaries or not. @@ -677,14 +677,13 @@ def get_platform(): pass else: try: - m = re.search( - r'<key>ProductUserVisibleVersion</key>\s*' + - r'<string>(.*?)</string>', f.read()) - if m is not None: - macrelease = '.'.join(m.group(1).split('.')[:2]) - # else: fall back to the default behaviour + m = re.search(r'<key>ProductUserVisibleVersion</key>\s*' + r'<string>(.*?)</string>', f.read()) finally: f.close() + if m is not None: + macrelease = '.'.join(m.group(1).split('.')[:2]) + # else: fall back to the default behaviour if not macver: macver = macrelease @@ -693,8 +692,8 @@ def get_platform(): release = macver osname = "macosx" - if (macrelease + '.') >= '10.4.' and \ - '-arch' in get_config_vars().get('CFLAGS', '').strip(): + if ((macrelease + '.') >= '10.4.' and + '-arch' in get_config_vars().get('CFLAGS', '').strip()): # The universal build will build fat binaries, but not on # systems before 10.4 # @@ -721,7 +720,7 @@ def get_platform(): machine = 'universal' else: raise ValueError( - "Don't know machine value for archs=%r"%(archs,)) + "Don't know machine value for archs=%r" % (archs,)) elif machine == 'i386': # On OSX the machine type returned by uname is always the @@ -744,21 +743,24 @@ def get_platform(): def get_python_version(): return _PY_VERSION_SHORT + def _print_dict(title, data): for index, (key, value) in enumerate(sorted(data.items())): if index == 0: - print('{0}: '.format(title)) - print('\t{0} = "{1}"'.format(key, value)) + print('%s: ' % (title)) + print('\t%s = "%s"' % (key, value)) + def _main(): """Display all information sysconfig detains.""" - print('Platform: "{0}"'.format(get_platform())) - print('Python version: "{0}"'.format(get_python_version())) - print('Current installation scheme: "{0}"'.format(_get_default_scheme())) - print('') + print('Platform: "%s"' % get_platform()) + print('Python version: "%s"' % get_python_version()) + print('Current installation scheme: "%s"' % _get_default_scheme()) + print() _print_dict('Paths', get_paths()) - print('') + print() _print_dict('Variables', get_config_vars()) + if __name__ == '__main__': _main() diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index 46f8163e5f..4a54f89e82 100755 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -126,6 +126,9 @@ def check(file): else: print(file, badline, repr(line)) return + finally: + f.close() + if verbose: print("%r: Clean bill of health." % (file,)) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 3d52a81b9e..6bdbf36e83 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -29,8 +29,6 @@ """Read from and write to tar format archives. """ -__version__ = "$Revision$" - version = "0.9.0" __author__ = "Lars Gust\u00e4bel (lars@gustaebel.de)" __date__ = "$Date: 2011-02-25 17:42:01 +0200 (Fri, 25 Feb 2011) $" @@ -1084,7 +1082,7 @@ class TarInfo(object): def create_pax_global_header(cls, pax_headers): """Return the object as a pax global header block sequence. """ - return cls._create_pax_generic_header(pax_headers, XGLTYPE, "utf8") + return cls._create_pax_generic_header(pax_headers, XGLTYPE, "utf-8") def _posix_split_name(self, name): """Split a name longer than 100 chars into a prefix @@ -1167,7 +1165,7 @@ class TarInfo(object): binary = False for keyword, value in pax_headers.items(): try: - value.encode("utf8", "strict") + value.encode("utf-8", "strict") except UnicodeEncodeError: binary = True break @@ -1178,13 +1176,13 @@ class TarInfo(object): records += b"21 hdrcharset=BINARY\n" for keyword, value in pax_headers.items(): - keyword = keyword.encode("utf8") + keyword = keyword.encode("utf-8") if binary: # Try to restore the original byte representation of `value'. # Needless to say, that the encoding must match the string. value = value.encode(encoding, "surrogateescape") else: - value = value.encode("utf8") + value = value.encode("utf-8") l = len(keyword) + len(value) + 3 # ' ' + '=' + '\n' n = p = 0 @@ -1393,7 +1391,7 @@ class TarInfo(object): # the translation to UTF-8 fails. match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) if match is not None: - pax_headers["hdrcharset"] = match.group(1).decode("utf8") + pax_headers["hdrcharset"] = match.group(1).decode("utf-8") # For the time being, we don't care about anything other than "BINARY". # The only other value that is currently allowed by the standard is @@ -1402,7 +1400,7 @@ class TarInfo(object): if hdrcharset == "BINARY": encoding = tarfile.encoding else: - encoding = "utf8" + encoding = "utf-8" # Parse pax header information. A record looks like that: # "%d %s=%s\n" % (length, keyword, value). length is the size @@ -1419,20 +1417,20 @@ class TarInfo(object): length = int(length) value = buf[match.end(2) + 1:match.start(1) + length - 1] - # Normally, we could just use "utf8" as the encoding and "strict" + # Normally, we could just use "utf-8" as the encoding and "strict" # as the error handler, but we better not take the risk. For # example, GNU tar <= 1.23 is known to store filenames it cannot # translate to UTF-8 as raw strings (unfortunately without a # hdrcharset=BINARY header). # We first try the strict standard encoding, and if that fails we # fall back on the user's encoding and error handler. - keyword = self._decode_pax_field(keyword, "utf8", "utf8", + keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", tarfile.errors) if keyword in PAX_NAME_FIELDS: value = self._decode_pax_field(value, encoding, tarfile.encoding, tarfile.errors) else: - value = self._decode_pax_field(value, "utf8", "utf8", + value = self._decode_pax_field(value, "utf-8", "utf-8", tarfile.errors) pax_headers[keyword] = value diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 48b77a87e2..a45000384f 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -10,8 +10,6 @@ This module also provides some data items to the user: TMP_MAX - maximum number of names that will be tried before giving up. - template - the default prefix for all temporary names. - You may change this to control the default prefix. tempdir - If this is set to a string before the first use of any routine from this module, it will be considered as another candidate location to store temporary files. @@ -74,6 +72,8 @@ if hasattr(_os, 'TMP_MAX'): else: TMP_MAX = 10000 +# Although it does not have an underscore for historical reasons, this +# variable is an internal implementation detail (see issue 10354). template = "tmp" # Internal routines. diff --git a/Lib/test/crashers/README b/Lib/test/crashers/README index 2a73e1bbd9..0259a0688c 100644 --- a/Lib/test/crashers/README +++ b/Lib/test/crashers/README @@ -14,3 +14,7 @@ note if the cause is system or environment dependent and what the variables are. Once the crash is fixed, the test case should be moved into an appropriate test (even if it was originally from the test suite). This ensures the regression doesn't happen again. And if it does, it should be easier to track down. + +Also see Lib/test_crashers.py which exercises the crashers in this directory. +In particular, make sure to add any new infinite loop crashers to the black +list so it doesn't try to run them. diff --git a/Lib/test/crashers/compiler_recursion.py b/Lib/test/crashers/compiler_recursion.py index 4954bdd8f0..31f28a956f 100644 --- a/Lib/test/crashers/compiler_recursion.py +++ b/Lib/test/crashers/compiler_recursion.py @@ -1,5 +1,13 @@ """ -The compiler (>= 2.5) recurses happily. +The compiler (>= 2.5) recurses happily until it blows the stack. + +Recorded on the tracker as http://bugs.python.org/issue11383 """ -compile('()'*9**5, '?', 'exec') +# The variant below blows up in compiler_call, but there are assorted +# other variations that blow up in other functions +# e.g. '1*'*10**5+'1' will die in compiler_visit_expr + +# The exact limit to destroy the stack will vary by platform +# but 10M should do the trick even with huge stack allocations +compile('()'*10**7, '?', 'exec') diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 38f3b8f19b..f91e8fc492 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1289,12 +1289,10 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): self.assertTrue(self.theclass.min) self.assertTrue(self.theclass.max) - def test_strftime_out_of_range(self): - # For nasty technical reasons, we can't handle years before 1000. - cls = self.theclass - self.assertEqual(cls(1000, 1, 1).strftime("%Y"), "1000") - for y in 1, 49, 51, 99, 100, 999: - self.assertRaises(ValueError, cls(y, 1, 1).strftime, "%Y") + def test_strftime_y2k(self): + for y in (1, 49, 70, 99, 100, 999, 1000, 1970): + self.assertIn(self.theclass(y, 1, 1).strftime("%Y"), + [str(y),'%04d' % y]) def test_replace(self): cls = self.theclass diff --git a/Lib/test/decimaltestdata/extra.decTest b/Lib/test/decimaltestdata/extra.decTest index fe8b77a6dd..b630d8e3f9 100644 --- a/Lib/test/decimaltestdata/extra.decTest +++ b/Lib/test/decimaltestdata/extra.decTest @@ -222,12 +222,25 @@ extr1700 power 10 1e-999999999 -> 1.000000000000000 Inexact Rounded extr1701 power 100.0 -557.71e-742888888 -> 1.000000000000000 Inexact Rounded extr1702 power 10 1e-100 -> 1.000000000000000 Inexact Rounded +-- Another one (see issue #12080). Thanks again to Stefan Krah. +extr1703 power 4 -1.2e-999999999 -> 1.000000000000000 Inexact Rounded + -- A couple of interesting exact cases for power. Note that the specification -- requires these to be reported as Inexact. extr1710 power 1e375 56e-3 -> 1.000000000000000E+21 Inexact Rounded extr1711 power 10000 0.75 -> 1000.000000000000 Inexact Rounded extr1712 power 1e-24 0.875 -> 1.000000000000000E-21 Inexact Rounded +-- Some more exact cases, exercising power with negative second argument. +extr1720 power 400 -0.5 -> 0.05000000000000000 Inexact Rounded +extr1721 power 4096 -0.75 -> 0.001953125000000000 Inexact Rounded +extr1722 power 625e4 -0.25 -> 0.02000000000000000 Inexact Rounded + +-- Nonexact cases, to exercise some of the early exit conditions from +-- _power_exact. +extr1730 power 2048 -0.75 -> 0.003284751622084822 Inexact Rounded + + -- Tests for the is_* boolean operations precision: 9 maxExponent: 999 diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index 1caab1c491..88527df25e 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -43,6 +43,7 @@ class ForkWait(unittest.TestCase): self.assertEqual(spid, cpid) self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8)) + @support.reap_threads def test_wait(self): for i in range(NUM_THREADS): _thread.start_new(self.f, (i,)) @@ -69,7 +70,8 @@ class ForkWait(unittest.TestCase): os._exit(n) else: # Parent - self.wait_impl(cpid) - # Tell threads to die - self.stop = 1 - time.sleep(2*SHORTSLEEP) # Wait for threads to die + try: + self.wait_impl(cpid) + finally: + # Tell threads to die + self.stop = 1 diff --git a/Lib/test/test_future1.py b/Lib/test/future_test1.py index 297c2e087c..297c2e087c 100644 --- a/Lib/test/test_future1.py +++ b/Lib/test/future_test1.py diff --git a/Lib/test/test_future2.py b/Lib/test/future_test2.py index 3d7fc860a3..3d7fc860a3 100644 --- a/Lib/test/test_future2.py +++ b/Lib/test/future_test2.py diff --git a/Lib/test/json_tests/test_dump.py b/Lib/test/json_tests/test_dump.py index 083c11f08c..f1c1b0478e 100644 --- a/Lib/test/json_tests/test_dump.py +++ b/Lib/test/json_tests/test_dump.py @@ -1,6 +1,7 @@ from io import StringIO from test.json_tests import PyTest, CTest +from test.support import precisionbigmemtest, _1G class TestDump: def test_dump(self): @@ -21,4 +22,20 @@ class TestDump: class TestPyDump(TestDump, PyTest): pass -class TestCDump(TestDump, CTest): pass + +class TestCDump(TestDump, CTest): + + # The size requirement here is hopefully over-estimated (actual + # memory consumption depending on implementation details, and also + # system memory management, since this may allocate a lot of + # small objects). + + @precisionbigmemtest(size=_1G, memuse=1) + def test_large_list(self, size): + N = int(30 * 1024 * 1024 * (size / _1G)) + l = [1] * N + encoded = self.dumps(l) + self.assertEqual(len(encoded), N * 3) + self.assertEqual(encoded[:1], "[") + self.assertEqual(encoded[-2:], "1]") + self.assertEqual(encoded[1:-2], "1, " * (N - 1)) diff --git a/Lib/test/keycert.passwd.pem b/Lib/test/keycert.passwd.pem new file mode 100644 index 0000000000..e90574881d --- /dev/null +++ b/Lib/test/keycert.passwd.pem @@ -0,0 +1,33 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index be054eadf6..42e118ba8f 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -418,6 +418,47 @@ class CommonTest(seq_tests.CommonTest): self.assertRaises(TypeError, u.reverse, 42) + def test_clear(self): + u = self.type2test([2, 3, 4]) + u.clear() + self.assertEqual(u, []) + + u = self.type2test([]) + u.clear() + self.assertEqual(u, []) + + u = self.type2test([]) + u.append(1) + u.clear() + u.append(2) + self.assertEqual(u, [2]) + + self.assertRaises(TypeError, u.clear, None) + + def test_copy(self): + u = self.type2test([1, 2, 3]) + v = u.copy() + self.assertEqual(v, [1, 2, 3]) + + u = self.type2test([]) + v = u.copy() + self.assertEqual(v, []) + + # test that it's indeed a copy and not a reference + u = self.type2test(['a', 'b']) + v = u.copy() + v.append('i') + self.assertEqual(u, ['a', 'b']) + self.assertEqual(v, u + ['i']) + + # test that it's a shallow, not a deep copy + u = self.type2test([1, 2, [3, 4], 5]) + v = u.copy() + self.assertEqual(u, v) + self.assertIs(v[3], u[3]) + + self.assertRaises(TypeError, u.copy, None) + def test_sort(self): u = self.type2test([1, 0]) u.sort() diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 30148e638a..12871c10f0 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -4,7 +4,7 @@ Various tests for synchronization primitives. import sys import time -from _thread import start_new_thread, get_ident, TIMEOUT_MAX +from _thread import start_new_thread, TIMEOUT_MAX import threading import unittest @@ -31,7 +31,7 @@ class Bunch(object): self.finished = [] self._can_exit = not wait_before_exit def task(): - tid = get_ident() + tid = threading.get_ident() self.started.append(tid) try: f() @@ -247,6 +247,7 @@ class RLockTests(BaseLockTests): # Cannot release an unacquired lock lock = self.locktype() self.assertRaises(RuntimeError, lock.release) + self.assertRaises(RuntimeError, lock._release_save) lock.acquire() lock.acquire() lock.release() @@ -254,6 +255,7 @@ class RLockTests(BaseLockTests): lock.release() lock.release() self.assertRaises(RuntimeError, lock.release) + self.assertRaises(RuntimeError, lock._release_save) def test_different_thread(self): # Cannot release from a different thread diff --git a/Lib/test/mailcap.txt b/Lib/test/mailcap.txt new file mode 100644 index 0000000000..f61135d8e5 --- /dev/null +++ b/Lib/test/mailcap.txt @@ -0,0 +1,39 @@ +# Mailcap file for test_mailcap; based on RFC 1524 +# Referred to by test_mailcap.py + +# +# This is a comment. +# + +application/frame; showframe %s; print="cat %s | lp" +application/postscript; ps-to-terminal %s;\ + needsterminal +application/postscript; ps-to-terminal %s; \ + compose=idraw %s +application/x-dvi; xdvi %s +application/x-movie; movieplayer %s; compose=moviemaker %s; \ + description="Movie"; \ + x11-bitmap="/usr/lib/Zmail/bitmaps/movie.xbm" +application/*; echo "This is \"%t\" but \ + is 50 \% Greek to me" \; cat %s; copiousoutput + +audio/basic; showaudio %s; compose=audiocompose %s; edit=audiocompose %s;\ +description="An audio fragment" +audio/* ; /usr/local/bin/showaudio %t + +image/rgb; display %s +#image/gif; display %s +image/x-xwindowdump; display %s + +# The continuation char shouldn't \ +# make a difference in a comment. + +message/external-body; showexternal %s %{access-type} %{name} %{site} \ + %{directory} %{mode} %{server}; needsterminal; composetyped = extcompose %s; \ + description="A reference to data stored in an external location" + +text/richtext; shownonascii iso-8859-8 -e richtext -p %s; test=test "`echo \ + %{charset} | tr '[A-Z]' '[a-z]'`" = iso-8859-8; copiousoutput + +video/mpeg; mpeg_play %s +video/*; animate %s diff --git a/Lib/test/math_testcases.txt b/Lib/test/math_testcases.txt index 5e24335a87..9585188243 100644 --- a/Lib/test/math_testcases.txt +++ b/Lib/test/math_testcases.txt @@ -517,3 +517,117 @@ expm10306 expm1 1.79e308 -> inf overflow -- weaker version of expm10302 expm10307 expm1 709.5 -> 1.3549863193146328e+308 + +------------------------- +-- log2: log to base 2 -- +------------------------- + +-- special values +log20000 log2 0.0 -> -inf divide-by-zero +log20001 log2 -0.0 -> -inf divide-by-zero +log20002 log2 inf -> inf +log20003 log2 -inf -> nan invalid +log20004 log2 nan -> nan + +-- exact value at 1.0 +log20010 log2 1.0 -> 0.0 + +-- negatives +log20020 log2 -5e-324 -> nan invalid +log20021 log2 -1.0 -> nan invalid +log20022 log2 -1.7e-308 -> nan invalid + +-- exact values at powers of 2 +log20100 log2 2.0 -> 1.0 +log20101 log2 4.0 -> 2.0 +log20102 log2 8.0 -> 3.0 +log20103 log2 16.0 -> 4.0 +log20104 log2 32.0 -> 5.0 +log20105 log2 64.0 -> 6.0 +log20106 log2 128.0 -> 7.0 +log20107 log2 256.0 -> 8.0 +log20108 log2 512.0 -> 9.0 +log20109 log2 1024.0 -> 10.0 +log20110 log2 2048.0 -> 11.0 + +log20200 log2 0.5 -> -1.0 +log20201 log2 0.25 -> -2.0 +log20202 log2 0.125 -> -3.0 +log20203 log2 0.0625 -> -4.0 + +-- values close to 1.0 +log20300 log2 1.0000000000000002 -> 3.2034265038149171e-16 +log20301 log2 1.0000000001 -> 1.4426951601859516e-10 +log20302 log2 1.00001 -> 1.4426878274712997e-5 + +log20310 log2 0.9999999999999999 -> -1.6017132519074588e-16 +log20311 log2 0.9999999999 -> -1.4426951603302210e-10 +log20312 log2 0.99999 -> -1.4427022544056922e-5 + +-- tiny values +log20400 log2 5e-324 -> -1074.0 +log20401 log2 1e-323 -> -1073.0 +log20402 log2 1.5e-323 -> -1072.4150374992789 +log20403 log2 2e-323 -> -1072.0 + +log20410 log2 1e-308 -> -1023.1538532253076 +log20411 log2 2.2250738585072014e-308 -> -1022.0 +log20412 log2 4.4501477170144028e-308 -> -1021.0 +log20413 log2 1e-307 -> -1019.8319251304202 + +-- huge values +log20500 log2 1.7976931348623157e+308 -> 1024.0 +log20501 log2 1.7e+308 -> 1023.9193879716706 +log20502 log2 8.9884656743115795e+307 -> 1023.0 + +-- selection of random values +log20600 log2 -7.2174324841039838e+289 -> nan invalid +log20601 log2 -2.861319734089617e+265 -> nan invalid +log20602 log2 -4.3507646894008962e+257 -> nan invalid +log20603 log2 -6.6717265307520224e+234 -> nan invalid +log20604 log2 -3.9118023786619294e+229 -> nan invalid +log20605 log2 -1.5478221302505161e+206 -> nan invalid +log20606 log2 -1.4380485131364602e+200 -> nan invalid +log20607 log2 -3.7235198730382645e+185 -> nan invalid +log20608 log2 -1.0472242235095724e+184 -> nan invalid +log20609 log2 -5.0141781956163884e+160 -> nan invalid +log20610 log2 -2.1157958031160324e+124 -> nan invalid +log20611 log2 -7.9677558612567718e+90 -> nan invalid +log20612 log2 -5.5553906194063732e+45 -> nan invalid +log20613 log2 -16573900952607.953 -> nan invalid +log20614 log2 -37198371019.888618 -> nan invalid +log20615 log2 -6.0727115121422674e-32 -> nan invalid +log20616 log2 -2.5406841656526057e-38 -> nan invalid +log20617 log2 -4.9056766703267657e-43 -> nan invalid +log20618 log2 -2.1646786075228305e-71 -> nan invalid +log20619 log2 -2.470826790488573e-78 -> nan invalid +log20620 log2 -3.8661709303489064e-165 -> nan invalid +log20621 log2 -1.0516496976649986e-182 -> nan invalid +log20622 log2 -1.5935458614317996e-255 -> nan invalid +log20623 log2 -2.8750977267336654e-293 -> nan invalid +log20624 log2 -7.6079466794732585e-296 -> nan invalid +log20625 log2 3.2073253539988545e-307 -> -1018.1505544209213 +log20626 log2 1.674937885472249e-244 -> -809.80634755783126 +log20627 log2 1.0911259044931283e-214 -> -710.76679472274213 +log20628 log2 2.0275372624809709e-154 -> -510.55719818383272 +log20629 log2 7.3926087369631841e-115 -> -379.13564735312292 +log20630 log2 1.3480198206342423e-86 -> -285.25497445094436 +log20631 log2 8.9927384655719947e-83 -> -272.55127136401637 +log20632 log2 3.1452398713597487e-60 -> -197.66251564496875 +log20633 log2 7.0706573215457351e-55 -> -179.88420087782217 +log20634 log2 3.1258285390731669e-49 -> -161.13023800505653 +log20635 log2 8.2253046627829942e-41 -> -133.15898277355879 +log20636 log2 7.8691367397519897e+49 -> 165.75068202732419 +log20637 log2 2.9920561983925013e+64 -> 214.18453534573757 +log20638 log2 4.7827254553946841e+77 -> 258.04629628445673 +log20639 log2 3.1903566496481868e+105 -> 350.47616767491166 +log20640 log2 5.6195082449502419e+113 -> 377.86831861008250 +log20641 log2 9.9625658250651047e+125 -> 418.55752921228753 +log20642 log2 2.7358945220961532e+145 -> 483.13158636923413 +log20643 log2 2.785842387926931e+174 -> 579.49360214860280 +log20644 log2 2.4169172507252751e+193 -> 642.40529039289652 +log20645 log2 3.1689091206395632e+205 -> 682.65924573798395 +log20646 log2 2.535995592365391e+208 -> 692.30359597460460 +log20647 log2 6.2011236566089916e+233 -> 776.64177576730913 +log20648 log2 2.1843274820677632e+253 -> 841.57499717289647 +log20649 log2 8.7493931063474791e+297 -> 989.74182713073981 diff --git a/Lib/test/mock_socket.py b/Lib/test/mock_socket.py index 803693283b..d09e78c1d5 100644 --- a/Lib/test/mock_socket.py +++ b/Lib/test/mock_socket.py @@ -106,7 +106,8 @@ def socket(family=None, type=None, proto=None): return MockSocket() -def create_connection(address, timeout=socket_module._GLOBAL_DEFAULT_TIMEOUT): +def create_connection(address, timeout=socket_module._GLOBAL_DEFAULT_TIMEOUT, + source_address=None): try: int_port = int(address[1]) except ValueError: diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index c2ed0d2849..d79aab7cef 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -4,10 +4,11 @@ import pickle import pickletools import sys import copyreg +import weakref from http.cookies import SimpleCookie from test.support import ( - TestFailed, TESTFN, run_with_locale, + TestFailed, TESTFN, run_with_locale, no_tracing, _2G, _4G, precisionbigmemtest, ) @@ -848,6 +849,25 @@ class AbstractPickleTests(unittest.TestCase): self.assertEqual(B(x), B(y), detail) self.assertEqual(x.__dict__, y.__dict__, detail) + def test_newobj_proxies(self): + # NEWOBJ should use the __class__ rather than the raw type + classes = myclasses[:] + # Cannot create weakproxies to these classes + for c in (MyInt, MyTuple): + classes.remove(c) + for proto in protocols: + for C in classes: + B = C.__base__ + x = C(C.sample) + x.foo = 42 + p = weakref.proxy(x) + s = self.dumps(p, proto) + y = self.loads(s) + self.assertEqual(type(y), type(x)) # rather than type(p) + detail = (proto, C, B, x, y, type(y)) + self.assertEqual(B(x), B(y), detail) + self.assertEqual(x.__dict__, y.__dict__, detail) + # Register a type with copyreg, with extension code extcode. Pickle # an object of that type. Check that the resulting pickle uses opcode # (EXT[124]) under proto 2, and not in proto 1. @@ -1008,13 +1028,13 @@ class AbstractPickleTests(unittest.TestCase): y = self.loads(s) self.assertEqual(y._reduce_called, 1) + @no_tracing def test_bad_getattr(self): x = BadGetattr() for proto in 0, 1: self.assertRaises(RuntimeError, self.dumps, x, proto) # protocol 2 don't raise a RuntimeError. d = self.dumps(x, 2) - self.assertRaises(RuntimeError, self.loads, d) def test_reduce_bad_iterator(self): # Issue4176: crash when 4th and 5th items of __reduce__() @@ -1104,6 +1124,15 @@ class AbstractPickleTests(unittest.TestCase): empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') self.assertEqual(empty, '') + def test_int_pickling_efficiency(self): + # Test compacity of int representation (see issue #12744) + for proto in protocols: + sizes = [len(self.dumps(2**n, proto)) for n in range(70)] + # the size function is monotonic + self.assertEqual(sorted(sizes), sizes) + if proto >= 2: + self.assertLessEqual(sizes[-1], 14) + def check_negative_32b_binXXX(self, dumped): if sys.maxsize > 2**32: self.skipTest("test is only meaningful on 32-bit builds") diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 10b515b5ab..65438aff5a 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -20,6 +20,11 @@ python -E -Wd -m test [options] [test_name1 ...] Options: -h/--help -- print this text and exit +--timeout TIMEOUT + -- dump the traceback and exit if a test takes more + than TIMEOUT seconds; disabled if TIMEOUT is negative + or equals to zero +--wait -- wait for user input, e.g., allow a debugger to be attached Verbosity @@ -44,6 +49,9 @@ Selecting tests -- specify which special resource intensive tests to run -M/--memlimit LIMIT -- run very large memory-consuming tests + --testdir DIR + -- execute test files in the specified directory (instead + of the Python stdlib test suite) Special runs @@ -125,6 +133,8 @@ resources to test. Currently only the following are defined: all - Enable all special resources. + none - Disable all special resources (this is the default). + audio - Tests that use the audio device. (There are known cases of broken audio drivers that can crash Python or even the Linux kernel.) @@ -157,6 +167,7 @@ option '-uall,-gui'. import builtins import errno +import faulthandler import getopt import io import json @@ -165,6 +176,7 @@ import os import platform import random import re +import signal import sys import sysconfig import tempfile @@ -224,6 +236,7 @@ ENV_CHANGED = -1 SKIPPED = -2 RESOURCE_DENIED = -3 INTERRUPTED = -4 +CHILD_ERROR = -5 # error in a child process from test import support @@ -267,6 +280,18 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, on the command line. """ + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + replace_stdout() support.record_original_stdout(sys.stdout) @@ -277,7 +302,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, 'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir', 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=', 'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug', - 'start=', 'nowindows', 'header', 'failfast', 'match']) + 'start=', 'nowindows', 'header', 'testdir=', 'timeout=', 'wait', + 'failfast', 'match']) except getopt.error as msg: usage(msg) @@ -288,6 +314,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, use_resources = [] debug = False start = None + timeout = None for o, a in opts: if o in ('-h', '--help'): print(__doc__) @@ -331,7 +358,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, elif o in ('-T', '--coverage'): trace = True elif o in ('-D', '--coverdir'): - coverdir = os.path.join(os.getcwd(), a) + # CWD is replaced with a temporary dir before calling main(), so we + # need join it with the saved CWD so it goes where the user expects. + coverdir = os.path.join(support.SAVEDCWD, a) elif o in ('-N', '--nocoverdir'): coverdir = None elif o in ('-R', '--huntrleaks'): @@ -360,6 +389,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if r == 'all': use_resources[:] = RESOURCE_NAMES continue + if r == 'none': + del use_resources[:] + continue remove = False if r[0] == '-': remove = True @@ -390,6 +422,15 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, forever = True elif o in ('-j', '--multiprocess'): use_mp = int(a) + if use_mp <= 0: + try: + import multiprocessing + # Use all cores + extras for tests that like to sleep + use_mp = 2 + multiprocessing.cpu_count() + except (ImportError, NotImplementedError): + use_mp = 3 + if use_mp == 1: + use_mp = None elif o == '--header': header = True elif o == '--slaveargs': @@ -402,6 +443,21 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, print() # Force a newline (just in case) print(json.dumps(result)) sys.exit(0) + elif o == '--testdir': + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + testdir = os.path.join(support.SAVEDCWD, a) + elif o == '--timeout': + if hasattr(faulthandler, 'dump_tracebacks_later'): + timeout = float(a) + if timeout <= 0: + timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_tracebacks_later") + timeout = None + elif o == '--wait': + input("Press any key to continue...") else: print(("No handler for option {}. Please report this as a bug " "at http://bugs.python.org.").format(o), file=sys.stderr) @@ -480,7 +536,13 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, print("== ", os.getcwd()) print("Testing with flags:", sys.flags) - alltests = findtests(testdir, stdtests, nottests) + # if testdir is set, then we are not running the python tests suite, so + # don't add default tests to be executed or skipped (pass empty values) + if testdir: + alltests = findtests(testdir, list(), set()) + else: + alltests = findtests(testdir, stdtests, nottests) + selected = tests or args or alltests if single: selected = selected[:1] @@ -555,7 +617,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, (test, verbose, quiet), dict(huntrleaks=huntrleaks, use_resources=use_resources, debug=debug, output_on_failure=verbose3, - failfast=failfast, match_tests=match_tests) + timeout=timeout, failfast=failfast, + match_tests=match_tests) ) yield (test, args_tuple) pending = tests_and_args() @@ -576,10 +639,15 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, universal_newlines=True, close_fds=(os.name != 'nt')) stdout, stderr = popen.communicate() + retcode = popen.wait() # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + return if not result: output.put((None, None, None, None)) return @@ -612,6 +680,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if result[0] == INTERRUPTED: assert result[1] == 'KeyboardInterrupt' raise KeyboardInterrupt # What else? + if result[0] == CHILD_ERROR: + raise Exception("Child error on {}: {}".format(test, result[1])) test_index += 1 except KeyboardInterrupt: interrupted = True @@ -628,13 +698,14 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - tracer.runctx('runtest(test, verbose, quiet)', + tracer.runctx('runtest(test, verbose, quiet, timeout=timeout)', globals=globals(), locals=vars()) else: try: result = runtest(test, verbose, quiet, huntrleaks, debug, output_on_failure=verbose3, - failfast=failfast, match_tests=match_tests) + timeout=timeout, failfast=failfast, + match_tests=match_tests) accumulate_result(test, result) except KeyboardInterrupt: interrupted = True @@ -705,7 +776,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, sys.stdout.flush() try: verbose = True - ok = runtest(test, True, quiet, huntrleaks, debug) + ok = runtest(test, True, quiet, huntrleaks, debug, timeout=timeout) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -730,6 +801,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, sys.exit(len(bad) > 0 or interrupted) +# small set of tests to determine if we have a basically functioning interpreter +# (i.e. if any of these fail, then anything else is likely to follow) STDTESTS = [ 'test_grammar', 'test_opcodes', @@ -740,12 +813,11 @@ STDTESTS = [ 'test_unittest', 'test_doctest', 'test_doctest2', + 'test_support' ] -NOTTESTS = { - 'test_future1', - 'test_future2', -} +# set of tests that we don't want to be executed when using regrtest +NOTTESTS = set() def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): """Return a list of all applicable test modules.""" @@ -754,25 +826,22 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): tests = [] others = set(stdtests) | nottests for name in names: - modname, ext = os.path.splitext(name) - if modname[:5] == "test_" and ext == ".py" and modname not in others: - tests.append(modname) + mod, ext = os.path.splitext(name) + if mod[:5] == "test_" and ext in (".py", "") and mod not in others: + tests.append(mod) return stdtests + sorted(tests) def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error handler) to avoid UnicodeEncodeError when printing a traceback""" - if os.name == "nt": - # Replace sys.stdout breaks the stdout newlines on Windows: issue #8533 - return - import atexit stdout = sys.stdout sys.stdout = open(stdout.fileno(), 'w', encoding=stdout.encoding, errors="backslashreplace", - closefd=False) + closefd=False, + newline='\n') def restore_stdout(): sys.stdout.close() @@ -781,7 +850,8 @@ def replace_stdout(): def runtest(test, verbose, quiet, huntrleaks=False, debug=False, use_resources=None, - output_on_failure=False, failfast=False, match_tests=None): + output_on_failure=False, failfast=False, match_tests=None, + timeout=None): """Run a single test. test -- the name of the test @@ -791,6 +861,8 @@ def runtest(test, verbose, quiet, huntrleaks -- run multiple times to test for leaks; requires a debug build; a triple corresponding to -R's three arguments output_on_failure -- if true, display test output on failure + timeout -- dump the traceback and exit if a test takes more than + timeout seconds Returns one of the test result constants: INTERRUPTED KeyboardInterrupt when run under -j @@ -803,6 +875,9 @@ def runtest(test, verbose, quiet, if use_resources is not None: support.use_resources = use_resources + use_timeout = (timeout is not None) + if use_timeout: + faulthandler.dump_tracebacks_later(timeout, exit=True) try: support.match_tests = match_tests if failfast: @@ -841,6 +916,8 @@ def runtest(test, verbose, quiet, display_failure=not verbose) return result finally: + if use_timeout: + faulthandler.cancel_dump_tracebacks_later() cleanup_test_droppings(test, verbose) runtest.stringio = None @@ -886,7 +963,7 @@ class saved_test_environment: resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', 'warnings.filters', 'asyncore.socket_map', - 'logging._handlers', 'logging._handlerList', + 'logging._handlers', 'logging._handlerList', 'sys.gettrace', 'sys.warnoptions', 'threading._dangling', 'multiprocessing.process._dangling') @@ -935,6 +1012,11 @@ class saved_test_environment: sys.path_hooks = saved_hooks[1] sys.path_hooks[:] = saved_hooks[2] + def get_sys_gettrace(self): + return sys.gettrace() + def restore_sys_gettrace(self, trace_fxn): + sys.settrace(trace_fxn) + def get___import__(self): return builtins.__import__ def restore___import__(self, import_): @@ -1140,7 +1222,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): False if the test didn't leak references; True if we detected refleaks. """ # This code is hackish and inelegant, but it seems to do the job. - import copyreg, _abcoll + import copyreg + import collections.abc if not hasattr(sys, 'gettotalrefcount'): raise Exception("Tracking reference leaks requires a debug build " @@ -1157,7 +1240,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks): else: zdc = zipimport._zip_directory_cache.copy() abcs = {} - for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]: + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): continue for obj in abc.__subclasses__() + [abc]: @@ -1203,7 +1286,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): import gc, copyreg import _strptime, linecache import urllib.parse, urllib.request, mimetypes, doctest - import struct, filecmp, _abcoll + import struct, filecmp, collections.abc from distutils.dir_util import _path_created from weakref import WeakSet @@ -1230,7 +1313,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]: + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): continue for obj in abc.__subclasses__() + [abc]: @@ -1309,8 +1392,8 @@ def printlist(x, width=70, indent=4): # Tests that are expected to be skipped everywhere except on one platform # are also handled separately. -_expectations = { - 'win32': +_expectations = ( + ('win32', """ test__locale test_crypt @@ -1338,15 +1421,15 @@ _expectations = { test_threadsignals test_wait3 test_wait4 - """, - 'linux2': + """), + ('linux', """ test_curses test_largefile test_kqueue test_ossaudiodev - """, - 'unixware7': + """), + ('unixware', """ test_epoll test_largefile @@ -1356,8 +1439,8 @@ _expectations = { test_pyexpat test_sax test_sundry - """, - 'openunix8': + """), + ('openunix', """ test_epoll test_largefile @@ -1367,8 +1450,8 @@ _expectations = { test_pyexpat test_sax test_sundry - """, - 'sco_sv3': + """), + ('sco_sv', """ test_asynchat test_fork1 @@ -1387,8 +1470,8 @@ _expectations = { test_threaded_import test_threadedtempfile test_threading - """, - 'darwin': + """), + ('darwin', """ test__locale test_curses @@ -1400,8 +1483,8 @@ _expectations = { test_minidom test_ossaudiodev test_poll - """, - 'sunos5': + """), + ('sunos', """ test_curses test_dbm @@ -1412,8 +1495,8 @@ _expectations = { test_openpty test_zipfile test_zlib - """, - 'hp-ux11': + """), + ('hp-ux', """ test_curses test_epoll @@ -1428,8 +1511,8 @@ _expectations = { test_sax test_zipfile test_zlib - """, - 'cygwin': + """), + ('cygwin', """ test_curses test_dbm @@ -1440,8 +1523,8 @@ _expectations = { test_locale test_ossaudiodev test_socketserver - """, - 'os2emx': + """), + ('os2emx', """ test_audioop test_curses @@ -1454,8 +1537,8 @@ _expectations = { test_pty test_resource test_signal - """, - 'freebsd4': + """), + ('freebsd', """ test_epoll test_dbm_gnu @@ -1471,8 +1554,8 @@ _expectations = { test_timeout test_urllibnet test_multiprocessing - """, - 'aix5': + """), + ('aix', """ test_bz2 test_epoll @@ -1486,8 +1569,8 @@ _expectations = { test_ttk_textonly test_zipimport test_zlib - """, - 'openbsd3': + """), + ('openbsd', """ test_ctypes test_epoll @@ -1501,8 +1584,8 @@ _expectations = { test_ttk_guionly test_ttk_textonly test_multiprocessing - """, - 'netbsd3': + """), + ('netbsd', """ test_ctypes test_curses @@ -1516,12 +1599,8 @@ _expectations = { test_ttk_guionly test_ttk_textonly test_multiprocessing - """, -} -_expectations['freebsd5'] = _expectations['freebsd4'] -_expectations['freebsd6'] = _expectations['freebsd4'] -_expectations['freebsd7'] = _expectations['freebsd4'] -_expectations['freebsd8'] = _expectations['freebsd4'] + """), +) class _ExpectedSkips: def __init__(self): @@ -1529,9 +1608,13 @@ class _ExpectedSkips: from test import test_timeout self.valid = False - if sys.platform in _expectations: - s = _expectations[sys.platform] - self.expected = set(s.split()) + expected = None + for item in _expectations: + if sys.platform.startswith(item[0]): + expected = item[1] + break + if expected is not None: + self.expected = set(expected.split()) # These are broken tests, for now skipped on every platform. # XXX Fix these! diff --git a/Lib/test/script_helper.py b/Lib/test/script_helper.py index 371c33d138..e556eca52a 100644 --- a/Lib/test/script_helper.py +++ b/Lib/test/script_helper.py @@ -56,11 +56,12 @@ def assert_python_failure(*args, **env_vars): """ return _assert_python(False, *args, **env_vars) -def spawn_python(*args): +def spawn_python(*args, **kw): cmd_line = [sys.executable, '-E'] cmd_line.extend(args) return subprocess.Popen(cmd_line, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + **kw) def kill_python(p): p.stdin.close() diff --git a/Lib/test/ssl_key.passwd.pem b/Lib/test/ssl_key.passwd.pem new file mode 100644 index 0000000000..2524672e70 --- /dev/null +++ b/Lib/test/ssl_key.passwd.pem @@ -0,0 +1,18 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- diff --git a/Lib/test/support.py b/Lib/test/support.py index 8e6ca2a541..03af378efe 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -15,7 +15,7 @@ import shutil import warnings import unittest import importlib -import collections +import collections.abc import re import subprocess import imp @@ -35,25 +35,28 @@ except ImportError: multiprocessing = None +try: + import zlib +except ImportError: + zlib = None + __all__ = [ "Error", "TestFailed", "ResourceDenied", "import_module", "verbose", "use_resources", "max_memuse", "record_original_stdout", "get_original_stdout", "unload", "unlink", "rmtree", "forget", - "is_resource_enabled", "requires", "requires_mac_ver", - "find_unused_port", "bind_port", - "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd", - "findfile", "sortdict", "check_syntax_error", "open_urlresource", - "check_warnings", "CleanImport", "EnvironmentVarGuard", - "TransientResource", "captured_output", "captured_stdout", - "captured_stdin", "captured_stderr", - "time_out", "socket_peer_reset", "ioerror_peer_reset", - "run_with_locale", 'temp_umask', "transient_internet", - "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner", - "run_unittest", "run_doctest", "threading_setup", "threading_cleanup", - "reap_children", "cpython_only", "check_impl_detail", "get_attribute", - "swap_item", "swap_attr", "requires_IEEE_754", + "is_resource_enabled", "requires", "requires_linux_version", + "requires_mac_ver", "find_unused_port", "bind_port", + "IPV6_ENABLED", "is_jython", "TESTFN", "HOST", "SAVEDCWD", "temp_cwd", + "findfile", "create_empty_file", "sortdict", "check_syntax_error", "open_urlresource", + "check_warnings", "CleanImport", "EnvironmentVarGuard", "TransientResource", + "captured_stdout", "captured_stdin", "captured_stderr", "time_out", + "socket_peer_reset", "ioerror_peer_reset", "run_with_locale", 'temp_umask', + "transient_internet", "set_memlimit", "bigmemtest", "bigaddrspacetest", + "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup", + "threading_cleanup", "reap_children", "cpython_only", "check_impl_detail", + "get_attribute", "swap_item", "swap_attr", "requires_IEEE_754", "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink", - "import_fresh_module", "failfast", + "import_fresh_module", "requires_zlib", "PIPE_MAX_SIZE", "failfast", ] class Error(Exception): @@ -168,7 +171,7 @@ def get_attribute(obj, name): attribute = getattr(obj, name) except AttributeError: raise unittest.SkipTest("module %s has no attribute %s" % ( - obj.__name__, name)) + repr(obj), name)) else: return attribute @@ -209,8 +212,7 @@ def rmtree(path): try: shutil.rmtree(path) except OSError as error: - # Unix returns ENOENT, Windows returns ESRCH. - if error.errno not in (errno.ENOENT, errno.ESRCH): + if error.errno != errno.ENOENT: raise def make_legacy_pyc(source): @@ -295,9 +297,36 @@ def requires(resource, msg=None): return if not is_resource_enabled(resource): if msg is None: - msg = "Use of the `%s' resource not enabled" % resource + msg = "Use of the %r resource not enabled" % resource raise ResourceDenied(msg) +def requires_linux_version(*min_version): + """Decorator raising SkipTest if the OS is Linux and the kernel version is + less than min_version. + + For example, @requires_linux_version(2, 6, 35) raises SkipTest if the Linux + kernel version is less than 2.6.35. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kw): + if sys.platform == 'linux': + version_txt = platform.release().split('-', 1)[0] + try: + version = tuple(map(int, version_txt.split('.'))) + except ValueError: + pass + else: + if version < min_version: + min_version_txt = '.'.join(map(str, min_version)) + raise unittest.SkipTest( + "Linux kernel %s or higher required, not %s" + % (min_version_txt, version_txt)) + return func(*args, **kw) + wrapper.min_version = min_version + return wrapper + return decorator + def requires_mac_ver(*min_version): """Decorator raising SkipTest if the OS is Mac OS X and the OS X version if less than min_version. @@ -325,6 +354,7 @@ def requires_mac_ver(*min_version): return wrapper return decorator + HOST = 'localhost' def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): @@ -420,29 +450,35 @@ def bind_port(sock, host=HOST): port = sock.getsockname()[1] return port -FUZZ = 1e-6 - -def fcmp(x, y): # fuzzy comparison function - if isinstance(x, float) or isinstance(y, float): +def _is_ipv6_enabled(): + """Check whether IPv6 is enabled on this host.""" + if socket.has_ipv6: try: - fuzz = (abs(x) + abs(y)) * FUZZ - if abs(x-y) <= fuzz: - return 0 - except: + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + sock.bind(('::1', 0)) + except (socket.error, socket.gaierror): pass - elif type(x) == type(y) and isinstance(x, (tuple, list)): - for i in range(min(len(x), len(y))): - outcome = fcmp(x[i], y[i]) - if outcome != 0: - return outcome - return (len(x) > len(y)) - (len(x) < len(y)) - return (x > y) - (x < y) + else: + sock.close() + return True + return False + +IPV6_ENABLED = _is_ipv6_enabled() + + +# A constant likely larger than the underlying OS pipe buffer size. +# Windows limit seems to be around 512B, and most Unix kernels have a 64K pipe +# buffer size: take 1M to be sure. +PIPE_MAX_SIZE = 1024 * 1024 + # decorator for skipping tests on non-IEEE 754 platforms requires_IEEE_754 = unittest.skipUnless( float.__getformat__("double").startswith("IEEE"), "test requires IEEE 754 doubles") +requires_zlib = unittest.skipUnless(zlib, 'requires zlib') + is_jython = sys.platform.startswith('java') # Filename used for testing @@ -543,14 +579,15 @@ def temp_cwd(name='tempcwd', quiet=False, path=None): rmtree(name) -@contextlib.contextmanager -def temp_umask(umask): - """Context manager that temporarily sets the process umask.""" - oldmask = os.umask(umask) - try: - yield - finally: - os.umask(oldmask) +if hasattr(os, "umask"): + @contextlib.contextmanager + def temp_umask(umask): + """Context manager that temporarily sets the process umask.""" + oldmask = os.umask(umask) + try: + yield + finally: + os.umask(oldmask) def findfile(file, here=__file__, subdir=None): @@ -568,6 +605,11 @@ def findfile(file, here=__file__, subdir=None): if os.path.exists(fn): return fn return file +def create_empty_file(filename): + """Create an empty file. If the file already exists, truncate it.""" + fd = os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + os.close(fd) + def sortdict(dict): "Like repr(dict), but in sorted order." items = sorted(dict.items()) @@ -632,7 +674,7 @@ def open_urlresource(url, *args, **kw): f = check_valid_file(fn) if f is not None: return f - raise TestFailed('invalid resource "%s"' % fn) + raise TestFailed('invalid resource %r' % fn) class WarningsRecorder(object): @@ -753,7 +795,7 @@ class CleanImport(object): sys.modules.update(self.original_modules) -class EnvironmentVarGuard(collections.MutableMapping): +class EnvironmentVarGuard(collections.abc.MutableMapping): """Class to help protect the environment variable properly. Can be used as a context manager.""" @@ -883,7 +925,7 @@ def transient_internet(resource_name, *, timeout=30.0, errnos=()): ('WSANO_DATA', 11004), ] - denied = ResourceDenied("Resource '%s' is not available" % resource_name) + denied = ResourceDenied("Resource %r is not available" % resource_name) captured_errnos = errnos gai_errnos = [] if not captured_errnos: @@ -972,6 +1014,16 @@ def gc_collect(): gc.collect() gc.collect() +@contextlib.contextmanager +def disable_gc(): + have_gc = gc.isenabled() + gc.disable() + try: + yield + finally: + if have_gc: + gc.enable() + def python_is_optimized(): """Find if Python was built with optimizations.""" @@ -980,7 +1032,7 @@ def python_is_optimized(): for opt in cflags.split(): if opt.startswith('-O'): final_opt = opt - return final_opt and final_opt != '-O0' + return final_opt != '' and final_opt != '-O0' #======================================================================= @@ -1090,6 +1142,11 @@ def bigmemtest(minsize, memuse): return decorator def precisionbigmemtest(size, memuse, dry_run=True): + """Decorator for bigmem tests that need exact sizes. + + Like bigmemtest, but without the size scaling upward to fill available + memory. + """ def decorator(f): def wrapper(self): size = wrapper.size @@ -1186,6 +1243,33 @@ def check_impl_detail(**guards): return guards.get(platform.python_implementation().lower(), default) +def no_tracing(func): + """Decorator to temporarily turn off tracing for the duration of a test.""" + if not hasattr(sys, 'gettrace'): + return func + else: + @functools.wraps(func) + def wrapper(*args, **kwargs): + original_trace = sys.gettrace() + try: + sys.settrace(None) + return func(*args, **kwargs) + finally: + sys.settrace(original_trace) + return wrapper + + +def refcount_test(test): + """Decorator for tests which involve reference counting. + + To start, the decorator does not run the test if is not run by CPython. + After that, any trace function is unset during the test to prevent + unexpected refcounts caused by the trace function. + + """ + return no_tracing(cpython_only(test)) + + def _filter_suite(suite, pred): """Recursively filter test cases in a suite based on a predicate.""" newtests = [] @@ -1198,7 +1282,6 @@ def _filter_suite(suite, pred): newtests.append(test) suite._tests = newtests - def _run_suite(suite): """Run tests from a unittest.TestSuite-derived class.""" if verbose: @@ -1425,7 +1508,7 @@ def strip_python_stderr(stderr): def args_from_interpreter_flags(): """Return a list of command-line arguments reproducing the current - settings in sys.flags.""" + settings in sys.flags and sys.warnoptions.""" flag_opt_map = { 'bytes_warning': 'b', 'dont_write_bytecode': 'B', @@ -1440,6 +1523,8 @@ def args_from_interpreter_flags(): v = getattr(sys.flags, flag) if v > 0: args.append('-' + opt * v) + for opt in sys.warnoptions: + args.append('-W' + opt) return args #============================================================ diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index d86f97c068..1319a64ef7 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -121,11 +121,32 @@ class TestABC(unittest.TestCase): self.assertFalse(issubclass(B, (A,))) self.assertNotIsInstance(b, A) self.assertNotIsInstance(b, (A,)) - A.register(B) + B1 = A.register(B) + self.assertTrue(issubclass(B, A)) + self.assertTrue(issubclass(B, (A,))) + self.assertIsInstance(b, A) + self.assertIsInstance(b, (A,)) + self.assertIs(B1, B) + class C(B): + pass + c = C() + self.assertTrue(issubclass(C, A)) + self.assertTrue(issubclass(C, (A,))) + self.assertIsInstance(c, A) + self.assertIsInstance(c, (A,)) + + def test_register_as_class_deco(self): + class A(metaclass=abc.ABCMeta): + pass + @A.register + class B(object): + pass + b = B() self.assertTrue(issubclass(B, A)) self.assertTrue(issubclass(B, (A,))) self.assertIsInstance(b, A) self.assertIsInstance(b, (A,)) + @A.register class C(B): pass c = C() @@ -133,6 +154,7 @@ class TestABC(unittest.TestCase): self.assertTrue(issubclass(C, (A,))) self.assertIsInstance(c, A) self.assertIsInstance(c, (A,)) + self.assertIs(C, A.register(C)) def test_isinstance_invalidation(self): class A(metaclass=abc.ABCMeta): diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 2a396cdad1..253e6f082c 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -14,6 +14,7 @@ class TestNumbers(unittest.TestCase): self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) self.assertEqual(7, int(7).conjugate()) + self.assertEqual(-7, int(-7).conjugate()) self.assertEqual(7, int(7).numerator) self.assertEqual(1, int(7).denominator) @@ -24,6 +25,7 @@ class TestNumbers(unittest.TestCase): self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) self.assertEqual(7.3, float(7.3).conjugate()) + self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): self.assertFalse(issubclass(complex, Real)) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 5ecfdc72ea..b0e9a03ea8 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4017,6 +4017,37 @@ class TestHelpSubparsersWithHelpOrdering(HelpTestCase): ''' + +class TestHelpMetavarTypeFormatter(HelpTestCase): + """""" + + def custom_type(string): + return string + + parser_signature = Sig(prog='PROG', description='description', + formatter_class=argparse.MetavarTypeHelpFormatter) + argument_signatures = [Sig('a', type=int), + Sig('-b', type=custom_type), + Sig('-c', type=float, metavar='SOME FLOAT')] + argument_group_signatures = [] + usage = '''\ + usage: PROG [-h] [-b custom_type] [-c SOME FLOAT] int + ''' + help = usage + '''\ + + description + + positional arguments: + int + + optional arguments: + -h, --help show this help message and exit + -b custom_type + -c SOME FLOAT + ''' + version = '' + + # ===================================== # Optional/Positional constructor tests # ===================================== @@ -4407,7 +4438,7 @@ class TestEncoding(TestCase): def _test_module_encoding(self, path): path, _ = os.path.splitext(path) path += ".py" - with codecs.open(path, 'r', 'utf8') as f: + with codecs.open(path, 'r', 'utf-8') as f: f.read() def test_argparse_module_encoding(self): @@ -4449,6 +4480,67 @@ class TestArgumentTypeError(TestCase): else: self.fail() +# ========================= +# MessageContentError tests +# ========================= + +class TestMessageContentError(TestCase): + + def test_missing_argument_name_in_message(self): + parser = ErrorRaisingArgumentParser(prog='PROG', usage='') + parser.add_argument('req_pos', type=str) + parser.add_argument('-req_opt', type=int, required=True) + parser.add_argument('need_one', type=str, nargs='+') + + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args([]) + msg = str(cm.exception) + self.assertRegex(msg, 'req_pos') + self.assertRegex(msg, 'req_opt') + self.assertRegex(msg, 'need_one') + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args(['myXargument']) + msg = str(cm.exception) + self.assertNotIn(msg, 'req_pos') + self.assertRegex(msg, 'req_opt') + self.assertRegex(msg, 'need_one') + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args(['myXargument', '-req_opt=1']) + msg = str(cm.exception) + self.assertNotIn(msg, 'req_pos') + self.assertNotIn(msg, 'req_opt') + self.assertRegex(msg, 'need_one') + + def test_optional_optional_not_in_message(self): + parser = ErrorRaisingArgumentParser(prog='PROG', usage='') + parser.add_argument('req_pos', type=str) + parser.add_argument('--req_opt', type=int, required=True) + parser.add_argument('--opt_opt', type=bool, nargs='?', + default=True) + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args([]) + msg = str(cm.exception) + self.assertRegex(msg, 'req_pos') + self.assertRegex(msg, 'req_opt') + self.assertNotIn(msg, 'opt_opt') + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args(['--req_opt=1']) + msg = str(cm.exception) + self.assertRegex(msg, 'req_pos') + self.assertNotIn(msg, 'req_opt') + self.assertNotIn(msg, 'opt_opt') + + def test_optional_positional_not_in_message(self): + parser = ErrorRaisingArgumentParser(prog='PROG', usage='') + parser.add_argument('req_pos') + parser.add_argument('optional_positional', nargs='?', default='eggs') + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args([]) + msg = str(cm.exception) + self.assertRegex(msg, 'req_pos') + self.assertNotIn(msg, 'optional_positional') + + # ====================== # parse_known_args tests # ====================== diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 7d1649c115..273185c3b8 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1,4 +1,6 @@ -import sys, unittest +import os +import sys +import unittest from test import support import ast @@ -52,6 +54,9 @@ exec_tests = [ "while v:pass", # If "if v:pass", + # With + "with x as y: pass", + "with x as y, z as q: pass", # Raise "raise Exception('string')", # TryExcept @@ -487,8 +492,412 @@ class ASTHelpers_Test(unittest.TestCase): self.assertEqual(ast.literal_eval('1.5 - 2j'), 1.5 - 2j) +class ASTValidatorTests(unittest.TestCase): + + def mod(self, mod, msg=None, mode="exec", *, exc=ValueError): + mod.lineno = mod.col_offset = 0 + ast.fix_missing_locations(mod) + with self.assertRaises(exc) as cm: + compile(mod, "<test>", mode) + if msg is not None: + self.assertIn(msg, str(cm.exception)) + + def expr(self, node, msg=None, *, exc=ValueError): + mod = ast.Module([ast.Expr(node)]) + self.mod(mod, msg, exc=exc) + + def stmt(self, stmt, msg=None): + mod = ast.Module([stmt]) + self.mod(mod, msg) + + def test_module(self): + m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))]) + self.mod(m, "must have Load context", "single") + m = ast.Expression(ast.Name("x", ast.Store())) + self.mod(m, "must have Load context", "eval") + + def _check_arguments(self, fac, check): + def arguments(args=None, vararg=None, varargannotation=None, + kwonlyargs=None, kwarg=None, kwargannotation=None, + defaults=None, kw_defaults=None): + if args is None: + args = [] + if kwonlyargs is None: + kwonlyargs = [] + if defaults is None: + defaults = [] + if kw_defaults is None: + kw_defaults = [] + args = ast.arguments(args, vararg, varargannotation, kwonlyargs, + kwarg, kwargannotation, defaults, kw_defaults) + return fac(args) + args = [ast.arg("x", ast.Name("x", ast.Store()))] + check(arguments(args=args), "must have Load context") + check(arguments(varargannotation=ast.Num(3)), + "varargannotation but no vararg") + check(arguments(varargannotation=ast.Name("x", ast.Store()), vararg="x"), + "must have Load context") + check(arguments(kwonlyargs=args), "must have Load context") + check(arguments(kwargannotation=ast.Num(42)), + "kwargannotation but no kwarg") + check(arguments(kwargannotation=ast.Name("x", ast.Store()), + kwarg="x"), "must have Load context") + check(arguments(defaults=[ast.Num(3)]), + "more positional defaults than args") + check(arguments(kw_defaults=[ast.Num(4)]), + "length of kwonlyargs is not the same as kw_defaults") + args = [ast.arg("x", ast.Name("x", ast.Load()))] + check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]), + "must have Load context") + args = [ast.arg("a", ast.Name("x", ast.Load())), + ast.arg("b", ast.Name("y", ast.Load()))] + check(arguments(kwonlyargs=args, + kw_defaults=[None, ast.Name("x", ast.Store())]), + "must have Load context") + + def test_funcdef(self): + a = ast.arguments([], None, None, [], None, None, [], []) + f = ast.FunctionDef("x", a, [], [], None) + self.stmt(f, "empty body on FunctionDef") + f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())], + None) + self.stmt(f, "must have Load context") + f = ast.FunctionDef("x", a, [ast.Pass()], [], + ast.Name("x", ast.Store())) + self.stmt(f, "must have Load context") + def fac(args): + return ast.FunctionDef("x", args, [ast.Pass()], [], None) + self._check_arguments(fac, self.stmt) + + def test_classdef(self): + def cls(bases=None, keywords=None, starargs=None, kwargs=None, + body=None, decorator_list=None): + if bases is None: + bases = [] + if keywords is None: + keywords = [] + if body is None: + body = [ast.Pass()] + if decorator_list is None: + decorator_list = [] + return ast.ClassDef("myclass", bases, keywords, starargs, + kwargs, body, decorator_list) + self.stmt(cls(bases=[ast.Name("x", ast.Store())]), + "must have Load context") + self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]), + "must have Load context") + self.stmt(cls(starargs=ast.Name("x", ast.Store())), + "must have Load context") + self.stmt(cls(kwargs=ast.Name("x", ast.Store())), + "must have Load context") + self.stmt(cls(body=[]), "empty body on ClassDef") + self.stmt(cls(body=[None]), "None disallowed") + self.stmt(cls(decorator_list=[ast.Name("x", ast.Store())]), + "must have Load context") + + def test_delete(self): + self.stmt(ast.Delete([]), "empty targets on Delete") + self.stmt(ast.Delete([None]), "None disallowed") + self.stmt(ast.Delete([ast.Name("x", ast.Load())]), + "must have Del context") + + def test_assign(self): + self.stmt(ast.Assign([], ast.Num(3)), "empty targets on Assign") + self.stmt(ast.Assign([None], ast.Num(3)), "None disallowed") + self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Num(3)), + "must have Store context") + self.stmt(ast.Assign([ast.Name("x", ast.Store())], + ast.Name("y", ast.Store())), + "must have Load context") + + def test_augassign(self): + aug = ast.AugAssign(ast.Name("x", ast.Load()), ast.Add(), + ast.Name("y", ast.Load())) + self.stmt(aug, "must have Store context") + aug = ast.AugAssign(ast.Name("x", ast.Store()), ast.Add(), + ast.Name("y", ast.Store())) + self.stmt(aug, "must have Load context") + + def test_for(self): + x = ast.Name("x", ast.Store()) + y = ast.Name("y", ast.Load()) + p = ast.Pass() + self.stmt(ast.For(x, y, [], []), "empty body on For") + self.stmt(ast.For(ast.Name("x", ast.Load()), y, [p], []), + "must have Store context") + self.stmt(ast.For(x, ast.Name("y", ast.Store()), [p], []), + "must have Load context") + e = ast.Expr(ast.Name("x", ast.Store())) + self.stmt(ast.For(x, y, [e], []), "must have Load context") + self.stmt(ast.For(x, y, [p], [e]), "must have Load context") + + def test_while(self): + self.stmt(ast.While(ast.Num(3), [], []), "empty body on While") + self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), + "must have Load context") + self.stmt(ast.While(ast.Num(3), [ast.Pass()], + [ast.Expr(ast.Name("x", ast.Store()))]), + "must have Load context") + + def test_if(self): + self.stmt(ast.If(ast.Num(3), [], []), "empty body on If") + i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) + self.stmt(i, "must have Load context") + i = ast.If(ast.Num(3), [ast.Expr(ast.Name("x", ast.Store()))], []) + self.stmt(i, "must have Load context") + i = ast.If(ast.Num(3), [ast.Pass()], + [ast.Expr(ast.Name("x", ast.Store()))]) + self.stmt(i, "must have Load context") + + def test_with(self): + p = ast.Pass() + self.stmt(ast.With([], [p]), "empty items on With") + i = ast.withitem(ast.Num(3), None) + self.stmt(ast.With([i], []), "empty body on With") + i = ast.withitem(ast.Name("x", ast.Store()), None) + self.stmt(ast.With([i], [p]), "must have Load context") + i = ast.withitem(ast.Num(3), ast.Name("x", ast.Load())) + self.stmt(ast.With([i], [p]), "must have Store context") + + def test_raise(self): + r = ast.Raise(None, ast.Num(3)) + self.stmt(r, "Raise with cause but no exception") + r = ast.Raise(ast.Name("x", ast.Store()), None) + self.stmt(r, "must have Load context") + r = ast.Raise(ast.Num(4), ast.Name("x", ast.Store())) + self.stmt(r, "must have Load context") + + def test_try(self): + p = ast.Pass() + t = ast.Try([], [], [], [p]) + self.stmt(t, "empty body on Try") + t = ast.Try([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) + self.stmt(t, "must have Load context") + t = ast.Try([p], [], [], []) + self.stmt(t, "Try has neither except handlers nor finalbody") + t = ast.Try([p], [], [p], [p]) + self.stmt(t, "Try has orelse but no except handlers") + t = ast.Try([p], [ast.ExceptHandler(None, "x", [])], [], []) + self.stmt(t, "empty body on ExceptHandler") + e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] + self.stmt(ast.Try([p], e, [], []), "must have Load context") + e = [ast.ExceptHandler(None, "x", [p])] + t = ast.Try([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) + self.stmt(t, "must have Load context") + t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) + self.stmt(t, "must have Load context") + + def test_assert(self): + self.stmt(ast.Assert(ast.Name("x", ast.Store()), None), + "must have Load context") + assrt = ast.Assert(ast.Name("x", ast.Load()), + ast.Name("y", ast.Store())) + self.stmt(assrt, "must have Load context") + + def test_import(self): + self.stmt(ast.Import([]), "empty names on Import") + + def test_importfrom(self): + imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) + self.stmt(imp, "level less than -1") + self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom") + + def test_global(self): + self.stmt(ast.Global([]), "empty names on Global") + + def test_nonlocal(self): + self.stmt(ast.Nonlocal([]), "empty names on Nonlocal") + + def test_expr(self): + e = ast.Expr(ast.Name("x", ast.Store())) + self.stmt(e, "must have Load context") + + def test_boolop(self): + b = ast.BoolOp(ast.And(), []) + self.expr(b, "less than 2 values") + b = ast.BoolOp(ast.And(), [ast.Num(3)]) + self.expr(b, "less than 2 values") + b = ast.BoolOp(ast.And(), [ast.Num(4), None]) + self.expr(b, "None disallowed") + b = ast.BoolOp(ast.And(), [ast.Num(4), ast.Name("x", ast.Store())]) + self.expr(b, "must have Load context") + + def test_unaryop(self): + u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store())) + self.expr(u, "must have Load context") + + def test_lambda(self): + a = ast.arguments([], None, None, [], None, None, [], []) + self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), + "must have Load context") + def fac(args): + return ast.Lambda(args, ast.Name("x", ast.Load())) + self._check_arguments(fac, self.expr) + + def test_ifexp(self): + l = ast.Name("x", ast.Load()) + s = ast.Name("y", ast.Store()) + for args in (s, l, l), (l, s, l), (l, l, s): + self.expr(ast.IfExp(*args), "must have Load context") + + def test_dict(self): + d = ast.Dict([], [ast.Name("x", ast.Load())]) + self.expr(d, "same number of keys as values") + d = ast.Dict([None], [ast.Name("x", ast.Load())]) + self.expr(d, "None disallowed") + d = ast.Dict([ast.Name("x", ast.Load())], [None]) + self.expr(d, "None disallowed") + + def test_set(self): + self.expr(ast.Set([None]), "None disallowed") + s = ast.Set([ast.Name("x", ast.Store())]) + self.expr(s, "must have Load context") + + def _check_comprehension(self, fac): + self.expr(fac([]), "comprehension with no generators") + g = ast.comprehension(ast.Name("x", ast.Load()), + ast.Name("x", ast.Load()), []) + self.expr(fac([g]), "must have Store context") + g = ast.comprehension(ast.Name("x", ast.Store()), + ast.Name("x", ast.Store()), []) + self.expr(fac([g]), "must have Load context") + x = ast.Name("x", ast.Store()) + y = ast.Name("y", ast.Load()) + g = ast.comprehension(x, y, [None]) + self.expr(fac([g]), "None disallowed") + g = ast.comprehension(x, y, [ast.Name("x", ast.Store())]) + self.expr(fac([g]), "must have Load context") + + def _simple_comp(self, fac): + g = ast.comprehension(ast.Name("x", ast.Store()), + ast.Name("x", ast.Load()), []) + self.expr(fac(ast.Name("x", ast.Store()), [g]), + "must have Load context") + def wrap(gens): + return fac(ast.Name("x", ast.Store()), gens) + self._check_comprehension(wrap) + + def test_listcomp(self): + self._simple_comp(ast.ListComp) + + def test_setcomp(self): + self._simple_comp(ast.SetComp) + + def test_generatorexp(self): + self._simple_comp(ast.GeneratorExp) + + def test_dictcomp(self): + g = ast.comprehension(ast.Name("y", ast.Store()), + ast.Name("p", ast.Load()), []) + c = ast.DictComp(ast.Name("x", ast.Store()), + ast.Name("y", ast.Load()), [g]) + self.expr(c, "must have Load context") + c = ast.DictComp(ast.Name("x", ast.Load()), + ast.Name("y", ast.Store()), [g]) + self.expr(c, "must have Load context") + def factory(comps): + k = ast.Name("x", ast.Load()) + v = ast.Name("y", ast.Load()) + return ast.DictComp(k, v, comps) + self._check_comprehension(factory) + + def test_yield(self): + self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") + + def test_compare(self): + left = ast.Name("x", ast.Load()) + comp = ast.Compare(left, [ast.In()], []) + self.expr(comp, "no comparators") + comp = ast.Compare(left, [ast.In()], [ast.Num(4), ast.Num(5)]) + self.expr(comp, "different number of comparators and operands") + comp = ast.Compare(ast.Num("blah"), [ast.In()], [left]) + self.expr(comp, "non-numeric", exc=TypeError) + comp = ast.Compare(left, [ast.In()], [ast.Num("blah")]) + self.expr(comp, "non-numeric", exc=TypeError) + + def test_call(self): + func = ast.Name("x", ast.Load()) + args = [ast.Name("y", ast.Load())] + keywords = [ast.keyword("w", ast.Name("z", ast.Load()))] + stararg = ast.Name("p", ast.Load()) + kwarg = ast.Name("q", ast.Load()) + call = ast.Call(ast.Name("x", ast.Store()), args, keywords, stararg, + kwarg) + self.expr(call, "must have Load context") + call = ast.Call(func, [None], keywords, stararg, kwarg) + self.expr(call, "None disallowed") + bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store()))] + call = ast.Call(func, args, bad_keywords, stararg, kwarg) + self.expr(call, "must have Load context") + call = ast.Call(func, args, keywords, ast.Name("z", ast.Store()), kwarg) + self.expr(call, "must have Load context") + call = ast.Call(func, args, keywords, stararg, + ast.Name("w", ast.Store())) + self.expr(call, "must have Load context") + + def test_num(self): + class subint(int): + pass + class subfloat(float): + pass + class subcomplex(complex): + pass + for obj in "0", "hello", subint(), subfloat(), subcomplex(): + self.expr(ast.Num(obj), "non-numeric", exc=TypeError) + + def test_attribute(self): + attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) + self.expr(attr, "must have Load context") + + def test_subscript(self): + sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Index(ast.Num(3)), + ast.Load()) + self.expr(sub, "must have Load context") + x = ast.Name("x", ast.Load()) + sub = ast.Subscript(x, ast.Index(ast.Name("y", ast.Store())), + ast.Load()) + self.expr(sub, "must have Load context") + s = ast.Name("x", ast.Store()) + for args in (s, None, None), (None, s, None), (None, None, s): + sl = ast.Slice(*args) + self.expr(ast.Subscript(x, sl, ast.Load()), + "must have Load context") + sl = ast.ExtSlice([]) + self.expr(ast.Subscript(x, sl, ast.Load()), "empty dims on ExtSlice") + sl = ast.ExtSlice([ast.Index(s)]) + self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") + + def test_starred(self): + left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], + ast.Store()) + assign = ast.Assign([left], ast.Num(4)) + self.stmt(assign, "must have Store context") + + def _sequence(self, fac): + self.expr(fac([None], ast.Load()), "None disallowed") + self.expr(fac([ast.Name("x", ast.Store())], ast.Load()), + "must have Load context") + + def test_list(self): + self._sequence(ast.List) + + def test_tuple(self): + self._sequence(ast.Tuple) + + def test_stdlib_validates(self): + stdlib = os.path.dirname(ast.__file__) + tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] + tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) + for module in tests: + fn = os.path.join(stdlib, module) + with open(fn, "r", encoding="utf-8") as fp: + source = fp.read() + mod = ast.parse(source) + compile(mod, fn, "exec") + + def test_main(): - support.run_unittest(AST_Tests, ASTHelpers_Test) + support.run_unittest(AST_Tests, ASTHelpers_Test, ASTValidatorTests) def main(): if __name__ != '__main__': @@ -522,9 +931,11 @@ exec_results = [ ('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]), ('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]), ('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]), +('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]), +('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]), ('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]), -('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]), -('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]), +('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]), +('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]), ('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]), ('Module', [('Import', (1, 0), [('alias', 'sys', None)])]), ('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]), diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index 53c49a803b..2c84a271f5 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -20,6 +20,8 @@ except ImportError: HOST = support.HOST +HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX') + class dummysocket: def __init__(self): self.closed = False @@ -87,6 +89,13 @@ def capture_server(evt, buf, serv): serv.close() evt.set() +def bind_af_aware(sock, addr): + """Helper function to bind a socket according to its family.""" + if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX: + # Make sure the path doesn't exist. + unlink(addr) + sock.bind(addr) + class HelperFunctionTests(unittest.TestCase): def test_readwriteexc(self): @@ -352,7 +361,7 @@ class DispatcherWithSendTests(unittest.TestCase): @support.reap_threads def test_send(self): evt = threading.Event() - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock = socket.socket() sock.settimeout(3) port = support.bind_port(sock) @@ -367,7 +376,7 @@ class DispatcherWithSendTests(unittest.TestCase): data = b"Suppose there isn't a 16-ton weight?" d = dispatcherwithsend_noread() - d.create_socket(socket.AF_INET, socket.SOCK_STREAM) + d.create_socket() d.connect((HOST, port)) # give time for socket to connect @@ -467,22 +476,22 @@ class BaseTestHandler(asyncore.dispatcher): raise -class TCPServer(asyncore.dispatcher): +class BaseServer(asyncore.dispatcher): """A server which listens on an address and dispatches the connection to a handler. """ - def __init__(self, handler=BaseTestHandler, host=HOST, port=0): + def __init__(self, family, addr, handler=BaseTestHandler): asyncore.dispatcher.__init__(self) - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.create_socket(family) self.set_reuse_addr() - self.bind((host, port)) + bind_af_aware(self.socket, addr) self.listen(5) self.handler = handler @property def address(self): - return self.socket.getsockname()[:2] + return self.socket.getsockname() def handle_accepted(self, sock, addr): self.handler(sock) @@ -493,9 +502,9 @@ class TCPServer(asyncore.dispatcher): class BaseClient(BaseTestHandler): - def __init__(self, address): + def __init__(self, family, address): BaseTestHandler.__init__(self) - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.create_socket(family) self.connect(address) def handle_connect(self): @@ -525,8 +534,8 @@ class BaseTestAPI(unittest.TestCase): def handle_connect(self): self.flag = True - server = TCPServer() - client = TestClient(server.address) + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) def test_handle_accept(self): @@ -534,18 +543,18 @@ class BaseTestAPI(unittest.TestCase): class TestListener(BaseTestHandler): - def __init__(self): + def __init__(self, family, addr): BaseTestHandler.__init__(self) - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.bind((HOST, 0)) + self.create_socket(family) + bind_af_aware(self.socket, addr) self.listen(5) - self.address = self.socket.getsockname()[:2] + self.address = self.socket.getsockname() def handle_accept(self): self.flag = True - server = TestListener() - client = BaseClient(server.address) + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) self.loop_waiting_for_flag(server) def test_handle_accepted(self): @@ -553,12 +562,12 @@ class BaseTestAPI(unittest.TestCase): class TestListener(BaseTestHandler): - def __init__(self): + def __init__(self, family, addr): BaseTestHandler.__init__(self) - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.bind((HOST, 0)) + self.create_socket(family) + bind_af_aware(self.socket, addr) self.listen(5) - self.address = self.socket.getsockname()[:2] + self.address = self.socket.getsockname() def handle_accept(self): asyncore.dispatcher.handle_accept(self) @@ -567,8 +576,8 @@ class BaseTestAPI(unittest.TestCase): sock.close() self.flag = True - server = TestListener() - client = BaseClient(server.address) + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) self.loop_waiting_for_flag(server) @@ -584,8 +593,8 @@ class BaseTestAPI(unittest.TestCase): BaseTestHandler.__init__(self, conn) self.send(b'x' * 1024) - server = TCPServer(TestHandler) - client = TestClient(server.address) + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) def test_handle_write(self): @@ -595,8 +604,8 @@ class BaseTestAPI(unittest.TestCase): def handle_write(self): self.flag = True - server = TCPServer() - client = TestClient(server.address) + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) def test_handle_close(self): @@ -619,8 +628,8 @@ class BaseTestAPI(unittest.TestCase): BaseTestHandler.__init__(self, conn) self.close() - server = TCPServer(TestHandler) - client = TestClient(server.address) + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) @unittest.skipIf(sys.platform.startswith("sunos"), @@ -629,6 +638,8 @@ class BaseTestAPI(unittest.TestCase): # Make sure handle_expt is called on OOB data received. # Note: this might fail on some platforms as OOB data is # tenuously supported and rarely used. + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") class TestClient(BaseClient): def handle_expt(self): @@ -639,8 +650,8 @@ class BaseTestAPI(unittest.TestCase): BaseTestHandler.__init__(self, conn) self.socket.send(bytes(chr(244), 'latin-1'), socket.MSG_OOB) - server = TCPServer(TestHandler) - client = TestClient(server.address) + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) def test_handle_error(self): @@ -657,13 +668,13 @@ class BaseTestAPI(unittest.TestCase): else: raise Exception("exception not raised") - server = TCPServer() - client = TestClient(server.address) + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) def test_connection_attributes(self): - server = TCPServer() - client = BaseClient(server.address) + server = BaseServer(self.family, self.addr) + client = BaseClient(self.family, server.address) # we start disconnected self.assertFalse(server.connected) @@ -693,25 +704,29 @@ class BaseTestAPI(unittest.TestCase): def test_create_socket(self): s = asyncore.dispatcher() - s.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.assertEqual(s.socket.family, socket.AF_INET) + s.create_socket(self.family) + self.assertEqual(s.socket.family, self.family) SOCK_NONBLOCK = getattr(socket, 'SOCK_NONBLOCK', 0) self.assertEqual(s.socket.type, socket.SOCK_STREAM | SOCK_NONBLOCK) def test_bind(self): + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") s1 = asyncore.dispatcher() - s1.create_socket(socket.AF_INET, socket.SOCK_STREAM) - s1.bind((HOST, 0)) + s1.create_socket(self.family) + s1.bind(self.addr) s1.listen(5) port = s1.socket.getsockname()[1] s2 = asyncore.dispatcher() - s2.create_socket(socket.AF_INET, socket.SOCK_STREAM) + s2.create_socket(self.family) # EADDRINUSE indicates the socket was correctly bound - self.assertRaises(socket.error, s2.bind, (HOST, port)) + self.assertRaises(socket.error, s2.bind, (self.addr[0], port)) def test_set_reuse_addr(self): - sock = socket.socket() + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + sock = socket.socket(self.family) try: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) except socket.error: @@ -719,11 +734,11 @@ class BaseTestAPI(unittest.TestCase): else: # if SO_REUSEADDR succeeded for sock we expect asyncore # to do the same - s = asyncore.dispatcher(socket.socket()) + s = asyncore.dispatcher(socket.socket(self.family)) self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR)) s.socket.close() - s.create_socket(socket.AF_INET, socket.SOCK_STREAM) + s.create_socket(self.family) s.set_reuse_addr() self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR)) @@ -731,18 +746,45 @@ class BaseTestAPI(unittest.TestCase): sock.close() -class TestAPI_UseSelect(BaseTestAPI): +class TestAPI_UseIPv4Sockets(BaseTestAPI): + family = socket.AF_INET + addr = (HOST, 0) + +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 support required') +class TestAPI_UseIPv6Sockets(BaseTestAPI): + family = socket.AF_INET6 + addr = ('::1', 0) + +@unittest.skipUnless(HAS_UNIX_SOCKETS, 'Unix sockets required') +class TestAPI_UseUnixSockets(BaseTestAPI): + if HAS_UNIX_SOCKETS: + family = socket.AF_UNIX + addr = support.TESTFN + use_poll = False + + def tearDown(self): + unlink(self.addr) + BaseTestAPI.tearDown(self) + +class TestAPI_UseIPv4Select(TestAPI_UseIPv4Sockets): use_poll = False @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') -class TestAPI_UsePoll(BaseTestAPI): +class TestAPI_UseIPv4Poll(TestAPI_UseIPv4Sockets): use_poll = True +class TestAPI_UseIPv6Select(TestAPI_UseIPv6Sockets): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseIPv6Poll(TestAPI_UseIPv6Sockets): + use_poll = True def test_main(): tests = [HelperFunctionTests, DispatcherTests, DispatcherWithSendTests, - DispatcherWithSendTests_UsePoll, TestAPI_UseSelect, - TestAPI_UsePoll, FileWrapperTest] + DispatcherWithSendTests_UsePoll, FileWrapperTest, + TestAPI_UseIPv4Select, TestAPI_UseIPv4Poll, TestAPI_UseIPv6Select, + TestAPI_UseIPv6Poll, TestAPI_UseUnixSockets] run_unittest(*tests) if __name__ == "__main__": diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index ac6b1096e1..91c62af9e3 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -1,3 +1,13 @@ +"""Bigmem tests - tests for the 32-bit boundary in containers. + +These tests try to exercise the 32-bit boundary that is sometimes, if +rarely, exceeded in practice, but almost never tested. They are really only +meaningful on 64-bit builds on machines with a *lot* of memory, but the +tests are always run, usually with very low memory limits to make sure the +tests themselves don't suffer from bitrot. To run them for real, pass a +high memory limit to regrtest, with the -M option. +""" + from test import support from test.support import bigmemtest, _1G, _2G, _4G, precisionbigmemtest @@ -6,30 +16,45 @@ import operator import sys import functools +# These tests all use one of the bigmemtest decorators to indicate how much +# memory they use and how much memory they need to be even meaningful. The +# decorators take two arguments: a 'memuse' indicator declaring +# (approximate) bytes per size-unit the test will use (at peak usage), and a +# 'minsize' indicator declaring a minimum *useful* size. A test that +# allocates a bytestring to test various operations near the end will have a +# minsize of at least 2Gb (or it wouldn't reach the 32-bit limit, so the +# test wouldn't be very useful) and a memuse of 1 (one byte per size-unit, +# if it allocates only one big string at a time.) +# +# When run with a memory limit set, both decorators skip tests that need +# more memory than available to be meaningful. The precisionbigmemtest will +# always pass minsize as size, even if there is much more memory available. +# The bigmemtest decorator will scale size upward to fill available memory. +# # Bigmem testing houserules: # # - Try not to allocate too many large objects. It's okay to rely on -# refcounting semantics, but don't forget that 's = create_largestring()' +# refcounting semantics, and don't forget that 's = create_largestring()' # doesn't release the old 's' (if it exists) until well after its new # value has been created. Use 'del s' before the create_largestring call. # -# - Do *not* compare large objects using assertEqual or similar. It's a -# lengthy operation and the errormessage will be utterly useless due to -# its size. To make sure whether a result has the right contents, better -# to use the strip or count methods, or compare meaningful slices. +# - Do *not* compare large objects using assertEqual, assertIn or similar. +# It's a lengthy operation and the errormessage will be utterly useless +# due to its size. To make sure whether a result has the right contents, +# better to use the strip or count methods, or compare meaningful slices. # # - Don't forget to test for large indices, offsets and results and such, -# in addition to large sizes. +# in addition to large sizes. Anything that probes the 32-bit boundary. # # - When repeating an object (say, a substring, or a small list) to create # a large object, make the subobject of a length that is not a power of # 2. That way, int-wrapping problems are more easily detected. # -# - While the bigmemtest decorator speaks of 'minsize', all tests will -# actually be called with a much smaller number too, in the normal -# test run (5Kb currently.) This is so the tests themselves get frequent -# testing. Consequently, always make all large allocations based on the -# passed-in 'size', and don't rely on the size being very large. Also, +# - While the bigmem decorators speak of 'minsize', all tests will actually +# be called with a much smaller number too, in the normal test run (5Kb +# currently.) This is so the tests themselves get frequent testing. +# Consequently, always make all large allocations based on the passed-in +# 'size', and don't rely on the size being very large. Also, # memuse-per-size should remain sane (less than a few thousand); if your # test uses more, adjust 'size' upward, instead. @@ -92,7 +117,7 @@ class BaseStrTest: _ = self.from_latin1 s = _('-') * size tabsize = 8 - self.assertEqual(s.expandtabs(), s) + self.assertTrue(s.expandtabs() == s) del s slen, remainder = divmod(size, tabsize) s = _(' \t') * slen @@ -519,19 +544,19 @@ class BaseStrTest: edge = _('-') * (size // 2) s = _('').join([edge, SUBSTR, edge]) del edge - self.assertIn(SUBSTR, s) - self.assertNotIn(SUBSTR * 2, s) - self.assertIn(_('-'), s) - self.assertNotIn(_('a'), s) + self.assertTrue(SUBSTR in s) + self.assertFalse(SUBSTR * 2 in s) + self.assertTrue(_('-') in s) + self.assertFalse(_('a') in s) s += _('a') - self.assertIn(_('a'), s) + self.assertTrue(_('a') in s) @bigmemtest(minsize=_2G + 10, memuse=2) def test_compare(self, size): _ = self.from_latin1 s1 = _('-') * size s2 = _('-') * size - self.assertEqual(s1, s2) + self.assertTrue(s1 == s2) del s2 s2 = s1 + _('a') self.assertFalse(s1 == s2) @@ -552,7 +577,7 @@ class BaseStrTest: h1 = hash(s) del s s = _('\x00') * (size + 1) - self.assertFalse(h1 == hash(s)) + self.assertNotEqual(h1, hash(s)) class StrTest(unittest.TestCase, BaseStrTest): @@ -633,7 +658,7 @@ class StrTest(unittest.TestCase, BaseStrTest): def test_format(self, size): s = '-' * size sf = '%s' % (s,) - self.assertEqual(s, sf) + self.assertTrue(s == sf) del sf sf = '..%s..' % (s,) self.assertEqual(len(sf), len(s) + 4) @@ -707,7 +732,7 @@ class StrTest(unittest.TestCase, BaseStrTest): class BytesTest(unittest.TestCase, BaseStrTest): def from_latin1(self, s): - return s.encode("latin1") + return s.encode("latin-1") @bigmemtest(minsize=_2G + 2, memuse=1 + character_size) def test_decode(self, size): @@ -718,7 +743,7 @@ class BytesTest(unittest.TestCase, BaseStrTest): class BytearrayTest(unittest.TestCase, BaseStrTest): def from_latin1(self, s): - return bytearray(s.encode("latin1")) + return bytearray(s.encode("latin-1")) @bigmemtest(minsize=_2G + 2, memuse=1 + character_size) def test_decode(self, size): @@ -743,7 +768,7 @@ class TupleTest(unittest.TestCase): def test_compare(self, size): t1 = ('',) * size t2 = ('',) * size - self.assertEqual(t1, t2) + self.assertTrue(t1 == t2) del t2 t2 = ('',) * (size + 1) self.assertFalse(t1 == t2) @@ -774,9 +799,9 @@ class TupleTest(unittest.TestCase): def test_contains(self, size): t = (1, 2, 3, 4, 5) * size self.assertEqual(len(t), size * 5) - self.assertIn(5, t) - self.assertNotIn((1, 2, 3, 4, 5), t) - self.assertNotIn(0, t) + self.assertTrue(5 in t) + self.assertFalse((1, 2, 3, 4, 5) in t) + self.assertFalse(0 in t) @bigmemtest(minsize=_2G + 10, memuse=8) def test_hash(self, size): @@ -879,7 +904,7 @@ class ListTest(unittest.TestCase): def test_compare(self, size): l1 = [''] * size l2 = [''] * size - self.assertEqual(l1, l2) + self.assertTrue(l1 == l2) del l2 l2 = [''] * (size + 1) self.assertFalse(l1 == l2) @@ -925,9 +950,9 @@ class ListTest(unittest.TestCase): def test_contains(self, size): l = [1, 2, 3, 4, 5] * size self.assertEqual(len(l), size * 5) - self.assertIn(5, l) - self.assertNotIn([1, 2, 3, 4, 5], l) - self.assertNotIn(0, l) + self.assertTrue(5 in l) + self.assertFalse([1, 2, 3, 4, 5] in l) + self.assertFalse(0 in l) @bigmemtest(minsize=_2G + 10, memuse=8) def test_hash(self, size): diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index b296870b09..4bab28be77 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -330,6 +330,16 @@ class BoolTest(unittest.TestCase): except (Exception) as e_len: self.assertEqual(str(e_bool), str(e_len)) + def test_real_and_imag(self): + self.assertEqual(True.real, 1) + self.assertEqual(True.imag, 0) + self.assertIs(type(True.real), int) + self.assertIs(type(True.imag), int) + self.assertEqual(False.real, 0) + self.assertEqual(False.imag, 0) + self.assertIs(type(False.real), int) + self.assertIs(type(False.imag), int) + def test_main(): support.run_unittest(BoolTest) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 1469e36847..4e33c23409 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -10,7 +10,7 @@ import ast import types import builtins import random -from test.support import fcmp, TESTFN, unlink, run_unittest, check_warnings +from test.support import TESTFN, unlink, run_unittest, check_warnings from operator import neg @@ -372,7 +372,15 @@ class BuiltinTest(unittest.TestCase): f = Foo() self.assertTrue(dir(f) == ["ga", "kan", "roo"]) - # dir(obj__dir__not_list) + # dir(obj__dir__tuple) + class Foo(object): + def __dir__(self): + return ("b", "c", "a") + res = dir(Foo()) + self.assertIsInstance(res, list) + self.assertTrue(res == ["a", "b", "c"]) + + # dir(obj__dir__not_sequence) class Foo(object): def __dir__(self): return 7 @@ -385,6 +393,8 @@ class BuiltinTest(unittest.TestCase): except: self.assertEqual(len(dir(sys.exc_info()[2])), 4) + # test that object has a __dir__() + self.assertEqual(sorted([].__dir__()), dir([])) def test_divmod(self): self.assertEqual(divmod(12, 7), (1, 5)) @@ -394,10 +404,13 @@ class BuiltinTest(unittest.TestCase): self.assertEqual(divmod(-sys.maxsize-1, -1), (sys.maxsize+1, 0)) - self.assertTrue(not fcmp(divmod(3.25, 1.0), (3.0, 0.25))) - self.assertTrue(not fcmp(divmod(-3.25, 1.0), (-4.0, 0.75))) - self.assertTrue(not fcmp(divmod(3.25, -1.0), (-4.0, -0.75))) - self.assertTrue(not fcmp(divmod(-3.25, -1.0), (3.0, -0.25))) + for num, denom, exp_result in [ (3.25, 1.0, (3.0, 0.25)), + (-3.25, 1.0, (-4.0, 0.75)), + (3.25, -1.0, (-4.0, -0.75)), + (-3.25, -1.0, (3.0, -0.25))]: + result = divmod(num, denom) + self.assertAlmostEqual(result[0], exp_result[0]) + self.assertAlmostEqual(result[1], exp_result[1]) self.assertRaises(TypeError, divmod) @@ -1115,6 +1128,9 @@ class BuiltinTest(unittest.TestCase): self.assertRaises(TypeError, sum, 42) self.assertRaises(TypeError, sum, ['a', 'b', 'c']) self.assertRaises(TypeError, sum, ['a', 'b', 'c'], '') + self.assertRaises(TypeError, sum, [b'a', b'c'], b'') + values = [bytearray(b'a'), bytearray(b'b')] + self.assertRaises(TypeError, sum, values, bytearray(b'')) self.assertRaises(TypeError, sum, [[1], [2], [3]]) self.assertRaises(TypeError, sum, [{2:3}]) self.assertRaises(TypeError, sum, [{2:3}]*2, {2:3}) @@ -1276,14 +1292,14 @@ class BuiltinTest(unittest.TestCase): # -------------------------------------------------------------------- # Issue #7994: object.__format__ with a non-empty format string is - # pending deprecated + # deprecated def test_deprecated_format_string(obj, fmt_str, should_raise_warning): with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always", PendingDeprecationWarning) + warnings.simplefilter("always", DeprecationWarning) format(obj, fmt_str) if should_raise_warning: self.assertEqual(len(w), 1) - self.assertIsInstance(w[0].message, PendingDeprecationWarning) + self.assertIsInstance(w[0].message, DeprecationWarning) self.assertIn('object.__format__ with a non-empty format ' 'string', str(w[0].message)) else: @@ -1327,6 +1343,13 @@ class BuiltinTest(unittest.TestCase): self.assertRaises(ValueError, x.translate, b"1", 1) self.assertRaises(TypeError, x.translate, b"1"*256, 1) + def test_construct_singletons(self): + for const in None, Ellipsis, NotImplemented: + tp = type(const) + self.assertIs(tp(), const) + self.assertRaises(TypeError, tp, 1, 2) + self.assertRaises(TypeError, tp, a=1, b=2) + class TestSorted(unittest.TestCase): def test_basic(self): diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 5eab8f5561..d32a44b7b4 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -188,24 +188,26 @@ class BaseBytesTest(unittest.TestCase): def test_encoding(self): sample = "Hello world\n\u1234\u5678\u9abc" - for enc in ("utf8", "utf16"): + for enc in ("utf-8", "utf-16"): b = self.type2test(sample, enc) self.assertEqual(b, self.type2test(sample.encode(enc))) - self.assertRaises(UnicodeEncodeError, self.type2test, sample, "latin1") - b = self.type2test(sample, "latin1", "ignore") + self.assertRaises(UnicodeEncodeError, self.type2test, sample, "latin-1") + b = self.type2test(sample, "latin-1", "ignore") self.assertEqual(b, self.type2test(sample[:-3], "utf-8")) def test_decode(self): sample = "Hello world\n\u1234\u5678\u9abc\def0\def0" - for enc in ("utf8", "utf16"): + for enc in ("utf-8", "utf-16"): b = self.type2test(sample, enc) self.assertEqual(b.decode(enc), sample) sample = "Hello world\n\x80\x81\xfe\xff" - b = self.type2test(sample, "latin1") - self.assertRaises(UnicodeDecodeError, b.decode, "utf8") - self.assertEqual(b.decode("utf8", "ignore"), "Hello world\n") - self.assertEqual(b.decode(errors="ignore", encoding="utf8"), + b = self.type2test(sample, "latin-1") + self.assertRaises(UnicodeDecodeError, b.decode, "utf-8") + self.assertEqual(b.decode("utf-8", "ignore"), "Hello world\n") + self.assertEqual(b.decode(errors="ignore", encoding="utf-8"), "Hello world\n") + # Default encoding is utf-8 + self.assertEqual(self.type2test(b'\xe2\x98\x83').decode(), '\u2603') def test_from_int(self): b = self.type2test(0) @@ -473,6 +475,27 @@ class BaseBytesTest(unittest.TestCase): self.assertRaises(TypeError, self.type2test(b'abc').lstrip, 'b') self.assertRaises(TypeError, self.type2test(b'abc').rstrip, 'b') + def test_center(self): + # Fill character can be either bytes or bytearray (issue 12380) + b = self.type2test(b'abc') + for fill_type in (bytes, bytearray): + self.assertEqual(b.center(7, fill_type(b'-')), + self.type2test(b'--abc--')) + + def test_ljust(self): + # Fill character can be either bytes or bytearray (issue 12380) + b = self.type2test(b'abc') + for fill_type in (bytes, bytearray): + self.assertEqual(b.ljust(7, fill_type(b'-')), + self.type2test(b'abc----')) + + def test_rjust(self): + # Fill character can be either bytes or bytearray (issue 12380) + b = self.type2test(b'abc') + for fill_type in (bytes, bytearray): + self.assertEqual(b.rjust(7, fill_type(b'-')), + self.type2test(b'----abc')) + def test_ord(self): b = self.type2test(b'\0A\x7f\x80\xff') self.assertEqual([ord(b[i:i+1]) for i in range(len(b))], @@ -634,6 +657,39 @@ class ByteArrayTest(BaseBytesTest): b.reverse() self.assertFalse(b) + def test_clear(self): + b = bytearray(b'python') + b.clear() + self.assertEqual(b, b'') + + b = bytearray(b'') + b.clear() + self.assertEqual(b, b'') + + b = bytearray(b'') + b.append(ord('r')) + b.clear() + b.append(ord('p')) + self.assertEqual(b, b'p') + + def test_copy(self): + b = bytearray(b'abc') + bb = b.copy() + self.assertEqual(bb, b'abc') + + b = bytearray(b'') + bb = b.copy() + self.assertEqual(bb, b'') + + # test that it's indeed a copy and not a reference + b = bytearray(b'abc') + bb = b.copy() + self.assertEqual(b, bb) + self.assertIsNot(b, bb) + bb.append(ord('d')) + self.assertEqual(bb, b'abcd') + self.assertEqual(b, b'abc') + def test_regexps(self): def by(s): return bytearray(map(ord, s)) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index be3558050b..5a7d342afd 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 from test import support -from test.support import TESTFN +from test.support import TESTFN, precisionbigmemtest, _4G import unittest from io import BytesIO import os +import random import subprocess import sys @@ -21,9 +22,31 @@ has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx") class BaseTest(unittest.TestCase): "Base for other testcases." - TEXT = b'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + TEXT_LINES = [ + b'root:x:0:0:root:/root:/bin/bash\n', + b'bin:x:1:1:bin:/bin:\n', + b'daemon:x:2:2:daemon:/sbin:\n', + b'adm:x:3:4:adm:/var/adm:\n', + b'lp:x:4:7:lp:/var/spool/lpd:\n', + b'sync:x:5:0:sync:/sbin:/bin/sync\n', + b'shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\n', + b'halt:x:7:0:halt:/sbin:/sbin/halt\n', + b'mail:x:8:12:mail:/var/spool/mail:\n', + b'news:x:9:13:news:/var/spool/news:\n', + b'uucp:x:10:14:uucp:/var/spool/uucp:\n', + b'operator:x:11:0:operator:/root:\n', + b'games:x:12:100:games:/usr/games:\n', + b'gopher:x:13:30:gopher:/usr/lib/gopher-data:\n', + b'ftp:x:14:50:FTP User:/var/ftp:/bin/bash\n', + b'nobody:x:65534:65534:Nobody:/home:\n', + b'postfix:x:100:101:postfix:/var/spool/postfix:\n', + b'niemeyer:x:500:500::/home/niemeyer:/bin/bash\n', + b'postgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\n', + b'mysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\n', + b'www:x:103:104::/var/www:/bin/false\n', + ] + TEXT = b''.join(TEXT_LINES) DATA = b'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2<Q\xb5\x0fH\xd3\xd4\xdd\xd5\x87\xbb\xf8\x94\r\x8f\xafI\x12\xe1\xc9\xf8/E\x00pu\x89\x12]\xc9\xbbDL\nQ\x0e\t1\x12\xdf\xa0\xc0\x97\xac2O9\x89\x13\x94\x0e\x1c7\x0ed\x95I\x0c\xaaJ\xa4\x18L\x10\x05#\x9c\xaf\xba\xbc/\x97\x8a#C\xc8\xe1\x8cW\xf9\xe2\xd0\xd6M\xa7\x8bXa<e\x84t\xcbL\xb3\xa7\xd9\xcd\xd1\xcb\x84.\xaf\xb3\xab\xab\xad`n}\xa0lh\tE,\x8eZ\x15\x17VH>\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = b'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1<l\xba\xcb_\xc00xY\x17r\x17\x88\x08\x08@\xa0\ry@\x10\x04$)`\xf2\xce\x89z\xb0s\xec\x9b.iW\x9d\x81\xb5-+t\x9f\x1a\'\x97dB\xf5x\xb5\xbe.[.\xd7\x0e\x81\xe7\x08\x1cN`\x88\x10\xca\x87\xc3!"\x80\x92R\xa1/\xd1\xc0\xe6mf\xac\xbd\x99\xcca\xb3\x8780>\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' if has_cmdline_bunzip2: def decompress(self, data): @@ -54,83 +77,135 @@ class BZ2FileTest(BaseTest): if os.path.isfile(self.filename): os.unlink(self.filename) - def createTempFile(self, crlf=0): + def createTempFile(self, streams=1): with open(self.filename, "wb") as f: - if crlf: - data = self.DATA_CRLF - else: - data = self.DATA - f.write(data) + f.write(self.DATA * streams) def testRead(self): - # "Test BZ2File.read()" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.read, None) self.assertEqual(bz2f.read(), self.TEXT) + def testReadMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.read, None) + self.assertEqual(bz2f.read(), self.TEXT * 5) + + def testReadMonkeyMultiStream(self): + # Test BZ2File.read() on a multi-stream archive where a stream + # boundary coincides with the end of the raw read buffer. + buffer_size = bz2._BUFFER_SIZE + bz2._BUFFER_SIZE = len(self.DATA) + try: + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.read, None) + self.assertEqual(bz2f.read(), self.TEXT * 5) + finally: + bz2._BUFFER_SIZE = buffer_size + def testRead0(self): - # Test BBZ2File.read(0)" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.read, None) self.assertEqual(bz2f.read(0), b"") def testReadChunk10(self): - # "Test BZ2File.read() in chunks of 10 bytes" self.createTempFile() with BZ2File(self.filename) as bz2f: text = b'' - while 1: + while True: str = bz2f.read(10) if not str: break text += str self.assertEqual(text, self.TEXT) + def testReadChunk10MultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + text = b'' + while True: + str = bz2f.read(10) + if not str: + break + text += str + self.assertEqual(text, self.TEXT * 5) + def testRead100(self): - # "Test BZ2File.read(100)" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertEqual(bz2f.read(100), self.TEXT[:100]) + def testPeek(self): + self.createTempFile() + with BZ2File(self.filename) as bz2f: + pdata = bz2f.peek() + self.assertNotEqual(len(pdata), 0) + self.assertTrue(self.TEXT.startswith(pdata)) + self.assertEqual(bz2f.read(), self.TEXT) + + def testReadInto(self): + self.createTempFile() + with BZ2File(self.filename) as bz2f: + n = 128 + b = bytearray(n) + self.assertEqual(bz2f.readinto(b), n) + self.assertEqual(b, self.TEXT[:n]) + n = len(self.TEXT) - n + b = bytearray(len(self.TEXT)) + self.assertEqual(bz2f.readinto(b), n) + self.assertEqual(b[:n], self.TEXT[-n:]) + def testReadLine(self): - # "Test BZ2File.readline()" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readline, None) - sio = BytesIO(self.TEXT) - for line in sio.readlines(): + for line in self.TEXT_LINES: + self.assertEqual(bz2f.readline(), line) + + def testReadLineMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.readline, None) + for line in self.TEXT_LINES * 5: self.assertEqual(bz2f.readline(), line) def testReadLines(self): - # "Test BZ2File.readlines()" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readlines, None) - sio = BytesIO(self.TEXT) - self.assertEqual(bz2f.readlines(), sio.readlines()) + self.assertEqual(bz2f.readlines(), self.TEXT_LINES) + + def testReadLinesMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.readlines, None) + self.assertEqual(bz2f.readlines(), self.TEXT_LINES * 5) def testIterator(self): - # "Test iter(BZ2File)" self.createTempFile() with BZ2File(self.filename) as bz2f: - sio = BytesIO(self.TEXT) - self.assertEqual(list(iter(bz2f)), sio.readlines()) + self.assertEqual(list(iter(bz2f)), self.TEXT_LINES) + + def testIteratorMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertEqual(list(iter(bz2f)), self.TEXT_LINES * 5) def testClosedIteratorDeadlock(self): - # "Test that iteration on a closed bz2file releases the lock." - # http://bugs.python.org/issue3309 + # Issue #3309: Iteration on a closed BZ2File should release the lock. self.createTempFile() bz2f = BZ2File(self.filename) bz2f.close() self.assertRaises(ValueError, bz2f.__next__) - # This call will deadlock of the above .__next__ call failed to + # This call will deadlock if the above .__next__ call failed to # release the lock. self.assertRaises(ValueError, bz2f.readlines) def testWrite(self): - # "Test BZ2File.write()" with BZ2File(self.filename, "w") as bz2f: self.assertRaises(TypeError, bz2f.write) bz2f.write(self.TEXT) @@ -138,10 +213,9 @@ class BZ2FileTest(BaseTest): self.assertEqual(self.decompress(f.read()), self.TEXT) def testWriteChunks10(self): - # "Test BZ2File.write() with chunks of 10 bytes" with BZ2File(self.filename, "w") as bz2f: n = 0 - while 1: + while True: str = self.TEXT[n*10:(n+1)*10] if not str: break @@ -151,12 +225,11 @@ class BZ2FileTest(BaseTest): self.assertEqual(self.decompress(f.read()), self.TEXT) def testWriteLines(self): - # "Test BZ2File.writelines()" with BZ2File(self.filename, "w") as bz2f: self.assertRaises(TypeError, bz2f.writelines) - sio = BytesIO(self.TEXT) - bz2f.writelines(sio.readlines()) - # patch #1535500 + bz2f.writelines(self.TEXT_LINES) + # Issue #1535500: Calling writelines() on a closed BZ2File + # should raise an exception. self.assertRaises(ValueError, bz2f.writelines, ["a"]) with open(self.filename, 'rb') as f: self.assertEqual(self.decompress(f.read()), self.TEXT) @@ -169,39 +242,73 @@ class BZ2FileTest(BaseTest): self.assertRaises(IOError, bz2f.write, b"a") self.assertRaises(IOError, bz2f.writelines, [b"a"]) + def testAppend(self): + with BZ2File(self.filename, "w") as bz2f: + self.assertRaises(TypeError, bz2f.write) + bz2f.write(self.TEXT) + with BZ2File(self.filename, "a") as bz2f: + self.assertRaises(TypeError, bz2f.write) + bz2f.write(self.TEXT) + with open(self.filename, 'rb') as f: + self.assertEqual(self.decompress(f.read()), self.TEXT * 2) + def testSeekForward(self): - # "Test BZ2File.seek(150, 0)" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.seek) bz2f.seek(150) self.assertEqual(bz2f.read(), self.TEXT[150:]) + def testSeekForwardAcrossStreams(self): + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.seek) + bz2f.seek(len(self.TEXT) + 150) + self.assertEqual(bz2f.read(), self.TEXT[150:]) + def testSeekBackwards(self): - # "Test BZ2File.seek(-150, 1)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.read(500) bz2f.seek(-150, 1) self.assertEqual(bz2f.read(), self.TEXT[500-150:]) + def testSeekBackwardsAcrossStreams(self): + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + readto = len(self.TEXT) + 100 + while readto > 0: + readto -= len(bz2f.read(readto)) + bz2f.seek(-150, 1) + self.assertEqual(bz2f.read(), self.TEXT[100-150:] + self.TEXT) + def testSeekBackwardsFromEnd(self): - # "Test BZ2File.seek(-150, 2)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(-150, 2) self.assertEqual(bz2f.read(), self.TEXT[len(self.TEXT)-150:]) + def testSeekBackwardsFromEndAcrossStreams(self): + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + bz2f.seek(-1000, 2) + self.assertEqual(bz2f.read(), (self.TEXT * 2)[-1000:]) + def testSeekPostEnd(self): - # "Test BZ2File.seek(150000)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(150000) self.assertEqual(bz2f.tell(), len(self.TEXT)) self.assertEqual(bz2f.read(), b"") + def testSeekPostEndMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + bz2f.seek(150000) + self.assertEqual(bz2f.tell(), len(self.TEXT) * 5) + self.assertEqual(bz2f.read(), b"") + def testSeekPostEndTwice(self): - # "Test BZ2File.seek(150000) twice" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(150000) @@ -209,27 +316,45 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.tell(), len(self.TEXT)) self.assertEqual(bz2f.read(), b"") + def testSeekPostEndTwiceMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + bz2f.seek(150000) + bz2f.seek(150000) + self.assertEqual(bz2f.tell(), len(self.TEXT) * 5) + self.assertEqual(bz2f.read(), b"") + def testSeekPreStart(self): - # "Test BZ2File.seek(-150, 0)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(-150) self.assertEqual(bz2f.tell(), 0) self.assertEqual(bz2f.read(), self.TEXT) + def testSeekPreStartMultiStream(self): + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + bz2f.seek(-150) + self.assertEqual(bz2f.tell(), 0) + self.assertEqual(bz2f.read(), self.TEXT * 2) + + def testFileno(self): + self.createTempFile() + with open(self.filename, 'rb') as rawf: + with BZ2File(fileobj=rawf) as bz2f: + self.assertEqual(bz2f.fileno(), rawf.fileno()) + def testOpenDel(self): - # "Test opening and deleting a file many times" self.createTempFile() for i in range(10000): o = BZ2File(self.filename) del o def testOpenNonexistent(self): - # "Test opening a nonexistent file" self.assertRaises(IOError, BZ2File, "/non/existent") - def testBug1191043(self): - # readlines() for files containing no newline + def testReadlinesNoNewline(self): + # Issue #1191043: readlines() fails on a file containing no newline. data = b'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' with open(self.filename, "wb") as f: f.write(data) @@ -241,7 +366,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(xlines, [b'Test']) def testContextProtocol(self): - # BZ2File supports the context management protocol f = None with BZ2File(self.filename, "wb") as f: f.write(b"xxx") @@ -264,7 +388,7 @@ class BZ2FileTest(BaseTest): @unittest.skipUnless(threading, 'Threading required for this test.') def testThreading(self): - # Using a BZ2File from several threads doesn't deadlock (issue #7205). + # Issue #7205: Using a BZ2File from several threads shouldn't deadlock. data = b"1" * 2**20 nthreads = 10 with bz2.BZ2File(self.filename, 'wb') as f: @@ -277,22 +401,76 @@ class BZ2FileTest(BaseTest): for t in threads: t.join() - def testMixedIterationReads(self): - # Issue #8397: mixed iteration and reads should be forbidden. - with bz2.BZ2File(self.filename, 'wb') as f: - # The internal buffer size is hard-wired to 8192 bytes, we must - # write out more than that for the test to stop half through - # the buffer. - f.write(self.TEXT * 100) - with bz2.BZ2File(self.filename, 'rb') as f: - next(f) - self.assertRaises(ValueError, f.read) - self.assertRaises(ValueError, f.readline) - self.assertRaises(ValueError, f.readlines) + def testMixedIterationAndReads(self): + self.createTempFile() + linelen = len(self.TEXT_LINES[0]) + halflen = linelen // 2 + with bz2.BZ2File(self.filename) as bz2f: + bz2f.read(halflen) + self.assertEqual(next(bz2f), self.TEXT_LINES[0][halflen:]) + self.assertEqual(bz2f.read(), self.TEXT[linelen:]) + with bz2.BZ2File(self.filename) as bz2f: + bz2f.readline() + self.assertEqual(next(bz2f), self.TEXT_LINES[1]) + self.assertEqual(bz2f.readline(), self.TEXT_LINES[2]) + with bz2.BZ2File(self.filename) as bz2f: + bz2f.readlines() + with self.assertRaises(StopIteration): + next(bz2f) + self.assertEqual(bz2f.readlines(), []) + + def testMultiStreamOrdering(self): + # Test the ordering of streams when reading a multi-stream archive. + data1 = b"foo" * 1000 + data2 = b"bar" * 1000 + with BZ2File(self.filename, "w") as bz2f: + bz2f.write(data1) + with BZ2File(self.filename, "a") as bz2f: + bz2f.write(data2) + with BZ2File(self.filename) as bz2f: + self.assertEqual(bz2f.read(), data1 + data2) + + # Tests for a BZ2File wrapping another file object: + + def testReadBytesIO(self): + with BytesIO(self.DATA) as bio: + with BZ2File(fileobj=bio) as bz2f: + self.assertRaises(TypeError, bz2f.read, None) + self.assertEqual(bz2f.read(), self.TEXT) + self.assertFalse(bio.closed) + + def testPeekBytesIO(self): + with BytesIO(self.DATA) as bio: + with BZ2File(fileobj=bio) as bz2f: + pdata = bz2f.peek() + self.assertNotEqual(len(pdata), 0) + self.assertTrue(self.TEXT.startswith(pdata)) + self.assertEqual(bz2f.read(), self.TEXT) + + def testWriteBytesIO(self): + with BytesIO() as bio: + with BZ2File(fileobj=bio, mode="w") as bz2f: + self.assertRaises(TypeError, bz2f.write) + bz2f.write(self.TEXT) + self.assertEqual(self.decompress(bio.getvalue()), self.TEXT) + self.assertFalse(bio.closed) + + def testSeekForwardBytesIO(self): + with BytesIO(self.DATA) as bio: + with BZ2File(fileobj=bio) as bz2f: + self.assertRaises(TypeError, bz2f.seek) + bz2f.seek(150) + self.assertEqual(bz2f.read(), self.TEXT[150:]) + + def testSeekBackwardsBytesIO(self): + with BytesIO(self.DATA) as bio: + with BZ2File(fileobj=bio) as bz2f: + bz2f.read(500) + bz2f.seek(-150, 1) + self.assertEqual(bz2f.read(), self.TEXT[500-150:]) class BZ2CompressorTest(BaseTest): def testCompress(self): - # "Test BZ2Compressor.compress()/flush()" bz2c = BZ2Compressor() self.assertRaises(TypeError, bz2c.compress) data = bz2c.compress(self.TEXT) @@ -300,11 +478,10 @@ class BZ2CompressorTest(BaseTest): self.assertEqual(self.decompress(data), self.TEXT) def testCompressChunks10(self): - # "Test BZ2Compressor.compress()/flush() with chunks of 10 bytes" bz2c = BZ2Compressor() n = 0 data = b'' - while 1: + while True: str = self.TEXT[n*10:(n+1)*10] if not str: break @@ -313,23 +490,38 @@ class BZ2CompressorTest(BaseTest): data += bz2c.flush() self.assertEqual(self.decompress(data), self.TEXT) + @precisionbigmemtest(size=_4G + 100, memuse=2) + def testCompress4G(self, size): + # "Test BZ2Compressor.compress()/flush() with >4GiB input" + bz2c = BZ2Compressor() + data = b"x" * size + try: + compressed = bz2c.compress(data) + compressed += bz2c.flush() + finally: + data = None # Release memory + data = bz2.decompress(compressed) + try: + self.assertEqual(len(data), size) + self.assertEqual(len(data.strip(b"x")), 0) + finally: + data = None + class BZ2DecompressorTest(BaseTest): def test_Constructor(self): self.assertRaises(TypeError, BZ2Decompressor, 42) def testDecompress(self): - # "Test BZ2Decompressor.decompress()" bz2d = BZ2Decompressor() self.assertRaises(TypeError, bz2d.decompress) text = bz2d.decompress(self.DATA) self.assertEqual(text, self.TEXT) def testDecompressChunks10(self): - # "Test BZ2Decompressor.decompress() with chunks of 10 bytes" bz2d = BZ2Decompressor() text = b'' n = 0 - while 1: + while True: str = self.DATA[n*10:(n+1)*10] if not str: break @@ -338,7 +530,6 @@ class BZ2DecompressorTest(BaseTest): self.assertEqual(text, self.TEXT) def testDecompressUnusedData(self): - # "Test BZ2Decompressor.decompress() with unused data" bz2d = BZ2Decompressor() unused_data = b"this is unused data" text = bz2d.decompress(self.DATA+unused_data) @@ -346,34 +537,49 @@ class BZ2DecompressorTest(BaseTest): self.assertEqual(bz2d.unused_data, unused_data) def testEOFError(self): - # "Calling BZ2Decompressor.decompress() after EOS must raise EOFError" bz2d = BZ2Decompressor() text = bz2d.decompress(self.DATA) self.assertRaises(EOFError, bz2d.decompress, b"anything") + @precisionbigmemtest(size=_4G + 100, memuse=3) + def testDecompress4G(self, size): + # "Test BZ2Decompressor.decompress() with >4GiB input" + blocksize = 10 * 1024 * 1024 + block = random.getrandbits(blocksize * 8).to_bytes(blocksize, 'little') + try: + data = block * (size // blocksize + 1) + compressed = bz2.compress(data) + bz2d = BZ2Decompressor() + decompressed = bz2d.decompress(compressed) + self.assertTrue(decompressed == data) + finally: + data = None + compressed = None + decompressed = None + class FuncTest(BaseTest): "Test module functions" def testCompress(self): - # "Test compress() function" data = bz2.compress(self.TEXT) self.assertEqual(self.decompress(data), self.TEXT) def testDecompress(self): - # "Test decompress() function" text = bz2.decompress(self.DATA) self.assertEqual(text, self.TEXT) def testDecompressEmpty(self): - # "Test decompress() function with empty string" text = bz2.decompress(b"") self.assertEqual(text, b"") def testDecompressIncomplete(self): - # "Test decompress() function with incomplete data" self.assertRaises(ValueError, bz2.decompress, self.DATA[:-10]) + def testDecompressMultiStream(self): + text = bz2.decompress(self.DATA * 5) + self.assertEqual(text, self.TEXT * 5) + def test_main(): support.run_unittest( BZ2FileTest, @@ -385,5 +591,3 @@ def test_main(): if __name__ == '__main__': test_main() - -# vim:ts=4:sw=4 diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index ada393d9b4..c649d3e681 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -193,18 +193,17 @@ def test_main(): idents = [] def callback(): - idents.append(_thread.get_ident()) + idents.append(threading.get_ident()) _testcapi._test_thread_state(callback) a = b = callback time.sleep(1) # Check our main thread is in the list exactly 3 times. - if idents.count(_thread.get_ident()) != 3: + if idents.count(threading.get_ident()) != 3: raise support.TestFailed( "Couldn't find main thread correctly in the list") if threading: - import _thread import time TestThreadState() t = threading.Thread(target=TestThreadState) diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py index c42db4e70d..dba77276d2 100644 --- a/Lib/test/test_cgi.py +++ b/Lib/test/test_cgi.py @@ -155,13 +155,7 @@ class CgiTests(unittest.TestCase): cgi.logfp = None cgi.logfile = "/dev/null" cgi.initlog("%s", "Testing log 3") - def log_cleanup(): - """Restore the global state of the log vars.""" - cgi.logfile = '' - cgi.logfp.close() - cgi.logfp = None - cgi.log = cgi.initlog - self.addCleanup(log_cleanup) + self.addCleanup(cgi.closelog) cgi.log("Testing log 4") def test_fieldstorage_readline(self): diff --git a/Lib/test/test_cgitb.py b/Lib/test/test_cgitb.py new file mode 100644 index 0000000000..16a4b1a034 --- /dev/null +++ b/Lib/test/test_cgitb.py @@ -0,0 +1,55 @@ +from test.support import run_unittest +import unittest +import sys +import subprocess +import cgitb + +class TestCgitb(unittest.TestCase): + + def test_fonts(self): + text = "Hello Robbie!" + self.assertEqual(cgitb.small(text), "<small>{}</small>".format(text)) + self.assertEqual(cgitb.strong(text), "<strong>{}</strong>".format(text)) + self.assertEqual(cgitb.grey(text), + '<font color="#909090">{}</font>'.format(text)) + + def test_blanks(self): + self.assertEqual(cgitb.small(""), "") + self.assertEqual(cgitb.strong(""), "") + self.assertEqual(cgitb.grey(""), "") + + def test_html(self): + try: + raise ValueError("Hello World") + except ValueError as err: + # If the html was templated we could do a bit more here. + # At least check that we get details on what we just raised. + html = cgitb.html(sys.exc_info()) + self.assertIn("ValueError", html) + self.assertIn(str(err), html) + + def test_text(self): + try: + raise ValueError("Hello World") + except ValueError as err: + text = cgitb.text(sys.exc_info()) + self.assertIn("ValueError", text) + self.assertIn("Hello World", text) + + def test_hook(self): + proc = subprocess.Popen([sys.executable, '-c', + ('import cgitb;' + 'cgitb.enable();' + 'raise ValueError("Hello World")')], + stdout=subprocess.PIPE) + out = proc.stdout.read().decode(sys.getfilesystemencoding()) + self.addCleanup(proc.stdout.close) + self.assertIn("ValueError", out) + self.assertIn("Hello World", out) + + +def test_main(): + run_unittest(TestCgitb) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 56a8e39659..9738691693 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -31,12 +31,6 @@ class CmdLineTest(unittest.TestCase): self.verify_valid_flag('-O') self.verify_valid_flag('-OO') - def test_q(self): - self.verify_valid_flag('-Qold') - self.verify_valid_flag('-Qnew') - self.verify_valid_flag('-Qwarn') - self.verify_valid_flag('-Qwarnall') - def test_site_flag(self): self.verify_valid_flag('-S') @@ -151,7 +145,7 @@ class CmdLineTest(unittest.TestCase): @unittest.skipUnless(sys.platform == 'darwin', 'test specific to Mac OS X') def test_osx_utf8(self): def check_output(text): - decoded = text.decode('utf8', 'surrogateescape') + decoded = text.decode('utf-8', 'surrogateescape') expected = ascii(decoded).encode('ascii') + b'\n' env = os.environ.copy() @@ -223,7 +217,7 @@ class CmdLineTest(unittest.TestCase): self.assertIn(path2.encode('ascii'), out) def test_displayhook_unencodable(self): - for encoding in ('ascii', 'latin1', 'utf8'): + for encoding in ('ascii', 'latin-1', 'utf-8'): env = os.environ.copy() env['PYTHONIOENCODING'] = encoding p = subprocess.Popen( diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py index dca9f10b83..ee3d1650cb 100644 --- a/Lib/test/test_codecencodings_cn.py +++ b/Lib/test/test_codecencodings_cn.py @@ -15,8 +15,8 @@ class Test_GB2312(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x81\x81\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x81\x81\xc1\xc4", "replace", "abc\ufffd\u804a"), - (b"abc\x81\x81\xc1\xc4\xc8", "replace", "abc\ufffd\u804a\ufffd"), + (b"abc\x81\x81\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x81\x81\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), (b"abc\x81\x81\xc1\xc4", "ignore", "abc\u804a"), (b"\xc1\x64", "strict", None), ) @@ -28,8 +28,8 @@ class Test_GBK(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u804a"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u804a\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u804a"), (b"\x83\x34\x83\x31", "strict", None), ("\u30fb", "strict", None), @@ -42,11 +42,14 @@ class Test_GB18030(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u804a"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u804a\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u804a"), - (b"abc\x84\x39\x84\x39\xc1\xc4", "replace", "abc\ufffd\u804a"), + (b"abc\x84\x39\x84\x39\xc1\xc4", "replace", "abc\ufffd9\ufffd9\u804a"), ("\u30fb", "strict", b"\x819\xa79"), + (b"abc\x84\x32\x80\x80def", "replace", 'abc\ufffd2\ufffd\ufffddef'), + (b"abc\x81\x30\x81\x30def", "strict", 'abc\x80def'), + (b"abc\x86\x30\x81\x30def", "replace", 'abc\ufffd0\ufffd0def'), ) has_iso10646 = True @@ -74,9 +77,11 @@ class Test_HZ(test_multibytecodec_support.TestBase, unittest.TestCase): '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002' 'Bye.\n'), # invalid bytes - (b'ab~cd', 'replace', 'ab\uFFFDd'), + (b'ab~cd', 'replace', 'ab\uFFFDcd'), (b'ab\xffcd', 'replace', 'ab\uFFFDcd'), (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'), + (b'ab~{\x41\x44~}cd', 'replace', 'ab\u804Acd'), + (b"ab~{\x79\x79\x41\x44~}cd", "replace", "ab\ufffd\ufffd\u804acd"), ) def test_main(): diff --git a/Lib/test/test_codecencodings_hk.py b/Lib/test/test_codecencodings_hk.py index ccdc0b4c55..520df43494 100644 --- a/Lib/test/test_codecencodings_hk.py +++ b/Lib/test/test_codecencodings_hk.py @@ -15,8 +15,8 @@ class Test_Big5HKSCS(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u8b10"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u8b10"), ) diff --git a/Lib/test/test_codecencodings_jp.py b/Lib/test/test_codecencodings_jp.py index f56a373896..87e4812482 100644 --- a/Lib/test/test_codecencodings_jp.py +++ b/Lib/test/test_codecencodings_jp.py @@ -15,50 +15,57 @@ class Test_CP932(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x81\x00\x81\x00\x82\x84", "strict", None), (b"abc\xf8", "strict", None), - (b"abc\x81\x00\x82\x84", "replace", "abc\ufffd\uff44"), - (b"abc\x81\x00\x82\x84\x88", "replace", "abc\ufffd\uff44\ufffd"), - (b"abc\x81\x00\x82\x84", "ignore", "abc\uff44"), + (b"abc\x81\x00\x82\x84", "replace", "abc\ufffd\x00\uff44"), + (b"abc\x81\x00\x82\x84\x88", "replace", "abc\ufffd\x00\uff44\ufffd"), + (b"abc\x81\x00\x82\x84", "ignore", "abc\x00\uff44"), + (b"ab\xEBxy", "replace", "ab\uFFFDxy"), + (b"ab\xF0\x39xy", "replace", "ab\uFFFD9xy"), + (b"ab\xEA\xF0xy", "replace", 'ab\ufffd\ue038y'), # sjis vs cp932 (b"\\\x7e", "replace", "\\\x7e"), (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\uff3c\u2225\uff0d"), ) +euc_commontests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u7956"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u7956\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u7956"), + (b"abc\xc8", "strict", None), + (b"abc\x8f\x83\x83", "replace", "abc\ufffd\ufffd\ufffd"), + (b"\x82\xFCxy", "replace", "\ufffd\ufffdxy"), + (b"\xc1\x64", "strict", None), + (b"\xa1\xc0", "strict", "\uff3c"), + (b"\xa1\xc0\\", "strict", "\uff3c\\"), + (b"\x8eXY", "replace", "\ufffdXY"), +) + +class Test_EUC_JIS_2004(test_multibytecodec_support.TestBase, + unittest.TestCase): + encoding = 'euc_jis_2004' + tstring = test_multibytecodec_support.load_teststring('euc_jisx0213') + codectests = euc_commontests + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩" + ) + class Test_EUC_JISX0213(test_multibytecodec_support.TestBase, unittest.TestCase): encoding = 'euc_jisx0213' tstring = test_multibytecodec_support.load_teststring('euc_jisx0213') - codectests = ( - # invalid bytes - (b"abc\x80\x80\xc1\xc4", "strict", None), - (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u7956"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u7956\ufffd"), - (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u7956"), - (b"abc\x8f\x83\x83", "replace", "abc\ufffd"), - (b"\xc1\x64", "strict", None), - (b"\xa1\xc0", "strict", "\uff3c"), - ) + codectests = euc_commontests xmlcharnametest = ( "\xab\u211c\xbb = \u2329\u1234\u232a", b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩" ) -eucjp_commontests = ( - (b"abc\x80\x80\xc1\xc4", "strict", None), - (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u7956"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u7956\ufffd"), - (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u7956"), - (b"abc\x8f\x83\x83", "replace", "abc\ufffd"), - (b"\xc1\x64", "strict", None), -) - class Test_EUC_JP_COMPAT(test_multibytecodec_support.TestBase, unittest.TestCase): encoding = 'euc_jp' tstring = test_multibytecodec_support.load_teststring('euc_jp') - codectests = eucjp_commontests + ( - (b"\xa1\xc0\\", "strict", "\uff3c\\"), + codectests = euc_commontests + ( ("\xa5", "strict", b"\x5c"), ("\u203e", "strict", b"\x7e"), ) @@ -66,8 +73,6 @@ class Test_EUC_JP_COMPAT(test_multibytecodec_support.TestBase, shiftjis_commonenctests = ( (b"abc\x80\x80\x82\x84", "strict", None), (b"abc\xf8", "strict", None), - (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\uff44"), - (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\uff44\ufffd"), (b"abc\x80\x80\x82\x84def", "ignore", "abc\uff44def"), ) @@ -75,20 +80,41 @@ class Test_SJIS_COMPAT(test_multibytecodec_support.TestBase, unittest.TestCase): encoding = 'shift_jis' tstring = test_multibytecodec_support.load_teststring('shift_jis') codectests = shiftjis_commonenctests + ( + (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"), + (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"), + (b"\\\x7e", "strict", "\\\x7e"), (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\uff3c\u2016\u2212"), + (b"abc\x81\x39", "replace", "abc\ufffd9"), + (b"abc\xEA\xFC", "replace", "abc\ufffd\ufffd"), + (b"abc\xFF\x58", "replace", "abc\ufffdX"), + ) + +class Test_SJIS_2004(test_multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'shift_jis_2004' + tstring = test_multibytecodec_support.load_teststring('shift_jis') + codectests = shiftjis_commonenctests + ( + (b"\\\x7e", "strict", "\xa5\u203e"), + (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\\\u2016\u2212"), + (b"abc\xEA\xFC", "strict", "abc\u64bf"), + (b"\x81\x39xy", "replace", "\ufffd9xy"), + (b"\xFF\x58xy", "replace", "\ufffdXxy"), + (b"\x80\x80\x82\x84xy", "replace", "\ufffd\ufffd\uff44xy"), + (b"\x80\x80\x82\x84\x88xy", "replace", "\ufffd\ufffd\uff44\u5864y"), + (b"\xFC\xFBxy", "replace", '\ufffd\u95b4y'), + ) + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\x85Gℜ\x85Q = ⟨ሴ⟩" ) class Test_SJISX0213(test_multibytecodec_support.TestBase, unittest.TestCase): encoding = 'shift_jisx0213' tstring = test_multibytecodec_support.load_teststring('shift_jisx0213') - codectests = ( - # invalid bytes - (b"abc\x80\x80\x82\x84", "strict", None), - (b"abc\xf8", "strict", None), - (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\uff44"), - (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\uff44\ufffd"), - (b"abc\x80\x80\x82\x84def", "ignore", "abc\uff44def"), + codectests = shiftjis_commonenctests + ( + (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"), + (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"), + # sjis vs cp932 (b"\\\x7e", "replace", "\xa5\u203e"), (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\x5c\u2016\u2212"), diff --git a/Lib/test/test_codecencodings_kr.py b/Lib/test/test_codecencodings_kr.py index de4da7f5b6..4997e8349b 100644 --- a/Lib/test/test_codecencodings_kr.py +++ b/Lib/test/test_codecencodings_kr.py @@ -15,8 +15,8 @@ class Test_CP949(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\uc894"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\uc894\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\uc894"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"), ) @@ -27,8 +27,8 @@ class Test_EUCKR(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\uc894"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\uc894\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", 'abc\ufffd\ufffd\uc894'), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"), # composed make-up sequence errors @@ -40,13 +40,14 @@ class Test_EUCKR(test_multibytecodec_support.TestBase, unittest.TestCase): (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4", "strict", None), (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "strict", "\uc4d4"), (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4x", "strict", "\uc4d4x"), - (b"a\xa4\xd4\xa4\xb6\xa4", "replace", "a\ufffd"), + (b"a\xa4\xd4\xa4\xb6\xa4", "replace", 'a\ufffd'), (b"\xa4\xd4\xa3\xb6\xa4\xd0\xa4\xd4", "strict", None), (b"\xa4\xd4\xa4\xb6\xa3\xd0\xa4\xd4", "strict", None), (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa3\xd4", "strict", None), - (b"\xa4\xd4\xa4\xff\xa4\xd0\xa4\xd4", "replace", "\ufffd"), - (b"\xa4\xd4\xa4\xb6\xa4\xff\xa4\xd4", "replace", "\ufffd"), - (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xff", "replace", "\ufffd"), + (b"\xa4\xd4\xa4\xff\xa4\xd0\xa4\xd4", "replace", '\ufffd\u6e21\ufffd\u3160\ufffd'), + (b"\xa4\xd4\xa4\xb6\xa4\xff\xa4\xd4", "replace", '\ufffd\u6e21\ub544\ufffd\ufffd'), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xff", "replace", '\ufffd\u6e21\ub544\u572d\ufffd'), + (b"\xa4\xd4\xff\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "replace", '\ufffd\ufffd\ufffd\uc4d4'), (b"\xc1\xc4", "strict", "\uc894"), ) @@ -57,9 +58,13 @@ class Test_JOHAB(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ucd27"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ucd27\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\ucd27"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\ucd27\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\ucd27"), + (b"\xD8abc", "replace", "\uFFFDabc"), + (b"\xD8\xFFabc", "replace", "\uFFFD\uFFFDabc"), + (b"\x84bxy", "replace", "\uFFFDbxy"), + (b"\x8CBxy", "replace", "\uFFFDBxy"), ) def test_main(): diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py index 12d3c9fa04..f2f3c1802d 100644 --- a/Lib/test/test_codecencodings_tw.py +++ b/Lib/test/test_codecencodings_tw.py @@ -15,8 +15,8 @@ class Test_Big5(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u8b10"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u8b10"), ) diff --git a/Lib/test/test_codecmaps_tw.py b/Lib/test/test_codecmaps_tw.py index 6db5091fc3..412b9de8b2 100644 --- a/Lib/test/test_codecmaps_tw.py +++ b/Lib/test/test_codecmaps_tw.py @@ -23,6 +23,9 @@ class TestCP950Map(test_multibytecodec_support.TestBase_Mapping, (b'\xa2\xcc', '\u5341'), (b'\xa2\xce', '\u5345'), ] + codectests = ( + (b"\xFFxy", "replace", "\ufffdxy"), + ) def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 4899a59ef2..7a9e38cfd6 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1262,7 +1262,7 @@ class EncodedFileTest(unittest.TestCase): self.assertEqual(ef.read(), b'\\\xd5\n\x00\x00\xae') f = io.BytesIO() - ef = codecs.EncodedFile(f, 'utf-8', 'latin1') + ef = codecs.EncodedFile(f, 'utf-8', 'latin-1') ef.write(b'\xc3\xbc') self.assertEqual(f.getvalue(), b'\xfc') @@ -1623,7 +1623,7 @@ class SurrogateEscapeTest(unittest.TestCase): def test_latin1(self): # Issue6373 - self.assertEqual("\udce4\udceb\udcef\udcf6\udcfc".encode("latin1", "surrogateescape"), + self.assertEqual("\udce4\udceb\udcef\udcf6\udcfc".encode("latin-1", "surrogateescape"), b"\xe4\xeb\xef\xf6\xfc") diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index ccf93c51c1..04c4d97c55 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1,6 +1,7 @@ """Unit tests for collections.py.""" import unittest, doctest, operator +from test.support import TESTFN, forget, unlink import inspect from test import support from collections import namedtuple, Counter, OrderedDict, _count_elements @@ -10,21 +11,20 @@ from random import randrange, shuffle import keyword import re import sys -from collections import _ChainMap -from collections import Hashable, Iterable, Iterator -from collections import Sized, Container, Callable -from collections import Set, MutableSet -from collections import Mapping, MutableMapping, KeysView, ItemsView, UserDict -from collections import Sequence, MutableSequence -from collections import ByteString +from collections import UserDict +from collections import ChainMap +from collections.abc import Hashable, Iterable, Iterator +from collections.abc import Sized, Container, Callable +from collections.abc import Set, MutableSet +from collections.abc import Mapping, MutableMapping, KeysView, ItemsView +from collections.abc import Sequence, MutableSequence +from collections.abc import ByteString ################################################################################ -### _ChainMap (helper class for configparser) +### ChainMap (helper class for configparser and the string module) ################################################################################ -ChainMap = _ChainMap # rename to keep test code in sync with 3.3 version - class TestChainMap(unittest.TestCase): def test_basics(self): @@ -128,6 +128,7 @@ class TestNamedTuple(unittest.TestCase): self.assertEqual(Point.__module__, __name__) self.assertEqual(Point.__getitem__, tuple.__getitem__) self.assertEqual(Point._fields, ('x', 'y')) + self.assertIn('class Point(tuple)', Point._source) self.assertRaises(ValueError, namedtuple, 'abc%', 'efg ghi') # type has non-alpha char self.assertRaises(ValueError, namedtuple, 'class', 'efg ghi') # type has keyword @@ -327,6 +328,17 @@ class TestNamedTuple(unittest.TestCase): pass self.assertEqual(repr(B(1)), 'B(x=1)') + def test_source(self): + # verify that _source can be run through exec() + tmp = namedtuple('NTColor', 'red green blue') + globals().pop('NTColor', None) # remove artifacts from other tests + exec(tmp._source, globals()) + self.assertIn('NTColor', globals()) + c = NTColor(10, 20, 30) + self.assertEqual((c.red, c.green, c.blue), (10, 20, 30)) + self.assertEqual(NTColor._fields, ('red', 'green', 'blue')) + globals().pop('NTColor', None) # clean-up after this test + ################################################################################ ### Abstract Base Classes @@ -729,6 +741,44 @@ class TestCollectionABCs(ABCTestCase): self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') + def test_MutableSequence_mixins(self): + # Test the mixins of MutableSequence by creating a miminal concrete + # class inherited from it. + class MutableSequenceSubclass(MutableSequence): + def __init__(self): + self.lst = [] + + def __setitem__(self, index, value): + self.lst[index] = value + + def __getitem__(self, index): + return self.lst[index] + + def __len__(self): + return len(self.lst) + + def __delitem__(self, index): + del self.lst[index] + + def insert(self, index, value): + self.lst.insert(index, value) + + mss = MutableSequenceSubclass() + mss.append(0) + mss.extend((1, 2, 3, 4)) + self.assertEqual(len(mss), 5) + self.assertEqual(mss[3], 3) + mss.reverse() + self.assertEqual(mss[3], 1) + mss.pop() + self.assertEqual(len(mss), 4) + mss.remove(3) + self.assertEqual(len(mss), 3) + mss += (10, 20, 30) + self.assertEqual(len(mss), 6) + self.assertEqual(mss[-1], 30) + mss.clear() + self.assertEqual(len(mss), 0) ################################################################################ ### Counter @@ -893,6 +943,11 @@ class TestCounter(unittest.TestCase): c.subtract('aaaabbcce') self.assertEqual(c, Counter(a=-1, b=0, c=-1, d=1, e=-1)) + def test_unary(self): + c = Counter(a=-5, b=0, c=5, d=10, e=15,g=40) + self.assertEqual(dict(+c), dict(c=5, d=10, e=15, g=40)) + self.assertEqual(dict(-c), dict(a=5)) + def test_helper_function(self): # two paths, one for real dicts and one for other mappings elems = list('abracadabra') diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 58ef2979a0..c5f9189fcc 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,6 +1,7 @@ import unittest import sys import _ast +import types from test import support class TestSpecifics(unittest.TestCase): @@ -433,6 +434,14 @@ if 1: ast.body = [_ast.BoolOp()] self.assertRaises(TypeError, compile, ast, '<ast>', 'exec') + @support.cpython_only + def test_same_filename_used(self): + s = """def f(): pass\ndef g(): pass""" + c = compile(s, "myfile", "exec") + for obj in c.co_consts: + if isinstance(obj, types.CodeType): + self.assertIs(obj.co_filename, c.co_filename) + def test_main(): support.run_unittest(TestSpecifics) diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 6cc57f85d0..78a9906d34 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -19,7 +19,7 @@ import unittest from concurrent import futures from concurrent.futures._base import ( PENDING, RUNNING, CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED, Future) -import concurrent.futures.process +from concurrent.futures.process import BrokenProcessPool def create_future(state=PENDING, exception=None, result=None): @@ -154,7 +154,7 @@ class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest): processes = self.executor._processes self.executor.shutdown() - for p in processes: + for p in processes.values(): p.join() def test_context_manager_shutdown(self): @@ -163,7 +163,7 @@ class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest): self.assertEqual(list(e.map(abs, range(-5, 5))), [5, 4, 3, 2, 1, 0, 1, 2, 3, 4]) - for p in processes: + for p in processes.values(): p.join() def test_del_shutdown(self): @@ -174,7 +174,7 @@ class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest): del executor queue_management_thread.join() - for p in processes: + for p in processes.values(): p.join() class WaitTests(unittest.TestCase): @@ -260,14 +260,14 @@ class WaitTests(unittest.TestCase): def test_timeout(self): future1 = self.executor.submit(mul, 6, 7) - future2 = self.executor.submit(time.sleep, 3) + future2 = self.executor.submit(time.sleep, 6) finished, pending = futures.wait( [CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, SUCCESSFUL_FUTURE, future1, future2], - timeout=1.5, + timeout=5, return_when=futures.ALL_COMPLETED) self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE, @@ -357,8 +357,8 @@ class ExecutorTest(unittest.TestCase): results = [] try: for i in self.executor.map(time.sleep, - [0, 0, 3], - timeout=1.5): + [0, 0, 6], + timeout=5): results.append(i) except futures.TimeoutError: pass @@ -367,13 +367,38 @@ class ExecutorTest(unittest.TestCase): self.assertEqual([None, None], results) + def test_shutdown_race_issue12456(self): + # Issue #12456: race condition at shutdown where trying to post a + # sentinel in the call queue blocks (the queue is full while processes + # have exited). + self.executor.map(str, [2] * (self.worker_count + 1)) + self.executor.shutdown() + class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest): - pass + def test_map_submits_without_iteration(self): + """Tests verifying issue 11777.""" + finished = [] + def record_finished(n): + finished.append(n) + + self.executor.map(record_finished, range(10)) + self.executor.shutdown(wait=True) + self.assertCountEqual(finished, range(10)) class ProcessPoolExecutorTest(ProcessPoolMixin, ExecutorTest): - pass + def test_killed_child(self): + # When a child process is abruptly terminated, the whole pool gets + # "broken". + futures = [self.executor.submit(time.sleep, 3)] + # Get one of the processes, and terminate (kill) it + p = next(iter(self.executor._processes.values())) + p.terminate() + for fut in futures: + self.assertRaises(BrokenProcessPool, fut.result) + # Submitting other jobs fails as well. + self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8) class FutureTests(unittest.TestCase): @@ -609,7 +634,8 @@ def test_main(): ThreadPoolAsCompletedTests, FutureTests, ProcessPoolShutdownTest, - ThreadPoolShutdownTest) + ThreadPoolShutdownTest, + ) finally: test.support.reap_children() diff --git a/Lib/test/test_cfgparser.py b/Lib/test/test_configparser.py index 1db92176f8..1db92176f8 100644 --- a/Lib/test/test_cfgparser.py +++ b/Lib/test/test_configparser.py diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index a84c109b87..c4baae4861 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -17,7 +17,7 @@ class TestCopy(unittest.TestCase): # Attempt full line coverage of copy.py from top to bottom def test_exceptions(self): - self.assertTrue(copy.Error is copy.error) + self.assertIs(copy.Error, copy.error) self.assertTrue(issubclass(copy.Error, Exception)) # The copy() method @@ -54,20 +54,26 @@ class TestCopy(unittest.TestCase): def test_copy_reduce_ex(self): class C(object): def __reduce_ex__(self, proto): + c.append(1) return "" def __reduce__(self): - raise support.TestFailed("shouldn't call this") + self.fail("shouldn't call this") + c = [] x = C() y = copy.copy(x) - self.assertTrue(y is x) + self.assertIs(y, x) + self.assertEqual(c, [1]) def test_copy_reduce(self): class C(object): def __reduce__(self): + c.append(1) return "" + c = [] x = C() y = copy.copy(x) - self.assertTrue(y is x) + self.assertIs(y, x) + self.assertEqual(c, [1]) def test_copy_cant(self): class C(object): @@ -91,7 +97,7 @@ class TestCopy(unittest.TestCase): "hello", "hello\u1234", f.__code__, NewStyle, range(10), Classic, max] for x in tests: - self.assertTrue(copy.copy(x) is x, repr(x)) + self.assertIs(copy.copy(x), x) def test_copy_list(self): x = [1, 2, 3] @@ -185,9 +191,9 @@ class TestCopy(unittest.TestCase): x = [x, x] y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y[0] is not x[0]) - self.assertTrue(y[0] is y[1]) + self.assertIsNot(y, x) + self.assertIsNot(y[0], x[0]) + self.assertIs(y[0], y[1]) def test_deepcopy_issubclass(self): # XXX Note: there's no way to test the TypeError coming out of @@ -227,20 +233,26 @@ class TestCopy(unittest.TestCase): def test_deepcopy_reduce_ex(self): class C(object): def __reduce_ex__(self, proto): + c.append(1) return "" def __reduce__(self): - raise support.TestFailed("shouldn't call this") + self.fail("shouldn't call this") + c = [] x = C() y = copy.deepcopy(x) - self.assertTrue(y is x) + self.assertIs(y, x) + self.assertEqual(c, [1]) def test_deepcopy_reduce(self): class C(object): def __reduce__(self): + c.append(1) return "" + c = [] x = C() y = copy.deepcopy(x) - self.assertTrue(y is x) + self.assertIs(y, x) + self.assertEqual(c, [1]) def test_deepcopy_cant(self): class C(object): @@ -264,14 +276,14 @@ class TestCopy(unittest.TestCase): "hello", "hello\u1234", f.__code__, NewStyle, range(10), Classic, max] for x in tests: - self.assertTrue(copy.deepcopy(x) is x, repr(x)) + self.assertIs(copy.deepcopy(x), x) def test_deepcopy_list(self): x = [[1, 2], 3] y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(x is not y) - self.assertTrue(x[0] is not y[0]) + self.assertIsNot(x, y) + self.assertIsNot(x[0], y[0]) def test_deepcopy_reflexive_list(self): x = [] @@ -279,16 +291,26 @@ class TestCopy(unittest.TestCase): y = copy.deepcopy(x) for op in comparisons: self.assertRaises(RuntimeError, op, y, x) - self.assertTrue(y is not x) - self.assertTrue(y[0] is y) + self.assertIsNot(y, x) + self.assertIs(y[0], y) self.assertEqual(len(y), 1) + def test_deepcopy_empty_tuple(self): + x = () + y = copy.deepcopy(x) + self.assertIs(x, y) + def test_deepcopy_tuple(self): x = ([1, 2], 3) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(x is not y) - self.assertTrue(x[0] is not y[0]) + self.assertIsNot(x, y) + self.assertIsNot(x[0], y[0]) + + def test_deepcopy_tuple_of_immutables(self): + x = ((1, 2), 3) + y = copy.deepcopy(x) + self.assertIs(x, y) def test_deepcopy_reflexive_tuple(self): x = ([],) @@ -296,16 +318,16 @@ class TestCopy(unittest.TestCase): y = copy.deepcopy(x) for op in comparisons: self.assertRaises(RuntimeError, op, y, x) - self.assertTrue(y is not x) - self.assertTrue(y[0] is not x[0]) - self.assertTrue(y[0][0] is y) + self.assertIsNot(y, x) + self.assertIsNot(y[0], x[0]) + self.assertIs(y[0][0], y) def test_deepcopy_dict(self): x = {"foo": [1, 2], "bar": 3} y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(x is not y) - self.assertTrue(x["foo"] is not y["foo"]) + self.assertIsNot(x, y) + self.assertIsNot(x["foo"], y["foo"]) def test_deepcopy_reflexive_dict(self): x = {} @@ -315,15 +337,30 @@ class TestCopy(unittest.TestCase): self.assertRaises(TypeError, op, y, x) for op in equality_comparisons: self.assertRaises(RuntimeError, op, y, x) - self.assertTrue(y is not x) - self.assertTrue(y['foo'] is y) + self.assertIsNot(y, x) + self.assertIs(y['foo'], y) self.assertEqual(len(y), 1) def test_deepcopy_keepalive(self): memo = {} - x = 42 + x = [] + y = copy.deepcopy(x, memo) + self.assertIs(memo[id(memo)][0], x) + + def test_deepcopy_dont_memo_immutable(self): + memo = {} + x = [1, 2, 3, 4] y = copy.deepcopy(x, memo) - self.assertTrue(memo[id(x)] is x) + self.assertEqual(y, x) + # There's the entry for the new list, and the keep alive. + self.assertEqual(len(memo), 2) + + memo = {} + x = [(1, 2)] + y = copy.deepcopy(x, memo) + self.assertEqual(y, x) + # Tuples with immutable contents are immutable for deepcopy. + self.assertEqual(len(memo), 2) def test_deepcopy_inst_vanilla(self): class C: @@ -334,7 +371,7 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_inst_deepcopy(self): class C: @@ -347,8 +384,8 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y, x) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_inst_getinitargs(self): class C: @@ -361,8 +398,8 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y, x) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_inst_getstate(self): class C: @@ -375,8 +412,8 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y, x) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_inst_setstate(self): class C: @@ -389,8 +426,8 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y, x) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_inst_getstate_setstate(self): class C: @@ -405,8 +442,8 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y, x) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_reflexive_inst(self): class C: @@ -414,8 +451,8 @@ class TestCopy(unittest.TestCase): x = C() x.foo = x y = copy.deepcopy(x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is y) + self.assertIsNot(y, x) + self.assertIs(y.foo, y) # _reconstruct() @@ -425,9 +462,9 @@ class TestCopy(unittest.TestCase): return "" x = C() y = copy.copy(x) - self.assertTrue(y is x) + self.assertIs(y, x) y = copy.deepcopy(x) - self.assertTrue(y is x) + self.assertIs(y, x) def test_reconstruct_nostate(self): class C(object): @@ -436,9 +473,9 @@ class TestCopy(unittest.TestCase): x = C() x.foo = 42 y = copy.copy(x) - self.assertTrue(y.__class__ is x.__class__) + self.assertIs(y.__class__, x.__class__) y = copy.deepcopy(x) - self.assertTrue(y.__class__ is x.__class__) + self.assertIs(y.__class__, x.__class__) def test_reconstruct_state(self): class C(object): @@ -452,7 +489,7 @@ class TestCopy(unittest.TestCase): self.assertEqual(y, x) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y.foo, x.foo) def test_reconstruct_state_setstate(self): class C(object): @@ -468,7 +505,7 @@ class TestCopy(unittest.TestCase): self.assertEqual(y, x) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y.foo, x.foo) def test_reconstruct_reflexive(self): class C(object): @@ -476,8 +513,8 @@ class TestCopy(unittest.TestCase): x = C() x.foo = x y = copy.deepcopy(x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is y) + self.assertIsNot(y, x) + self.assertIs(y.foo, y) # Additions for Python 2.3 and pickle protocol 2 @@ -491,12 +528,12 @@ class TestCopy(unittest.TestCase): x = C([[1, 2], 3]) y = copy.copy(x) self.assertEqual(x, y) - self.assertTrue(x is not y) - self.assertTrue(x[0] is y[0]) + self.assertIsNot(x, y) + self.assertIs(x[0], y[0]) y = copy.deepcopy(x) self.assertEqual(x, y) - self.assertTrue(x is not y) - self.assertTrue(x[0] is not y[0]) + self.assertIsNot(x, y) + self.assertIsNot(x[0], y[0]) def test_reduce_5tuple(self): class C(dict): @@ -508,12 +545,12 @@ class TestCopy(unittest.TestCase): x = C([("foo", [1, 2]), ("bar", 3)]) y = copy.copy(x) self.assertEqual(x, y) - self.assertTrue(x is not y) - self.assertTrue(x["foo"] is y["foo"]) + self.assertIsNot(x, y) + self.assertIs(x["foo"], y["foo"]) y = copy.deepcopy(x) self.assertEqual(x, y) - self.assertTrue(x is not y) - self.assertTrue(x["foo"] is not y["foo"]) + self.assertIsNot(x, y) + self.assertIsNot(x["foo"], y["foo"]) def test_copy_slots(self): class C(object): @@ -521,7 +558,7 @@ class TestCopy(unittest.TestCase): x = C() x.foo = [42] y = copy.copy(x) - self.assertTrue(x.foo is y.foo) + self.assertIs(x.foo, y.foo) def test_deepcopy_slots(self): class C(object): @@ -530,7 +567,7 @@ class TestCopy(unittest.TestCase): x.foo = [42] y = copy.deepcopy(x) self.assertEqual(x.foo, y.foo) - self.assertTrue(x.foo is not y.foo) + self.assertIsNot(x.foo, y.foo) def test_deepcopy_dict_subclass(self): class C(dict): @@ -547,7 +584,7 @@ class TestCopy(unittest.TestCase): y = copy.deepcopy(x) self.assertEqual(x, y) self.assertEqual(x._keys, y._keys) - self.assertTrue(x is not y) + self.assertIsNot(x, y) x['bar'] = 1 self.assertNotEqual(x, y) self.assertNotEqual(x._keys, y._keys) @@ -560,8 +597,8 @@ class TestCopy(unittest.TestCase): y = copy.copy(x) self.assertEqual(list(x), list(y)) self.assertEqual(x.foo, y.foo) - self.assertTrue(x[0] is y[0]) - self.assertTrue(x.foo is y.foo) + self.assertIs(x[0], y[0]) + self.assertIs(x.foo, y.foo) def test_deepcopy_list_subclass(self): class C(list): @@ -571,8 +608,8 @@ class TestCopy(unittest.TestCase): y = copy.deepcopy(x) self.assertEqual(list(x), list(y)) self.assertEqual(x.foo, y.foo) - self.assertTrue(x[0] is not y[0]) - self.assertTrue(x.foo is not y.foo) + self.assertIsNot(x[0], y[0]) + self.assertIsNot(x.foo, y.foo) def test_copy_tuple_subclass(self): class C(tuple): @@ -589,8 +626,8 @@ class TestCopy(unittest.TestCase): self.assertEqual(tuple(x), ([1, 2], 3)) y = copy.deepcopy(x) self.assertEqual(tuple(y), ([1, 2], 3)) - self.assertTrue(x is not y) - self.assertTrue(x[0] is not y[0]) + self.assertIsNot(x, y) + self.assertIsNot(x[0], y[0]) def test_getstate_exc(self): class EvilState(object): @@ -618,10 +655,10 @@ class TestCopy(unittest.TestCase): obj = C() x = weakref.ref(obj) y = _copy(x) - self.assertTrue(y is x) + self.assertIs(y, x) del obj y = _copy(x) - self.assertTrue(y is x) + self.assertIs(y, x) def test_copy_weakref(self): self._check_weakref(copy.copy) @@ -637,7 +674,7 @@ class TestCopy(unittest.TestCase): u[a] = b u[c] = d v = copy.copy(u) - self.assertFalse(v is u) + self.assertIsNot(v, u) self.assertEqual(v, u) self.assertEqual(v[a], b) self.assertEqual(v[c], d) @@ -667,8 +704,8 @@ class TestCopy(unittest.TestCase): v = copy.deepcopy(u) self.assertNotEqual(v, u) self.assertEqual(len(v), 2) - self.assertFalse(v[a] is b) - self.assertFalse(v[c] is d) + self.assertIsNot(v[a], b) + self.assertIsNot(v[c], d) self.assertEqual(v[a].i, b.i) self.assertEqual(v[c].i, d.i) del c @@ -687,12 +724,12 @@ class TestCopy(unittest.TestCase): self.assertNotEqual(v, u) self.assertEqual(len(v), 2) (x, y), (z, t) = sorted(v.items(), key=lambda pair: pair[0].i) - self.assertFalse(x is a) + self.assertIsNot(x, a) self.assertEqual(x.i, a.i) - self.assertTrue(y is b) - self.assertFalse(z is c) + self.assertIs(y, b) + self.assertIsNot(z, c) self.assertEqual(z.i, c.i) - self.assertTrue(t is d) + self.assertIs(t, d) del x, y, z, t del d self.assertEqual(len(v), 1) @@ -705,7 +742,7 @@ class TestCopy(unittest.TestCase): f.b = f.m g = copy.deepcopy(f) self.assertEqual(g.m, g.b) - self.assertTrue(g.b.__self__ is g) + self.assertIs(g.b.__self__, g) g.b() diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index ae17c2b694..56766682b3 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -18,16 +18,19 @@ class CProfileTest(ProfileTest): def test_bad_counter_during_dealloc(self): import _lsprof # Must use a file as StringIO doesn't trigger the bug. - with open(TESTFN, 'w') as file: - sys.stderr = file - try: - obj = _lsprof.Profiler(lambda: int) - obj.enable() - obj = _lsprof.Profiler(1) - obj.disable() - finally: - sys.stderr = sys.__stderr__ - unlink(TESTFN) + orig_stderr = sys.stderr + try: + with open(TESTFN, 'w') as file: + sys.stderr = file + try: + obj = _lsprof.Profiler(lambda: int) + obj.enable() + obj = _lsprof.Profiler(1) + obj.disable() + finally: + sys.stderr = orig_stderr + finally: + unlink(TESTFN) def test_main(): diff --git a/Lib/test/test_crashers.py b/Lib/test/test_crashers.py new file mode 100644 index 0000000000..336ccbeaff --- /dev/null +++ b/Lib/test/test_crashers.py @@ -0,0 +1,38 @@ +# Tests that the crashers in the Lib/test/crashers directory actually +# do crash the interpreter as expected +# +# If a crasher is fixed, it should be moved elsewhere in the test suite to +# ensure it continues to work correctly. + +import unittest +import glob +import os.path +import test.support +from test.script_helper import assert_python_failure + +CRASHER_DIR = os.path.join(os.path.dirname(__file__), "crashers") +CRASHER_FILES = os.path.join(CRASHER_DIR, "*.py") + +infinite_loops = ["infinite_loop_re.py", "nasty_eq_vs_dict.py"] + +class CrasherTest(unittest.TestCase): + + @unittest.skip("these tests are too fragile") + @test.support.cpython_only + def test_crashers_crash(self): + for fname in glob.glob(CRASHER_FILES): + if os.path.basename(fname) in infinite_loops: + continue + # Some "crashers" only trigger an exception rather than a + # segfault. Consider that an acceptable outcome. + if test.support.verbose: + print("Checking crasher:", fname) + assert_python_failure(fname) + + +def test_main(): + test.support.run_unittest(CrasherTest) + test.support.reap_children() + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_crypt.py b/Lib/test/test_crypt.py index 2adb28d7ce..dc107d8970 100755..100644 --- a/Lib/test/test_crypt.py +++ b/Lib/test/test_crypt.py @@ -10,6 +10,25 @@ class CryptTestCase(unittest.TestCase): if support.verbose: print('Test encryption: ', c) + def test_salt(self): + self.assertEqual(len(crypt._saltchars), 64) + for method in crypt.methods: + salt = crypt.mksalt(method) + self.assertEqual(len(salt), + method.salt_chars + (3 if method.ident else 0)) + + def test_saltedcrypt(self): + for method in crypt.methods: + pw = crypt.crypt('assword', method) + self.assertEqual(len(pw), method.total_size) + pw = crypt.crypt('assword', crypt.mksalt(method)) + self.assertEqual(len(pw), method.total_size) + + def test_methods(self): + # Gurantee that METHOD_CRYPT is the last method in crypt.methods. + self.assertTrue(len(crypt.methods) >= 1) + self.assertEqual(crypt.METHOD_CRYPT, crypt.methods[-1]) + def test_main(): support.run_unittest(CryptTestCase) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index b9ff34644b..c767e9388f 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -276,11 +276,11 @@ def main(stdscr): curses.resetty() def test_main(): - if not sys.stdout.isatty(): - raise unittest.SkipTest("sys.stdout is not a tty") + if not sys.__stdout__.isatty(): + raise unittest.SkipTest("sys.__stdout__ is not a tty") # testing setupterm() inside initscr/endwin # causes terminal breakage - curses.setupterm(fd=sys.stdout.fileno()) + curses.setupterm(fd=sys.__stdout__.fileno()) try: stdscr = curses.initscr() main(stdscr) diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 26d4c14623..02df7e3cab 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -71,8 +71,8 @@ class AnyDBMTestCase(unittest.TestCase): f.close() def test_anydbm_creation_n_file_exists_with_invalid_contents(self): - with open(_fname, "w") as w: - pass # create an empty file + # create an empty file + test.support.create_empty_file(_fname) f = dbm.open(_fname, 'n') self.addCleanup(f.close) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index e46cd91e5e..96bbafe739 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -1970,6 +1970,17 @@ class ContextAPItests(unittest.TestCase): self.assertRaises(TypeError, c.fma, 2, '3', 4) self.assertRaises(TypeError, c.fma, 2, 3, '4') + # Issue 12079 for Context.fma ... + self.assertRaises(TypeError, c.fma, + Decimal('Infinity'), Decimal(0), "not a decimal") + self.assertRaises(TypeError, c.fma, + Decimal(1), Decimal('snan'), 1.222) + # ... and for Decimal.fma. + self.assertRaises(TypeError, Decimal('Infinity').fma, + Decimal(0), "not a decimal") + self.assertRaises(TypeError, Decimal(1).fma, + Decimal('snan'), 1.222) + def test_is_finite(self): c = Context() d = c.is_finite(Decimal(10)) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 587d792f51..2a9f88083b 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1631,12 +1631,7 @@ order (MRO) for bases """ for attr, obj in env.items(): setattr(X, attr, obj) setattr(X, name, ErrDescr()) - try: - runner(X()) - except MyException: - pass - else: - self.fail("{0!r} didn't raise".format(name)) + self.assertRaises(MyException, runner, X()) def test_specials(self): # Testing special operators... @@ -2073,9 +2068,6 @@ order (MRO) for bases """ # Two essentially featureless objects, just inheriting stuff from # object. self.assertEqual(dir(NotImplemented), dir(Ellipsis)) - if support.check_impl_detail(): - # None differs in PyPy: it has a __nonzero__ - self.assertEqual(dir(None), dir(Ellipsis)) # Nasty test case for proxied objects class Wrapper(object): @@ -4261,6 +4253,27 @@ order (MRO) for bases """ foo = Foo() str(foo) + def test_slot_shadows_class_variable(self): + with self.assertRaises(ValueError) as cm: + class X: + __slots__ = ["foo"] + foo = None + m = str(cm.exception) + self.assertEqual("'foo' in __slots__ conflicts with class variable", m) + + def test_set_doc(self): + class X: + "elephant" + X.__doc__ = "banana" + self.assertEqual(X.__doc__, "banana") + with self.assertRaises(TypeError) as cm: + type(list).__dict__["__doc__"].__set__(list, "blah") + self.assertIn("can't set list.__doc__", str(cm.exception)) + with self.assertRaises(TypeError) as cm: + type(X).__dict__["__doc__"].__delete__(X) + self.assertIn("can't delete X.__doc__", str(cm.exception)) + self.assertEqual(X.__doc__, "banana") + class DictProxyTests(unittest.TestCase): def setUp(self): class C(object): @@ -4268,6 +4281,8 @@ class DictProxyTests(unittest.TestCase): pass self.C = C + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __local__') def test_iter_keys(self): # Testing dict-proxy keys... it = self.C.__dict__.keys() @@ -4277,6 +4292,8 @@ class DictProxyTests(unittest.TestCase): self.assertEqual(keys, ['__dict__', '__doc__', '__module__', '__weakref__', 'meth']) + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __local__') def test_iter_values(self): # Testing dict-proxy values... it = self.C.__dict__.values() @@ -4284,6 +4301,8 @@ class DictProxyTests(unittest.TestCase): values = list(it) self.assertEqual(len(values), 5) + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __local__') def test_iter_items(self): # Testing dict-proxy iteritems... it = self.C.__dict__.items() diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index 2db3d33226..f495e1816b 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -170,6 +170,7 @@ You can get the information from the list type: '__contains__', '__delattr__', '__delitem__', + '__dir__', '__doc__', '__eq__', '__format__', @@ -199,6 +200,8 @@ You can get the information from the list type: '__str__', '__subclasshook__', 'append', + 'clear', + 'copy', 'count', 'extend', 'index', diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 7a61493c10..4b0b02af26 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1,11 +1,35 @@ # Minimal tests for dis module from test.support import run_unittest, captured_stdout +import difflib import unittest import sys import dis import io +class _C: + def __init__(self, x): + self.x = x == 1 + +dis_c_instance_method = """\ + %-4d 0 LOAD_FAST 1 (x) + 3 LOAD_CONST 1 (1) + 6 COMPARE_OP 2 (==) + 9 LOAD_FAST 0 (self) + 12 STORE_ATTR 0 (x) + 15 LOAD_CONST 0 (None) + 18 RETURN_VALUE +""" % (_C.__init__.__code__.co_firstlineno + 1,) + +dis_c_instance_method_bytes = """\ + 0 LOAD_FAST 1 (1) + 3 LOAD_CONST 1 (1) + 6 COMPARE_OP 2 (==) + 9 LOAD_FAST 0 (0) + 12 STORE_ATTR 0 (0) + 15 LOAD_CONST 0 (0) + 18 RETURN_VALUE +""" def _f(a): print(a) @@ -23,6 +47,16 @@ dis_f = """\ _f.__code__.co_firstlineno + 2) +dis_f_co_code = """\ + 0 LOAD_GLOBAL 0 (0) + 3 LOAD_FAST 0 (0) + 6 CALL_FUNCTION 1 + 9 POP_TOP + 10 LOAD_CONST 1 (1) + 13 RETURN_VALUE +""" + + def bug708901(): for res in range(1, 10): @@ -138,18 +172,27 @@ dis_compound_stmt_str = """\ """ class DisTests(unittest.TestCase): - def do_disassembly_test(self, func, expected): + + def get_disassembly(self, func, lasti=-1, wrapper=True): s = io.StringIO() save_stdout = sys.stdout sys.stdout = s - dis.dis(func) - sys.stdout = save_stdout - got = s.getvalue() + try: + if wrapper: + dis.dis(func) + else: + dis.disassemble(func, lasti) + finally: + sys.stdout = save_stdout # Trim trailing blanks (if any). - lines = got.split('\n') - lines = [line.rstrip() for line in lines] - expected = expected.split("\n") - import difflib + return [line.rstrip() for line in s.getvalue().splitlines()] + + def get_disassemble_as_string(self, func, lasti=-1): + return '\n'.join(self.get_disassembly(func, lasti, False)) + + def do_disassembly_test(self, func, expected): + lines = self.get_disassembly(func) + expected = expected.splitlines() if expected != lines: self.fail( "events did not match expectation:\n" + @@ -157,7 +200,7 @@ class DisTests(unittest.TestCase): lines))) def test_opmap(self): - self.assertEqual(dis.opmap["STOP_CODE"], 0) + self.assertEqual(dis.opmap["NOP"], 9) self.assertIn(dis.opmap["LOAD_CONST"], dis.hasconst) self.assertIn(dis.opmap["STORE_NAME"], dis.hasname) @@ -211,6 +254,44 @@ class DisTests(unittest.TestCase): self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str) self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str) + def test_disassemble_bytes(self): + self.do_disassembly_test(_f.__code__.co_code, dis_f_co_code) + + def test_disassemble_method(self): + self.do_disassembly_test(_C(1).__init__, dis_c_instance_method) + + def test_disassemble_method_bytes(self): + method_bytecode = _C(1).__init__.__code__.co_code + self.do_disassembly_test(method_bytecode, dis_c_instance_method_bytes) + + def test_dis_none(self): + try: + del sys.last_traceback + except AttributeError: + pass + self.assertRaises(RuntimeError, dis.dis, None) + + def test_dis_object(self): + self.assertRaises(TypeError, dis.dis, object()) + + def test_dis_traceback(self): + try: + del sys.last_traceback + except AttributeError: + pass + + try: + 1/0 + except Exception as e: + tb = e.__traceback__ + sys.last_traceback = tb + + tb_dis = self.get_disassemble_as_string(tb.tb_frame.f_code, tb.tb_lasti) + self.do_disassembly_test(None, tb_dis) + + def test_dis_object(self): + self.assertRaises(TypeError, dis.dis, object()) + code_info_code_info = """\ Name: code_info Filename: (.*) @@ -363,6 +444,13 @@ class CodeInfoTests(unittest.TestCase): dis.show_code(x) self.assertRegex(output.getvalue(), expected+"\n") + def test_code_info_object(self): + self.assertRaises(TypeError, dis.code_info, object()) + + def test_pretty_flags_no_flags(self): + self.assertEqual(dis.pretty_flags(0), '0x0') + + def test_main(): run_unittest(DisTests, CodeInfoTests) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 13836bafa0..cd871790d5 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -5,6 +5,7 @@ Test script for doctest. from test import support import doctest import os +import sys # NOTE: There are some additional tests relating to interaction with @@ -373,7 +374,7 @@ We'll simulate a __file__ attr that ends in pyc: >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [<DocTest sample_func from ...:17 (1 example)>] + [<DocTest sample_func from ...:18 (1 example)>] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -1686,226 +1687,227 @@ Run the debugger on the docstring, and then restore sys.stdin. """ -def test_pdb_set_trace(): - """Using pdb.set_trace from a doctest. - - You can use pdb.set_trace from a doctest. To do so, you must - retrieve the set_trace function from the pdb module at the time - you use it. The doctest module changes sys.stdout so that it can - capture program output. It also temporarily replaces pdb.set_trace - with a version that restores stdout. This is necessary for you to - see debugger output. - - >>> doc = ''' - ... >>> x = 42 - ... >>> raise Exception('clé') - ... Traceback (most recent call last): - ... Exception: clé - ... >>> import pdb; pdb.set_trace() - ... ''' - >>> parser = doctest.DocTestParser() - >>> test = parser.get_doctest(doc, {}, "foo-bar@baz", "foo-bar@baz.py", 0) - >>> runner = doctest.DocTestRunner(verbose=False) - - To demonstrate this, we'll create a fake standard input that - captures our debugger input: - - >>> import tempfile - >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ - ... 'print(x)', # print data defined by the example - ... 'continue', # stop debugging - ... '']) - - >>> try: runner.run(test) - ... finally: sys.stdin = real_stdin - --Return-- - > <doctest foo-bar@baz[2]>(1)<module>()->None - -> import pdb; pdb.set_trace() - (Pdb) print(x) - 42 - (Pdb) continue - TestResults(failed=0, attempted=3) - - You can also put pdb.set_trace in a function called from a test: - - >>> def calls_set_trace(): - ... y=2 - ... import pdb; pdb.set_trace() - - >>> doc = ''' - ... >>> x=1 - ... >>> calls_set_trace() - ... ''' - >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) - >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ - ... 'print(y)', # print data defined in the function - ... 'up', # out of function - ... 'print(x)', # print data defined by the example - ... 'continue', # stop debugging - ... '']) - - >>> try: - ... runner.run(test) - ... finally: - ... sys.stdin = real_stdin - --Return-- - > <doctest test.test_doctest.test_pdb_set_trace[8]>(3)calls_set_trace()->None - -> import pdb; pdb.set_trace() - (Pdb) print(y) - 2 - (Pdb) up - > <doctest foo-bar@baz[1]>(1)<module>() - -> calls_set_trace() - (Pdb) print(x) - 1 - (Pdb) continue - TestResults(failed=0, attempted=2) - - During interactive debugging, source code is shown, even for - doctest examples: - - >>> doc = ''' - ... >>> def f(x): - ... ... g(x*2) - ... >>> def g(x): - ... ... print(x+3) - ... ... import pdb; pdb.set_trace() - ... >>> f(3) - ... ''' - >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) - >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ - ... 'list', # list source from example 2 - ... 'next', # return from g() - ... 'list', # list source from example 1 - ... 'next', # return from f() - ... 'list', # list source from example 3 - ... 'continue', # stop debugging - ... '']) - >>> try: runner.run(test) - ... finally: sys.stdin = real_stdin - ... # doctest: +NORMALIZE_WHITESPACE - --Return-- - > <doctest foo-bar@baz[1]>(3)g()->None - -> import pdb; pdb.set_trace() - (Pdb) list - 1 def g(x): - 2 print(x+3) - 3 -> import pdb; pdb.set_trace() - [EOF] - (Pdb) next - --Return-- - > <doctest foo-bar@baz[0]>(2)f()->None - -> g(x*2) - (Pdb) list - 1 def f(x): - 2 -> g(x*2) - [EOF] - (Pdb) next - --Return-- - > <doctest foo-bar@baz[2]>(1)<module>()->None - -> f(3) - (Pdb) list - 1 -> f(3) - [EOF] - (Pdb) continue - ********************************************************************** - File "foo-bar@baz.py", line 7, in foo-bar@baz - Failed example: - f(3) - Expected nothing - Got: - 9 - TestResults(failed=1, attempted=3) - """ - -def test_pdb_set_trace_nested(): - """This illustrates more-demanding use of set_trace with nested functions. - - >>> class C(object): - ... def calls_set_trace(self): - ... y = 1 - ... import pdb; pdb.set_trace() - ... self.f1() - ... y = 2 - ... def f1(self): - ... x = 1 - ... self.f2() - ... x = 2 - ... def f2(self): - ... z = 1 - ... z = 2 - - >>> calls_set_trace = C().calls_set_trace - - >>> doc = ''' - ... >>> a = 1 - ... >>> calls_set_trace() - ... ''' - >>> parser = doctest.DocTestParser() - >>> runner = doctest.DocTestRunner(verbose=False) - >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) - >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ - ... 'print(y)', # print data defined in the function - ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', - ... 'up', 'print(x)', - ... 'up', 'print(y)', - ... 'up', 'print(foo)', - ... 'continue', # stop debugging - ... '']) - - >>> try: - ... runner.run(test) - ... finally: - ... sys.stdin = real_stdin - ... # doctest: +REPORT_NDIFF - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace() - -> self.f1() - (Pdb) print(y) - 1 - (Pdb) step - --Call-- - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(7)f1() - -> def f1(self): - (Pdb) step - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(8)f1() - -> x = 1 - (Pdb) step - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1() - -> self.f2() - (Pdb) step - --Call-- - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(11)f2() - -> def f2(self): - (Pdb) step - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(12)f2() - -> z = 1 - (Pdb) step - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(13)f2() - -> z = 2 - (Pdb) print(z) - 1 - (Pdb) up - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1() - -> self.f2() - (Pdb) print(x) - 1 - (Pdb) up - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace() - -> self.f1() - (Pdb) print(y) - 1 - (Pdb) up - > <doctest foo-bar@baz[1]>(1)<module>() - -> calls_set_trace() - (Pdb) print(foo) - *** NameError: name 'foo' is not defined - (Pdb) continue - TestResults(failed=0, attempted=2) -""" +if not hasattr(sys, 'gettrace') or not sys.gettrace(): + def test_pdb_set_trace(): + """Using pdb.set_trace from a doctest. + + You can use pdb.set_trace from a doctest. To do so, you must + retrieve the set_trace function from the pdb module at the time + you use it. The doctest module changes sys.stdout so that it can + capture program output. It also temporarily replaces pdb.set_trace + with a version that restores stdout. This is necessary for you to + see debugger output. + + >>> doc = ''' + ... >>> x = 42 + ... >>> raise Exception('clé') + ... Traceback (most recent call last): + ... Exception: clé + ... >>> import pdb; pdb.set_trace() + ... ''' + >>> parser = doctest.DocTestParser() + >>> test = parser.get_doctest(doc, {}, "foo-bar@baz", "foo-bar@baz.py", 0) + >>> runner = doctest.DocTestRunner(verbose=False) + + To demonstrate this, we'll create a fake standard input that + captures our debugger input: + + >>> import tempfile + >>> real_stdin = sys.stdin + >>> sys.stdin = _FakeInput([ + ... 'print(x)', # print data defined by the example + ... 'continue', # stop debugging + ... '']) + + >>> try: runner.run(test) + ... finally: sys.stdin = real_stdin + --Return-- + > <doctest foo-bar@baz[2]>(1)<module>()->None + -> import pdb; pdb.set_trace() + (Pdb) print(x) + 42 + (Pdb) continue + TestResults(failed=0, attempted=3) + + You can also put pdb.set_trace in a function called from a test: + + >>> def calls_set_trace(): + ... y=2 + ... import pdb; pdb.set_trace() + + >>> doc = ''' + ... >>> x=1 + ... >>> calls_set_trace() + ... ''' + >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> real_stdin = sys.stdin + >>> sys.stdin = _FakeInput([ + ... 'print(y)', # print data defined in the function + ... 'up', # out of function + ... 'print(x)', # print data defined by the example + ... 'continue', # stop debugging + ... '']) + + >>> try: + ... runner.run(test) + ... finally: + ... sys.stdin = real_stdin + --Return-- + > <doctest test.test_doctest.test_pdb_set_trace[8]>(3)calls_set_trace()->None + -> import pdb; pdb.set_trace() + (Pdb) print(y) + 2 + (Pdb) up + > <doctest foo-bar@baz[1]>(1)<module>() + -> calls_set_trace() + (Pdb) print(x) + 1 + (Pdb) continue + TestResults(failed=0, attempted=2) + + During interactive debugging, source code is shown, even for + doctest examples: + + >>> doc = ''' + ... >>> def f(x): + ... ... g(x*2) + ... >>> def g(x): + ... ... print(x+3) + ... ... import pdb; pdb.set_trace() + ... >>> f(3) + ... ''' + >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> real_stdin = sys.stdin + >>> sys.stdin = _FakeInput([ + ... 'list', # list source from example 2 + ... 'next', # return from g() + ... 'list', # list source from example 1 + ... 'next', # return from f() + ... 'list', # list source from example 3 + ... 'continue', # stop debugging + ... '']) + >>> try: runner.run(test) + ... finally: sys.stdin = real_stdin + ... # doctest: +NORMALIZE_WHITESPACE + --Return-- + > <doctest foo-bar@baz[1]>(3)g()->None + -> import pdb; pdb.set_trace() + (Pdb) list + 1 def g(x): + 2 print(x+3) + 3 -> import pdb; pdb.set_trace() + [EOF] + (Pdb) next + --Return-- + > <doctest foo-bar@baz[0]>(2)f()->None + -> g(x*2) + (Pdb) list + 1 def f(x): + 2 -> g(x*2) + [EOF] + (Pdb) next + --Return-- + > <doctest foo-bar@baz[2]>(1)<module>()->None + -> f(3) + (Pdb) list + 1 -> f(3) + [EOF] + (Pdb) continue + ********************************************************************** + File "foo-bar@baz.py", line 7, in foo-bar@baz + Failed example: + f(3) + Expected nothing + Got: + 9 + TestResults(failed=1, attempted=3) + """ + + def test_pdb_set_trace_nested(): + """This illustrates more-demanding use of set_trace with nested functions. + + >>> class C(object): + ... def calls_set_trace(self): + ... y = 1 + ... import pdb; pdb.set_trace() + ... self.f1() + ... y = 2 + ... def f1(self): + ... x = 1 + ... self.f2() + ... x = 2 + ... def f2(self): + ... z = 1 + ... z = 2 + + >>> calls_set_trace = C().calls_set_trace + + >>> doc = ''' + ... >>> a = 1 + ... >>> calls_set_trace() + ... ''' + >>> parser = doctest.DocTestParser() + >>> runner = doctest.DocTestRunner(verbose=False) + >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> real_stdin = sys.stdin + >>> sys.stdin = _FakeInput([ + ... 'print(y)', # print data defined in the function + ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', + ... 'up', 'print(x)', + ... 'up', 'print(y)', + ... 'up', 'print(foo)', + ... 'continue', # stop debugging + ... '']) + + >>> try: + ... runner.run(test) + ... finally: + ... sys.stdin = real_stdin + ... # doctest: +REPORT_NDIFF + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace() + -> self.f1() + (Pdb) print(y) + 1 + (Pdb) step + --Call-- + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(7)f1() + -> def f1(self): + (Pdb) step + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(8)f1() + -> x = 1 + (Pdb) step + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1() + -> self.f2() + (Pdb) step + --Call-- + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(11)f2() + -> def f2(self): + (Pdb) step + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(12)f2() + -> z = 1 + (Pdb) step + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(13)f2() + -> z = 2 + (Pdb) print(z) + 1 + (Pdb) up + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1() + -> self.f2() + (Pdb) print(x) + 1 + (Pdb) up + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace() + -> self.f1() + (Pdb) print(y) + 1 + (Pdb) up + > <doctest foo-bar@baz[1]>(1)<module>() + -> calls_set_trace() + (Pdb) print(foo) + *** NameError: name 'foo' is not defined + (Pdb) continue + TestResults(failed=0, attempted=2) + """ def test_DocTestSuite(): """DocTestSuite creates a unittest test suite from a doctest. diff --git a/Lib/test/test_dummy_thread.py b/Lib/test/test_dummy_thread.py index c61078d620..2fafe1df85 100644 --- a/Lib/test/test_dummy_thread.py +++ b/Lib/test/test_dummy_thread.py @@ -35,8 +35,8 @@ class LockTests(unittest.TestCase): "Lock object did not release properly.") def test_improper_release(self): - #Make sure release of an unlocked thread raises _thread.error - self.assertRaises(_thread.error, self.lock.release) + #Make sure release of an unlocked thread raises RuntimeError + self.assertRaises(RuntimeError, self.lock.release) def test_cond_acquire_success(self): #Make sure the conditional acquiring of the lock works. diff --git a/Lib/test/test_email.py b/Lib/test/test_email.py deleted file mode 100644 index 5eebba5590..0000000000 --- a/Lib/test/test_email.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (C) 2001-2007 Python Software Foundation -# email package unit tests - -# The specific tests now live in Lib/email/test -from email.test.test_email import suite -from email.test.test_email_codecs import suite as codecs_suite -from test import support - -def test_main(): - support.run_unittest(suite()) - support.run_unittest(codecs_suite()) - -if __name__ == '__main__': - test_main() diff --git a/Lib/test/test_email/__init__.py b/Lib/test/test_email/__init__.py new file mode 100644 index 0000000000..04fdf89973 --- /dev/null +++ b/Lib/test/test_email/__init__.py @@ -0,0 +1,45 @@ +import os +import sys +import unittest +import test.support +import email +from test.test_email import __file__ as landmark + +# used by regrtest and __main__. +def test_main(): + here = os.path.dirname(__file__) + # Unittest mucks with the path, so we have to save and restore + # it to keep regrtest happy. + savepath = sys.path[:] + test.support._run_suite(unittest.defaultTestLoader.discover(here)) + sys.path[:] = savepath + + +# helper code used by a number of test modules. + +def openfile(filename, *args, **kws): + path = os.path.join(os.path.dirname(landmark), 'data', filename) + return open(path, *args, **kws) + + +# Base test class +class TestEmailBase(unittest.TestCase): + + maxDiff = None + + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self.addTypeEqualityFunc(bytes, self.assertBytesEqual) + + ndiffAssertEqual = unittest.TestCase.assertEqual + + def _msgobj(self, filename): + with openfile(filename) as fp: + return email.message_from_file(fp) + + def _bytes_repr(self, b): + return [repr(x) for x in b.splitlines(True)] + + def assertBytesEqual(self, first, second, msg): + """Our byte strings are really encoded strings; improve diff output""" + self.assertEqual(self._bytes_repr(first), self._bytes_repr(second)) diff --git a/Lib/test/test_email/__main__.py b/Lib/test/test_email/__main__.py new file mode 100644 index 0000000000..98af9ecbad --- /dev/null +++ b/Lib/test/test_email/__main__.py @@ -0,0 +1,3 @@ +from test.test_email import test_main + +test_main() diff --git a/Lib/email/test/data/PyBanner048.gif b/Lib/test/test_email/data/PyBanner048.gif Binary files differindex 1a5c87f647..1a5c87f647 100644 --- a/Lib/email/test/data/PyBanner048.gif +++ b/Lib/test/test_email/data/PyBanner048.gif diff --git a/Lib/email/test/data/audiotest.au b/Lib/test/test_email/data/audiotest.au Binary files differindex f76b0501b8..f76b0501b8 100644 --- a/Lib/email/test/data/audiotest.au +++ b/Lib/test/test_email/data/audiotest.au diff --git a/Lib/email/test/data/msg_01.txt b/Lib/test/test_email/data/msg_01.txt index 7e33bcf96a..7e33bcf96a 100644 --- a/Lib/email/test/data/msg_01.txt +++ b/Lib/test/test_email/data/msg_01.txt diff --git a/Lib/email/test/data/msg_02.txt b/Lib/test/test_email/data/msg_02.txt index 43f248038a..43f248038a 100644 --- a/Lib/email/test/data/msg_02.txt +++ b/Lib/test/test_email/data/msg_02.txt diff --git a/Lib/email/test/data/msg_03.txt b/Lib/test/test_email/data/msg_03.txt index c748ebf117..c748ebf117 100644 --- a/Lib/email/test/data/msg_03.txt +++ b/Lib/test/test_email/data/msg_03.txt diff --git a/Lib/email/test/data/msg_04.txt b/Lib/test/test_email/data/msg_04.txt index 1f633c4496..1f633c4496 100644 --- a/Lib/email/test/data/msg_04.txt +++ b/Lib/test/test_email/data/msg_04.txt diff --git a/Lib/email/test/data/msg_05.txt b/Lib/test/test_email/data/msg_05.txt index 87d5e9cbf8..87d5e9cbf8 100644 --- a/Lib/email/test/data/msg_05.txt +++ b/Lib/test/test_email/data/msg_05.txt diff --git a/Lib/email/test/data/msg_06.txt b/Lib/test/test_email/data/msg_06.txt index 69f3a47ff4..69f3a47ff4 100644 --- a/Lib/email/test/data/msg_06.txt +++ b/Lib/test/test_email/data/msg_06.txt diff --git a/Lib/email/test/data/msg_07.txt b/Lib/test/test_email/data/msg_07.txt index 721f3a0d31..721f3a0d31 100644 --- a/Lib/email/test/data/msg_07.txt +++ b/Lib/test/test_email/data/msg_07.txt diff --git a/Lib/email/test/data/msg_08.txt b/Lib/test/test_email/data/msg_08.txt index b5630836c5..b5630836c5 100644 --- a/Lib/email/test/data/msg_08.txt +++ b/Lib/test/test_email/data/msg_08.txt diff --git a/Lib/email/test/data/msg_09.txt b/Lib/test/test_email/data/msg_09.txt index 575c4c205a..575c4c205a 100644 --- a/Lib/email/test/data/msg_09.txt +++ b/Lib/test/test_email/data/msg_09.txt diff --git a/Lib/email/test/data/msg_10.txt b/Lib/test/test_email/data/msg_10.txt index 07903960f9..07903960f9 100644 --- a/Lib/email/test/data/msg_10.txt +++ b/Lib/test/test_email/data/msg_10.txt diff --git a/Lib/email/test/data/msg_11.txt b/Lib/test/test_email/data/msg_11.txt index 8f7f1991cb..8f7f1991cb 100644 --- a/Lib/email/test/data/msg_11.txt +++ b/Lib/test/test_email/data/msg_11.txt diff --git a/Lib/email/test/data/msg_12.txt b/Lib/test/test_email/data/msg_12.txt index 4bec8d9444..4bec8d9444 100644 --- a/Lib/email/test/data/msg_12.txt +++ b/Lib/test/test_email/data/msg_12.txt diff --git a/Lib/email/test/data/msg_12a.txt b/Lib/test/test_email/data/msg_12a.txt index e94224ecfe..e94224ecfe 100644 --- a/Lib/email/test/data/msg_12a.txt +++ b/Lib/test/test_email/data/msg_12a.txt diff --git a/Lib/email/test/data/msg_13.txt b/Lib/test/test_email/data/msg_13.txt index 8e6d52d5be..8e6d52d5be 100644 --- a/Lib/email/test/data/msg_13.txt +++ b/Lib/test/test_email/data/msg_13.txt diff --git a/Lib/email/test/data/msg_14.txt b/Lib/test/test_email/data/msg_14.txt index 5d98d2fd14..5d98d2fd14 100644 --- a/Lib/email/test/data/msg_14.txt +++ b/Lib/test/test_email/data/msg_14.txt diff --git a/Lib/email/test/data/msg_15.txt b/Lib/test/test_email/data/msg_15.txt index 0025624e75..0025624e75 100644 --- a/Lib/email/test/data/msg_15.txt +++ b/Lib/test/test_email/data/msg_15.txt diff --git a/Lib/email/test/data/msg_16.txt b/Lib/test/test_email/data/msg_16.txt index 56167e9f5b..56167e9f5b 100644 --- a/Lib/email/test/data/msg_16.txt +++ b/Lib/test/test_email/data/msg_16.txt diff --git a/Lib/email/test/data/msg_17.txt b/Lib/test/test_email/data/msg_17.txt index 8d86e4180d..8d86e4180d 100644 --- a/Lib/email/test/data/msg_17.txt +++ b/Lib/test/test_email/data/msg_17.txt diff --git a/Lib/email/test/data/msg_18.txt b/Lib/test/test_email/data/msg_18.txt index f9f4904d36..f9f4904d36 100644 --- a/Lib/email/test/data/msg_18.txt +++ b/Lib/test/test_email/data/msg_18.txt diff --git a/Lib/email/test/data/msg_19.txt b/Lib/test/test_email/data/msg_19.txt index 49bf7fccdd..49bf7fccdd 100644 --- a/Lib/email/test/data/msg_19.txt +++ b/Lib/test/test_email/data/msg_19.txt diff --git a/Lib/email/test/data/msg_20.txt b/Lib/test/test_email/data/msg_20.txt index 1a6a88783e..1a6a88783e 100644 --- a/Lib/email/test/data/msg_20.txt +++ b/Lib/test/test_email/data/msg_20.txt diff --git a/Lib/email/test/data/msg_21.txt b/Lib/test/test_email/data/msg_21.txt index 23590b255d..23590b255d 100644 --- a/Lib/email/test/data/msg_21.txt +++ b/Lib/test/test_email/data/msg_21.txt diff --git a/Lib/email/test/data/msg_22.txt b/Lib/test/test_email/data/msg_22.txt index af9de5fa27..af9de5fa27 100644 --- a/Lib/email/test/data/msg_22.txt +++ b/Lib/test/test_email/data/msg_22.txt diff --git a/Lib/email/test/data/msg_23.txt b/Lib/test/test_email/data/msg_23.txt index bb2e8ec36b..bb2e8ec36b 100644 --- a/Lib/email/test/data/msg_23.txt +++ b/Lib/test/test_email/data/msg_23.txt diff --git a/Lib/email/test/data/msg_24.txt b/Lib/test/test_email/data/msg_24.txt index 4e52339e86..4e52339e86 100644 --- a/Lib/email/test/data/msg_24.txt +++ b/Lib/test/test_email/data/msg_24.txt diff --git a/Lib/email/test/data/msg_25.txt b/Lib/test/test_email/data/msg_25.txt index 9e35275fe0..9e35275fe0 100644 --- a/Lib/email/test/data/msg_25.txt +++ b/Lib/test/test_email/data/msg_25.txt diff --git a/Lib/email/test/data/msg_26.txt b/Lib/test/test_email/data/msg_26.txt index 58efaa9c9a..58efaa9c9a 100644 --- a/Lib/email/test/data/msg_26.txt +++ b/Lib/test/test_email/data/msg_26.txt diff --git a/Lib/email/test/data/msg_27.txt b/Lib/test/test_email/data/msg_27.txt index d0191769d7..d0191769d7 100644 --- a/Lib/email/test/data/msg_27.txt +++ b/Lib/test/test_email/data/msg_27.txt diff --git a/Lib/email/test/data/msg_28.txt b/Lib/test/test_email/data/msg_28.txt index 1e4824cabb..1e4824cabb 100644 --- a/Lib/email/test/data/msg_28.txt +++ b/Lib/test/test_email/data/msg_28.txt diff --git a/Lib/email/test/data/msg_29.txt b/Lib/test/test_email/data/msg_29.txt index 1fab561617..1fab561617 100644 --- a/Lib/email/test/data/msg_29.txt +++ b/Lib/test/test_email/data/msg_29.txt diff --git a/Lib/email/test/data/msg_30.txt b/Lib/test/test_email/data/msg_30.txt index 4334bb6e60..4334bb6e60 100644 --- a/Lib/email/test/data/msg_30.txt +++ b/Lib/test/test_email/data/msg_30.txt diff --git a/Lib/email/test/data/msg_31.txt b/Lib/test/test_email/data/msg_31.txt index 1e58e56cf5..1e58e56cf5 100644 --- a/Lib/email/test/data/msg_31.txt +++ b/Lib/test/test_email/data/msg_31.txt diff --git a/Lib/email/test/data/msg_32.txt b/Lib/test/test_email/data/msg_32.txt index 07ec5af9a3..07ec5af9a3 100644 --- a/Lib/email/test/data/msg_32.txt +++ b/Lib/test/test_email/data/msg_32.txt diff --git a/Lib/email/test/data/msg_33.txt b/Lib/test/test_email/data/msg_33.txt index 042787a4fd..042787a4fd 100644 --- a/Lib/email/test/data/msg_33.txt +++ b/Lib/test/test_email/data/msg_33.txt diff --git a/Lib/email/test/data/msg_34.txt b/Lib/test/test_email/data/msg_34.txt index 055dfea531..055dfea531 100644 --- a/Lib/email/test/data/msg_34.txt +++ b/Lib/test/test_email/data/msg_34.txt diff --git a/Lib/email/test/data/msg_35.txt b/Lib/test/test_email/data/msg_35.txt index be7d5a2f7b..be7d5a2f7b 100644 --- a/Lib/email/test/data/msg_35.txt +++ b/Lib/test/test_email/data/msg_35.txt diff --git a/Lib/email/test/data/msg_36.txt b/Lib/test/test_email/data/msg_36.txt index 5632c3062c..5632c3062c 100644 --- a/Lib/email/test/data/msg_36.txt +++ b/Lib/test/test_email/data/msg_36.txt diff --git a/Lib/email/test/data/msg_37.txt b/Lib/test/test_email/data/msg_37.txt index 038d34a1a4..038d34a1a4 100644 --- a/Lib/email/test/data/msg_37.txt +++ b/Lib/test/test_email/data/msg_37.txt diff --git a/Lib/email/test/data/msg_38.txt b/Lib/test/test_email/data/msg_38.txt index 006df81cb5..006df81cb5 100644 --- a/Lib/email/test/data/msg_38.txt +++ b/Lib/test/test_email/data/msg_38.txt diff --git a/Lib/email/test/data/msg_39.txt b/Lib/test/test_email/data/msg_39.txt index 124b269192..124b269192 100644 --- a/Lib/email/test/data/msg_39.txt +++ b/Lib/test/test_email/data/msg_39.txt diff --git a/Lib/email/test/data/msg_40.txt b/Lib/test/test_email/data/msg_40.txt index 1435fa1e1a..1435fa1e1a 100644 --- a/Lib/email/test/data/msg_40.txt +++ b/Lib/test/test_email/data/msg_40.txt diff --git a/Lib/email/test/data/msg_41.txt b/Lib/test/test_email/data/msg_41.txt index 76cdd1cb7f..76cdd1cb7f 100644 --- a/Lib/email/test/data/msg_41.txt +++ b/Lib/test/test_email/data/msg_41.txt diff --git a/Lib/email/test/data/msg_42.txt b/Lib/test/test_email/data/msg_42.txt index a75f8f4a02..a75f8f4a02 100644 --- a/Lib/email/test/data/msg_42.txt +++ b/Lib/test/test_email/data/msg_42.txt diff --git a/Lib/email/test/data/msg_43.txt b/Lib/test/test_email/data/msg_43.txt index 797d12c568..797d12c568 100644 --- a/Lib/email/test/data/msg_43.txt +++ b/Lib/test/test_email/data/msg_43.txt diff --git a/Lib/email/test/data/msg_44.txt b/Lib/test/test_email/data/msg_44.txt index 15a225287b..15a225287b 100644 --- a/Lib/email/test/data/msg_44.txt +++ b/Lib/test/test_email/data/msg_44.txt diff --git a/Lib/email/test/data/msg_45.txt b/Lib/test/test_email/data/msg_45.txt index 58fde956e7..58fde956e7 100644 --- a/Lib/email/test/data/msg_45.txt +++ b/Lib/test/test_email/data/msg_45.txt diff --git a/Lib/email/test/data/msg_46.txt b/Lib/test/test_email/data/msg_46.txt index 1e22c4f600..1e22c4f600 100644 --- a/Lib/email/test/data/msg_46.txt +++ b/Lib/test/test_email/data/msg_46.txt diff --git a/Lib/email/test/test_email_codecs.py b/Lib/test/test_email/test_asian_codecs.py index ca85f5731e..a4dd9a9235 100644 --- a/Lib/email/test/test_email_codecs.py +++ b/Lib/test/test_email/test_asian_codecs.py @@ -5,7 +5,7 @@ import unittest from test.support import run_unittest -from email.test.test_email import TestEmailBase +from test.test_email.test_email import TestEmailBase from email.charset import Charset from email.header import Header, decode_header from email.message import Message @@ -78,16 +78,5 @@ Hello World! =?iso-2022-jp?b?GyRCJU8lbSE8JW8hPCVrJUkhKhsoQg==?= -def suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(TestEmailAsianCodecs)) - return suite - - -def test_main(): - run_unittest(TestEmailAsianCodecs) - - - if __name__ == '__main__': - unittest.main(defaultTest='suite') + unittest.main() diff --git a/Lib/email/test/test_email.py b/Lib/test/test_email/test_email.py index 102e15b9ff..121c939561 100644 --- a/Lib/email/test/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -36,40 +36,14 @@ from email import iterators from email import base64mime from email import quoprimime -from test.support import findfile, run_unittest, unlink -from email.test import __file__ as landmark - +from test.support import run_unittest, unlink +from test.test_email import openfile, TestEmailBase NL = '\n' EMPTYSTRING = '' SPACE = ' ' - -def openfile(filename, *args, **kws): - path = os.path.join(os.path.dirname(landmark), 'data', filename) - return open(path, *args, **kws) - - - -# Base test class -class TestEmailBase(unittest.TestCase): - def ndiffAssertEqual(self, first, second): - """Like assertEqual except use ndiff for readable output.""" - if first != second: - sfirst = str(first) - ssecond = str(second) - rfirst = [repr(line) for line in sfirst.splitlines()] - rsecond = [repr(line) for line in ssecond.splitlines()] - diff = difflib.ndiff(rfirst, rsecond) - raise self.failureException(NL + NL.join(diff)) - - def _msgobj(self, filename): - with openfile(findfile(filename)) as fp: - return email.message_from_file(fp) - - - # Test various aspects of the Message class's API class TestMessageAPI(TestEmailBase): def test_get_all(self): @@ -194,7 +168,7 @@ class TestMessageAPI(TestEmailBase): def test_message_rfc822_only(self): # Issue 7970: message/rfc822 not in multipart parsed by # HeaderParser caused an exception when flattened. - with openfile(findfile('msg_46.txt')) as fp: + with openfile('msg_46.txt') as fp: msgdata = fp.read() parser = HeaderParser() msg = parser.parsestr(msgdata) @@ -203,6 +177,17 @@ class TestMessageAPI(TestEmailBase): gen.flatten(msg, False) self.assertEqual(out.getvalue(), msgdata) + def test_byte_message_rfc822_only(self): + # Make sure new bytes header parser also passes this. + with openfile('msg_46.txt', 'rb') as fp: + msgdata = fp.read() + parser = email.parser.BytesHeaderParser() + msg = parser.parsebytes(msgdata) + out = BytesIO() + gen = email.generator.BytesGenerator(out) + gen.flatten(msg) + self.assertEqual(out.getvalue(), msgdata) + def test_get_decoded_payload(self): eq = self.assertEqual msg = self._msgobj('msg_10.txt') @@ -1264,13 +1249,7 @@ Blah blah blah # Test the basic MIMEAudio class class TestMIMEAudio(unittest.TestCase): def setUp(self): - # Make sure we pick up the audiotest.au that lives in email/test/data. - # In Python, there's an audiotest.au living in Lib/test but that isn't - # included in some binary distros that don't include the test - # package. The trailing empty string on the .join() is significant - # since findfile() will do a dirname(). - datadir = os.path.join(os.path.dirname(landmark), 'data', '') - with open(findfile('audiotest.au', datadir), 'rb') as fp: + with openfile('audiotest.au', 'rb') as fp: self._audiodata = fp.read() self._au = MIMEAudio(self._audiodata) @@ -1797,7 +1776,12 @@ YXNkZg== # Test some badly formatted messages -class TestNonConformant(TestEmailBase): +class TestNonConformantBase: + + def _msgobj(self, filename): + with openfile(filename) as fp: + return email.message_from_file(fp, policy=self.policy) + def test_parse_missing_minor_type(self): eq = self.assertEqual msg = self._msgobj('msg_14.txt') @@ -1811,19 +1795,65 @@ class TestNonConformant(TestEmailBase): # XXX We can probably eventually do better inner = msg.get_payload(0) unless(hasattr(inner, 'defects')) - self.assertEqual(len(inner.defects), 1) - unless(isinstance(inner.defects[0], + self.assertEqual(len(self.get_defects(inner)), 1) + unless(isinstance(self.get_defects(inner)[0], errors.StartBoundaryNotFoundDefect)) def test_multipart_no_boundary(self): unless = self.assertTrue msg = self._msgobj('msg_25.txt') unless(isinstance(msg.get_payload(), str)) - self.assertEqual(len(msg.defects), 2) - unless(isinstance(msg.defects[0], errors.NoBoundaryInMultipartDefect)) - unless(isinstance(msg.defects[1], + self.assertEqual(len(self.get_defects(msg)), 2) + unless(isinstance(self.get_defects(msg)[0], + errors.NoBoundaryInMultipartDefect)) + unless(isinstance(self.get_defects(msg)[1], errors.MultipartInvariantViolationDefect)) + multipart_msg = textwrap.dedent("""\ + Date: Wed, 14 Nov 2007 12:56:23 GMT + From: foo@bar.invalid + To: foo@bar.invalid + Subject: Content-Transfer-Encoding: base64 and multipart + MIME-Version: 1.0 + Content-Type: multipart/mixed; + boundary="===============3344438784458119861=="{} + + --===============3344438784458119861== + Content-Type: text/plain + + Test message + + --===============3344438784458119861== + Content-Type: application/octet-stream + Content-Transfer-Encoding: base64 + + YWJj + + --===============3344438784458119861==-- + """) + + def test_multipart_invalid_cte(self): + msg = email.message_from_string( + self.multipart_msg.format("\nContent-Transfer-Encoding: base64"), + policy = self.policy) + self.assertEqual(len(self.get_defects(msg)), 1) + self.assertIsInstance(self.get_defects(msg)[0], + errors.InvalidMultipartContentTransferEncodingDefect) + + def test_multipart_no_cte_no_defect(self): + msg = email.message_from_string( + self.multipart_msg.format(''), + policy = self.policy) + self.assertEqual(len(self.get_defects(msg)), 0) + + def test_multipart_valid_cte_no_defect(self): + for cte in ('7bit', '8bit', 'BINary'): + msg = email.message_from_string( + self.multipart_msg.format( + "\nContent-Transfer-Encoding: {}".format(cte)), + policy = self.policy) + self.assertEqual(len(self.get_defects(msg)), 0) + def test_invalid_content_type(self): eq = self.assertEqual neq = self.ndiffAssertEqual @@ -1877,9 +1907,10 @@ counter to RFC 2822, there's no separating newline here unless = self.assertTrue msg = self._msgobj('msg_41.txt') unless(hasattr(msg, 'defects')) - self.assertEqual(len(msg.defects), 2) - unless(isinstance(msg.defects[0], errors.NoBoundaryInMultipartDefect)) - unless(isinstance(msg.defects[1], + self.assertEqual(len(self.get_defects(msg)), 2) + unless(isinstance(self.get_defects(msg)[0], + errors.NoBoundaryInMultipartDefect)) + unless(isinstance(self.get_defects(msg)[1], errors.MultipartInvariantViolationDefect)) def test_missing_start_boundary(self): @@ -1893,21 +1924,71 @@ counter to RFC 2822, there's no separating newline here # # [*] This message is missing its start boundary bad = outer.get_payload(1).get_payload(0) - self.assertEqual(len(bad.defects), 1) - self.assertTrue(isinstance(bad.defects[0], + self.assertEqual(len(self.get_defects(bad)), 1) + self.assertTrue(isinstance(self.get_defects(bad)[0], errors.StartBoundaryNotFoundDefect)) def test_first_line_is_continuation_header(self): eq = self.assertEqual m = ' Line 1\nLine 2\nLine 3' - msg = email.message_from_string(m) + msg = email.message_from_string(m, policy=self.policy) eq(msg.keys(), []) eq(msg.get_payload(), 'Line 2\nLine 3') - eq(len(msg.defects), 1) - self.assertTrue(isinstance(msg.defects[0], + eq(len(self.get_defects(msg)), 1) + self.assertTrue(isinstance(self.get_defects(msg)[0], errors.FirstHeaderLineIsContinuationDefect)) - eq(msg.defects[0].line, ' Line 1\n') + eq(self.get_defects(msg)[0].line, ' Line 1\n') + + +class TestNonConformant(TestNonConformantBase, TestEmailBase): + + policy=email.policy.default + + def get_defects(self, obj): + return obj.defects + + +class TestNonConformantCapture(TestNonConformantBase, TestEmailBase): + + class CapturePolicy(email.policy.Policy): + captured = None + def register_defect(self, obj, defect): + self.captured.append(defect) + + def setUp(self): + self.policy = self.CapturePolicy(captured=list()) + + def get_defects(self, obj): + return self.policy.captured + +class TestRaisingDefects(TestEmailBase): + + def _msgobj(self, filename): + with openfile(filename) as fp: + return email.message_from_file(fp, policy=email.policy.strict) + + def test_same_boundary_inner_outer(self): + with self.assertRaises(errors.StartBoundaryNotFoundDefect): + self._msgobj('msg_15.txt') + + def test_multipart_no_boundary(self): + with self.assertRaises(errors.NoBoundaryInMultipartDefect): + self._msgobj('msg_25.txt') + + def test_lying_multipart(self): + with self.assertRaises(errors.NoBoundaryInMultipartDefect): + self._msgobj('msg_41.txt') + + + def test_missing_start_boundary(self): + with self.assertRaises(errors.StartBoundaryNotFoundDefect): + self._msgobj('msg_42.txt') + + def test_first_line_is_continuation_header(self): + m = ' Line 1\nLine 2\nLine 3' + with self.assertRaises(errors.FirstHeaderLineIsContinuationDefect): + msg = email.message_from_string(m, policy=email.policy.strict) # Test RFC 2047 header encoding and decoding @@ -2561,6 +2642,13 @@ class TestMiscellaneous(TestEmailBase): (2002, 4, 3, 14, 58, 26, 0, 1, -1, -28800)) + def test_parsedate_accepts_time_with_dots(self): + eq = self.assertEqual + eq(utils.parsedate_tz('5 Feb 2003 13.47.26 -0800'), + (2003, 2, 5, 13, 47, 26, 0, 1, -1, -28800)) + eq(utils.parsedate_tz('5 Feb 2003 13.47 -0800'), + (2003, 2, 5, 13, 47, 0, 0, 1, -1, -28800)) + def test_parsedate_acceptable_to_time_functions(self): eq = self.assertEqual timetup = utils.parsedate('5 Feb 2003 13:47:26 -0800') @@ -2610,6 +2698,46 @@ class TestMiscellaneous(TestEmailBase): b = 'person@dom.ain' self.assertEqual(utils.parseaddr(utils.formataddr((a, b))), (a, b)) + def test_quotes_unicode_names(self): + # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + name = "H\u00e4ns W\u00fcrst" + addr = 'person@dom.ain' + utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= <person@dom.ain>" + latin1_quopri = "=?iso-8859-1?q?H=E4ns_W=FCrst?= <person@dom.ain>" + self.assertEqual(utils.formataddr((name, addr)), utf8_base64) + self.assertEqual(utils.formataddr((name, addr), 'iso-8859-1'), + latin1_quopri) + + def test_accepts_any_charset_like_object(self): + # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + name = "H\u00e4ns W\u00fcrst" + addr = 'person@dom.ain' + utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= <person@dom.ain>" + foobar = "FOOBAR" + class CharsetMock: + def header_encode(self, string): + return foobar + mock = CharsetMock() + mock_expected = "%s <%s>" % (foobar, addr) + self.assertEqual(utils.formataddr((name, addr), mock), mock_expected) + self.assertEqual(utils.formataddr((name, addr), Charset('utf-8')), + utf8_base64) + + def test_invalid_charset_like_object_raises_error(self): + # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + name = "H\u00e4ns W\u00fcrst" + addr = 'person@dom.ain' + # A object without a header_encode method: + bad_charset = object() + self.assertRaises(AttributeError, utils.formataddr, (name, addr), + bad_charset) + + def test_unicode_address_raises_error(self): + # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + addr = 'pers\u00f6n@dom.in' + self.assertRaises(UnicodeError, utils.formataddr, (None, addr)) + self.assertRaises(UnicodeError, utils.formataddr, ("Name", addr)) + def test_name_with_dot(self): x = 'John X. Doe <jxd@example.com>' y = '"John X. Doe" <jxd@example.com>' @@ -2890,6 +3018,7 @@ Do you like this message? class TestParsers(TestEmailBase): + def test_header_parser(self): eq = self.assertEqual # Parse only the headers of a complex multipart MIME document @@ -2901,6 +3030,18 @@ class TestParsers(TestEmailBase): self.assertFalse(msg.is_multipart()) self.assertTrue(isinstance(msg.get_payload(), str)) + def test_bytes_header_parser(self): + eq = self.assertEqual + # Parse only the headers of a complex multipart MIME document + with openfile('msg_02.txt', 'rb') as fp: + msg = email.parser.BytesHeaderParser().parse(fp) + eq(msg['from'], 'ppp-request@zzz.org') + eq(msg['to'], 'ppp@zzz.org') + eq(msg.get_content_type(), 'multipart/mixed') + self.assertFalse(msg.is_multipart()) + self.assertTrue(isinstance(msg.get_payload(), str)) + self.assertTrue(isinstance(msg.get_payload(decode=True), bytes)) + def test_whitespace_continuation(self): eq = self.assertEqual # This message contains a line after the Subject: header that has only @@ -2958,6 +3099,25 @@ Here's the message body g.flatten(msg, linesep='\r\n') self.assertEqual(s.getvalue(), text) + def test_crlf_control_via_policy(self): + with openfile('msg_26.txt', newline='\n') as fp: + text = fp.read() + msg = email.message_from_string(text) + s = StringIO() + g = email.generator.Generator(s, policy=email.policy.SMTP) + g.flatten(msg) + self.assertEqual(s.getvalue(), text) + + def test_flatten_linesep_overrides_policy(self): + # msg_27 is lf separated + with openfile('msg_27.txt', newline='\n') as fp: + text = fp.read() + msg = email.message_from_string(text) + s = StringIO() + g = email.generator.Generator(s, policy=email.policy.SMTP) + g.flatten(msg, linesep='\n') + self.assertEqual(s.getvalue(), text) + maxDiff = None def test_multipart_digest_with_extra_mime_headers(self): @@ -3424,6 +3584,44 @@ class Test8BitBytesHandling(unittest.TestCase): g.flatten(msg) self.assertEqual(s.getvalue(), source) + def test_crlf_control_via_policy(self): + # msg_26 is crlf terminated + with openfile('msg_26.txt', 'rb') as fp: + text = fp.read() + msg = email.message_from_bytes(text) + s = BytesIO() + g = email.generator.BytesGenerator(s, policy=email.policy.SMTP) + g.flatten(msg) + self.assertEqual(s.getvalue(), text) + + def test_flatten_linesep_overrides_policy(self): + # msg_27 is lf separated + with openfile('msg_27.txt', 'rb') as fp: + text = fp.read() + msg = email.message_from_bytes(text) + s = BytesIO() + g = email.generator.BytesGenerator(s, policy=email.policy.SMTP) + g.flatten(msg, linesep='\n') + self.assertEqual(s.getvalue(), text) + + def test_must_be_7bit_handles_unknown_8bit(self): + msg = email.message_from_bytes(self.non_latin_bin_msg) + out = BytesIO() + g = email.generator.BytesGenerator(out, + policy=email.policy.default.clone(must_be_7bit=True)) + g.flatten(msg) + self.assertEqual(out.getvalue(), + self.non_latin_bin_msg_as7bit_wrapped.encode('ascii')) + + def test_must_be_7bit_transforms_8bit_cte(self): + msg = email.message_from_bytes(self.latin_bin_msg) + out = BytesIO() + g = email.generator.BytesGenerator(out, + policy=email.policy.default.clone(must_be_7bit=True)) + g.flatten(msg) + self.assertEqual(out.getvalue(), + self.latin_bin_msg_as7bit.encode('ascii')) + maxDiff = None @@ -3442,12 +3640,7 @@ class BaseTestBytesGeneratorIdempotent: b = BytesIO() g = email.generator.BytesGenerator(b, maxheaderlen=0) g.flatten(msg, unixfrom=unixfrom, linesep=self.linesep) - self.assertByteStringsEqual(data, b.getvalue()) - - def assertByteStringsEqual(self, str1, str2): - # Not using self.blinesep here is intentional. This way the output - # is more useful when the failure results in mixed line endings. - self.assertListEqual(str1.split(b'\n'), str2.split(b'\n')) + self.assertEqual(data, b.getvalue()) class TestBytesGeneratorIdempotentNL(BaseTestBytesGeneratorIdempotent, @@ -4599,7 +4792,7 @@ Content-Type: application/x-foo; class TestSigned(TestEmailBase): def _msg_and_obj(self, filename): - with openfile(findfile(filename)) as fp: + with openfile(filename) as fp: original = fp.read() msg = email.message_from_string(original) return original, msg @@ -4631,23 +4824,5 @@ class TestSigned(TestEmailBase): -def _testclasses(): - mod = sys.modules[__name__] - return [getattr(mod, name) for name in dir(mod) if name.startswith('Test')] - - -def suite(): - suite = unittest.TestSuite() - for testclass in _testclasses(): - suite.addTest(unittest.makeSuite(testclass)) - return suite - - -def test_main(): - for testclass in _testclasses(): - run_unittest(testclass) - - - if __name__ == '__main__': - unittest.main(defaultTest='suite') + unittest.main() diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py new file mode 100644 index 0000000000..35ca6c5080 --- /dev/null +++ b/Lib/test/test_email/test_generator.py @@ -0,0 +1,136 @@ +import io +import textwrap +import unittest +from email import message_from_string, message_from_bytes +from email.generator import Generator, BytesGenerator +from email import policy +from test.test_email import TestEmailBase + +# XXX: move generator tests from test_email into here at some point. + + +class TestGeneratorBase(): + + long_subject = { + 0: textwrap.dedent("""\ + To: whom_it_may_concern@example.com + From: nobody_you_want_to_know@example.com + Subject: We the willing led by the unknowing are doing the + impossible for the ungrateful. We have done so much for so long with so little + we are now qualified to do anything with nothing. + + None + """), + 40: textwrap.dedent("""\ + To: whom_it_may_concern@example.com + From:\x20 + nobody_you_want_to_know@example.com + Subject: We the willing led by the + unknowing are doing the + impossible for the ungrateful. We have + done so much for so long with so little + we are now qualified to do anything + with nothing. + + None + """), + 20: textwrap.dedent("""\ + To:\x20 + whom_it_may_concern@example.com + From:\x20 + nobody_you_want_to_know@example.com + Subject: We the + willing led by the + unknowing are doing + the + impossible for the + ungrateful. We have + done so much for so + long with so little + we are now + qualified to do + anything with + nothing. + + None + """), + } + long_subject[100] = long_subject[0] + + def maxheaderlen_parameter_test(self, n): + msg = self.msgmaker(self.long_subject[0]) + s = self.ioclass() + g = self.genclass(s, maxheaderlen=n) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.long_subject[n]) + + def test_maxheaderlen_parameter_0(self): + self.maxheaderlen_parameter_test(0) + + def test_maxheaderlen_parameter_100(self): + self.maxheaderlen_parameter_test(100) + + def test_maxheaderlen_parameter_40(self): + self.maxheaderlen_parameter_test(40) + + def test_maxheaderlen_parameter_20(self): + self.maxheaderlen_parameter_test(20) + + def maxheaderlen_policy_test(self, n): + msg = self.msgmaker(self.long_subject[0]) + s = self.ioclass() + g = self.genclass(s, policy=policy.default.clone(max_line_length=n)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.long_subject[n]) + + def test_maxheaderlen_policy_0(self): + self.maxheaderlen_policy_test(0) + + def test_maxheaderlen_policy_100(self): + self.maxheaderlen_policy_test(100) + + def test_maxheaderlen_policy_40(self): + self.maxheaderlen_policy_test(40) + + def test_maxheaderlen_policy_20(self): + self.maxheaderlen_policy_test(20) + + def maxheaderlen_parm_overrides_policy_test(self, n): + msg = self.msgmaker(self.long_subject[0]) + s = self.ioclass() + g = self.genclass(s, maxheaderlen=n, + policy=policy.default.clone(max_line_length=10)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.long_subject[n]) + + def test_maxheaderlen_parm_overrides_policy_0(self): + self.maxheaderlen_parm_overrides_policy_test(0) + + def test_maxheaderlen_parm_overrides_policy_100(self): + self.maxheaderlen_parm_overrides_policy_test(100) + + def test_maxheaderlen_parm_overrides_policy_40(self): + self.maxheaderlen_parm_overrides_policy_test(40) + + def test_maxheaderlen_parm_overrides_policy_20(self): + self.maxheaderlen_parm_overrides_policy_test(20) + + +class TestGenerator(TestGeneratorBase, TestEmailBase): + + msgmaker = staticmethod(message_from_string) + genclass = Generator + ioclass = io.StringIO + + +class TestBytesGenerator(TestGeneratorBase, TestEmailBase): + + msgmaker = staticmethod(message_from_bytes) + genclass = BytesGenerator + ioclass = io.BytesIO + long_subject = {key: x.encode('ascii') + for key, x in TestGeneratorBase.long_subject.items()} + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py new file mode 100644 index 0000000000..086ce40eac --- /dev/null +++ b/Lib/test/test_email/test_policy.py @@ -0,0 +1,148 @@ +import types +import unittest +import email.policy + +class PolicyAPITests(unittest.TestCase): + + longMessage = True + + # These default values are the ones set on email.policy.default. + # If any of these defaults change, the docs must be updated. + policy_defaults = { + 'max_line_length': 78, + 'linesep': '\n', + 'must_be_7bit': False, + 'raise_on_defect': False, + } + + # For each policy under test, we give here the values of the attributes + # that are different from the defaults for that policy. + policies = { + email.policy.Policy(): {}, + email.policy.default: {}, + email.policy.SMTP: {'linesep': '\r\n'}, + email.policy.HTTP: {'linesep': '\r\n', 'max_line_length': None}, + email.policy.strict: {'raise_on_defect': True}, + } + + def test_defaults(self): + for policy, changed_defaults in self.policies.items(): + expected = self.policy_defaults.copy() + expected.update(changed_defaults) + for attr, value in expected.items(): + self.assertEqual(getattr(policy, attr), value, + ("change {} docs/docstrings if defaults have " + "changed").format(policy)) + + def test_all_attributes_covered(self): + for attr in dir(email.policy.default): + if (attr.startswith('_') or + isinstance(getattr(email.policy.Policy, attr), + types.FunctionType)): + continue + else: + self.assertIn(attr, self.policy_defaults, + "{} is not fully tested".format(attr)) + + def test_policy_is_immutable(self): + for policy in self.policies: + for attr in self.policy_defaults: + with self.assertRaisesRegex(AttributeError, attr+".*read-only"): + setattr(policy, attr, None) + with self.assertRaisesRegex(AttributeError, 'no attribute.*foo'): + policy.foo = None + + def test_set_policy_attrs_when_calledl(self): + testattrdict = { attr: None for attr in self.policy_defaults } + for policyclass in self.policies: + policy = policyclass.clone(**testattrdict) + for attr in self.policy_defaults: + self.assertIsNone(getattr(policy, attr)) + + def test_reject_non_policy_keyword_when_called(self): + for policyclass in self.policies: + with self.assertRaises(TypeError): + policyclass(this_keyword_should_not_be_valid=None) + with self.assertRaises(TypeError): + policyclass(newtline=None) + + def test_policy_addition(self): + expected = self.policy_defaults.copy() + p1 = email.policy.default.clone(max_line_length=100) + p2 = email.policy.default.clone(max_line_length=50) + added = p1 + p2 + expected.update(max_line_length=50) + for attr, value in expected.items(): + self.assertEqual(getattr(added, attr), value) + added = p2 + p1 + expected.update(max_line_length=100) + for attr, value in expected.items(): + self.assertEqual(getattr(added, attr), value) + added = added + email.policy.default + for attr, value in expected.items(): + self.assertEqual(getattr(added, attr), value) + + def test_register_defect(self): + class Dummy: + def __init__(self): + self.defects = [] + obj = Dummy() + defect = object() + policy = email.policy.Policy() + policy.register_defect(obj, defect) + self.assertEqual(obj.defects, [defect]) + defect2 = object() + policy.register_defect(obj, defect2) + self.assertEqual(obj.defects, [defect, defect2]) + + class MyObj: + def __init__(self): + self.defects = [] + + class MyDefect(Exception): + pass + + def test_handle_defect_raises_on_strict(self): + foo = self.MyObj() + defect = self.MyDefect("the telly is broken") + with self.assertRaisesRegex(self.MyDefect, "the telly is broken"): + email.policy.strict.handle_defect(foo, defect) + + def test_handle_defect_registers_defect(self): + foo = self.MyObj() + defect1 = self.MyDefect("one") + email.policy.default.handle_defect(foo, defect1) + self.assertEqual(foo.defects, [defect1]) + defect2 = self.MyDefect("two") + email.policy.default.handle_defect(foo, defect2) + self.assertEqual(foo.defects, [defect1, defect2]) + + class MyPolicy(email.policy.Policy): + defects = [] + def register_defect(self, obj, defect): + self.defects.append(defect) + + def test_overridden_register_defect_still_raises(self): + foo = self.MyObj() + defect = self.MyDefect("the telly is broken") + with self.assertRaisesRegex(self.MyDefect, "the telly is broken"): + self.MyPolicy(raise_on_defect=True).handle_defect(foo, defect) + + def test_overriden_register_defect_works(self): + foo = self.MyObj() + defect1 = self.MyDefect("one") + my_policy = self.MyPolicy() + my_policy.handle_defect(foo, defect1) + self.assertEqual(my_policy.defects, [defect1]) + self.assertEqual(foo.defects, []) + defect2 = self.MyDefect("two") + my_policy.handle_defect(foo, defect2) + self.assertEqual(my_policy.defects, [defect1, defect2]) + self.assertEqual(foo.defects, []) + + # XXX: Need subclassing tests. + # For adding subclassed objects, make sure the usual rules apply (subclass + # wins), but that the order still works (right overrides left). + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py new file mode 100644 index 0000000000..e003a64895 --- /dev/null +++ b/Lib/test/test_email/test_utils.py @@ -0,0 +1,45 @@ +import datetime +from email import utils +import unittest + +class DateTimeTests(unittest.TestCase): + + datestring = 'Sun, 23 Sep 2001 20:10:55' + dateargs = (2001, 9, 23, 20, 10, 55) + offsetstring = ' -0700' + utcoffset = datetime.timedelta(hours=-7) + tz = datetime.timezone(utcoffset) + naive_dt = datetime.datetime(*dateargs) + aware_dt = datetime.datetime(*dateargs, tzinfo=tz) + + def test_naive_datetime(self): + self.assertEqual(utils.format_datetime(self.naive_dt), + self.datestring + ' -0000') + + def test_aware_datetime(self): + self.assertEqual(utils.format_datetime(self.aware_dt), + self.datestring + self.offsetstring) + + def test_usegmt(self): + utc_dt = datetime.datetime(*self.dateargs, + tzinfo=datetime.timezone.utc) + self.assertEqual(utils.format_datetime(utc_dt, usegmt=True), + self.datestring + ' GMT') + + def test_usegmt_with_naive_datetime_raises(self): + with self.assertRaises(ValueError): + utils.format_datetime(self.naive_dt, usegmt=True) + + def test_usegmt_with_non_utc_datetime_raises(self): + with self.assertRaises(ValueError): + utils.format_datetime(self.aware_dt, usegmt=True) + + def test_parsedate_to_datetime(self): + self.assertEqual( + utils.parsedate_to_datetime(self.datestring + self.offsetstring), + self.aware_dt) + + def test_parsedate_to_datetime_naive(self): + self.assertEqual( + utils.parsedate_to_datetime(self.datestring + ' -0000'), + self.naive_dt) diff --git a/Lib/email/test/test_email_torture.py b/Lib/test/test_email/torture_test.py index 544b1bbb39..544b1bbb39 100644 --- a/Lib/email/test/test_email_torture.py +++ b/Lib/test/test_email/torture_test.py diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 0a7ddd4c5b..9be6958820 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -8,7 +8,7 @@ import weakref import errno from test.support import (TESTFN, unlink, run_unittest, captured_output, - gc_collect, cpython_only) + gc_collect, cpython_only, no_tracing) # XXX This is not really enough, each *operation* should be tested! @@ -389,6 +389,7 @@ class ExceptionTests(unittest.TestCase): x = DerivedException(fancy_arg=42) self.assertEqual(x.fancy_arg, 42) + @no_tracing def testInfiniteRecursion(self): def f(): return f() @@ -721,6 +722,7 @@ class ExceptionTests(unittest.TestCase): u.start = 1000 self.assertEqual(str(u), "can't translate characters in position 1000-4: 965230951443685724997") + @no_tracing def test_badisinstance(self): # Bug #2542: if issubclass(e, MyException) raises an exception, # it should be ignored @@ -831,6 +833,7 @@ class ExceptionTests(unittest.TestCase): self.fail("MemoryError not raised") self.assertEqual(wr(), None) + @no_tracing def test_recursion_error_cleanup(self): # Same test as above, but with "recursion exceeded" errors class C: diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py index 1f7f63042e..6b6c12de2e 100644 --- a/Lib/test/test_extcall.py +++ b/Lib/test/test_extcall.py @@ -66,17 +66,17 @@ Verify clearing of SF bug #733667 >>> g() Traceback (most recent call last): ... - TypeError: g() takes at least 1 argument (0 given) + TypeError: g() missing 1 required positional argument: 'x' >>> g(*()) Traceback (most recent call last): ... - TypeError: g() takes at least 1 argument (0 given) + TypeError: g() missing 1 required positional argument: 'x' >>> g(*(), **{}) Traceback (most recent call last): ... - TypeError: g() takes at least 1 argument (0 given) + TypeError: g() missing 1 required positional argument: 'x' >>> g(1) 1 () {} @@ -151,7 +151,7 @@ What about willful misconduct? >>> g(1, 2, 3, **{'x': 4, 'y': 5}) Traceback (most recent call last): ... - TypeError: g() got multiple values for keyword argument 'x' + TypeError: g() got multiple values for argument 'x' >>> f(**{1:2}) Traceback (most recent call last): @@ -263,29 +263,80 @@ the function call setup. See <http://bugs.python.org/issue2016>. >>> f(**x) 1 2 -A obscure message: +Too many arguments: - >>> def f(a, b): - ... pass - >>> f(b=1) + >>> def f(): pass + >>> f(1) + Traceback (most recent call last): + ... + TypeError: f() takes 0 positional arguments but 1 was given + >>> def f(a): pass + >>> f(1, 2) Traceback (most recent call last): ... - TypeError: f() takes exactly 2 arguments (1 given) + TypeError: f() takes 1 positional argument but 2 were given + >>> def f(a, b=1): pass + >>> f(1, 2, 3) + Traceback (most recent call last): + ... + TypeError: f() takes from 1 to 2 positional arguments but 3 were given + >>> def f(*, kw): pass + >>> f(1, kw=3) + Traceback (most recent call last): + ... + TypeError: f() takes 0 positional arguments but 1 positional argument (and 1 keyword-only argument) were given + >>> def f(*, kw, b): pass + >>> f(1, 2, 3, b=3, kw=3) + Traceback (most recent call last): + ... + TypeError: f() takes 0 positional arguments but 3 positional arguments (and 2 keyword-only arguments) were given + >>> def f(a, b=2, *, kw): pass + >>> f(2, 3, 4, kw=4) + Traceback (most recent call last): + ... + TypeError: f() takes from 1 to 2 positional arguments but 3 positional arguments (and 1 keyword-only argument) were given -The number of arguments passed in includes keywords: +Too few and missing arguments: - >>> def f(a): - ... pass - >>> f(6, a=4, *(1, 2, 3)) + >>> def f(a): pass + >>> f() Traceback (most recent call last): ... - TypeError: f() takes exactly 1 positional argument (5 given) - >>> def f(a, *, kw): - ... pass - >>> f(6, 4, kw=4) + TypeError: f() missing 1 required positional argument: 'a' + >>> def f(a, b): pass + >>> f() Traceback (most recent call last): ... - TypeError: f() takes exactly 1 positional argument (3 given) + TypeError: f() missing 2 required positional arguments: 'a' and 'b' + >>> def f(a, b, c): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 3 required positional arguments: 'a', 'b', and 'c' + >>> def f(a, b, c, d, e): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 5 required positional arguments: 'a', 'b', 'c', 'd', and 'e' + >>> def f(a, b=4, c=5, d=5): pass + >>> f(c=12, b=9) + Traceback (most recent call last): + ... + TypeError: f() missing 1 required positional argument: 'a' + +Same with keyword only args: + + >>> def f(*, w): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 1 required keyword-only argument: 'w' + >>> def f(*, a, b, c, d, e): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 5 required keyword-only arguments: 'a', 'b', 'c', 'd', and 'e' + """ import sys diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py new file mode 100644 index 0000000000..977cb3928b --- /dev/null +++ b/Lib/test/test_faulthandler.py @@ -0,0 +1,554 @@ +from contextlib import contextmanager +import datetime +import faulthandler +import os +import re +import signal +import subprocess +import sys +from test import support, script_helper +import tempfile +import unittest + +try: + import threading + HAVE_THREADS = True +except ImportError: + HAVE_THREADS = False + +TIMEOUT = 0.5 + +try: + from resource import setrlimit, RLIMIT_CORE, error as resource_error +except ImportError: + prepare_subprocess = None +else: + def prepare_subprocess(): + # don't create core file + try: + setrlimit(RLIMIT_CORE, (0, 0)) + except (ValueError, resource_error): + pass + +def expected_traceback(lineno1, lineno2, header, min_count=1): + regex = header + regex += ' File "<string>", line %s in func\n' % lineno1 + regex += ' File "<string>", line %s in <module>' % lineno2 + if 1 < min_count: + return '^' + (regex + '\n') * (min_count - 1) + regex + else: + return '^' + regex + '$' + +@contextmanager +def temporary_filename(): + filename = tempfile.mktemp() + try: + yield filename + finally: + support.unlink(filename) + +class FaultHandlerTests(unittest.TestCase): + def get_output(self, code, filename=None): + """ + Run the specified code in Python (in a new child process) and read the + output from the standard error or from a file (if filename is set). + Return the output lines as a list. + + Strip the reference count from the standard error for Python debug + build, and replace "Current thread 0x00007f8d8fbd9700" by "Current + thread XXX". + """ + options = {} + if prepare_subprocess: + options['preexec_fn'] = prepare_subprocess + process = script_helper.spawn_python('-c', code, **options) + stdout, stderr = process.communicate() + exitcode = process.wait() + output = support.strip_python_stderr(stdout) + output = output.decode('ascii', 'backslashreplace') + if filename: + self.assertEqual(output, '') + with open(filename, "rb") as fp: + output = fp.read() + output = output.decode('ascii', 'backslashreplace') + output = re.sub('Current thread 0x[0-9a-f]+', + 'Current thread XXX', + output) + return output.splitlines(), exitcode + + def check_fatal_error(self, code, line_number, name_regex, + filename=None, all_threads=True, other_regex=None): + """ + Check that the fault handler for fatal errors is enabled and check the + traceback from the child process output. + + Raise an error if the output doesn't match the expected format. + """ + if all_threads: + header = 'Current thread XXX' + else: + header = 'Traceback (most recent call first)' + regex = """ +^Fatal Python error: {name} + +{header}: + File "<string>", line {lineno} in <module>$ +""".strip() + regex = regex.format( + lineno=line_number, + name=name_regex, + header=re.escape(header)) + if other_regex: + regex += '|' + other_regex + output, exitcode = self.get_output(code, filename) + output = '\n'.join(output) + self.assertRegex(output, regex) + self.assertNotEqual(exitcode, 0) + + def test_read_null(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._read_null() +""".strip(), + 3, + # Issue #12700: Read NULL raises SIGILL on Mac OS X Lion + '(?:Segmentation fault|Bus error|Illegal instruction)') + + def test_sigsegv(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._sigsegv() +""".strip(), + 3, + 'Segmentation fault') + + def test_sigabrt(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._sigabrt() +""".strip(), + 3, + 'Aborted') + + @unittest.skipIf(sys.platform == 'win32', + "SIGFPE cannot be caught on Windows") + def test_sigfpe(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._sigfpe() +""".strip(), + 3, + 'Floating point exception') + + @unittest.skipIf(not hasattr(faulthandler, '_sigbus'), + "need faulthandler._sigbus()") + def test_sigbus(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._sigbus() +""".strip(), + 3, + 'Bus error') + + @unittest.skipIf(not hasattr(faulthandler, '_sigill'), + "need faulthandler._sigill()") + def test_sigill(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._sigill() +""".strip(), + 3, + 'Illegal instruction') + + def test_fatal_error(self): + self.check_fatal_error(""" +import faulthandler +faulthandler._fatal_error(b'xyz') +""".strip(), + 2, + 'xyz') + + @unittest.skipIf(sys.platform.startswith('openbsd') and HAVE_THREADS, + "Issue #12868: sigaltstack() doesn't work on " + "OpenBSD if Python is compiled with pthread") + @unittest.skipIf(not hasattr(faulthandler, '_stack_overflow'), + 'need faulthandler._stack_overflow()') + def test_stack_overflow(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._stack_overflow() +""".strip(), + 3, + '(?:Segmentation fault|Bus error)', + other_regex='unable to raise a stack overflow') + + def test_gil_released(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._read_null(True) +""".strip(), + 3, + '(?:Segmentation fault|Bus error|Illegal instruction)') + + def test_enable_file(self): + with temporary_filename() as filename: + self.check_fatal_error(""" +import faulthandler +output = open({filename}, 'wb') +faulthandler.enable(output) +faulthandler._read_null() +""".strip().format(filename=repr(filename)), + 4, + '(?:Segmentation fault|Bus error|Illegal instruction)', + filename=filename) + + def test_enable_single_thread(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable(all_threads=False) +faulthandler._read_null() +""".strip(), + 3, + '(?:Segmentation fault|Bus error|Illegal instruction)', + all_threads=False) + + def test_disable(self): + code = """ +import faulthandler +faulthandler.enable() +faulthandler.disable() +faulthandler._read_null() +""".strip() + not_expected = 'Fatal Python error' + stderr, exitcode = self.get_output(code) + stder = '\n'.join(stderr) + self.assertTrue(not_expected not in stderr, + "%r is present in %r" % (not_expected, stderr)) + self.assertNotEqual(exitcode, 0) + + def test_is_enabled(self): + orig_stderr = sys.stderr + try: + # regrtest may replace sys.stderr by io.StringIO object, but + # faulthandler.enable() requires that sys.stderr has a fileno() + # method + sys.stderr = sys.__stderr__ + + was_enabled = faulthandler.is_enabled() + try: + faulthandler.enable() + self.assertTrue(faulthandler.is_enabled()) + faulthandler.disable() + self.assertFalse(faulthandler.is_enabled()) + finally: + if was_enabled: + faulthandler.enable() + else: + faulthandler.disable() + finally: + sys.stderr = orig_stderr + + def check_dump_traceback(self, filename): + """ + Explicitly call dump_traceback() function and check its output. + Raise an error if the output doesn't match the expected format. + """ + code = """ +import faulthandler + +def funcB(): + if {has_filename}: + with open({filename}, "wb") as fp: + faulthandler.dump_traceback(fp, all_threads=False) + else: + faulthandler.dump_traceback(all_threads=False) + +def funcA(): + funcB() + +funcA() +""".strip() + code = code.format( + filename=repr(filename), + has_filename=bool(filename), + ) + if filename: + lineno = 6 + else: + lineno = 8 + expected = [ + 'Traceback (most recent call first):', + ' File "<string>", line %s in funcB' % lineno, + ' File "<string>", line 11 in funcA', + ' File "<string>", line 13 in <module>' + ] + trace, exitcode = self.get_output(code, filename) + self.assertEqual(trace, expected) + self.assertEqual(exitcode, 0) + + def test_dump_traceback(self): + self.check_dump_traceback(None) + + def test_dump_traceback_file(self): + with temporary_filename() as filename: + self.check_dump_traceback(filename) + + @unittest.skipIf(not HAVE_THREADS, 'need threads') + def check_dump_traceback_threads(self, filename): + """ + Call explicitly dump_traceback(all_threads=True) and check the output. + Raise an error if the output doesn't match the expected format. + """ + code = """ +import faulthandler +from threading import Thread, Event +import time + +def dump(): + if {filename}: + with open({filename}, "wb") as fp: + faulthandler.dump_traceback(fp, all_threads=True) + else: + faulthandler.dump_traceback(all_threads=True) + +class Waiter(Thread): + # avoid blocking if the main thread raises an exception. + daemon = True + + def __init__(self): + Thread.__init__(self) + self.running = Event() + self.stop = Event() + + def run(self): + self.running.set() + self.stop.wait() + +waiter = Waiter() +waiter.start() +waiter.running.wait() +dump() +waiter.stop.set() +waiter.join() +""".strip() + code = code.format(filename=repr(filename)) + output, exitcode = self.get_output(code, filename) + output = '\n'.join(output) + if filename: + lineno = 8 + else: + lineno = 10 + regex = """ +^Thread 0x[0-9a-f]+: +(?: File ".*threading.py", line [0-9]+ in [_a-z]+ +){{1,3}} File "<string>", line 23 in run + File ".*threading.py", line [0-9]+ in _bootstrap_inner + File ".*threading.py", line [0-9]+ in _bootstrap + +Current thread XXX: + File "<string>", line {lineno} in dump + File "<string>", line 28 in <module>$ +""".strip() + regex = regex.format(lineno=lineno) + self.assertRegex(output, regex) + self.assertEqual(exitcode, 0) + + def test_dump_traceback_threads(self): + self.check_dump_traceback_threads(None) + + def test_dump_traceback_threads_file(self): + with temporary_filename() as filename: + self.check_dump_traceback_threads(filename) + + def _check_dump_tracebacks_later(self, repeat, cancel, filename, loops): + """ + Check how many times the traceback is written in timeout x 2.5 seconds, + or timeout x 3.5 seconds if cancel is True: 1, 2 or 3 times depending + on repeat and cancel options. + + Raise an error if the output doesn't match the expect format. + """ + timeout_str = str(datetime.timedelta(seconds=TIMEOUT)) + code = """ +import faulthandler +import time + +def func(timeout, repeat, cancel, file, loops): + for loop in range(loops): + faulthandler.dump_tracebacks_later(timeout, repeat=repeat, file=file) + if cancel: + faulthandler.cancel_dump_tracebacks_later() + time.sleep(timeout * 5) + faulthandler.cancel_dump_tracebacks_later() + +timeout = {timeout} +repeat = {repeat} +cancel = {cancel} +loops = {loops} +if {has_filename}: + file = open({filename}, "wb") +else: + file = None +func(timeout, repeat, cancel, file, loops) +if file is not None: + file.close() +""".strip() + code = code.format( + timeout=TIMEOUT, + repeat=repeat, + cancel=cancel, + loops=loops, + has_filename=bool(filename), + filename=repr(filename), + ) + trace, exitcode = self.get_output(code, filename) + trace = '\n'.join(trace) + + if not cancel: + count = loops + if repeat: + count *= 2 + header = r'Timeout \(%s\)!\nThread 0x[0-9a-f]+:\n' % timeout_str + regex = expected_traceback(9, 20, header, min_count=count) + self.assertRegex(trace, regex) + else: + self.assertEqual(trace, '') + self.assertEqual(exitcode, 0) + + @unittest.skipIf(not hasattr(faulthandler, 'dump_tracebacks_later'), + 'need faulthandler.dump_tracebacks_later()') + def check_dump_tracebacks_later(self, repeat=False, cancel=False, + file=False, twice=False): + if twice: + loops = 2 + else: + loops = 1 + if file: + with temporary_filename() as filename: + self._check_dump_tracebacks_later(repeat, cancel, + filename, loops) + else: + self._check_dump_tracebacks_later(repeat, cancel, None, loops) + + def test_dump_tracebacks_later(self): + self.check_dump_tracebacks_later() + + def test_dump_tracebacks_later_repeat(self): + self.check_dump_tracebacks_later(repeat=True) + + def test_dump_tracebacks_later_cancel(self): + self.check_dump_tracebacks_later(cancel=True) + + def test_dump_tracebacks_later_file(self): + self.check_dump_tracebacks_later(file=True) + + def test_dump_tracebacks_later_twice(self): + self.check_dump_tracebacks_later(twice=True) + + @unittest.skipIf(not hasattr(faulthandler, "register"), + "need faulthandler.register") + def check_register(self, filename=False, all_threads=False, + unregister=False, chain=False): + """ + Register a handler displaying the traceback on a user signal. Raise the + signal and check the written traceback. + + If chain is True, check that the previous signal handler is called. + + Raise an error if the output doesn't match the expected format. + """ + signum = signal.SIGUSR1 + code = """ +import faulthandler +import os +import signal +import sys + +def func(signum): + os.kill(os.getpid(), signum) + +def handler(signum, frame): + handler.called = True +handler.called = False + +exitcode = 0 +signum = {signum} +unregister = {unregister} +chain = {chain} + +if {has_filename}: + file = open({filename}, "wb") +else: + file = None +if chain: + signal.signal(signum, handler) +faulthandler.register(signum, file=file, + all_threads={all_threads}, chain={chain}) +if unregister: + faulthandler.unregister(signum) +func(signum) +if chain and not handler.called: + if file is not None: + output = file + else: + output = sys.stderr + print("Error: signal handler not called!", file=output) + exitcode = 1 +if file is not None: + file.close() +sys.exit(exitcode) +""".strip() + code = code.format( + filename=repr(filename), + has_filename=bool(filename), + all_threads=all_threads, + signum=signum, + unregister=unregister, + chain=chain, + ) + trace, exitcode = self.get_output(code, filename) + trace = '\n'.join(trace) + if not unregister: + if all_threads: + regex = 'Current thread XXX:\n' + else: + regex = 'Traceback \(most recent call first\):\n' + regex = expected_traceback(7, 28, regex) + self.assertRegex(trace, regex) + else: + self.assertEqual(trace, '') + if unregister: + self.assertNotEqual(exitcode, 0) + else: + self.assertEqual(exitcode, 0) + + def test_register(self): + self.check_register() + + def test_unregister(self): + self.check_register(unregister=True) + + def test_register_file(self): + with temporary_filename() as filename: + self.check_register(filename=filename) + + def test_register_threads(self): + self.check_register(all_threads=True) + + def test_register_chain(self): + self.check_register(chain=True) + + +def test_main(): + support.run_unittest(FaultHandlerTests) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index f312882b2a..a96d48a561 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -2,18 +2,34 @@ Tests for fileinput module. Nick Mathewson ''' - +import os +import sys +import re +import fileinput +import collections +import types +import codecs import unittest -from test.support import verbose, TESTFN, run_unittest -from test.support import unlink as safe_unlink -import sys, re + +try: + import bz2 +except ImportError: + bz2 = None +try: + import gzip +except ImportError: + gzip = None + from io import StringIO from fileinput import FileInput, hook_encoded +from test.support import verbose, TESTFN, run_unittest +from test.support import unlink as safe_unlink + + # The fileinput module has 2 interfaces: the FileInput class which does # all the work, and a few functions (input, etc.) that use a global _state -# variable. We only test the FileInput class, since the other functions -# only provide a thin facade over FileInput. +# variable. # Write lines (a list of lines) to temp file number i, and return the # temp file's name. @@ -121,7 +137,16 @@ class BufferSizesTests(unittest.TestCase): self.assertEqual(int(m.group(1)), fi.filelineno()) fi.close() +class UnconditionallyRaise: + def __init__(self, exception_type): + self.exception_type = exception_type + self.invoked = False + def __call__(self, *args, **kwargs): + self.invoked = True + raise self.exception_type() + class FileInputTests(unittest.TestCase): + def test_zero_byte_files(self): t1 = t2 = t3 = t4 = None try: @@ -219,17 +244,20 @@ class FileInputTests(unittest.TestCase): self.fail("FileInput should check openhook for being callable") except ValueError: pass - # XXX The rot13 codec was removed. - # So this test needs to be changed to use something else. - # (Or perhaps the API needs to change so we can just pass - # an encoding rather than using a hook?) -## try: -## t1 = writeTmp(1, ["A\nB"], mode="wb") -## fi = FileInput(files=t1, openhook=hook_encoded("rot13")) -## lines = list(fi) -## self.assertEqual(lines, ["N\n", "O"]) -## finally: -## remove_tempfiles(t1) + + class CustomOpenHook: + def __init__(self): + self.invoked = False + def __call__(self, *args): + self.invoked = True + return open(*args) + + t = writeTmp(1, ["\n"]) + self.addCleanup(remove_tempfiles, t) + custom_open_hook = CustomOpenHook() + with FileInput([t], openhook=custom_open_hook) as fi: + fi.readline() + self.assertTrue(custom_open_hook.invoked, "openhook not invoked") def test_context_manager(self): try: @@ -254,9 +282,585 @@ class FileInputTests(unittest.TestCase): finally: remove_tempfiles(t1) + def test_empty_files_list_specified_to_constructor(self): + with FileInput(files=[]) as fi: + self.assertEqual(fi._files, ('-',)) + + def test__getitem__(self): + """Tests invoking FileInput.__getitem__() with the current + line number""" + t = writeTmp(1, ["line1\n", "line2\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t]) as fi: + retval1 = fi[0] + self.assertEqual(retval1, "line1\n") + retval2 = fi[1] + self.assertEqual(retval2, "line2\n") + + def test__getitem__invalid_key(self): + """Tests invoking FileInput.__getitem__() with an index unequal to + the line number""" + t = writeTmp(1, ["line1\n", "line2\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t]) as fi: + with self.assertRaises(RuntimeError) as cm: + fi[1] + self.assertEqual(cm.exception.args, ("accessing lines out of order",)) + + def test__getitem__eof(self): + """Tests invoking FileInput.__getitem__() with the line number but at + end-of-input""" + t = writeTmp(1, []) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t]) as fi: + with self.assertRaises(IndexError) as cm: + fi[0] + self.assertEqual(cm.exception.args, ("end of input reached",)) + + def test_nextfile_oserror_deleting_backup(self): + """Tests invoking FileInput.nextfile() when the attempt to delete + the backup file would raise OSError. This error is expected to be + silently ignored""" + + os_unlink_orig = os.unlink + os_unlink_replacement = UnconditionallyRaise(OSError) + try: + t = writeTmp(1, ["\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t], inplace=True) as fi: + next(fi) # make sure the file is opened + os.unlink = os_unlink_replacement + fi.nextfile() + finally: + os.unlink = os_unlink_orig + + # sanity check to make sure that our test scenario was actually hit + self.assertTrue(os_unlink_replacement.invoked, + "os.unlink() was not invoked") + + def test_readline_os_fstat_raises_OSError(self): + """Tests invoking FileInput.readline() when os.fstat() raises OSError. + This exception should be silently discarded.""" + + os_fstat_orig = os.fstat + os_fstat_replacement = UnconditionallyRaise(OSError) + try: + t = writeTmp(1, ["\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t], inplace=True) as fi: + os.fstat = os_fstat_replacement + fi.readline() + finally: + os.fstat = os_fstat_orig + + # sanity check to make sure that our test scenario was actually hit + self.assertTrue(os_fstat_replacement.invoked, + "os.fstat() was not invoked") + + @unittest.skipIf(not hasattr(os, "chmod"), "os.chmod does not exist") + def test_readline_os_chmod_raises_OSError(self): + """Tests invoking FileInput.readline() when os.chmod() raises OSError. + This exception should be silently discarded.""" + + os_chmod_orig = os.chmod + os_chmod_replacement = UnconditionallyRaise(OSError) + try: + t = writeTmp(1, ["\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t], inplace=True) as fi: + os.chmod = os_chmod_replacement + fi.readline() + finally: + os.chmod = os_chmod_orig + + # sanity check to make sure that our test scenario was actually hit + self.assertTrue(os_chmod_replacement.invoked, + "os.fstat() was not invoked") + + def test_fileno_when_ValueError_raised(self): + class FilenoRaisesValueError(UnconditionallyRaise): + def __init__(self): + UnconditionallyRaise.__init__(self, ValueError) + def fileno(self): + self.__call__() + + unconditionally_raise_ValueError = FilenoRaisesValueError() + t = writeTmp(1, ["\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t]) as fi: + file_backup = fi._file + try: + fi._file = unconditionally_raise_ValueError + result = fi.fileno() + finally: + fi._file = file_backup # make sure the file gets cleaned up + + # sanity check to make sure that our test scenario was actually hit + self.assertTrue(unconditionally_raise_ValueError.invoked, + "_file.fileno() was not invoked") + + self.assertEqual(result, -1, "fileno() should return -1") + +class MockFileInput: + """A class that mocks out fileinput.FileInput for use during unit tests""" + + def __init__(self, files=None, inplace=False, backup="", bufsize=0, + mode="r", openhook=None): + self.files = files + self.inplace = inplace + self.backup = backup + self.bufsize = bufsize + self.mode = mode + self.openhook = openhook + self._file = None + self.invocation_counts = collections.defaultdict(lambda: 0) + self.return_values = {} + + def close(self): + self.invocation_counts["close"] += 1 + + def nextfile(self): + self.invocation_counts["nextfile"] += 1 + return self.return_values["nextfile"] + + def filename(self): + self.invocation_counts["filename"] += 1 + return self.return_values["filename"] + + def lineno(self): + self.invocation_counts["lineno"] += 1 + return self.return_values["lineno"] + + def filelineno(self): + self.invocation_counts["filelineno"] += 1 + return self.return_values["filelineno"] + + def fileno(self): + self.invocation_counts["fileno"] += 1 + return self.return_values["fileno"] + + def isfirstline(self): + self.invocation_counts["isfirstline"] += 1 + return self.return_values["isfirstline"] + + def isstdin(self): + self.invocation_counts["isstdin"] += 1 + return self.return_values["isstdin"] + +class BaseFileInputGlobalMethodsTest(unittest.TestCase): + """Base class for unit tests for the global function of + the fileinput module.""" + + def setUp(self): + self._orig_state = fileinput._state + self._orig_FileInput = fileinput.FileInput + fileinput.FileInput = MockFileInput + + def tearDown(self): + fileinput.FileInput = self._orig_FileInput + fileinput._state = self._orig_state + + def assertExactlyOneInvocation(self, mock_file_input, method_name): + # assert that the method with the given name was invoked once + actual_count = mock_file_input.invocation_counts[method_name] + self.assertEqual(actual_count, 1, method_name) + # assert that no other unexpected methods were invoked + actual_total_count = len(mock_file_input.invocation_counts) + self.assertEqual(actual_total_count, 1) + +class Test_fileinput_input(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.input()""" + + def test_state_is_not_None_and_state_file_is_not_None(self): + """Tests invoking fileinput.input() when fileinput._state is not None + and its _file attribute is also not None. Expect RuntimeError to + be raised with a meaningful error message and for fileinput._state + to *not* be modified.""" + instance = MockFileInput() + instance._file = object() + fileinput._state = instance + with self.assertRaises(RuntimeError) as cm: + fileinput.input() + self.assertEqual(("input() already active",), cm.exception.args) + self.assertIs(instance, fileinput._state, "fileinput._state") + + def test_state_is_not_None_and_state_file_is_None(self): + """Tests invoking fileinput.input() when fileinput._state is not None + but its _file attribute *is* None. Expect it to create and return + a new fileinput.FileInput object with all method parameters passed + explicitly to the __init__() method; also ensure that + fileinput._state is set to the returned instance.""" + instance = MockFileInput() + instance._file = None + fileinput._state = instance + self.do_test_call_input() + + def test_state_is_None(self): + """Tests invoking fileinput.input() when fileinput._state is None + Expect it to create and return a new fileinput.FileInput object + with all method parameters passed explicitly to the __init__() + method; also ensure that fileinput._state is set to the returned + instance.""" + fileinput._state = None + self.do_test_call_input() + + def do_test_call_input(self): + """Tests that fileinput.input() creates a new fileinput.FileInput + object, passing the given parameters unmodified to + fileinput.FileInput.__init__(). Note that this test depends on the + monkey patching of fileinput.FileInput done by setUp().""" + files = object() + inplace = object() + backup = object() + bufsize = object() + mode = object() + openhook = object() + + # call fileinput.input() with different values for each argument + result = fileinput.input(files=files, inplace=inplace, backup=backup, + bufsize=bufsize, + mode=mode, openhook=openhook) + + # ensure fileinput._state was set to the returned object + self.assertIs(result, fileinput._state, "fileinput._state") + + # ensure the parameters to fileinput.input() were passed directly + # to FileInput.__init__() + self.assertIs(files, result.files, "files") + self.assertIs(inplace, result.inplace, "inplace") + self.assertIs(backup, result.backup, "backup") + self.assertIs(bufsize, result.bufsize, "bufsize") + self.assertIs(mode, result.mode, "mode") + self.assertIs(openhook, result.openhook, "openhook") + +class Test_fileinput_close(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.close()""" + + def test_state_is_None(self): + """Tests that fileinput.close() does nothing if fileinput._state + is None""" + fileinput._state = None + fileinput.close() + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests that fileinput.close() invokes close() on fileinput._state + and sets _state=None""" + instance = MockFileInput() + fileinput._state = instance + fileinput.close() + self.assertExactlyOneInvocation(instance, "close") + self.assertIsNone(fileinput._state) + +class Test_fileinput_nextfile(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.nextfile()""" + + def test_state_is_None(self): + """Tests fileinput.nextfile() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.nextfile() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.nextfile() when fileinput._state is not None. + Ensure that it invokes fileinput._state.nextfile() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + nextfile_retval = object() + instance = MockFileInput() + instance.return_values["nextfile"] = nextfile_retval + fileinput._state = instance + retval = fileinput.nextfile() + self.assertExactlyOneInvocation(instance, "nextfile") + self.assertIs(retval, nextfile_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_filename(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.filename()""" + + def test_state_is_None(self): + """Tests fileinput.filename() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.filename() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.filename() when fileinput._state is not None. + Ensure that it invokes fileinput._state.filename() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + filename_retval = object() + instance = MockFileInput() + instance.return_values["filename"] = filename_retval + fileinput._state = instance + retval = fileinput.filename() + self.assertExactlyOneInvocation(instance, "filename") + self.assertIs(retval, filename_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_lineno(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.lineno()""" + + def test_state_is_None(self): + """Tests fileinput.lineno() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.lineno() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.lineno() when fileinput._state is not None. + Ensure that it invokes fileinput._state.lineno() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + lineno_retval = object() + instance = MockFileInput() + instance.return_values["lineno"] = lineno_retval + fileinput._state = instance + retval = fileinput.lineno() + self.assertExactlyOneInvocation(instance, "lineno") + self.assertIs(retval, lineno_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_filelineno(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.filelineno()""" + + def test_state_is_None(self): + """Tests fileinput.filelineno() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.filelineno() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.filelineno() when fileinput._state is not None. + Ensure that it invokes fileinput._state.filelineno() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + filelineno_retval = object() + instance = MockFileInput() + instance.return_values["filelineno"] = filelineno_retval + fileinput._state = instance + retval = fileinput.filelineno() + self.assertExactlyOneInvocation(instance, "filelineno") + self.assertIs(retval, filelineno_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_fileno(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.fileno()""" + + def test_state_is_None(self): + """Tests fileinput.fileno() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.fileno() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.fileno() when fileinput._state is not None. + Ensure that it invokes fileinput._state.fileno() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + fileno_retval = object() + instance = MockFileInput() + instance.return_values["fileno"] = fileno_retval + instance.fileno_retval = fileno_retval + fileinput._state = instance + retval = fileinput.fileno() + self.assertExactlyOneInvocation(instance, "fileno") + self.assertIs(retval, fileno_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_isfirstline(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.isfirstline()""" + + def test_state_is_None(self): + """Tests fileinput.isfirstline() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.isfirstline() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.isfirstline() when fileinput._state is not None. + Ensure that it invokes fileinput._state.isfirstline() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + isfirstline_retval = object() + instance = MockFileInput() + instance.return_values["isfirstline"] = isfirstline_retval + fileinput._state = instance + retval = fileinput.isfirstline() + self.assertExactlyOneInvocation(instance, "isfirstline") + self.assertIs(retval, isfirstline_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_isstdin(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.isstdin()""" + + def test_state_is_None(self): + """Tests fileinput.isstdin() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.isstdin() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.isstdin() when fileinput._state is not None. + Ensure that it invokes fileinput._state.isstdin() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + isstdin_retval = object() + instance = MockFileInput() + instance.return_values["isstdin"] = isstdin_retval + fileinput._state = instance + retval = fileinput.isstdin() + self.assertExactlyOneInvocation(instance, "isstdin") + self.assertIs(retval, isstdin_retval) + self.assertIs(fileinput._state, instance) + +class InvocationRecorder: + def __init__(self): + self.invocation_count = 0 + def __call__(self, *args, **kwargs): + self.invocation_count += 1 + self.last_invocation = (args, kwargs) + +class Test_hook_compressed(unittest.TestCase): + """Unit tests for fileinput.hook_compressed()""" + + def setUp(self): + self.fake_open = InvocationRecorder() + + def test_empty_string(self): + self.do_test_use_builtin_open("", 1) + + def test_no_ext(self): + self.do_test_use_builtin_open("abcd", 2) + + @unittest.skipUnless(gzip, "Requires gzip and zlib") + def test_gz_ext_fake(self): + original_open = gzip.open + gzip.open = self.fake_open + try: + result = fileinput.hook_compressed("test.gz", 3) + finally: + gzip.open = original_open + + self.assertEqual(self.fake_open.invocation_count, 1) + self.assertEqual(self.fake_open.last_invocation, (("test.gz", 3), {})) + + @unittest.skipUnless(bz2, "Requires bz2") + def test_bz2_ext_fake(self): + original_open = bz2.BZ2File + bz2.BZ2File = self.fake_open + try: + result = fileinput.hook_compressed("test.bz2", 4) + finally: + bz2.BZ2File = original_open + + self.assertEqual(self.fake_open.invocation_count, 1) + self.assertEqual(self.fake_open.last_invocation, (("test.bz2", 4), {})) + + def test_blah_ext(self): + self.do_test_use_builtin_open("abcd.blah", 5) + + def test_gz_ext_builtin(self): + self.do_test_use_builtin_open("abcd.Gz", 6) + + def test_bz2_ext_builtin(self): + self.do_test_use_builtin_open("abcd.Bz2", 7) + + def do_test_use_builtin_open(self, filename, mode): + original_open = self.replace_builtin_open(self.fake_open) + try: + result = fileinput.hook_compressed(filename, mode) + finally: + self.replace_builtin_open(original_open) + + self.assertEqual(self.fake_open.invocation_count, 1) + self.assertEqual(self.fake_open.last_invocation, + ((filename, mode), {})) + + @staticmethod + def replace_builtin_open(new_open_func): + builtins_type = type(__builtins__) + if builtins_type is dict: + original_open = __builtins__["open"] + __builtins__["open"] = new_open_func + elif builtins_type is types.ModuleType: + original_open = __builtins__.open + __builtins__.open = new_open_func + else: + raise RuntimeError( + "unknown __builtins__ type: %r (unable to replace open)" % + builtins_type) + + return original_open + +class Test_hook_encoded(unittest.TestCase): + """Unit tests for fileinput.hook_encoded()""" + + def test(self): + encoding = object() + result = fileinput.hook_encoded(encoding) + + fake_open = InvocationRecorder() + original_open = codecs.open + codecs.open = fake_open + try: + filename = object() + mode = object() + open_result = result(filename, mode) + finally: + codecs.open = original_open + + self.assertEqual(fake_open.invocation_count, 1) + + args = fake_open.last_invocation[0] + self.assertIs(args[0], filename) + self.assertIs(args[1], mode) + self.assertIs(args[2], encoding) def test_main(): - run_unittest(BufferSizesTests, FileInputTests) + run_unittest( + BufferSizesTests, + FileInputTests, + Test_fileinput_input, + Test_fileinput_close, + Test_fileinput_nextfile, + Test_fileinput_filename, + Test_fileinput_lineno, + Test_fileinput_filelineno, + Test_fileinput_fileno, + Test_fileinput_isfirstline, + Test_fileinput_isstdin, + Test_hook_compressed, + Test_hook_encoded, + ) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 1968b8ae78..4e6f854697 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -88,7 +88,7 @@ class GeneralFloatCases(unittest.TestCase): self.assertRaises(ValueError, float, " -0x3.p-1 ") self.assertRaises(ValueError, float, " +0x3.p-1 ") self.assertEqual(float(" 25.e-1 "), 2.5) - self.assertEqual(support.fcmp(float(" .25e-1 "), .025), 0) + self.assertAlmostEqual(float(" .25e-1 "), .025) def test_floatconversion(self): # Make sure that calls to __float__() work properly diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 5b69b0abbc..f73b8af517 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -22,10 +22,25 @@ from test.support import HOST threading = support.import_module('threading') # the dummy data returned by server over the data channel when -# RETR, LIST and NLST commands are issued +# RETR, LIST, NLST, MLSD commands are issued RETR_DATA = 'abcde12345\r\n' * 1000 LIST_DATA = 'foo\r\nbar\r\n' NLST_DATA = 'foo\r\nbar\r\n' +MLSD_DATA = ("type=cdir;perm=el;unique==keVO1+ZF4; test\r\n" + "type=pdir;perm=e;unique==keVO1+d?3; ..\r\n" + "type=OS.unix=slink:/foobar;perm=;unique==keVO1+4G4; foobar\r\n" + "type=OS.unix=chr-13/29;perm=;unique==keVO1+5G4; device\r\n" + "type=OS.unix=blk-11/108;perm=;unique==keVO1+6G4; block\r\n" + "type=file;perm=awr;unique==keVO1+8G4; writable\r\n" + "type=dir;perm=cpmel;unique==keVO1+7G4; promiscuous\r\n" + "type=dir;perm=;unique==keVO1+1t2; no-exec\r\n" + "type=file;perm=r;unique==keVO1+EG4; two words\r\n" + "type=file;perm=r;unique==keVO1+IH4; leading space\r\n" + "type=file;perm=r;unique==keVO1+1G4; file1\r\n" + "type=dir;perm=cpmel;unique==keVO1+7G4; incoming\r\n" + "type=file;perm=r;unique==keVO1+1G4; file2\r\n" + "type=file;perm=r;unique==keVO1+1G4; file3\r\n" + "type=file;perm=r;unique==keVO1+1G4; file4\r\n") class DummyDTPHandler(asynchat.async_chat): @@ -49,6 +64,11 @@ class DummyDTPHandler(asynchat.async_chat): self.dtp_conn_closed = True def push(self, what): + if self.baseclass.next_data is not None: + what = self.baseclass.next_data + self.baseclass.next_data = None + if not what: + return self.close_when_done() super(DummyDTPHandler, self).push(what.encode('ascii')) def handle_error(self): @@ -69,6 +89,7 @@ class DummyFTPHandler(asynchat.async_chat): self.last_received_cmd = None self.last_received_data = '' self.next_response = '' + self.next_data = None self.rest = None self.push('220 welcome') @@ -104,7 +125,7 @@ class DummyFTPHandler(asynchat.async_chat): addr = list(map(int, arg.split(','))) ip = '%d.%d.%d.%d' %tuple(addr[:4]) port = (addr[4] * 256) + addr[5] - s = socket.create_connection((ip, port), timeout=10) + s = socket.create_connection((ip, port), timeout=2) self.dtp = self.dtp_handler(s, baseclass=self) self.push('200 active data connection established') @@ -122,7 +143,7 @@ class DummyFTPHandler(asynchat.async_chat): def cmd_eprt(self, arg): af, ip, port = arg.split(arg[0])[1:-1] port = int(port) - s = socket.create_connection((ip, port), timeout=10) + s = socket.create_connection((ip, port), timeout=2) self.dtp = self.dtp_handler(s, baseclass=self) self.push('200 active data connection established') @@ -213,6 +234,14 @@ class DummyFTPHandler(asynchat.async_chat): self.dtp.push(NLST_DATA) self.dtp.close_when_done() + def cmd_opts(self, arg): + self.push('200 opts ok') + + def cmd_mlsd(self, arg): + self.push('125 mlsd ok') + self.dtp.push(MLSD_DATA) + self.dtp.close_when_done() + class DummyFTPServer(asyncore.dispatcher, threading.Thread): @@ -274,11 +303,11 @@ if ssl is not None: _ssl_closing = False def secure_connection(self): - self.del_channel() socket = ssl.wrap_socket(self.socket, suppress_ragged_eofs=False, certfile=CERTFILE, server_side=True, do_handshake_on_connect=False, ssl_version=ssl.PROTOCOL_SSLv23) + self.del_channel() self.set_socket(socket) self._ssl_accepting = True @@ -313,7 +342,10 @@ if ssl is not None: # http://www.mail-archive.com/openssl-users@openssl.org/msg60710.html pass self._ssl_closing = False - super(SSLConnection, self).close() + if getattr(self, '_ccc', False) == False: + super(SSLConnection, self).close() + else: + pass def handle_read_event(self): if self._ssl_accepting: @@ -381,12 +413,18 @@ if ssl is not None: def __init__(self, conn): DummyFTPHandler.__init__(self, conn) self.secure_data_channel = False + self._ccc = False def cmd_auth(self, line): """Set up secure control channel.""" self.push('234 AUTH TLS successful') self.secure_connection() + def cmd_ccc(self, line): + self.push('220 Reverting back to clear-text') + self._ccc = True + self._do_ssl_shutdown() + def cmd_pbsz(self, line): """Negotiate size of buffer for secure data transfer. For TLS/SSL the only valid value for the parameter is '0'. @@ -416,7 +454,7 @@ class TestFTPClass(TestCase): def setUp(self): self.server = DummyFTPServer((HOST, 0)) self.server.start() - self.client = ftplib.FTP(timeout=10) + self.client = ftplib.FTP(timeout=2) self.client.connect(self.server.host, self.server.port) def tearDown(self): @@ -558,6 +596,64 @@ class TestFTPClass(TestCase): self.client.dir(lambda x: l.append(x)) self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', '')) + def test_mlsd(self): + list(self.client.mlsd()) + list(self.client.mlsd(path='/')) + list(self.client.mlsd(path='/', facts=['size', 'type'])) + + ls = list(self.client.mlsd()) + for name, facts in ls: + self.assertIsInstance(name, str) + self.assertIsInstance(facts, dict) + self.assertTrue(name) + self.assertIn('type', facts) + self.assertIn('perm', facts) + self.assertIn('unique', facts) + + def set_data(data): + self.server.handler_instance.next_data = data + + def test_entry(line, type=None, perm=None, unique=None, name=None): + type = 'type' if type is None else type + perm = 'perm' if perm is None else perm + unique = 'unique' if unique is None else unique + name = 'name' if name is None else name + set_data(line) + _name, facts = next(self.client.mlsd()) + self.assertEqual(_name, name) + self.assertEqual(facts['type'], type) + self.assertEqual(facts['perm'], perm) + self.assertEqual(facts['unique'], unique) + + # plain + test_entry('type=type;perm=perm;unique=unique; name\r\n') + # "=" in fact value + test_entry('type=ty=pe;perm=perm;unique=unique; name\r\n', type="ty=pe") + test_entry('type==type;perm=perm;unique=unique; name\r\n', type="=type") + test_entry('type=t=y=pe;perm=perm;unique=unique; name\r\n', type="t=y=pe") + test_entry('type=====;perm=perm;unique=unique; name\r\n', type="====") + # spaces in name + test_entry('type=type;perm=perm;unique=unique; na me\r\n', name="na me") + test_entry('type=type;perm=perm;unique=unique; name \r\n', name="name ") + test_entry('type=type;perm=perm;unique=unique; name\r\n', name=" name") + test_entry('type=type;perm=perm;unique=unique; n am e\r\n', name="n am e") + # ";" in name + test_entry('type=type;perm=perm;unique=unique; na;me\r\n', name="na;me") + test_entry('type=type;perm=perm;unique=unique; ;name\r\n', name=";name") + test_entry('type=type;perm=perm;unique=unique; ;name;\r\n', name=";name;") + test_entry('type=type;perm=perm;unique=unique; ;;;;\r\n', name=";;;;") + # case sensitiveness + set_data('Type=type;TyPe=perm;UNIQUE=unique; name\r\n') + _name, facts = next(self.client.mlsd()) + for x in facts: + self.assertTrue(x.islower()) + # no data (directory empty) + set_data('') + self.assertRaises(StopIteration, next, self.client.mlsd()) + set_data('') + for x in self.client.mlsd(): + self.fail("unexpected data %s" % data) + def test_makeport(self): with self.client.makeport(): # IPv4 is in use, just make sure send_eprt has not been used @@ -584,7 +680,7 @@ class TestFTPClass(TestCase): return True # base test - with ftplib.FTP(timeout=10) as self.client: + with ftplib.FTP(timeout=2) as self.client: self.client.connect(self.server.host, self.server.port) self.client.sendcmd('noop') self.assertTrue(is_client_connected()) @@ -592,7 +688,7 @@ class TestFTPClass(TestCase): self.assertFalse(is_client_connected()) # QUIT sent inside the with block - with ftplib.FTP(timeout=10) as self.client: + with ftplib.FTP(timeout=2) as self.client: self.client.connect(self.server.host, self.server.port) self.client.sendcmd('noop') self.client.quit() @@ -602,7 +698,7 @@ class TestFTPClass(TestCase): # force a wrong response code to be sent on QUIT: error_perm # is expected and the connection is supposed to be closed try: - with ftplib.FTP(timeout=10) as self.client: + with ftplib.FTP(timeout=2) as self.client: self.client.connect(self.server.host, self.server.port) self.client.sendcmd('noop') self.server.handler_instance.next_response = '550 error on quit' @@ -616,6 +712,30 @@ class TestFTPClass(TestCase): self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit') self.assertFalse(is_client_connected()) + def test_source_address(self): + self.client.quit() + port = support.find_unused_port() + try: + self.client.connect(self.server.host, self.server.port, + source_address=(HOST, port)) + self.assertEqual(self.client.sock.getsockname()[1], port) + self.client.quit() + except IOError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to port %d" % port) + raise + + def test_source_address_passive_connection(self): + port = support.find_unused_port() + self.client.source_address = (HOST, port) + try: + with self.client.transfercmd('list') as sock: + self.assertEqual(sock.getsockname()[1], port) + except IOError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to port %d" % port) + raise + def test_parse257(self): self.assertEqual(ftplib.parse257('257 "/foo/bar"'), '/foo/bar') self.assertEqual(ftplib.parse257('257 "/foo/bar" created'), '/foo/bar') @@ -632,7 +752,7 @@ class TestFTPClass(TestCase): class TestIPv6Environment(TestCase): def setUp(self): - self.server = DummyFTPServer((HOST, 0), af=socket.AF_INET6) + self.server = DummyFTPServer(('::1', 0), af=socket.AF_INET6) self.server.start() self.client = ftplib.FTP() self.client.connect(self.server.host, self.server.port) @@ -676,7 +796,7 @@ class TestTLS_FTPClassMixin(TestFTPClass): def setUp(self): self.server = DummyTLS_FTPServer((HOST, 0)) self.server.start() - self.client = ftplib.FTP_TLS(timeout=10) + self.client = ftplib.FTP_TLS(timeout=2) self.client.connect(self.server.host, self.server.port) # enable TLS self.client.auth() @@ -689,7 +809,7 @@ class TestTLS_FTPClass(TestCase): def setUp(self): self.server = DummyTLS_FTPServer((HOST, 0)) self.server.start() - self.client = ftplib.FTP_TLS(timeout=10) + self.client = ftplib.FTP_TLS(timeout=2) self.client.connect(self.server.host, self.server.port) def tearDown(self): @@ -749,7 +869,7 @@ class TestTLS_FTPClass(TestCase): self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE, keyfile=CERTFILE, context=ctx) - self.client = ftplib.FTP_TLS(context=ctx, timeout=10) + self.client = ftplib.FTP_TLS(context=ctx, timeout=2) self.client.connect(self.server.host, self.server.port) self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) self.client.auth() @@ -761,6 +881,13 @@ class TestTLS_FTPClass(TestCase): self.assertIs(sock.context, ctx) self.assertIsInstance(sock, ssl.SSLSocket) + def test_ccc(self): + self.assertRaises(ValueError, self.client.ccc) + self.client.login(secure=True) + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + self.client.ccc() + self.assertRaises(ValueError, self.client.sock.unwrap) + class TestTimeouts(TestCase): @@ -857,13 +984,8 @@ class TestTimeouts(TestCase): def test_main(): tests = [TestFTPClass, TestTimeouts] - if socket.has_ipv6: - try: - DummyFTPServer((HOST, 0), af=socket.AF_INET6) - except socket.error: - pass - else: - tests.append(TestIPv6Environment) + if support.IPV6_ENABLED: + tests.append(TestIPv6Environment) if ssl is not None: tests.extend([TestTLS_FTPClassMixin, TestTLS_FTPClass]) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 7d11b53a71..97d7524f59 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -436,19 +436,82 @@ class TestReduce(unittest.TestCase): self.assertEqual(self.func(add, d), "".join(d.keys())) class TestCmpToKey(unittest.TestCase): + def test_cmp_to_key(self): + def cmp1(x, y): + return (x > y) - (x < y) + key = functools.cmp_to_key(cmp1) + self.assertEqual(key(3), key(3)) + self.assertGreater(key(3), key(1)) + def cmp2(x, y): + return int(x) - int(y) + key = functools.cmp_to_key(cmp2) + self.assertEqual(key(4.0), key('4')) + self.assertLess(key(2), key('35')) + + def test_cmp_to_key_arguments(self): + def cmp1(x, y): + return (x > y) - (x < y) + key = functools.cmp_to_key(mycmp=cmp1) + self.assertEqual(key(obj=3), key(obj=3)) + self.assertGreater(key(obj=3), key(obj=1)) + with self.assertRaises((TypeError, AttributeError)): + key(3) > 1 # rhs is not a K object + with self.assertRaises((TypeError, AttributeError)): + 1 < key(3) # lhs is not a K object + with self.assertRaises(TypeError): + key = functools.cmp_to_key() # too few args + with self.assertRaises(TypeError): + key = functools.cmp_to_key(cmp1, None) # too many args + key = functools.cmp_to_key(cmp1) + with self.assertRaises(TypeError): + key() # too few args + with self.assertRaises(TypeError): + key(None, None) # too many args + + def test_bad_cmp(self): + def cmp1(x, y): + raise ZeroDivisionError + key = functools.cmp_to_key(cmp1) + with self.assertRaises(ZeroDivisionError): + key(3) > key(1) + + class BadCmp: + def __lt__(self, other): + raise ZeroDivisionError + def cmp1(x, y): + return BadCmp() + with self.assertRaises(ZeroDivisionError): + key(3) > key(1) + + def test_obj_field(self): + def cmp1(x, y): + return (x > y) - (x < y) + key = functools.cmp_to_key(mycmp=cmp1) + self.assertEqual(key(50).obj, 50) + + def test_sort_int(self): def mycmp(x, y): return y - x self.assertEqual(sorted(range(5), key=functools.cmp_to_key(mycmp)), [4, 3, 2, 1, 0]) + def test_sort_int_str(self): + def mycmp(x, y): + x, y = int(x), int(y) + return (x > y) - (x < y) + values = [5, '3', 7, 2, '0', '1', 4, '10', 1] + values = sorted(values, key=functools.cmp_to_key(mycmp)) + self.assertEqual([int(value) for value in values], + [0, 1, 1, 2, 3, 4, 5, 7, 10]) + def test_hash(self): def mycmp(x, y): return y - x key = functools.cmp_to_key(mycmp) k = key(10) self.assertRaises(TypeError, hash, k) - self.assertFalse(isinstance(k, collections.Hashable)) + self.assertNotIsInstance(k, collections.Hashable) class TestTotalOrdering(unittest.TestCase): diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index c6689a1a18..3a25eb1565 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -13,14 +13,14 @@ def get_error_location(msg): class FutureTest(unittest.TestCase): def test_future1(self): - support.unload('test_future1') - from test import test_future1 - self.assertEqual(test_future1.result, 6) + support.unload('future_test1') + from test import future_test1 + self.assertEqual(future_test1.result, 6) def test_future2(self): - support.unload('test_future2') - from test import test_future2 - self.assertEqual(test_future2.result, 6) + support.unload('future_test2') + from test import future_test2 + self.assertEqual(future_test2.result, 6) def test_future3(self): support.unload('test_future3') diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 100c76729c..19313db796 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,5 +1,6 @@ import unittest -from test.support import verbose, run_unittest, strip_python_stderr +from test.support import (verbose, refcount_test, run_unittest, + strip_python_stderr) import sys import gc import weakref @@ -175,6 +176,7 @@ class GCTests(unittest.TestCase): del d self.assertEqual(gc.collect(), 2) + @refcount_test def test_frame(self): def f(): frame = sys._getframe() @@ -242,6 +244,7 @@ class GCTests(unittest.TestCase): # For example, disposed tuples are not freed, but reused. # To minimize variations, though, we first store the get_count() results # and check them at the end. + @refcount_test def test_get_count(self): gc.collect() a, b, c = gc.get_count() @@ -255,6 +258,7 @@ class GCTests(unittest.TestCase): # created (the list). self.assertGreater(d, a) + @refcount_test def test_collect_generations(self): gc.collect() # This object will "trickle" into generation N + 1 after diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py index 1f46af13e3..413043c5a3 100644 --- a/Lib/test/test_genexps.py +++ b/Lib/test/test_genexps.py @@ -257,11 +257,15 @@ Verify that genexps are weakly referencable """ +import sys -__test__ = {'doctests' : doctests} +# Trace function can throw off the tuple reuse test. +if hasattr(sys, 'gettrace') and sys.gettrace(): + __test__ = {} +else: + __test__ = {'doctests' : doctests} def test_main(verbose=None): - import sys from test import support from test import test_genexps support.run_doctest(test_genexps, verbose) diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py index 3d9c06a460..768ea8d486 100644 --- a/Lib/test/test_getargs2.py +++ b/Lib/test/test_getargs2.py @@ -294,6 +294,15 @@ class Keywords_TestCase(unittest.TestCase): self.fail('TypeError should have been raised') class Bytes_TestCase(unittest.TestCase): + def test_c(self): + from _testcapi import getargs_c + self.assertRaises(TypeError, getargs_c, b'abc') # len > 1 + self.assertEqual(getargs_c(b'a'), b'a') + self.assertEqual(getargs_c(bytearray(b'a')), b'a') + self.assertRaises(TypeError, getargs_c, memoryview(b'a')) + self.assertRaises(TypeError, getargs_c, 's') + self.assertRaises(TypeError, getargs_c, None) + def test_s(self): from _testcapi import getargs_s self.assertEqual(getargs_s('abc\xe9'), b'abc\xc3\xa9') diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 1560a6bbf0..6ee08db0b6 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -1,5 +1,6 @@ import unittest -from test.support import run_unittest, TESTFN, skip_unless_symlink, can_symlink +from test.support import (run_unittest, TESTFN, skip_unless_symlink, + can_symlink, create_empty_file) import glob import os import shutil @@ -14,8 +15,7 @@ class GlobTests(unittest.TestCase): base, file = os.path.split(filename) if not os.path.exists(base): os.makedirs(base) - f = open(filename, 'w') - f.close() + create_empty_file(filename) def setUp(self): self.tempdir = TESTFN+"_dir" diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 329b25807f..32dc15ed32 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -10,7 +10,7 @@ from sys import * class TokenTests(unittest.TestCase): - def testBackslash(self): + def test_backslash(self): # Backslash means line continuation: x = 1 \ + 1 @@ -20,7 +20,7 @@ class TokenTests(unittest.TestCase): x = 0 self.assertEqual(x, 0, 'backslash ending comment') - def testPlainIntegers(self): + def test_plain_integers(self): self.assertEqual(type(000), type(0)) self.assertEqual(0xff, 255) self.assertEqual(0o377, 255) @@ -56,7 +56,7 @@ class TokenTests(unittest.TestCase): else: self.fail('Weird maxsize value %r' % maxsize) - def testLongIntegers(self): + def test_long_integers(self): x = 0 x = 0xffffffffffffffff x = 0Xffffffffffffffff @@ -66,7 +66,7 @@ class TokenTests(unittest.TestCase): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 - def testFloats(self): + def test_floats(self): x = 3.14 x = 314. x = 0.314 @@ -80,7 +80,7 @@ class TokenTests(unittest.TestCase): x = .3e14 x = 3.1e4 - def testStringLiterals(self): + def test_string_literals(self): x = ''; y = ""; self.assertTrue(len(x) == 0 and x == y) x = '\''; y = "'"; self.assertTrue(len(x) == 1 and x == y and ord(x) == 39) x = '"'; y = "\""; self.assertTrue(len(x) == 1 and x == y and ord(x) == 34) @@ -120,11 +120,18 @@ the \'lazy\' dog.\n\ ' self.assertEqual(x, y) - def testEllipsis(self): + def test_ellipsis(self): x = ... self.assertTrue(x is Ellipsis) self.assertRaises(SyntaxError, eval, ".. .") + def test_eof_error(self): + samples = ("def foo(", "\ndef foo(", "def foo(\n") + for s in samples: + with self.assertRaises(SyntaxError) as cm: + compile(s, "<test>", "exec") + self.assertIn("unexpected EOF", str(cm.exception)) + class GrammarTests(unittest.TestCase): # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE @@ -136,11 +143,11 @@ class GrammarTests(unittest.TestCase): # expr_input: testlist NEWLINE # XXX Hard to test -- used only in calls to input() - def testEvalInput(self): + def test_eval_input(self): # testlist ENDMARKER x = eval('1, 0 or 1') - def testFuncdef(self): + def test_funcdef(self): ### [decorators] 'def' NAME parameters ['->' test] ':' suite ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE ### decorators: decorator+ @@ -324,7 +331,7 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "f(*g(1=2))") check_syntax_error(self, "f(**g(1=2))") - def testLambdef(self): + def test_lambdef(self): ### lambdef: 'lambda' [varargslist] ':' test l1 = lambda : 0 self.assertEqual(l1(), 0) @@ -346,7 +353,7 @@ class GrammarTests(unittest.TestCase): ### stmt: simple_stmt | compound_stmt # Tested below - def testSimpleStmt(self): + def test_simple_stmt(self): ### simple_stmt: small_stmt (';' small_stmt)* [';'] x = 1; pass; del x def foo(): @@ -357,7 +364,7 @@ class GrammarTests(unittest.TestCase): ### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt # Tested below - def testExprStmt(self): + def test_expr_stmt(self): # (exprlist '=')* exprlist 1 1, 2, 3 @@ -370,7 +377,7 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "x + 1 = 1") check_syntax_error(self, "a + 1 = b + 2") - def testDelStmt(self): + def test_del_stmt(self): # 'del' exprlist abc = [1,2,3] x, y, z = abc @@ -379,18 +386,18 @@ class GrammarTests(unittest.TestCase): del abc del x, y, (z, xyz) - def testPassStmt(self): + def test_pass_stmt(self): # 'pass' pass # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt # Tested below - def testBreakStmt(self): + def test_break_stmt(self): # 'break' while 1: break - def testContinueStmt(self): + def test_continue_stmt(self): # 'continue' i = 1 while i: i = 0; continue @@ -442,7 +449,7 @@ class GrammarTests(unittest.TestCase): self.fail("continue then break in try/except in loop broken!") test_inner() - def testReturn(self): + def test_return(self): # 'return' [testlist] def g1(): return def g2(): return 1 @@ -450,17 +457,17 @@ class GrammarTests(unittest.TestCase): x = g2() check_syntax_error(self, "class foo:return 1") - def testYield(self): + def test_yield(self): check_syntax_error(self, "class foo:yield 1") - def testRaise(self): + def test_raise(self): # 'raise' test [',' test] try: raise RuntimeError('just testing') except RuntimeError: pass try: raise KeyboardInterrupt except KeyboardInterrupt: pass - def testImport(self): + def test_import(self): # 'import' dotted_as_names import sys import time, sys @@ -473,13 +480,13 @@ class GrammarTests(unittest.TestCase): from sys import (path, argv) from sys import (path, argv,) - def testGlobal(self): + def test_global(self): # 'global' NAME (',' NAME)* global a global a, b global one, two, three, four, five, six, seven, eight, nine, ten - def testNonlocal(self): + def test_nonlocal(self): # 'nonlocal' NAME (',' NAME)* x = 0 y = 0 @@ -487,7 +494,7 @@ class GrammarTests(unittest.TestCase): nonlocal x nonlocal x, y - def testAssert(self): + def test_assert(self): # assertTruestmt: 'assert' test [',' test] assert 1 assert 1, 1 @@ -504,7 +511,7 @@ class GrammarTests(unittest.TestCase): ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef # Tested below - def testIf(self): + def test_if(self): # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] if 1: pass if 1: pass @@ -517,7 +524,7 @@ class GrammarTests(unittest.TestCase): elif 0: pass else: pass - def testWhile(self): + def test_while(self): # 'while' test ':' suite ['else' ':' suite] while 0: pass while 0: pass @@ -532,7 +539,7 @@ class GrammarTests(unittest.TestCase): x = 2 self.assertEqual(x, 2) - def testFor(self): + def test_for(self): # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] for i in 1, 2, 3: pass for i, j, k in (): pass @@ -559,7 +566,7 @@ class GrammarTests(unittest.TestCase): result.append(x) self.assertEqual(result, [1, 2, 3]) - def testTry(self): + def test_try(self): ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] ### | 'try' ':' suite 'finally' ':' suite ### except_clause: 'except' [expr ['as' expr]] @@ -582,7 +589,7 @@ class GrammarTests(unittest.TestCase): try: pass finally: pass - def testSuite(self): + def test_suite(self): # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT if 1: pass if 1: @@ -597,7 +604,7 @@ class GrammarTests(unittest.TestCase): pass # - def testTest(self): + def test_test(self): ### and_test ('or' and_test)* ### and_test: not_test ('and' not_test)* ### not_test: 'not' not_test | comparison @@ -608,7 +615,7 @@ class GrammarTests(unittest.TestCase): if not 1 and 1 and 1: pass if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass - def testComparison(self): + def test_comparison(self): ### comparison: expr (comp_op expr)* ### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not' if 1: pass @@ -625,36 +632,36 @@ class GrammarTests(unittest.TestCase): if 1 not in (): pass if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass - def testBinaryMaskOps(self): + def test_binary_mask_ops(self): x = 1 & 1 x = 1 ^ 1 x = 1 | 1 - def testShiftOps(self): + def test_shift_ops(self): x = 1 << 1 x = 1 >> 1 x = 1 << 1 >> 1 - def testAdditiveOps(self): + def test_additive_ops(self): x = 1 x = 1 + 1 x = 1 - 1 - 1 x = 1 - 1 + 1 - 1 + 1 - def testMultiplicativeOps(self): + def test_multiplicative_ops(self): x = 1 * 1 x = 1 / 1 x = 1 % 1 x = 1 / 1 * 1 % 1 - def testUnaryOps(self): + def test_unary_ops(self): x = +1 x = -1 x = ~1 x = ~1 ^ 1 & 1 | 1 & 1 ^ -1 x = -1*1/1 + 1*1 - ---1*1 - def testSelectors(self): + def test_selectors(self): ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME ### subscript: expr | [expr] ':' [expr] @@ -684,7 +691,7 @@ class GrammarTests(unittest.TestCase): L.sort(key=lambda x: x if isinstance(x, tuple) else ()) self.assertEqual(str(L), '[1, (1,), (1, 2), (1, 2, 3)]') - def testAtoms(self): + def test_atoms(self): ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING ### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [',']) @@ -719,7 +726,7 @@ class GrammarTests(unittest.TestCase): ### testlist: test (',' test)* [','] # These have been exercised enough above - def testClassdef(self): + def test_classdef(self): # 'class' NAME ['(' [testlist] ')'] ':' suite class B: pass class B2(): pass @@ -738,14 +745,14 @@ class GrammarTests(unittest.TestCase): @class_decorator class G: pass - def testDictcomps(self): + def test_dictcomps(self): # dictorsetmaker: ( (test ':' test (comp_for | # (',' test ':' test)* [','])) | # (test (comp_for | (',' test)* [','])) ) nums = [1, 2, 3] self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4}) - def testListcomps(self): + def test_listcomps(self): # list comprehension tests nums = [1, 2, 3, 4, 5] strs = ["Apple", "Banana", "Coconut"] @@ -808,7 +815,7 @@ class GrammarTests(unittest.TestCase): self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'), ('Macdonalds', 'Cheeseburger')]) - def testGenexps(self): + def test_genexps(self): # generator expression tests g = ([x for x in range(10)] for x in range(1)) self.assertEqual(next(g), [x for x in range(10)]) @@ -843,7 +850,7 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "foo(x for x in range(10), 100)") check_syntax_error(self, "foo(100, x for x in range(10))") - def testComprehensionSpecials(self): + def test_comprehension_specials(self): # test for outmost iterable precomputation x = 10; g = (i for i in range(x)); x = 5 self.assertEqual(len(list(g)), 10) @@ -882,7 +889,7 @@ class GrammarTests(unittest.TestCase): with manager() as x, manager(): pass - def testIfElseExpr(self): + def test_if_else_expr(self): # Test ifelse expressions in various cases def _checkeval(msg, ret): "helper to check that evaluation of expressions is done correctly" diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 2b0ac36c23..3ea5c41147 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -64,6 +64,21 @@ class TestGzip(unittest.TestCase): d = f.read() self.assertEqual(d, data1*50) + def test_read1(self): + self.test_write() + blocks = [] + nread = 0 + with gzip.GzipFile(self.filename, 'r') as f: + while True: + d = f.read1() + if not d: + break + blocks.append(d) + nread += len(d) + # Check that position was updated correctly (see issue10791). + self.assertEqual(f.tell(), nread) + self.assertEqual(b''.join(blocks), data1 * 50) + def test_io_on_closed_object(self): # Test that I/O operations on closed GzipFile objects raise a # ValueError, just like the corresponding functions on file objects. @@ -323,6 +338,14 @@ class TestGzip(unittest.TestCase): self.assertEqual(f.read(100), b'') self.assertEqual(nread, len(uncompressed)) + def test_textio_readlines(self): + # Issue #10791: TextIOWrapper.readlines() fails when wrapping GzipFile. + lines = (data1 * 50).decode("ascii").splitlines(True) + self.test_write() + with gzip.GzipFile(self.filename, 'r') as f: + with io.TextIOWrapper(f, encoding="ascii") as t: + self.assertEqual(t.readlines(), lines) + # Testing compress/decompress shortcut functions def test_compress(self): diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index ce9e34638a..066a6b7e92 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -506,8 +506,7 @@ class HTTPSTest(TestCase): def test_local_good_hostname(self): # The (valid) cert validates the HTTP hostname import ssl - from test.ssl_servers import make_https_server - server = make_https_server(self, CERT_localhost) + server = self.make_server(CERT_localhost) context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) context.verify_mode = ssl.CERT_REQUIRED context.load_verify_locations(CERT_localhost) @@ -515,12 +514,12 @@ class HTTPSTest(TestCase): h.request('GET', '/nonexistent') resp = h.getresponse() self.assertEqual(resp.status, 404) + del server def test_local_bad_hostname(self): # The (valid) cert doesn't validate the HTTP hostname import ssl - from test.ssl_servers import make_https_server - server = make_https_server(self, CERT_fakehostname) + server = self.make_server(CERT_fakehostname) context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) context.verify_mode = ssl.CERT_REQUIRED context.load_verify_locations(CERT_fakehostname) @@ -538,6 +537,7 @@ class HTTPSTest(TestCase): h.request('GET', '/nonexistent') resp = h.getresponse() self.assertEqual(resp.status, 404) + del server class RequestBodyTest(TestCase): diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 39ebc26338..1bbaf0ece3 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -461,6 +461,23 @@ class RejectingSocketlessRequestHandler(SocketlessRequestHandler): self.send_error(417) return False + +class AuditableBytesIO: + + def __init__(self): + self.datas = [] + + def write(self, data): + self.datas.append(data) + + def getData(self): + return b''.join(self.datas) + + @property + def numWrites(self): + return len(self.datas) + + class BaseHTTPRequestHandlerTestCase(unittest.TestCase): """Test the functionality of the BaseHTTPServer. @@ -527,27 +544,49 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): self.verify_get_called() self.assertEqual(result[-1], b'<html><body>Data</body></html>\r\n') - def test_header_buffering(self): + def test_header_buffering_of_send_error(self): - def _readAndReseek(f): - pos = f.tell() - f.seek(0) - data = f.read() - f.seek(pos) - return data + input = BytesIO(b'GET / HTTP/1.1\r\n\r\n') + output = AuditableBytesIO() + handler = SocketlessRequestHandler() + handler.rfile = input + handler.wfile = output + handler.request_version = 'HTTP/1.1' + handler.requestline = '' + handler.command = None + + handler.send_error(418) + self.assertEqual(output.numWrites, 2) + + def test_header_buffering_of_send_response_only(self): input = BytesIO(b'GET / HTTP/1.1\r\n\r\n') - output = BytesIO() - self.handler.rfile = input - self.handler.wfile = output - self.handler.request_version = 'HTTP/1.1' + output = AuditableBytesIO() + handler = SocketlessRequestHandler() + handler.rfile = input + handler.wfile = output + handler.request_version = 'HTTP/1.1' - self.handler.send_header('Foo', 'foo') - self.handler.send_header('bar', 'bar') - self.assertEqual(_readAndReseek(output), b'') - self.handler.end_headers() - self.assertEqual(_readAndReseek(output), - b'Foo: foo\r\nbar: bar\r\n\r\n') + handler.send_response_only(418) + self.assertEqual(output.numWrites, 0) + handler.end_headers() + self.assertEqual(output.numWrites, 1) + + def test_header_buffering_of_send_header(self): + + input = BytesIO(b'GET / HTTP/1.1\r\n\r\n') + output = AuditableBytesIO() + handler = SocketlessRequestHandler() + handler.rfile = input + handler.wfile = output + handler.request_version = 'HTTP/1.1' + + handler.send_header('Foo', 'foo') + handler.send_header('bar', 'bar') + self.assertEqual(output.numWrites, 0) + handler.end_headers() + self.assertEqual(output.getData(), b'Foo: foo\r\nbar: bar\r\n\r\n') + self.assertEqual(output.numWrites, 1) def test_header_unbuffered_when_continue(self): diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 80340004d7..c4c7eccb00 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -258,11 +258,58 @@ class RemoteIMAP_SSLTest(RemoteIMAPTest): port = 993 imap_class = IMAP4_SSL + def setUp(self): + pass + + def tearDown(self): + pass + + def create_ssl_context(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ssl_context.load_cert_chain(CERTFILE) + return ssl_context + + def check_logincapa(self, server): + try: + for cap in server.capabilities: + self.assertIsInstance(cap, str) + self.assertFalse('LOGINDISABLED' in server.capabilities) + self.assertTrue('AUTH=PLAIN' in server.capabilities) + rs = server.login(self.username, self.password) + self.assertEqual(rs[0], 'OK') + finally: + server.logout() + def test_logincapa(self): - for cap in self.server.capabilities: - self.assertIsInstance(cap, str) - self.assertFalse('LOGINDISABLED' in self.server.capabilities) - self.assertTrue('AUTH=PLAIN' in self.server.capabilities) + with transient_internet(self.host): + _server = self.imap_class(self.host, self.port) + self.check_logincapa(_server) + + def test_logincapa_with_client_certfile(self): + with transient_internet(self.host): + _server = self.imap_class(self.host, self.port, certfile=CERTFILE) + self.check_logincapa(_server) + + def test_logincapa_with_client_ssl_context(self): + with transient_internet(self.host): + _server = self.imap_class(self.host, self.port, ssl_context=self.create_ssl_context()) + self.check_logincapa(_server) + + def test_logout(self): + with transient_internet(self.host): + _server = self.imap_class(self.host, self.port) + rs = _server.logout() + self.assertEqual(rs[0], 'BYE') + + def test_ssl_context_certfile_exclusive(self): + with transient_internet(self.host): + self.assertRaises(ValueError, self.imap_class, self.host, self.port, + certfile=CERTFILE, ssl_context=self.create_ssl_context()) + + def test_ssl_context_keyfile_exclusive(self): + with transient_internet(self.host): + self.assertRaises(ValueError, self.imap_class, self.host, self.port, + keyfile=CERTFILE, ssl_context=self.create_ssl_context()) def test_main(): diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 551ad1b85c..3041218083 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -58,6 +58,12 @@ class ImportTests(unittest.TestCase): with imp.find_module('module_' + mod, self.test_path)[0] as fd: self.assertEqual(fd.encoding, encoding) + path = [os.path.dirname(__file__)] + self.assertRaisesRegex(SyntaxError, + r"Non-UTF-8 code starting with '\\xf6'" + r" in file .*badsyntax_pep3120.py", + imp.find_module, 'badsyntax_pep3120', path) + def test_issue1267(self): for mod, encoding, _ in self.test_strings: fp, filename, info = imp.find_module('module_' + mod, @@ -215,6 +221,10 @@ class PEP3147Tests(unittest.TestCase): self.assertEqual( imp.cache_from_source('/foo/bar/baz/qux.py', True), '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)) + # Directory with a dot, filename without dot + self.assertEqual( + imp.cache_from_source('/foo.bar/file', True), + '/foo.bar/__pycache__/file{}.pyc'.format(self.tag)) def test_cache_from_source_optimized(self): # Given the path to a .py file, return the path to its PEP 3147 @@ -314,8 +324,7 @@ class PEP3147Tests(unittest.TestCase): shutil.rmtree('pep3147') self.addCleanup(cleanup) # Touch the __init__.py file. - with open('pep3147/__init__.py', 'w'): - pass + support.create_empty_file('pep3147/__init__.py') m = __import__('pep3147') # Ensure we load the pyc file. support.forget('pep3147') diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py index 95a5f48fce..a1cebf91e5 100644 --- a/Lib/test/test_import.py +++ b/Lib/test/test_import.py @@ -14,7 +14,7 @@ import textwrap from test.support import ( EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython, make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask, - unlink, unload) + unlink, unload, create_empty_file) from test import script_helper @@ -103,7 +103,7 @@ class ImportTests(unittest.TestCase): sys.path.insert(0, os.curdir) try: fname = TESTFN + os.extsep + "py" - open(fname, 'w').close() + create_empty_file(fname) os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) __import__(TESTFN) @@ -286,8 +286,6 @@ class ImportTests(unittest.TestCase): self.skipTest('path is not encodable to {}'.format(encoding)) with self.assertRaises(ImportError) as c: __import__(path) - self.assertEqual("Import by filename is not supported.", - c.exception.args[0]) def test_import_in_del_does_not_crash(self): # Issue 4236 diff --git a/Lib/test/test_importhooks.py b/Lib/test/test_importhooks.py index ec6730e9ba..7a256570cb 100644 --- a/Lib/test/test_importhooks.py +++ b/Lib/test/test_importhooks.py @@ -51,7 +51,7 @@ class TestImporter: def __init__(self, path=test_path): if path != test_path: - # if out class is on sys.path_hooks, we must raise + # if our class is on sys.path_hooks, we must raise # ImportError for any path item that we can't handle. raise ImportError self.path = path @@ -229,7 +229,9 @@ class ImportHooksTestCase(ImportHooksBaseTestCase): i = ImpWrapper() sys.meta_path.append(i) sys.path_hooks.append(ImpWrapper) - mnames = ("colorsys", "urllib.parse", "distutils.core") + mnames = ( + "colorsys", "urllib.parse", "distutils.core", "sys", + ) for mname in mnames: parent = mname.split(".")[0] for n in list(sys.modules): @@ -237,7 +239,8 @@ class ImportHooksTestCase(ImportHooksBaseTestCase): del sys.modules[n] for mname in mnames: m = __import__(mname, globals(), locals(), ["__dummy__"]) - m.__loader__ # to make sure we actually handled the import + # to make sure we actually handled the import + self.assertTrue(hasattr(m, "__loader__")) def test_main(): diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 72c9a2d8a7..1129484533 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -46,7 +46,7 @@ except ImportError: def _default_chunk_size(): """Get the default TextIOWrapper chunk size""" - with open(__file__, "r", encoding="latin1") as f: + with open(__file__, "r", encoding="latin-1") as f: return f._CHUNK_SIZE @@ -811,6 +811,12 @@ class BufferedReaderTest(unittest.TestCase, CommonBufferedTests): self.assertEqual(b, b"gf") self.assertEqual(bufio.readinto(b), 0) self.assertEqual(b, b"gf") + rawio = self.MockRawIO((b"abc", None)) + bufio = self.tp(rawio) + self.assertEqual(bufio.readinto(b), 2) + self.assertEqual(b, b"ab") + self.assertEqual(bufio.readinto(b), 1) + self.assertEqual(b, b"cb") def test_readlines(self): def bufio(): @@ -1770,11 +1776,11 @@ class TextIOWrapperTest(unittest.TestCase): r = self.BytesIO(b"\xc3\xa9\n\n") b = self.BufferedReader(r, 1000) t = self.TextIOWrapper(b) - t.__init__(b, encoding="latin1", newline="\r\n") - self.assertEqual(t.encoding, "latin1") + t.__init__(b, encoding="latin-1", newline="\r\n") + self.assertEqual(t.encoding, "latin-1") self.assertEqual(t.line_buffering, False) - t.__init__(b, encoding="utf8", line_buffering=True) - self.assertEqual(t.encoding, "utf8") + t.__init__(b, encoding="utf-8", line_buffering=True) + self.assertEqual(t.encoding, "utf-8") self.assertEqual(t.line_buffering, True) self.assertEqual("\xe9\n", t.readline()) self.assertRaises(TypeError, t.__init__, b, newline=42) @@ -1824,8 +1830,8 @@ class TextIOWrapperTest(unittest.TestCase): def test_encoding(self): # Check the encoding attribute is always set, and valid b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="utf8") - self.assertEqual(t.encoding, "utf8") + t = self.TextIOWrapper(b, encoding="utf-8") + self.assertEqual(t.encoding, "utf-8") t = self.TextIOWrapper(b) self.assertTrue(t.encoding is not None) codecs.lookup(t.encoding) @@ -2004,7 +2010,7 @@ class TextIOWrapperTest(unittest.TestCase): def test_basic_io(self): for chunksize in (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65): - for enc in "ascii", "latin1", "utf8" :# , "utf-16-be", "utf-16-le": + for enc in "ascii", "latin-1", "utf-8" :# , "utf-16-be", "utf-16-le": f = self.open(support.TESTFN, "w+", encoding=enc) f._CHUNK_SIZE = chunksize self.assertEqual(f.write("abc"), 3) @@ -2054,7 +2060,7 @@ class TextIOWrapperTest(unittest.TestCase): self.assertEqual(rlines, wlines) def test_telling(self): - f = self.open(support.TESTFN, "w+", encoding="utf8") + f = self.open(support.TESTFN, "w+", encoding="utf-8") p0 = f.tell() f.write("\xff\n") p1 = f.tell() @@ -2300,6 +2306,7 @@ class TextIOWrapperTest(unittest.TestCase): with self.open(support.TESTFN, "w", errors="replace") as f: self.assertEqual(f.errors, "replace") + @support.no_tracing @unittest.skipUnless(threading, 'Threading required for this test.') def test_threads_write(self): # Issue6750: concurrent writes could duplicate data @@ -2729,14 +2736,14 @@ class SignalsTest(unittest.TestCase): 1/0 @unittest.skipUnless(threading, 'Threading required for this test.') - @unittest.skipIf(sys.platform in ('freebsd5', 'freebsd6', 'freebsd7'), - 'issue #12429: skip test on FreeBSD <= 7') def check_interrupted_write(self, item, bytes, **fdopen_kwargs): """Check that a partial write, when it gets interrupted, properly invokes the signal handler, and bubbles up the exception raised in the latter.""" read_results = [] def _read(): + if hasattr(signal, 'pthread_sigmask'): + signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGALRM]) s = os.read(r, 1) read_results.append(s) t = threading.Thread(target=_read) @@ -2754,7 +2761,7 @@ class SignalsTest(unittest.TestCase): # The buffered IO layer must check for pending signal # handlers, which in this case will invoke alarm_interrupt(). self.assertRaises(ZeroDivisionError, - wio.write, item * (1024 * 1024)) + wio.write, item * (support.PIPE_MAX_SIZE // len(item))) t.join() # We got one byte, get another one and check that it isn't a # repeat of the first one. @@ -2781,6 +2788,7 @@ class SignalsTest(unittest.TestCase): def test_interrupted_write_text(self): self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii") + @support.no_tracing def check_reentrant_write(self, data, **fdopen_kwargs): def on_alarm(*args): # Will be called reentrantly from the same thread diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 8cdc597e58..957991cc09 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -69,11 +69,21 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(list(accumulate('abc')), ['a', 'ab', 'abc']) # works with non-numeric self.assertEqual(list(accumulate([])), []) # empty iterable self.assertEqual(list(accumulate([7])), [7]) # iterable of length one - self.assertRaises(TypeError, accumulate, range(10), 5) # too many args + self.assertRaises(TypeError, accumulate, range(10), 5, 6) # too many args self.assertRaises(TypeError, accumulate) # too few args self.assertRaises(TypeError, accumulate, x=range(10)) # unexpected kwd arg self.assertRaises(TypeError, list, accumulate([1, []])) # args that don't add + s = [2, 8, 9, 5, 7, 0, 3, 4, 1, 6] + self.assertEqual(list(accumulate(s, min)), + [2, 2, 2, 2, 2, 0, 0, 0, 0, 0]) + self.assertEqual(list(accumulate(s, max)), + [2, 8, 9, 9, 9, 9, 9, 9, 9, 9]) + self.assertEqual(list(accumulate(s, operator.mul)), + [2, 16, 144, 720, 5040, 0, 0, 0, 0, 0]) + with self.assertRaises(TypeError): + list(accumulate(s, chr)) # unary-operation + def test_chain(self): def chain2(*iterables): @@ -158,7 +168,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version - # Test implementation detail: tuple re-use + @support.impl_detail("tuple reuse is specific to CPython") + def test_combinations_tuple_reuse(self): self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1) @@ -228,7 +239,9 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(result, list(cwr1(values, r))) # matches first pure python version self.assertEqual(result, list(cwr2(values, r))) # matches second pure python version - # Test implementation detail: tuple re-use + @support.impl_detail("tuple reuse is specific to CPython") + def test_combinations_with_replacement_tuple_reuse(self): + cwr = combinations_with_replacement self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1) @@ -292,7 +305,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(result, list(permutations(values, None))) # test r as None self.assertEqual(result, list(permutations(values))) # test default r - # Test implementation detail: tuple re-use + @support.impl_detail("tuple resuse is CPython specific") + def test_permutations_tuple_reuse(self): self.assertEqual(len(set(map(id, permutations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(permutations('abcde', 3))))), 1) @@ -556,11 +570,13 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(list(zip()), lzip()) self.assertRaises(TypeError, zip, 3) self.assertRaises(TypeError, zip, range(3), 3) - # Check tuple re-use (implementation detail) self.assertEqual([tuple(list(pair)) for pair in zip('abc', 'def')], lzip('abc', 'def')) self.assertEqual([pair for pair in zip('abc', 'def')], lzip('abc', 'def')) + + @support.impl_detail("tuple reuse is specific to CPython") + def test_zip_tuple_reuse(self): ids = list(map(id, zip('abc', 'def'))) self.assertEqual(min(ids), max(ids)) ids = list(map(id, list(zip('abc', 'def')))) @@ -603,11 +619,13 @@ class TestBasicOps(unittest.TestCase): else: self.fail('Did not raise Type in: ' + stmt) - # Check tuple re-use (implementation detail) self.assertEqual([tuple(list(pair)) for pair in zip_longest('abc', 'def')], list(zip('abc', 'def'))) self.assertEqual([pair for pair in zip_longest('abc', 'def')], list(zip('abc', 'def'))) + + @support.impl_detail("tuple reuse is specific to CPython") + def test_zip_longest_tuple_reuse(self): ids = list(map(id, zip_longest('abc', 'def'))) self.assertEqual(min(ids), max(ids)) ids = list(map(id, list(zip_longest('abc', 'def')))) @@ -711,7 +729,8 @@ class TestBasicOps(unittest.TestCase): args = map(iter, args) self.assertEqual(len(list(product(*args))), expected_len) - # Test implementation detail: tuple re-use + @support.impl_detail("tuple reuse is specific to CPython") + def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) diff --git a/Lib/test/test_keywordonlyarg.py b/Lib/test/test_keywordonlyarg.py index d7f7541837..44be32c049 100644 --- a/Lib/test/test_keywordonlyarg.py +++ b/Lib/test/test_keywordonlyarg.py @@ -78,7 +78,7 @@ class KeywordOnlyArgTestCase(unittest.TestCase): pass with self.assertRaises(TypeError) as exc: f(1, 2, 3) - expected = "f() takes at most 2 positional arguments (3 given)" + expected = "f() takes from 1 to 2 positional arguments but 3 were given" self.assertEqual(str(exc.exception), expected) def testSyntaxErrorForFunctionCall(self): diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 5d4b5fb420..19abe596fd 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -404,6 +404,8 @@ class TestMiscellaneous(unittest.TestCase): # Unsupported locale on this system self.skipTest('test needs Turkish locale') loc = locale.getlocale(locale.LC_CTYPE) + if verbose: + print('got locale %a' % (loc,)) locale.setlocale(locale.LC_CTYPE, loc) self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 90d293e3ab..556bbba313 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -36,20 +36,38 @@ import queue import re import select import socket -from socketserver import ThreadingTCPServer, StreamRequestHandler import struct import sys import tempfile -from test.support import captured_stdout, run_with_locale, run_unittest +from test.support import captured_stdout, run_with_locale, run_unittest, patch from test.support import TestHandler, Matcher import textwrap +import time import unittest import warnings import weakref try: import threading + # The following imports are needed only for tests which + import asynchat + import asyncore + import errno + from http.server import HTTPServer, BaseHTTPRequestHandler + import smtpd + from urllib.parse import urlparse, parse_qs + from socketserver import (ThreadingUDPServer, DatagramRequestHandler, + ThreadingTCPServer, StreamRequestHandler) except ImportError: threading = None +try: + import win32evtlog +except ImportError: + win32evtlog = None +try: + import win32evtlogutil +except ImportError: + win32evtlogutil = None + win32evtlog = None class BaseTest(unittest.TestCase): @@ -140,8 +158,7 @@ class BaseTest(unittest.TestCase): except AttributeError: # StringIO.StringIO lacks a reset() method. actual_lines = stream.getvalue().splitlines() - self.assertEqual(len(actual_lines), len(expected_values), - '%s vs. %s' % (actual_lines, expected_values)) + self.assertEqual(len(actual_lines), len(expected_values)) for actual, expected in zip(actual_lines, expected_values): match = pat.search(actual) if not match: @@ -351,6 +368,10 @@ class BasicFilterTest(BaseTest): finally: handler.removeFilter(filterfunc) + def test_empty_filter(self): + f = logging.Filter() + r = logging.makeLogRecord({'name': 'spam.eggs'}) + self.assertTrue(f.filter(r)) # # First, we define our levels. There can be as many as you want - the only @@ -494,6 +515,438 @@ class CustomLevelsAndFiltersTest(BaseTest): handler.removeFilter(garr) +class HandlerTest(BaseTest): + def test_name(self): + h = logging.Handler() + h.name = 'generic' + self.assertEqual(h.name, 'generic') + h.name = 'anothergeneric' + self.assertEqual(h.name, 'anothergeneric') + self.assertRaises(NotImplementedError, h.emit, None) + + def test_builtin_handlers(self): + # We can't actually *use* too many handlers in the tests, + # but we can try instantiating them with various options + if sys.platform in ('linux', 'darwin'): + for existing in (True, False): + fd, fn = tempfile.mkstemp() + os.close(fd) + if not existing: + os.unlink(fn) + h = logging.handlers.WatchedFileHandler(fn, delay=True) + if existing: + dev, ino = h.dev, h.ino + self.assertNotEqual(dev, -1) + self.assertNotEqual(ino, -1) + r = logging.makeLogRecord({'msg': 'Test'}) + h.handle(r) + # Now remove the file. + os.unlink(fn) + self.assertFalse(os.path.exists(fn)) + # The next call should recreate the file. + h.handle(r) + self.assertTrue(os.path.exists(fn)) + else: + self.assertEqual(h.dev, -1) + self.assertEqual(h.ino, -1) + h.close() + if existing: + os.unlink(fn) + if sys.platform == 'darwin': + sockname = '/var/run/syslog' + else: + sockname = '/dev/log' + try: + h = logging.handlers.SysLogHandler(sockname) + self.assertEqual(h.facility, h.LOG_USER) + self.assertTrue(h.unixsocket) + h.close() + except socket.error: # syslogd might not be available + pass + for method in ('GET', 'POST', 'PUT'): + if method == 'PUT': + self.assertRaises(ValueError, logging.handlers.HTTPHandler, + 'localhost', '/log', method) + else: + h = logging.handlers.HTTPHandler('localhost', '/log', method) + h.close() + h = logging.handlers.BufferingHandler(0) + r = logging.makeLogRecord({}) + self.assertTrue(h.shouldFlush(r)) + h.close() + h = logging.handlers.BufferingHandler(1) + self.assertFalse(h.shouldFlush(r)) + h.close() + +class BadStream(object): + def write(self, data): + raise RuntimeError('deliberate mistake') + +class TestStreamHandler(logging.StreamHandler): + def handleError(self, record): + self.error_record = record + +class StreamHandlerTest(BaseTest): + def test_error_handling(self): + h = TestStreamHandler(BadStream()) + r = logging.makeLogRecord({}) + old_raise = logging.raiseExceptions + old_stderr = sys.stderr + try: + h.handle(r) + self.assertIs(h.error_record, r) + h = logging.StreamHandler(BadStream()) + sys.stderr = sio = io.StringIO() + h.handle(r) + self.assertIn('\nRuntimeError: deliberate mistake\n', + sio.getvalue()) + logging.raiseExceptions = False + sys.stderr = sio = io.StringIO() + h.handle(r) + self.assertEqual('', sio.getvalue()) + finally: + logging.raiseExceptions = old_raise + sys.stderr = old_stderr + +# -- The following section could be moved into a server_helper.py module +# -- if it proves to be of wider utility than just test_logging + +if threading: + class TestSMTPChannel(smtpd.SMTPChannel): + """ + This derived class has had to be created because smtpd does not + support use of custom channel maps, although they are allowed by + asyncore's design. Issue #11959 has been raised to address this, + and if resolved satisfactorily, some of this code can be removed. + """ + def __init__(self, server, conn, addr, sockmap): + asynchat.async_chat.__init__(self, conn, sockmap) + self.smtp_server = server + self.conn = conn + self.addr = addr + self.received_lines = [] + self.smtp_state = self.COMMAND + self.seen_greeting = '' + self.mailfrom = None + self.rcpttos = [] + self.received_data = '' + self.fqdn = socket.getfqdn() + self.num_bytes = 0 + try: + self.peer = conn.getpeername() + except socket.error as err: + # a race condition may occur if the other end is closing + # before we can get the peername + self.close() + if err.args[0] != errno.ENOTCONN: + raise + return + self.push('220 %s %s' % (self.fqdn, smtpd.__version__)) + self.set_terminator(b'\r\n') + + + class TestSMTPServer(smtpd.SMTPServer): + """ + This class implements a test SMTP server. + + :param addr: A (host, port) tuple which the server listens on. + You can specify a port value of zero: the server's + *port* attribute will hold the actual port number + used, which can be used in client connections. + :param handler: A callable which will be called to process + incoming messages. The handler will be passed + the client address tuple, who the message is from, + a list of recipients and the message data. + :param poll_interval: The interval, in seconds, used in the underlying + :func:`select` or :func:`poll` call by + :func:`asyncore.loop`. + :param sockmap: A dictionary which will be used to hold + :class:`asyncore.dispatcher` instances used by + :func:`asyncore.loop`. This avoids changing the + :mod:`asyncore` module's global state. + """ + channel_class = TestSMTPChannel + + def __init__(self, addr, handler, poll_interval, sockmap): + self._localaddr = addr + self._remoteaddr = None + self.sockmap = sockmap + asyncore.dispatcher.__init__(self, map=sockmap) + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(0) + self.set_socket(sock, map=sockmap) + # try to re-use a server port if possible + self.set_reuse_addr() + self.bind(addr) + self.port = sock.getsockname()[1] + self.listen(5) + except: + self.close() + raise + self._handler = handler + self._thread = None + self.poll_interval = poll_interval + + def handle_accepted(self, conn, addr): + """ + Redefined only because the base class does not pass in a + map, forcing use of a global in :mod:`asyncore`. + """ + channel = self.channel_class(self, conn, addr, self.sockmap) + + def process_message(self, peer, mailfrom, rcpttos, data): + """ + Delegates to the handler passed in to the server's constructor. + + Typically, this will be a test case method. + :param peer: The client (host, port) tuple. + :param mailfrom: The address of the sender. + :param rcpttos: The addresses of the recipients. + :param data: The message. + """ + self._handler(peer, mailfrom, rcpttos, data) + + def start(self): + """ + Start the server running on a separate daemon thread. + """ + self._thread = t = threading.Thread(target=self.serve_forever, + args=(self.poll_interval,)) + t.setDaemon(True) + t.start() + + def serve_forever(self, poll_interval): + """ + Run the :mod:`asyncore` loop until normal termination + conditions arise. + :param poll_interval: The interval, in seconds, used in the underlying + :func:`select` or :func:`poll` call by + :func:`asyncore.loop`. + """ + try: + asyncore.loop(poll_interval, map=self.sockmap) + except select.error: + # On FreeBSD 8, closing the server repeatably + # raises this error. We swallow it if the + # server has been closed. + if self.connected or self.accepting: + raise + + def stop(self, timeout=None): + """ + Stop the thread by closing the server instance. + Wait for the server thread to terminate. + + :param timeout: How long to wait for the server thread + to terminate. + """ + self.close() + self._thread.join(timeout) + self._thread = None + + class ControlMixin(object): + """ + This mixin is used to start a server on a separate thread, and + shut it down programmatically. Request handling is simplified - instead + of needing to derive a suitable RequestHandler subclass, you just + provide a callable which will be passed each received request to be + processed. + + :param handler: A handler callable which will be called with a + single parameter - the request - in order to + process the request. This handler is called on the + server thread, effectively meaning that requests are + processed serially. While not quite Web scale ;-), + this should be fine for testing applications. + :param poll_interval: The polling interval in seconds. + """ + def __init__(self, handler, poll_interval): + self._thread = None + self.poll_interval = poll_interval + self._handler = handler + self.ready = threading.Event() + + def start(self): + """ + Create a daemon thread to run the server, and start it. + """ + self._thread = t = threading.Thread(target=self.serve_forever, + args=(self.poll_interval,)) + t.setDaemon(True) + t.start() + + def serve_forever(self, poll_interval): + """ + Run the server. Set the ready flag before entering the + service loop. + """ + self.ready.set() + super(ControlMixin, self).serve_forever(poll_interval) + + def stop(self, timeout=None): + """ + Tell the server thread to stop, and wait for it to do so. + + :param timeout: How long to wait for the server thread + to terminate. + """ + self.shutdown() + if self._thread is not None: + self._thread.join(timeout) + self._thread = None + self.server_close() + self.ready.clear() + + class TestHTTPServer(ControlMixin, HTTPServer): + """ + An HTTP server which is controllable using :class:`ControlMixin`. + + :param addr: A tuple with the IP address and port to listen on. + :param handler: A handler callable which will be called with a + single parameter - the request - in order to + process the request. + :param poll_interval: The polling interval in seconds. + :param log: Pass ``True`` to enable log messages. + """ + def __init__(self, addr, handler, poll_interval=0.5, + log=False, sslctx=None): + class DelegatingHTTPRequestHandler(BaseHTTPRequestHandler): + def __getattr__(self, name, default=None): + if name.startswith('do_'): + return self.process_request + raise AttributeError(name) + + def process_request(self): + self.server._handler(self) + + def log_message(self, format, *args): + if log: + super(DelegatingHTTPRequestHandler, + self).log_message(format, *args) + HTTPServer.__init__(self, addr, DelegatingHTTPRequestHandler) + ControlMixin.__init__(self, handler, poll_interval) + self.sslctx = sslctx + + def get_request(self): + try: + sock, addr = self.socket.accept() + if self.sslctx: + sock = self.sslctx.wrap_socket(sock, server_side=True) + except socket.error as e: + # socket errors are silenced by the caller, print them here + sys.stderr.write("Got an error:\n%s\n" % e) + raise + return sock, addr + + class TestTCPServer(ControlMixin, ThreadingTCPServer): + """ + A TCP server which is controllable using :class:`ControlMixin`. + + :param addr: A tuple with the IP address and port to listen on. + :param handler: A handler callable which will be called with a single + parameter - the request - in order to process the request. + :param poll_interval: The polling interval in seconds. + :bind_and_activate: If True (the default), binds the server and starts it + listening. If False, you need to call + :meth:`server_bind` and :meth:`server_activate` at + some later time before calling :meth:`start`, so that + the server will set up the socket and listen on it. + """ + + allow_reuse_address = True + + def __init__(self, addr, handler, poll_interval=0.5, + bind_and_activate=True): + class DelegatingTCPRequestHandler(StreamRequestHandler): + + def handle(self): + self.server._handler(self) + ThreadingTCPServer.__init__(self, addr, DelegatingTCPRequestHandler, + bind_and_activate) + ControlMixin.__init__(self, handler, poll_interval) + + def server_bind(self): + super(TestTCPServer, self).server_bind() + self.port = self.socket.getsockname()[1] + + class TestUDPServer(ControlMixin, ThreadingUDPServer): + """ + A UDP server which is controllable using :class:`ControlMixin`. + + :param addr: A tuple with the IP address and port to listen on. + :param handler: A handler callable which will be called with a + single parameter - the request - in order to + process the request. + :param poll_interval: The polling interval for shutdown requests, + in seconds. + :bind_and_activate: If True (the default), binds the server and + starts it listening. If False, you need to + call :meth:`server_bind` and + :meth:`server_activate` at some later time + before calling :meth:`start`, so that the server will + set up the socket and listen on it. + """ + def __init__(self, addr, handler, poll_interval=0.5, + bind_and_activate=True): + class DelegatingUDPRequestHandler(DatagramRequestHandler): + + def handle(self): + self.server._handler(self) + + def finish(self): + data = self.wfile.getvalue() + if data: + try: + super(DelegatingUDPRequestHandler, self).finish() + except socket.error: + if not self.server._closed: + raise + + ThreadingUDPServer.__init__(self, addr, + DelegatingUDPRequestHandler, + bind_and_activate) + ControlMixin.__init__(self, handler, poll_interval) + self._closed = False + + def server_bind(self): + super(TestUDPServer, self).server_bind() + self.port = self.socket.getsockname()[1] + + def server_close(self): + super(TestUDPServer, self).server_close() + self._closed = True + +# - end of server_helper section + +@unittest.skipUnless(threading, 'Threading required for this test.') +class SMTPHandlerTest(BaseTest): + def test_basic(self): + sockmap = {} + server = TestSMTPServer(('localhost', 0), self.process_message, 0.001, + sockmap) + server.start() + addr = ('localhost', server.port) + h = logging.handlers.SMTPHandler(addr, 'me', 'you', 'Log') + self.assertEqual(h.toaddrs, ['you']) + self.messages = [] + r = logging.makeLogRecord({'msg': 'Hello'}) + self.handled = threading.Event() + h.handle(r) + self.handled.wait() + server.stop() + self.assertEqual(len(self.messages), 1) + peer, mailfrom, rcpttos, data = self.messages[0] + self.assertEqual(mailfrom, 'me') + self.assertEqual(rcpttos, ['you']) + self.assertIn('\nSubject: Log\n', data) + self.assertTrue(data.endswith('\n\nHello')) + h.close() + + def process_message(self, *args): + self.messages.append(args) + self.handled.set() + class MemoryHandlerTest(BaseTest): """Tests for the MemoryHandler.""" @@ -866,116 +1319,280 @@ class ConfigFileTest(BaseTest): # Original logger output is empty. self.assert_log_lines([]) -class LogRecordStreamHandler(StreamRequestHandler): - """Handler for a streaming logging request. It saves the log message in the - TCP server's 'log_output' attribute.""" +@unittest.skipUnless(threading, 'Threading required for this test.') +class SocketHandlerTest(BaseTest): + + """Test for SocketHandler objects.""" + + def setUp(self): + """Set up a TCP server to receive log messages, and a SocketHandler + pointing to that server's address and port.""" + BaseTest.setUp(self) + addr = ('localhost', 0) + self.server = server = TestTCPServer(addr, self.handle_socket, + 0.01) + server.start() + server.ready.wait() + self.sock_hdlr = logging.handlers.SocketHandler('localhost', + server.port) + self.log_output = '' + self.root_logger.removeHandler(self.root_logger.handlers[0]) + self.root_logger.addHandler(self.sock_hdlr) + self.handled = threading.Semaphore(0) - TCP_LOG_END = "!!!END!!!" + def tearDown(self): + """Shutdown the TCP server.""" + try: + self.server.stop(2.0) + self.root_logger.removeHandler(self.sock_hdlr) + self.sock_hdlr.close() + finally: + BaseTest.tearDown(self) - def handle(self): - """Handle multiple requests - each expected to be of 4-byte length, - followed by the LogRecord in pickle format. Logs the record - according to whatever policy is configured locally.""" + def handle_socket(self, request): + conn = request.connection while True: - chunk = self.connection.recv(4) + chunk = conn.recv(4) if len(chunk) < 4: break slen = struct.unpack(">L", chunk)[0] - chunk = self.connection.recv(slen) + chunk = conn.recv(slen) while len(chunk) < slen: - chunk = chunk + self.connection.recv(slen - len(chunk)) - obj = self.unpickle(chunk) + chunk = chunk + conn.recv(slen - len(chunk)) + obj = pickle.loads(chunk) record = logging.makeLogRecord(obj) - self.handle_log_record(record) + self.log_output += record.msg + '\n' + self.handled.release() - def unpickle(self, data): - return pickle.loads(data) + def test_output(self): + # The log message sent to the SocketHandler is properly received. + logger = logging.getLogger("tcp") + logger.error("spam") + self.handled.acquire() + logger.debug("eggs") + self.handled.acquire() + self.assertEqual(self.log_output, "spam\neggs\n") - def handle_log_record(self, record): - # If the end-of-messages sentinel is seen, tell the server to - # terminate. - if self.TCP_LOG_END in record.msg: - self.server.abort = 1 - return - self.server.log_output += record.msg + "\n" + def test_noserver(self): + # Kill the server + self.server.stop(2.0) + #The logging call should try to connect, which should fail + try: + raise RuntimeError('Deliberate mistake') + except RuntimeError: + self.root_logger.exception('Never sent') + self.root_logger.error('Never sent, either') + now = time.time() + self.assertTrue(self.sock_hdlr.retryTime > now) + time.sleep(self.sock_hdlr.retryTime - now + 0.001) + self.root_logger.error('Nor this') -class LogRecordSocketReceiver(ThreadingTCPServer): +@unittest.skipUnless(threading, 'Threading required for this test.') +class DatagramHandlerTest(BaseTest): - """A simple-minded TCP socket-based logging receiver suitable for test - purposes.""" + """Test for DatagramHandler.""" + + def setUp(self): + """Set up a UDP server to receive log messages, and a DatagramHandler + pointing to that server's address and port.""" + BaseTest.setUp(self) + addr = ('localhost', 0) + self.server = server = TestUDPServer(addr, self.handle_datagram, 0.01) + server.start() + server.ready.wait() + self.sock_hdlr = logging.handlers.DatagramHandler('localhost', + server.port) + self.log_output = '' + self.root_logger.removeHandler(self.root_logger.handlers[0]) + self.root_logger.addHandler(self.sock_hdlr) + self.handled = threading.Event() - allow_reuse_address = 1 - log_output = "" + def tearDown(self): + """Shutdown the UDP server.""" + try: + self.server.stop(2.0) + self.root_logger.removeHandler(self.sock_hdlr) + self.sock_hdlr.close() + finally: + BaseTest.tearDown(self) - def __init__(self, host='localhost', - port=logging.handlers.DEFAULT_TCP_LOGGING_PORT, - handler=LogRecordStreamHandler): - ThreadingTCPServer.__init__(self, (host, port), handler) - self.abort = False - self.timeout = 0.1 - self.finished = threading.Event() + def handle_datagram(self, request): + slen = struct.pack('>L', 0) # length of prefix + packet = request.packet[len(slen):] + obj = pickle.loads(packet) + record = logging.makeLogRecord(obj) + self.log_output += record.msg + '\n' + self.handled.set() - def serve_until_stopped(self): - while not self.abort: - rd, wr, ex = select.select([self.socket.fileno()], [], [], - self.timeout) - if rd: - self.handle_request() - # Notify the main thread that we're about to exit - self.finished.set() - # close the listen socket - self.server_close() + def test_output(self): + # The log message sent to the DatagramHandler is properly received. + logger = logging.getLogger("udp") + logger.error("spam") + self.handled.wait() + self.handled.clear() + logger.error("eggs") + self.handled.wait() + self.assertEqual(self.log_output, "spam\neggs\n") @unittest.skipUnless(threading, 'Threading required for this test.') -class SocketHandlerTest(BaseTest): +class SysLogHandlerTest(BaseTest): - """Test for SocketHandler objects.""" + """Test for SysLogHandler using UDP.""" def setUp(self): - """Set up a TCP server to receive log messages, and a SocketHandler + """Set up a UDP server to receive log messages, and a SysLogHandler pointing to that server's address and port.""" BaseTest.setUp(self) - self.tcpserver = LogRecordSocketReceiver(port=0) - self.port = self.tcpserver.socket.getsockname()[1] - self.threads = [ - threading.Thread(target=self.tcpserver.serve_until_stopped)] - for thread in self.threads: - thread.start() - - self.sock_hdlr = logging.handlers.SocketHandler('localhost', self.port) - self.sock_hdlr.setFormatter(self.root_formatter) + addr = ('localhost', 0) + self.server = server = TestUDPServer(addr, self.handle_datagram, + 0.01) + server.start() + server.ready.wait() + self.sl_hdlr = logging.handlers.SysLogHandler(('localhost', + server.port)) + self.log_output = '' self.root_logger.removeHandler(self.root_logger.handlers[0]) - self.root_logger.addHandler(self.sock_hdlr) + self.root_logger.addHandler(self.sl_hdlr) + self.handled = threading.Event() def tearDown(self): - """Shutdown the TCP server.""" + """Shutdown the UDP server.""" try: - self.tcpserver.abort = True - del self.tcpserver - self.root_logger.removeHandler(self.sock_hdlr) - self.sock_hdlr.close() - for thread in self.threads: - thread.join(2.0) + self.server.stop(2.0) + self.root_logger.removeHandler(self.sl_hdlr) + self.sl_hdlr.close() finally: BaseTest.tearDown(self) - def get_output(self): - """Get the log output as received by the TCP server.""" - # Signal the TCP receiver and wait for it to terminate. - self.root_logger.critical(LogRecordStreamHandler.TCP_LOG_END) - self.tcpserver.finished.wait(2.0) - return self.tcpserver.log_output + def handle_datagram(self, request): + self.log_output = request.packet + self.handled.set() def test_output(self): - # The log message sent to the SocketHandler is properly received. - logger = logging.getLogger("tcp") - logger.error("spam") - logger.debug("eggs") - self.assertEqual(self.get_output(), "spam\neggs\n") + # The log message sent to the SysLogHandler is properly received. + logger = logging.getLogger("slh") + logger.error("sp\xe4m") + self.handled.wait() + self.assertEqual(self.log_output, b'<11>\xef\xbb\xbfsp\xc3\xa4m\x00') + self.handled.clear() + self.sl_hdlr.append_nul = False + logger.error("sp\xe4m") + self.handled.wait() + self.assertEqual(self.log_output, b'<11>\xef\xbb\xbfsp\xc3\xa4m') + self.handled.clear() + self.sl_hdlr.ident = "h\xe4m-" + logger.error("sp\xe4m") + self.handled.wait() + self.assertEqual(self.log_output, b'<11>\xef\xbb\xbfh\xc3\xa4m-sp\xc3\xa4m') + + +@unittest.skipUnless(threading, 'Threading required for this test.') +class HTTPHandlerTest(BaseTest): + """Test for HTTPHandler.""" + + PEMFILE = """-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDGT4xS5r91rbLJQK2nUDenBhBG6qFk+bVOjuAGC/LSHlAoBnvG +zQG3agOG+e7c5z2XT8m2ktORLqG3E4mYmbxgyhDrzP6ei2Anc+pszmnxPoK3Puh5 +aXV+XKt0bU0C1m2+ACmGGJ0t3P408art82nOxBw8ZHgIg9Dtp6xIUCyOqwIDAQAB +AoGBAJFTnFboaKh5eUrIzjmNrKsG44jEyy+vWvHN/FgSC4l103HxhmWiuL5Lv3f7 +0tMp1tX7D6xvHwIG9VWvyKb/Cq9rJsDibmDVIOslnOWeQhG+XwJyitR0pq/KlJIB +5LjORcBw795oKWOAi6RcOb1ON59tysEFYhAGQO9k6VL621gRAkEA/Gb+YXULLpbs +piXN3q4zcHzeaVANo69tUZ6TjaQqMeTxE4tOYM0G0ZoSeHEdaP59AOZGKXXNGSQy +2z/MddcYGQJBAMkjLSYIpOLJY11ja8OwwswFG2hEzHe0cS9bzo++R/jc1bHA5R0Y +i6vA5iPi+wopPFvpytdBol7UuEBe5xZrxWMCQQCWxELRHiP2yWpEeLJ3gGDzoXMN +PydWjhRju7Bx3AzkTtf+D6lawz1+eGTuEss5i0JKBkMEwvwnN2s1ce+EuF4JAkBb +E96h1lAzkVW5OAfYOPY8RCPA90ZO/hoyg7PpSxR0ECuDrgERR8gXIeYUYfejBkEa +rab4CfRoVJKKM28Yq/xZAkBvuq670JRCwOgfUTdww7WpdOQBYPkzQccsKNCslQW8 +/DyW6y06oQusSENUvynT6dr3LJxt/NgZPhZX2+k1eYDV +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICGzCCAYSgAwIBAgIJAIq84a2Q/OvlMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV +BAMTCWxvY2FsaG9zdDAeFw0xMTA1MjExMDIzMzNaFw03NTAzMjEwMzU1MTdaMBQx +EjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA +xk+MUua/da2yyUCtp1A3pwYQRuqhZPm1To7gBgvy0h5QKAZ7xs0Bt2oDhvnu3Oc9 +l0/JtpLTkS6htxOJmJm8YMoQ68z+notgJ3PqbM5p8T6Ctz7oeWl1flyrdG1NAtZt +vgAphhidLdz+NPGq7fNpzsQcPGR4CIPQ7aesSFAsjqsCAwEAAaN1MHMwHQYDVR0O +BBYEFLWaUPO6N7efGiuoS9i3DVYcUwn0MEQGA1UdIwQ9MDuAFLWaUPO6N7efGiuo +S9i3DVYcUwn0oRikFjAUMRIwEAYDVQQDEwlsb2NhbGhvc3SCCQCKvOGtkPzr5TAM +BgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAMK5whPjLNQK1Ivvk88oqJqq +4f889OwikGP0eUhOBhbFlsZs+jq5YZC2UzHz+evzKBlgAP1u4lP/cB85CnjvWqM+ +1c/lywFHQ6HOdDeQ1L72tSYMrNOG4XNmLn0h7rx6GoTU7dcFRfseahBCq8mv0IDt +IRbTpvlHWPjsSvHz0ZOH +-----END CERTIFICATE-----""" + + def setUp(self): + """Set up an HTTP server to receive log messages, and a HTTPHandler + pointing to that server's address and port.""" + BaseTest.setUp(self) + self.handled = threading.Event() + + def handle_request(self, request): + self.command = request.command + self.log_data = urlparse(request.path) + if self.command == 'POST': + try: + rlen = int(request.headers['Content-Length']) + self.post_data = request.rfile.read(rlen) + except: + self.post_data = None + request.send_response(200) + request.end_headers() + self.handled.set() + def test_output(self): + # The log message sent to the HTTPHandler is properly received. + logger = logging.getLogger("http") + root_logger = self.root_logger + root_logger.removeHandler(self.root_logger.handlers[0]) + for secure in (False, True): + addr = ('localhost', 0) + if secure: + try: + import ssl + fd, fn = tempfile.mkstemp() + os.close(fd) + with open(fn, 'w') as f: + f.write(self.PEMFILE) + sslctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslctx.load_cert_chain(fn) + os.unlink(fn) + except ImportError: + sslctx = None + else: + sslctx = None + self.server = server = TestHTTPServer(addr, self.handle_request, + 0.01, sslctx=sslctx) + server.start() + server.ready.wait() + host = 'localhost:%d' % server.server_port + secure_client = secure and sslctx + self.h_hdlr = logging.handlers.HTTPHandler(host, '/frob', + secure=secure_client) + self.log_data = None + root_logger.addHandler(self.h_hdlr) + + for method in ('GET', 'POST'): + self.h_hdlr.method = method + self.handled.clear() + msg = "sp\xe4m" + logger.error(msg) + self.handled.wait() + self.assertEqual(self.log_data.path, '/frob') + self.assertEqual(self.command, method) + if method == 'GET': + d = parse_qs(self.log_data.query) + else: + d = parse_qs(self.post_data.decode('utf-8')) + self.assertEqual(d['name'], ['http']) + self.assertEqual(d['funcName'], ['test_output']) + self.assertEqual(d['msg'], [msg]) + + self.server.stop(2.0) + self.root_logger.removeHandler(self.h_hdlr) + self.h_hdlr.close() class MemoryTest(BaseTest): @@ -1083,28 +1700,39 @@ class WarningsTest(BaseTest): def test_warnings(self): with warnings.catch_warnings(): logging.captureWarnings(True) - try: - warnings.filterwarnings("always", category=UserWarning) - file = io.StringIO() - h = logging.StreamHandler(file) - logger = logging.getLogger("py.warnings") - logger.addHandler(h) - warnings.warn("I'm warning you...") - logger.removeHandler(h) - s = file.getvalue() - h.close() - self.assertTrue(s.find("UserWarning: I'm warning you...\n") > 0) - - #See if an explicit file uses the original implementation - file = io.StringIO() - warnings.showwarning("Explicit", UserWarning, "dummy.py", 42, - file, "Dummy line") - s = file.getvalue() - file.close() - self.assertEqual(s, - "dummy.py:42: UserWarning: Explicit\n Dummy line\n") - finally: - logging.captureWarnings(False) + self.addCleanup(logging.captureWarnings, False) + warnings.filterwarnings("always", category=UserWarning) + stream = io.StringIO() + h = logging.StreamHandler(stream) + logger = logging.getLogger("py.warnings") + logger.addHandler(h) + warnings.warn("I'm warning you...") + logger.removeHandler(h) + s = stream.getvalue() + h.close() + self.assertTrue(s.find("UserWarning: I'm warning you...\n") > 0) + + #See if an explicit file uses the original implementation + a_file = io.StringIO() + warnings.showwarning("Explicit", UserWarning, "dummy.py", 42, + a_file, "Dummy line") + s = a_file.getvalue() + a_file.close() + self.assertEqual(s, + "dummy.py:42: UserWarning: Explicit\n Dummy line\n") + + def test_warnings_no_handlers(self): + with warnings.catch_warnings(): + logging.captureWarnings(True) + self.addCleanup(logging.captureWarnings, False) + + # confirm our assumption: no loggers are set + logger = logging.getLogger("py.warnings") + self.assertEqual(logger.handlers, []) + + warnings.showwarning("Explicit", UserWarning, "dummy.py", 42) + self.assertEqual(len(logger.handlers), 1) + self.assertIsInstance(logger.handlers[0], logging.NullHandler) def formatFunc(format, datefmt=None): @@ -1957,6 +2585,7 @@ class ConfigDictTest(BaseTest): logging.config.stopListening() t.join(2.0) + @unittest.skipUnless(threading, 'Threading required for this test.') def test_listen_config_10_ok(self): with captured_stdout() as output: self.setup_via_listener(json.dumps(self.config10)) @@ -1976,6 +2605,7 @@ class ConfigDictTest(BaseTest): ('ERROR', '4'), ], stream=output) + @unittest.skipUnless(threading, 'Threading required for this test.') def test_listen_config_1_ok(self): with captured_stdout() as output: self.setup_via_listener(textwrap.dedent(ConfigFileTest.config1)) @@ -1990,6 +2620,27 @@ class ConfigDictTest(BaseTest): # Original logger output is empty. self.assert_log_lines([]) + def test_baseconfig(self): + d = { + 'atuple': (1, 2, 3), + 'alist': ['a', 'b', 'c'], + 'adict': {'d': 'e', 'f': 3 }, + 'nest1': ('g', ('h', 'i'), 'j'), + 'nest2': ['k', ['l', 'm'], 'n'], + 'nest3': ['o', 'cfg://alist', 'p'], + } + bc = logging.config.BaseConfigurator(d) + self.assertEqual(bc.convert('cfg://atuple[1]'), 2) + self.assertEqual(bc.convert('cfg://alist[1]'), 'b') + self.assertEqual(bc.convert('cfg://nest1[1][0]'), 'h') + self.assertEqual(bc.convert('cfg://nest2[1][1]'), 'm') + self.assertEqual(bc.convert('cfg://adict.d'), 'e') + self.assertEqual(bc.convert('cfg://adict[f]'), 3) + v = bc.convert('cfg://nest3') + self.assertEqual(v.pop(1), ['a', 'b', 'c']) + self.assertRaises(KeyError, bc.convert, 'cfg://nosuch') + self.assertRaises(ValueError, bc.convert, 'cfg://!') + self.assertRaises(KeyError, bc.convert, 'cfg://adict[2]') class ManagerTest(BaseTest): def test_manager_loggerclass(self): @@ -2008,6 +2659,11 @@ class ManagerTest(BaseTest): self.assertEqual(logged, ['should appear in logged']) + def test_set_log_record_factory(self): + man = logging.Manager(None) + expected = object() + man.setLogRecordFactory(expected) + self.assertEqual(man.logRecordFactory, expected) class ChildLoggerTest(BaseTest): def test_child_loggers(self): @@ -2109,6 +2765,18 @@ class QueueHandlerTest(BaseTest): self.assertTrue(handler.matches(levelno=logging.ERROR, message='2')) self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='3')) +ZERO = datetime.timedelta(0) + +class UTC(datetime.tzinfo): + def utcoffset(self, dt): + return ZERO + + dst = utcoffset + + def tzname(self, dt): + return 'UTC' + +utc = UTC() class FormatterTest(unittest.TestCase): def setUp(self): @@ -2182,6 +2850,69 @@ class FormatterTest(unittest.TestCase): f = logging.Formatter('asctime', style='$') self.assertFalse(f.usesTime()) + def test_invalid_style(self): + self.assertRaises(ValueError, logging.Formatter, None, None, 'x') + + def test_time(self): + r = self.get_record() + dt = datetime.datetime(1993,4,21,8,3,0,0,utc) + r.created = time.mktime(dt.timetuple()) - time.timezone + r.msecs = 123 + f = logging.Formatter('%(asctime)s %(message)s') + f.converter = time.gmtime + self.assertEqual(f.formatTime(r), '1993-04-21 08:03:00,123') + self.assertEqual(f.formatTime(r, '%Y:%d'), '1993:21') + f.format(r) + self.assertEqual(r.asctime, '1993-04-21 08:03:00,123') + +class TestBufferingFormatter(logging.BufferingFormatter): + def formatHeader(self, records): + return '[(%d)' % len(records) + + def formatFooter(self, records): + return '(%d)]' % len(records) + +class BufferingFormatterTest(unittest.TestCase): + def setUp(self): + self.records = [ + logging.makeLogRecord({'msg': 'one'}), + logging.makeLogRecord({'msg': 'two'}), + ] + + def test_default(self): + f = logging.BufferingFormatter() + self.assertEqual('', f.format([])) + self.assertEqual('onetwo', f.format(self.records)) + + def test_custom(self): + f = TestBufferingFormatter() + self.assertEqual('[(2)onetwo(2)]', f.format(self.records)) + lf = logging.Formatter('<%(message)s>') + f = TestBufferingFormatter(lf) + self.assertEqual('[(2)<one><two>(2)]', f.format(self.records)) + +class ExceptionTest(BaseTest): + def test_formatting(self): + r = self.root_logger + h = RecordingHandler() + r.addHandler(h) + try: + raise RuntimeError('deliberate mistake') + except: + logging.exception('failed', stack_info=True) + r.removeHandler(h) + h.close() + r = h.records[0] + self.assertTrue(r.exc_text.startswith('Traceback (most recent ' + 'call last):\n')) + self.assertTrue(r.exc_text.endswith('\nRuntimeError: ' + 'deliberate mistake')) + self.assertTrue(r.stack_info.startswith('Stack (most recent ' + 'call last):\n')) + self.assertTrue(r.stack_info.endswith('logging.exception(\'failed\', ' + 'stack_info=True)')) + + class LastResortTest(BaseTest): def test_last_resort(self): # Test the last resort handler @@ -2192,6 +2923,8 @@ class LastResortTest(BaseTest): old_raise_exceptions = logging.raiseExceptions try: sys.stderr = sio = io.StringIO() + root.debug('This should not appear') + self.assertEqual(sio.getvalue(), '') root.warning('This is your final chance!') self.assertEqual(sio.getvalue(), 'This is your final chance!\n') #No handlers and no last resort, so 'No handlers' message @@ -2216,6 +2949,570 @@ class LastResortTest(BaseTest): logging.raiseExceptions = old_raise_exceptions +class FakeHandler: + + def __init__(self, identifier, called): + for method in ('acquire', 'flush', 'close', 'release'): + setattr(self, method, self.record_call(identifier, method, called)) + + def record_call(self, identifier, method_name, called): + def inner(): + called.append('{} - {}'.format(identifier, method_name)) + return inner + + +class RecordingHandler(logging.NullHandler): + + def __init__(self, *args, **kwargs): + super(RecordingHandler, self).__init__(*args, **kwargs) + self.records = [] + + def handle(self, record): + """Keep track of all the emitted records.""" + self.records.append(record) + + +class ShutdownTest(BaseTest): + + """Test suite for the shutdown method.""" + + def setUp(self): + super(ShutdownTest, self).setUp() + self.called = [] + + raise_exceptions = logging.raiseExceptions + self.addCleanup(setattr, logging, 'raiseExceptions', raise_exceptions) + + def raise_error(self, error): + def inner(): + raise error() + return inner + + def test_no_failure(self): + # create some fake handlers + handler0 = FakeHandler(0, self.called) + handler1 = FakeHandler(1, self.called) + handler2 = FakeHandler(2, self.called) + + # create live weakref to those handlers + handlers = map(logging.weakref.ref, [handler0, handler1, handler2]) + + logging.shutdown(handlerList=list(handlers)) + + expected = ['2 - acquire', '2 - flush', '2 - close', '2 - release', + '1 - acquire', '1 - flush', '1 - close', '1 - release', + '0 - acquire', '0 - flush', '0 - close', '0 - release'] + self.assertEqual(expected, self.called) + + def _test_with_failure_in_method(self, method, error): + handler = FakeHandler(0, self.called) + setattr(handler, method, self.raise_error(error)) + handlers = [logging.weakref.ref(handler)] + + logging.shutdown(handlerList=list(handlers)) + + self.assertEqual('0 - release', self.called[-1]) + + def test_with_ioerror_in_acquire(self): + self._test_with_failure_in_method('acquire', IOError) + + def test_with_ioerror_in_flush(self): + self._test_with_failure_in_method('flush', IOError) + + def test_with_ioerror_in_close(self): + self._test_with_failure_in_method('close', IOError) + + def test_with_valueerror_in_acquire(self): + self._test_with_failure_in_method('acquire', ValueError) + + def test_with_valueerror_in_flush(self): + self._test_with_failure_in_method('flush', ValueError) + + def test_with_valueerror_in_close(self): + self._test_with_failure_in_method('close', ValueError) + + def test_with_other_error_in_acquire_without_raise(self): + logging.raiseExceptions = False + self._test_with_failure_in_method('acquire', IndexError) + + def test_with_other_error_in_flush_without_raise(self): + logging.raiseExceptions = False + self._test_with_failure_in_method('flush', IndexError) + + def test_with_other_error_in_close_without_raise(self): + logging.raiseExceptions = False + self._test_with_failure_in_method('close', IndexError) + + def test_with_other_error_in_acquire_with_raise(self): + logging.raiseExceptions = True + self.assertRaises(IndexError, self._test_with_failure_in_method, + 'acquire', IndexError) + + def test_with_other_error_in_flush_with_raise(self): + logging.raiseExceptions = True + self.assertRaises(IndexError, self._test_with_failure_in_method, + 'flush', IndexError) + + def test_with_other_error_in_close_with_raise(self): + logging.raiseExceptions = True + self.assertRaises(IndexError, self._test_with_failure_in_method, + 'close', IndexError) + + +class ModuleLevelMiscTest(BaseTest): + + """Test suite for some module level methods.""" + + def test_disable(self): + old_disable = logging.root.manager.disable + # confirm our assumptions are correct + self.assertEqual(old_disable, 0) + self.addCleanup(logging.disable, old_disable) + + logging.disable(83) + self.assertEqual(logging.root.manager.disable, 83) + + def _test_log(self, method, level=None): + called = [] + patch(self, logging, 'basicConfig', + lambda *a, **kw: called.append((a, kw))) + + recording = RecordingHandler() + logging.root.addHandler(recording) + + log_method = getattr(logging, method) + if level is not None: + log_method(level, "test me: %r", recording) + else: + log_method("test me: %r", recording) + + self.assertEqual(len(recording.records), 1) + record = recording.records[0] + self.assertEqual(record.getMessage(), "test me: %r" % recording) + + expected_level = level if level is not None else getattr(logging, method.upper()) + self.assertEqual(record.levelno, expected_level) + + # basicConfig was not called! + self.assertEqual(called, []) + + def test_log(self): + self._test_log('log', logging.ERROR) + + def test_debug(self): + self._test_log('debug') + + def test_info(self): + self._test_log('info') + + def test_warning(self): + self._test_log('warning') + + def test_error(self): + self._test_log('error') + + def test_critical(self): + self._test_log('critical') + + def test_set_logger_class(self): + self.assertRaises(TypeError, logging.setLoggerClass, object) + + class MyLogger(logging.Logger): + pass + + logging.setLoggerClass(MyLogger) + self.assertEqual(logging.getLoggerClass(), MyLogger) + + logging.setLoggerClass(logging.Logger) + self.assertEqual(logging.getLoggerClass(), logging.Logger) + +class LogRecordTest(BaseTest): + def test_str_rep(self): + r = logging.makeLogRecord({}) + s = str(r) + self.assertTrue(s.startswith('<LogRecord: ')) + self.assertTrue(s.endswith('>')) + + def test_dict_arg(self): + h = RecordingHandler() + r = logging.getLogger() + r.addHandler(h) + d = {'less' : 'more' } + logging.warning('less is %(less)s', d) + self.assertIs(h.records[0].args, d) + self.assertEqual(h.records[0].message, 'less is more') + r.removeHandler(h) + h.close() + + def test_multiprocessing(self): + r = logging.makeLogRecord({}) + self.assertEqual(r.processName, 'MainProcess') + try: + import multiprocessing as mp + r = logging.makeLogRecord({}) + self.assertEqual(r.processName, mp.current_process().name) + except ImportError: + pass + + def test_optional(self): + r = logging.makeLogRecord({}) + NOT_NONE = self.assertIsNotNone + if threading: + NOT_NONE(r.thread) + NOT_NONE(r.threadName) + NOT_NONE(r.process) + NOT_NONE(r.processName) + log_threads = logging.logThreads + log_processes = logging.logProcesses + log_multiprocessing = logging.logMultiprocessing + try: + logging.logThreads = False + logging.logProcesses = False + logging.logMultiprocessing = False + r = logging.makeLogRecord({}) + NONE = self.assertIsNone + NONE(r.thread) + NONE(r.threadName) + NONE(r.process) + NONE(r.processName) + finally: + logging.logThreads = log_threads + logging.logProcesses = log_processes + logging.logMultiprocessing = log_multiprocessing + +class BasicConfigTest(unittest.TestCase): + + """Test suite for logging.basicConfig.""" + + def setUp(self): + super(BasicConfigTest, self).setUp() + self.handlers = logging.root.handlers + self.saved_handlers = logging._handlers.copy() + self.saved_handler_list = logging._handlerList[:] + self.original_logging_level = logging.root.level + self.addCleanup(self.cleanup) + logging.root.handlers = [] + + def tearDown(self): + for h in logging.root.handlers[:]: + logging.root.removeHandler(h) + h.close() + super(BasicConfigTest, self).tearDown() + + def cleanup(self): + setattr(logging.root, 'handlers', self.handlers) + logging._handlers.clear() + logging._handlers.update(self.saved_handlers) + logging._handlerList[:] = self.saved_handler_list + logging.root.level = self.original_logging_level + + def test_no_kwargs(self): + logging.basicConfig() + + # handler defaults to a StreamHandler to sys.stderr + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.StreamHandler) + self.assertEqual(handler.stream, sys.stderr) + + formatter = handler.formatter + # format defaults to logging.BASIC_FORMAT + self.assertEqual(formatter._style._fmt, logging.BASIC_FORMAT) + # datefmt defaults to None + self.assertIsNone(formatter.datefmt) + # style defaults to % + self.assertIsInstance(formatter._style, logging.PercentStyle) + + # level is not explicitely set + self.assertEqual(logging.root.level, self.original_logging_level) + + def test_filename(self): + logging.basicConfig(filename='test.log') + + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.FileHandler) + + expected = logging.FileHandler('test.log', 'a') + self.addCleanup(expected.close) + self.assertEqual(handler.stream.mode, expected.stream.mode) + self.assertEqual(handler.stream.name, expected.stream.name) + + def test_filemode(self): + logging.basicConfig(filename='test.log', filemode='wb') + + handler = logging.root.handlers[0] + expected = logging.FileHandler('test.log', 'wb') + self.addCleanup(expected.close) + self.assertEqual(handler.stream.mode, expected.stream.mode) + + def test_stream(self): + stream = io.StringIO() + self.addCleanup(stream.close) + logging.basicConfig(stream=stream) + + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.StreamHandler) + self.assertEqual(handler.stream, stream) + + def test_format(self): + logging.basicConfig(format='foo') + + formatter = logging.root.handlers[0].formatter + self.assertEqual(formatter._style._fmt, 'foo') + + def test_datefmt(self): + logging.basicConfig(datefmt='bar') + + formatter = logging.root.handlers[0].formatter + self.assertEqual(formatter.datefmt, 'bar') + + def test_style(self): + logging.basicConfig(style='$') + + formatter = logging.root.handlers[0].formatter + self.assertIsInstance(formatter._style, logging.StringTemplateStyle) + + def test_level(self): + old_level = logging.root.level + self.addCleanup(logging.root.setLevel, old_level) + + logging.basicConfig(level=57) + self.assertEqual(logging.root.level, 57) + # Test that second call has no effect + logging.basicConfig(level=58) + self.assertEqual(logging.root.level, 57) + + def test_incompatible(self): + assertRaises = self.assertRaises + handlers = [logging.StreamHandler()] + stream = sys.stderr + assertRaises(ValueError, logging.basicConfig, filename='test.log', + stream=stream) + assertRaises(ValueError, logging.basicConfig, filename='test.log', + handlers=handlers) + assertRaises(ValueError, logging.basicConfig, stream=stream, + handlers=handlers) + + def test_handlers(self): + handlers = [ + logging.StreamHandler(), + logging.StreamHandler(sys.stdout), + logging.StreamHandler(), + ] + f = logging.Formatter() + handlers[2].setFormatter(f) + logging.basicConfig(handlers=handlers) + self.assertIs(handlers[0], logging.root.handlers[0]) + self.assertIs(handlers[1], logging.root.handlers[1]) + self.assertIs(handlers[2], logging.root.handlers[2]) + self.assertIsNotNone(handlers[0].formatter) + self.assertIsNotNone(handlers[1].formatter) + self.assertIs(handlers[2].formatter, f) + self.assertIs(handlers[0].formatter, handlers[1].formatter) + + def _test_log(self, method, level=None): + # logging.root has no handlers so basicConfig should be called + called = [] + + old_basic_config = logging.basicConfig + def my_basic_config(*a, **kw): + old_basic_config() + old_level = logging.root.level + logging.root.setLevel(100) # avoid having messages in stderr + self.addCleanup(logging.root.setLevel, old_level) + called.append((a, kw)) + + patch(self, logging, 'basicConfig', my_basic_config) + + log_method = getattr(logging, method) + if level is not None: + log_method(level, "test me") + else: + log_method("test me") + + # basicConfig was called with no arguments + self.assertEqual(called, [((), {})]) + + def test_log(self): + self._test_log('log', logging.WARNING) + + def test_debug(self): + self._test_log('debug') + + def test_info(self): + self._test_log('info') + + def test_warning(self): + self._test_log('warning') + + def test_error(self): + self._test_log('error') + + def test_critical(self): + self._test_log('critical') + + +class LoggerAdapterTest(unittest.TestCase): + + def setUp(self): + super(LoggerAdapterTest, self).setUp() + old_handler_list = logging._handlerList[:] + + self.recording = RecordingHandler() + self.logger = logging.root + self.logger.addHandler(self.recording) + self.addCleanup(self.logger.removeHandler, self.recording) + self.addCleanup(self.recording.close) + + def cleanup(): + logging._handlerList[:] = old_handler_list + + self.addCleanup(cleanup) + self.addCleanup(logging.shutdown) + self.adapter = logging.LoggerAdapter(logger=self.logger, extra=None) + + def test_exception(self): + msg = 'testing exception: %r' + exc = None + try: + 1 / 0 + except ZeroDivisionError as e: + exc = e + self.adapter.exception(msg, self.recording) + + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.levelno, logging.ERROR) + self.assertEqual(record.msg, msg) + self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.exc_info, + (exc.__class__, exc, exc.__traceback__)) + + def test_critical(self): + msg = 'critical test! %r' + self.adapter.critical(msg, self.recording) + + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.levelno, logging.CRITICAL) + self.assertEqual(record.msg, msg) + self.assertEqual(record.args, (self.recording,)) + + def test_is_enabled_for(self): + old_disable = self.adapter.logger.manager.disable + self.adapter.logger.manager.disable = 33 + self.addCleanup(setattr, self.adapter.logger.manager, 'disable', + old_disable) + self.assertFalse(self.adapter.isEnabledFor(32)) + + def test_has_handlers(self): + self.assertTrue(self.adapter.hasHandlers()) + + for handler in self.logger.handlers: + self.logger.removeHandler(handler) + + self.assertFalse(self.logger.hasHandlers()) + self.assertFalse(self.adapter.hasHandlers()) + + +class LoggerTest(BaseTest): + + def setUp(self): + super(LoggerTest, self).setUp() + self.recording = RecordingHandler() + self.logger = logging.Logger(name='blah') + self.logger.addHandler(self.recording) + self.addCleanup(self.logger.removeHandler, self.recording) + self.addCleanup(self.recording.close) + self.addCleanup(logging.shutdown) + + def test_set_invalid_level(self): + self.assertRaises(TypeError, self.logger.setLevel, object()) + + def test_exception(self): + msg = 'testing exception: %r' + exc = None + try: + 1 / 0 + except ZeroDivisionError as e: + exc = e + self.logger.exception(msg, self.recording) + + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.levelno, logging.ERROR) + self.assertEqual(record.msg, msg) + self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.exc_info, + (exc.__class__, exc, exc.__traceback__)) + + def test_log_invalid_level_with_raise(self): + old_raise = logging.raiseExceptions + self.addCleanup(setattr, logging, 'raiseExecptions', old_raise) + + logging.raiseExceptions = True + self.assertRaises(TypeError, self.logger.log, '10', 'test message') + + def test_log_invalid_level_no_raise(self): + old_raise = logging.raiseExceptions + self.addCleanup(setattr, logging, 'raiseExecptions', old_raise) + + logging.raiseExceptions = False + self.logger.log('10', 'test message') # no exception happens + + def test_find_caller_with_stack_info(self): + called = [] + patch(self, logging.traceback, 'print_stack', + lambda f, file: called.append(file.getvalue())) + + self.logger.findCaller(stack_info=True) + + self.assertEqual(len(called), 1) + self.assertEqual('Stack (most recent call last):\n', called[0]) + + def test_make_record_with_extra_overwrite(self): + name = 'my record' + level = 13 + fn = lno = msg = args = exc_info = func = sinfo = None + rv = logging._logRecordFactory(name, level, fn, lno, msg, args, + exc_info, func, sinfo) + + for key in ('message', 'asctime') + tuple(rv.__dict__.keys()): + extra = {key: 'some value'} + self.assertRaises(KeyError, self.logger.makeRecord, name, level, + fn, lno, msg, args, exc_info, + extra=extra, sinfo=sinfo) + + def test_make_record_with_extra_no_overwrite(self): + name = 'my record' + level = 13 + fn = lno = msg = args = exc_info = func = sinfo = None + extra = {'valid_key': 'some value'} + result = self.logger.makeRecord(name, level, fn, lno, msg, args, + exc_info, extra=extra, sinfo=sinfo) + self.assertIn('valid_key', result.__dict__) + + def test_has_handlers(self): + self.assertTrue(self.logger.hasHandlers()) + + for handler in self.logger.handlers: + self.logger.removeHandler(handler) + self.assertFalse(self.logger.hasHandlers()) + + def test_has_handlers_no_propagate(self): + child_logger = logging.getLogger('blah.child') + child_logger.propagate = False + self.assertFalse(child_logger.hasHandlers()) + + def test_is_enabled_for(self): + old_disable = self.logger.manager.disable + self.logger.manager.disable = 23 + self.addCleanup(setattr, self.logger.manager, 'disable', old_disable) + self.assertFalse(self.logger.isEnabledFor(22)) + + class BaseFileTest(BaseTest): "Base class for handler tests that write log files" @@ -2235,10 +3532,21 @@ class BaseFileTest(BaseTest): def assertLogFile(self, filename): "Assert a log file is there and register it for deletion" self.assertTrue(os.path.exists(filename), - msg="Log file %r does not exist") + msg="Log file %r does not exist" % filename) self.rmfiles.append(filename) +class FileHandlerTest(BaseFileTest): + def test_delay(self): + os.unlink(self.fn) + fh = logging.FileHandler(self.fn, delay=True) + self.assertIsNone(fh.stream) + self.assertFalse(os.path.exists(self.fn)) + fh.handle(logging.makeLogRecord({})) + self.assertIsNotNone(fh.stream) + self.assertTrue(os.path.exists(self.fn)) + fh.close() + class RotatingFileHandlerTest(BaseFileTest): def next_rec(self): return logging.LogRecord('n', logging.DEBUG, 'p', 1, @@ -2276,8 +3584,49 @@ class RotatingFileHandlerTest(BaseFileTest): rh.close() class TimedRotatingFileHandlerTest(BaseFileTest): - # test methods added below - pass + # other test methods added below + def test_rollover(self): + fh = logging.handlers.TimedRotatingFileHandler(self.fn, 'S', + backupCount=1) + r = logging.makeLogRecord({'msg': 'testing'}) + fh.emit(r) + self.assertLogFile(self.fn) + time.sleep(1.01) # just a little over a second ... + fh.emit(r) + fh.close() + # At this point, we should have a recent rotated file which we + # can test for the existence of. However, in practice, on some + # machines which run really slowly, we don't know how far back + # in time to go to look for the log file. So, we go back a fair + # bit, and stop as soon as we see a rotated file. In theory this + # could of course still fail, but the chances are lower. + found = False + now = datetime.datetime.now() + GO_BACK = 5 * 60 # seconds + for secs in range(GO_BACK): + prev = now - datetime.timedelta(seconds=secs) + fn = self.fn + prev.strftime(".%Y-%m-%d_%H-%M-%S") + found = os.path.exists(fn) + if found: + self.rmfiles.append(fn) + break + msg = 'No rotated files found, went back %d seconds' % GO_BACK + if not found: + #print additional diagnostics + dn, fn = os.path.split(self.fn) + files = [f for f in os.listdir(dn) if f.startswith(fn)] + print('Test time: %s' % now.strftime("%Y-%m-%d %H-%M-%S"), file=sys.stderr) + print('The only matching files are: %s' % files, file=sys.stderr) + self.assertTrue(found, msg=msg) + + def test_invalid(self): + assertRaises = self.assertRaises + assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, + self.fn, 'X', delay=True) + assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, + self.fn, 'W', delay=True) + assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, + self.fn, 'W7', delay=True) def secs(**kw): return datetime.timedelta(**kw) // datetime.timedelta(seconds=1) @@ -2326,19 +3675,51 @@ for when, exp in (('S', 1), rh.close() setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover) + +@unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil required for this test.') +class NTEventLogHandlerTest(BaseTest): + def test_basic(self): + logtype = 'Application' + elh = win32evtlog.OpenEventLog(None, logtype) + num_recs = win32evtlog.GetNumberOfEventLogRecords(elh) + h = logging.handlers.NTEventLogHandler('test_logging') + r = logging.makeLogRecord({'msg': 'Test Log Message'}) + h.handle(r) + h.close() + # Now see if the event is recorded + self.assertTrue(num_recs < win32evtlog.GetNumberOfEventLogRecords(elh)) + flags = win32evtlog.EVENTLOG_BACKWARDS_READ | \ + win32evtlog.EVENTLOG_SEQUENTIAL_READ + found = False + GO_BACK = 100 + events = win32evtlog.ReadEventLog(elh, flags, GO_BACK) + for e in events: + if e.SourceName != 'test_logging': + continue + msg = win32evtlogutil.SafeFormatMessage(e, logtype) + if msg != 'Test Log Message\r\n': + continue + found = True + break + msg = 'Record not found in event log, went back %d records' % GO_BACK + self.assertTrue(found, msg=msg) + # Set the locale to the platform-dependent default. I have no idea # why the test does this, but in any case we save the current locale # first and restore it at the end. @run_with_locale('LC_ALL', '') def test_main(): run_unittest(BuiltinLevelsTest, BasicFilterTest, - CustomLevelsAndFiltersTest, MemoryHandlerTest, - ConfigFileTest, SocketHandlerTest, MemoryTest, - EncodingTest, WarningsTest, ConfigDictTest, ManagerTest, - FormatterTest, - LogRecordFactoryTest, ChildLoggerTest, QueueHandlerTest, - RotatingFileHandlerTest, - LastResortTest, + CustomLevelsAndFiltersTest, HandlerTest, MemoryHandlerTest, + ConfigFileTest, SocketHandlerTest, DatagramHandlerTest, + MemoryTest, EncodingTest, WarningsTest, ConfigDictTest, + ManagerTest, FormatterTest, BufferingFormatterTest, + StreamHandlerTest, LogRecordFactoryTest, ChildLoggerTest, + QueueHandlerTest, ShutdownTest, ModuleLevelMiscTest, + BasicConfigTest, LoggerAdapterTest, LoggerTest, + SMTPHandlerTest, FileHandlerTest, RotatingFileHandlerTest, + LastResortTest, LogRecordTest, ExceptionTest, + SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest, TimedRotatingFileHandlerTest ) diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index fb4812dd99..8c8920a6c2 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -95,14 +95,14 @@ class TestMailbox(TestBase): """) def test_add_invalid_8bit_bytes_header(self): - key = self._box.add(self._nonascii_msg.encode('latin1')) + key = self._box.add(self._nonascii_msg.encode('latin-1')) self.assertEqual(len(self._box), 1) self.assertEqual(self._box.get_bytes(key), - self._nonascii_msg.encode('latin1')) + self._nonascii_msg.encode('latin-1')) def test_invalid_nonascii_header_as_string(self): subj = self._nonascii_msg.splitlines()[1] - key = self._box.add(subj.encode('latin1')) + key = self._box.add(subj.encode('latin-1')) self.assertEqual(self._box.get_string(key), 'Subject: =?unknown-8bit?b?RmFsaW5hcHThciBo4Xpob3pzeuFsbO104XNz' 'YWwuIE3hciByZW5kZWx06Ww/?=\n\n') @@ -902,8 +902,7 @@ class TestMaildir(TestMailbox): # Now, write something into cur and remove it. This changes # the mtime and should cause a re-read. filename = os.path.join(self._path, 'cur', 'stray-file') - f = open(filename, 'w') - f.close() + support.create_empty_file(filename) os.unlink(filename) self._box._refresh() self.assertTrue(refreshed()) diff --git a/Lib/test/test_mailcap.py b/Lib/test/test_mailcap.py new file mode 100644 index 0000000000..a4cd09c706 --- /dev/null +++ b/Lib/test/test_mailcap.py @@ -0,0 +1,221 @@ +import mailcap +import os +import shutil +import test.support +import unittest + +# Location of mailcap file +MAILCAPFILE = test.support.findfile("mailcap.txt") + +# Dict to act as mock mailcap entry for this test +# The keys and values should match the contents of MAILCAPFILE +MAILCAPDICT = { + 'application/x-movie': + [{'compose': 'moviemaker %s', + 'x11-bitmap': '"/usr/lib/Zmail/bitmaps/movie.xbm"', + 'description': '"Movie"', + 'view': 'movieplayer %s'}], + 'application/*': + [{'copiousoutput': '', + 'view': 'echo "This is \\"%t\\" but is 50 \\% Greek to me" \\; cat %s'}], + 'audio/basic': + [{'edit': 'audiocompose %s', + 'compose': 'audiocompose %s', + 'description': '"An audio fragment"', + 'view': 'showaudio %s'}], + 'video/mpeg': + [{'view': 'mpeg_play %s'}], + 'application/postscript': + [{'needsterminal': '', 'view': 'ps-to-terminal %s'}, + {'compose': 'idraw %s', 'view': 'ps-to-terminal %s'}], + 'application/x-dvi': + [{'view': 'xdvi %s'}], + 'message/external-body': + [{'composetyped': 'extcompose %s', + 'description': '"A reference to data stored in an external location"', + 'needsterminal': '', + 'view': 'showexternal %s %{access-type} %{name} %{site} %{directory} %{mode} %{server}'}], + 'text/richtext': + [{'test': 'test "`echo %{charset} | tr \'[A-Z]\' \'[a-z]\'`" = iso-8859-8', + 'copiousoutput': '', + 'view': 'shownonascii iso-8859-8 -e richtext -p %s'}], + 'image/x-xwindowdump': + [{'view': 'display %s'}], + 'audio/*': + [{'view': '/usr/local/bin/showaudio %t'}], + 'video/*': + [{'view': 'animate %s'}], + 'application/frame': + [{'print': '"cat %s | lp"', 'view': 'showframe %s'}], + 'image/rgb': + [{'view': 'display %s'}] +} + + +class HelperFunctionTest(unittest.TestCase): + + def test_listmailcapfiles(self): + # The return value for listmailcapfiles() will vary by system. + # So verify that listmailcapfiles() returns a list of strings that is of + # non-zero length. + mcfiles = mailcap.listmailcapfiles() + self.assertIsInstance(mcfiles, list) + for m in mcfiles: + self.assertIsInstance(m, str) + with test.support.EnvironmentVarGuard() as env: + # According to RFC 1524, if MAILCAPS env variable exists, use that + # and only that. + if "MAILCAPS" in env: + env_mailcaps = env["MAILCAPS"].split(os.pathsep) + else: + env_mailcaps = ["/testdir1/.mailcap", "/testdir2/mailcap"] + env["MAILCAPS"] = os.pathsep.join(env_mailcaps) + mcfiles = mailcap.listmailcapfiles() + self.assertEqual(env_mailcaps, mcfiles) + + def test_readmailcapfile(self): + # Test readmailcapfile() using test file. It should match MAILCAPDICT. + with open(MAILCAPFILE, 'r') as mcf: + d = mailcap.readmailcapfile(mcf) + self.assertDictEqual(d, MAILCAPDICT) + + def test_lookup(self): + # Test without key + expected = [{'view': 'mpeg_play %s'}, {'view': 'animate %s'}] + actual = mailcap.lookup(MAILCAPDICT, 'video/mpeg') + self.assertListEqual(expected, actual) + + # Test with key + key = 'compose' + expected = [{'edit': 'audiocompose %s', + 'compose': 'audiocompose %s', + 'description': '"An audio fragment"', + 'view': 'showaudio %s'}] + actual = mailcap.lookup(MAILCAPDICT, 'audio/basic', key) + self.assertListEqual(expected, actual) + + def test_subst(self): + plist = ['id=1', 'number=2', 'total=3'] + # test case: ([field, MIMEtype, filename, plist=[]], <expected string>) + test_cases = [ + (["", "audio/*", "foo.txt"], ""), + (["echo foo", "audio/*", "foo.txt"], "echo foo"), + (["echo %s", "audio/*", "foo.txt"], "echo foo.txt"), + (["echo %t", "audio/*", "foo.txt"], "echo audio/*"), + (["echo \%t", "audio/*", "foo.txt"], "echo %t"), + (["echo foo", "audio/*", "foo.txt", plist], "echo foo"), + (["echo %{total}", "audio/*", "foo.txt", plist], "echo 3") + ] + for tc in test_cases: + self.assertEqual(mailcap.subst(*tc[0]), tc[1]) + + +class GetcapsTest(unittest.TestCase): + + def test_mock_getcaps(self): + # Test mailcap.getcaps() using mock mailcap file in this dir. + # Temporarily override any existing system mailcap file by pointing the + # MAILCAPS environment variable to our mock file. + with test.support.EnvironmentVarGuard() as env: + env["MAILCAPS"] = MAILCAPFILE + caps = mailcap.getcaps() + self.assertDictEqual(caps, MAILCAPDICT) + + def test_system_mailcap(self): + # Test mailcap.getcaps() with mailcap file(s) on system, if any. + caps = mailcap.getcaps() + self.assertIsInstance(caps, dict) + mailcapfiles = mailcap.listmailcapfiles() + existingmcfiles = [mcf for mcf in mailcapfiles if os.path.exists(mcf)] + if existingmcfiles: + # At least 1 mailcap file exists, so test that. + for (k, v) in caps.items(): + self.assertIsInstance(k, str) + self.assertIsInstance(v, list) + for e in v: + self.assertIsInstance(e, dict) + else: + # No mailcap files on system. getcaps() should return empty dict. + self.assertEqual({}, caps) + + +class FindmatchTest(unittest.TestCase): + + def test_findmatch(self): + + # default findmatch arguments + c = MAILCAPDICT + fname = "foo.txt" + plist = ["access-type=default", "name=john", "site=python.org", + "directory=/tmp", "mode=foo", "server=bar"] + audio_basic_entry = { + 'edit': 'audiocompose %s', + 'compose': 'audiocompose %s', + 'description': '"An audio fragment"', + 'view': 'showaudio %s' + } + audio_entry = {"view": "/usr/local/bin/showaudio %t"} + video_entry = {'view': 'animate %s'} + message_entry = { + 'composetyped': 'extcompose %s', + 'description': '"A reference to data stored in an external location"', 'needsterminal': '', + 'view': 'showexternal %s %{access-type} %{name} %{site} %{directory} %{mode} %{server}' + } + + # test case: (findmatch args, findmatch keyword args, expected output) + # positional args: caps, MIMEtype + # keyword args: key="view", filename="/dev/null", plist=[] + # output: (command line, mailcap entry) + cases = [ + ([{}, "video/mpeg"], {}, (None, None)), + ([c, "foo/bar"], {}, (None, None)), + ([c, "video/mpeg"], {}, ('mpeg_play /dev/null', {'view': 'mpeg_play %s'})), + ([c, "audio/basic", "edit"], {}, ("audiocompose /dev/null", audio_basic_entry)), + ([c, "audio/basic", "compose"], {}, ("audiocompose /dev/null", audio_basic_entry)), + ([c, "audio/basic", "description"], {}, ('"An audio fragment"', audio_basic_entry)), + ([c, "audio/basic", "foobar"], {}, (None, None)), + ([c, "video/*"], {"filename": fname}, ("animate %s" % fname, video_entry)), + ([c, "audio/basic", "compose"], + {"filename": fname}, + ("audiocompose %s" % fname, audio_basic_entry)), + ([c, "audio/basic"], + {"key": "description", "filename": fname}, + ('"An audio fragment"', audio_basic_entry)), + ([c, "audio/*"], + {"filename": fname}, + ("/usr/local/bin/showaudio audio/*", audio_entry)), + ([c, "message/external-body"], + {"plist": plist}, + ("showexternal /dev/null default john python.org /tmp foo bar", message_entry)) + ] + self._run_cases(cases) + + @unittest.skipUnless(os.name == "posix", "Requires 'test' command on system") + def test_test(self): + # findmatch() will automatically check any "test" conditions and skip + # the entry if the check fails. + caps = {"test/pass": [{"test": "test 1 -eq 1"}], + "test/fail": [{"test": "test 1 -eq 0"}]} + # test case: (findmatch args, findmatch keyword args, expected output) + # positional args: caps, MIMEtype, key ("test") + # keyword args: N/A + # output: (command line, mailcap entry) + cases = [ + # findmatch will return the mailcap entry for test/pass because it evaluates to true + ([caps, "test/pass", "test"], {}, ("test 1 -eq 1", {"test": "test 1 -eq 1"})), + # findmatch will return None because test/fail evaluates to false + ([caps, "test/fail", "test"], {}, (None, None)) + ] + self._run_cases(cases) + + def _run_cases(self, cases): + for c in cases: + self.assertEqual(mailcap.findmatch(*c[0], **c[1]), c[2]) + + +def test_main(): + test.support.run_unittest(HelperFunctionTest, GetcapsTest, FindmatchTest) + + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index cd100f9be4..dec8129e12 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -5,6 +5,7 @@ import marshal import sys import unittest import os +import types class HelperMixin: def helper(self, sample, *extra): @@ -113,6 +114,22 @@ class CodeTestCase(unittest.TestCase): codes = (ExceptionTestCase.test_exceptions.__code__,) * count marshal.loads(marshal.dumps(codes)) + def test_different_filenames(self): + co1 = compile("x", "f1", "exec") + co2 = compile("y", "f2", "exec") + co1, co2 = marshal.loads(marshal.dumps((co1, co2))) + self.assertEqual(co1.co_filename, "f1") + self.assertEqual(co2.co_filename, "f2") + + @support.cpython_only + def test_same_filename_used(self): + s = """def f(): pass\ndef g(): pass""" + co = compile(s, "myfile", "exec") + co = marshal.loads(marshal.dumps(co)) + for obj in co.co_consts: + if isinstance(obj, types.CodeType): + self.assertIs(co.co_filename, obj.co_filename) + class ContainerTestCase(unittest.TestCase, HelperMixin): d = {'astring': 'foo@bar.baz.spam', 'afloat': 7283.43, diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index dddc889375..715003af05 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -2,11 +2,12 @@ # XXXX Should not do tests around zero only from test.support import run_unittest, verbose, requires_IEEE_754 +from test import support import unittest import math import os +import platform import sys -import random import struct import sysconfig @@ -457,12 +458,12 @@ class MathTests(unittest.TestCase): def testFmod(self): self.assertRaises(TypeError, math.fmod) - self.ftest('fmod(10,1)', math.fmod(10,1), 0) - self.ftest('fmod(10,0.5)', math.fmod(10,0.5), 0) - self.ftest('fmod(10,1.5)', math.fmod(10,1.5), 1) - self.ftest('fmod(-10,1)', math.fmod(-10,1), 0) - self.ftest('fmod(-10,0.5)', math.fmod(-10,0.5), 0) - self.ftest('fmod(-10,1.5)', math.fmod(-10,1.5), -1) + self.ftest('fmod(10, 1)', math.fmod(10, 1), 0.0) + self.ftest('fmod(10, 0.5)', math.fmod(10, 0.5), 0.0) + self.ftest('fmod(10, 1.5)', math.fmod(10, 1.5), 1.0) + self.ftest('fmod(-10, 1)', math.fmod(-10, 1), -0.0) + self.ftest('fmod(-10, 0.5)', math.fmod(-10, 0.5), -0.0) + self.ftest('fmod(-10, 1.5)', math.fmod(-10, 1.5), -1.0) self.assertTrue(math.isnan(math.fmod(NAN, 1.))) self.assertTrue(math.isnan(math.fmod(1., NAN))) self.assertTrue(math.isnan(math.fmod(NAN, NAN))) @@ -650,6 +651,33 @@ class MathTests(unittest.TestCase): n= 2**90 self.assertAlmostEqual(math.log1p(n), math.log1p(float(n))) + @requires_IEEE_754 + def testLog2(self): + self.assertRaises(TypeError, math.log2) + + # Check some integer values + self.assertEqual(math.log2(1), 0.0) + self.assertEqual(math.log2(2), 1.0) + self.assertEqual(math.log2(4), 2.0) + + # Large integer values + self.assertEqual(math.log2(2**1023), 1023.0) + self.assertEqual(math.log2(2**1024), 1024.0) + self.assertEqual(math.log2(2**2000), 2000.0) + + self.assertRaises(ValueError, math.log2, -1.5) + self.assertRaises(ValueError, math.log2, NINF) + self.assertTrue(math.isnan(math.log2(NAN))) + + @requires_IEEE_754 + # log2() is not accurate enough on Mac OS X Tiger (10.4) + @support.requires_mac_ver(10, 5) + def testLog2Exact(self): + # Check that we get exact equality for log2 of powers of 2. + actual = [math.log2(math.ldexp(1.0, n)) for n in range(-1074, 1024)] + expected = [float(n) for n in range(-1074, 1024)] + self.assertEqual(actual, expected) + def testLog10(self): self.assertRaises(TypeError, math.log10) self.ftest('log10(0.1)', math.log10(0.1), -1) @@ -1010,7 +1038,6 @@ class MathTests(unittest.TestCase): @requires_IEEE_754 def test_mtestfile(self): - ALLOWED_ERROR = 20 # permitted error, in ulps fail_fmt = "{}:{}({!r}): expected {!r}, got {!r}" failures = [] diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py index 219ab99840..6862900b01 100644 --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -246,7 +246,13 @@ Test failures in looking up the __prepare__ method work. """ -__test__ = {'doctests' : doctests} +import sys + +# Trace function introduces __locals__ which causes various tests to fail. +if hasattr(sys, 'gettrace') and sys.gettrace(): + __test__ = {} +else: + __test__ = {'doctests' : doctests} def test_main(verbose=False): from test import support diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 200b95db73..533258c8e7 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -4,9 +4,7 @@ import pickle from test.support import verbose, run_unittest, findfile import unittest -import xml.dom import xml.dom.minidom -import xml.parsers.expat from xml.dom.minidom import parse, Node, Document, parseString from xml.dom.minidom import getDOMImplementation @@ -14,7 +12,6 @@ from xml.dom.minidom import getDOMImplementation tstfile = findfile("test.xml", subdir="xmltestdata") - # The tests of DocumentType importing use these helpers to construct # the documents to work with, since not all DOM builders actually # create the DocumentType nodes. @@ -282,6 +279,7 @@ class MinidomTest(unittest.TestCase): child.setAttribute("def", "ghi") self.confirm(len(child.attributes) == 1) + self.assertRaises(xml.dom.NotFoundErr, child.removeAttribute, "foo") child.removeAttribute("def") self.confirm(len(child.attributes) == 0) dom.unlink() @@ -293,6 +291,8 @@ class MinidomTest(unittest.TestCase): child.setAttributeNS("http://www.w3.org", "xmlns:python", "http://www.python.org") child.setAttributeNS("http://www.python.org", "python:abcattr", "foo") + self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNS, + "foo", "http://www.python.org") self.confirm(len(child.attributes) == 2) child.removeAttributeNS("http://www.python.org", "abcattr") self.confirm(len(child.attributes) == 1) @@ -304,11 +304,23 @@ class MinidomTest(unittest.TestCase): child.setAttribute("spam", "jam") self.confirm(len(child.attributes) == 1) node = child.getAttributeNode("spam") + self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNode, + None) child.removeAttributeNode(node) self.confirm(len(child.attributes) == 0 and child.getAttributeNode("spam") is None) + dom2 = Document() + child2 = dom2.appendChild(dom.createElement("foo")) + self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNode, + node) dom.unlink() + def testHasAttribute(self): + dom = Document() + child = dom.appendChild(dom.createElement("foo")) + child.setAttribute("spam", "jam") + self.confirm(child.hasAttribute("spam")) + def testChangeAttr(self): dom = parseString("<abc/>") el = dom.documentElement @@ -356,7 +368,16 @@ class MinidomTest(unittest.TestCase): def testGetAttribute(self): pass - def testGetAttributeNS(self): pass + def testGetAttributeNS(self): + dom = Document() + child = dom.appendChild( + dom.createElementNS("http://www.python.org", "python:abc")) + child.setAttributeNS("http://www.w3.org", "xmlns:python", + "http://www.python.org") + self.assertEqual(child.getAttributeNS("http://www.w3.org", "python"), + 'http://www.python.org') + self.assertEqual(child.getAttributeNS("http://www.w3.org", "other"), + '') def testGetAttributeNode(self): pass @@ -540,7 +561,13 @@ class MinidomTest(unittest.TestCase): def testFirstChild(self): pass - def testHasChildNodes(self): pass + def testHasChildNodes(self): + dom = parseString("<doc><foo/></doc>") + doc = dom.documentElement + self.assertTrue(dom.hasChildNodes()) + dom2 = parseString("<doc/>") + doc2 = dom2.documentElement + self.assertFalse(doc2.hasChildNodes()) def _testCloneElementCopiesAttributes(self, e1, e2, test): attrs1 = e1.attributes @@ -989,41 +1016,6 @@ class MinidomTest(unittest.TestCase): "test NodeList.item()") doc.unlink() - def testSAX2DOM(self): - from xml.dom import pulldom - - sax2dom = pulldom.SAX2DOM() - sax2dom.startDocument() - sax2dom.startElement("doc", {}) - sax2dom.characters("text") - sax2dom.startElement("subelm", {}) - sax2dom.characters("text") - sax2dom.endElement("subelm") - sax2dom.characters("text") - sax2dom.endElement("doc") - sax2dom.endDocument() - - doc = sax2dom.document - root = doc.documentElement - (text1, elm1, text2) = root.childNodes - text3 = elm1.childNodes[0] - - self.confirm(text1.previousSibling is None and - text1.nextSibling is elm1 and - elm1.previousSibling is text1 and - elm1.nextSibling is text2 and - text2.previousSibling is elm1 and - text2.nextSibling is None and - text3.previousSibling is None and - text3.nextSibling is None, "testSAX2DOM - siblings") - - self.confirm(root.parentNode is doc and - text1.parentNode is root and - elm1.parentNode is root and - text2.parentNode is root and - text3.parentNode is elm1, "testSAX2DOM - parents") - doc.unlink() - def testEncodings(self): doc = parseString('<foo>€</foo>') self.assertEqual(doc.toxml(), @@ -1470,12 +1462,21 @@ class MinidomTest(unittest.TestCase): doc.appendChild(doc.createComment("foo--bar")) self.assertRaises(ValueError, doc.toxml) + def testEmptyXMLNSValue(self): doc = parseString("<element xmlns=''>\n" "<foo/>\n</element>") doc2 = parseString(doc.toxml()) self.confirm(doc2.namespaceURI == xml.dom.EMPTY_NAMESPACE) + def testDocRemoveChild(self): + doc = parse(tstfile) + title_tag = doc.documentElement.getElementsByTagName("TITLE")[0] + self.assertRaises( xml.dom.NotFoundErr, doc.removeChild, title_tag) + num_children_before = len(doc.childNodes) + doc.removeChild(doc.childNodes[0]) + num_children_after = len(doc.childNodes) + self.assertTrue(num_children_after == num_children_before - 1) def test_main(): run_unittest(MinidomTest) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 0e18aabd66..1cfcee67c5 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -417,6 +417,35 @@ class MmapTests(unittest.TestCase): m[x] = b self.assertEqual(m[x], b) + def test_read_all(self): + m = mmap.mmap(-1, 16) + self.addCleanup(m.close) + + # With no parameters, or None or a negative argument, reads all + m.write(bytes(range(16))) + m.seek(0) + self.assertEqual(m.read(), bytes(range(16))) + m.seek(8) + self.assertEqual(m.read(), bytes(range(8, 16))) + m.seek(16) + self.assertEqual(m.read(), b'') + m.seek(3) + self.assertEqual(m.read(None), bytes(range(3, 16))) + m.seek(4) + self.assertEqual(m.read(-1), bytes(range(4, 16))) + m.seek(5) + self.assertEqual(m.read(-2), bytes(range(5, 16))) + m.seek(9) + self.assertEqual(m.read(-42), bytes(range(9, 16))) + + def test_read_invalid_arg(self): + m = mmap.mmap(-1, 16) + self.addCleanup(m.close) + + self.assertRaises(TypeError, m.read, 'foo') + self.assertRaises(TypeError, m.read, 5.5) + self.assertRaises(TypeError, m.read, [1, 2, 3]) + def test_extended_getslice(self): # Test extended slicing by comparing with list slicing. s = bytes(reversed(range(256))) diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py index a1842178df..c5fc320f64 100644 --- a/Lib/test/test_modulefinder.py +++ b/Lib/test/test_modulefinder.py @@ -1,7 +1,7 @@ -import __future__ import os +import errno +import shutil import unittest -import distutils.dir_util import tempfile from test import support @@ -9,7 +9,7 @@ from test import support import modulefinder TEST_DIR = tempfile.mkdtemp() -TEST_PATH = [TEST_DIR, os.path.dirname(__future__.__file__)] +TEST_PATH = [TEST_DIR, os.path.dirname(tempfile.__file__)] # Each test description is a list of 5 items: # @@ -196,12 +196,17 @@ a/module.py from . import bar """] + def open_file(path): - ##print "#", os.path.abspath(path) dirname = os.path.dirname(path) - distutils.dir_util.mkpath(dirname) + try: + os.makedirs(dirname) + except OSError as e: + if e.errno != errno.EEXIST: + raise return open(path, "w") + def create_package(source): ofi = None try: @@ -216,6 +221,7 @@ def create_package(source): if ofi: ofi.close() + class ModuleFinderTest(unittest.TestCase): def _do_test(self, info, report=False): import_this, modules, missing, maybe_missing, source = info @@ -234,19 +240,17 @@ class ModuleFinderTest(unittest.TestCase): ## import traceback; traceback.print_exc() ## sys.path = opath ## return - modules = set(modules) - found = set(mf.modules.keys()) - more = list(found - modules) - less = list(modules - found) + modules = sorted(set(modules)) + found = sorted(mf.modules) # check if we found what we expected, not more, not less - self.assertEqual((more, less), ([], [])) + self.assertEqual(found, modules) # check for missing and maybe missing modules bad, maybe = mf.any_missing_maybe() self.assertEqual(bad, missing) self.assertEqual(maybe, maybe_missing) finally: - distutils.dir_util.remove_tree(TEST_DIR) + shutil.rmtree(TEST_DIR) def test_package(self): self._do_test(package_test) @@ -254,25 +258,23 @@ class ModuleFinderTest(unittest.TestCase): def test_maybe(self): self._do_test(maybe_test) - if getattr(__future__, "absolute_import", None): + def test_maybe_new(self): + self._do_test(maybe_test_new) - def test_maybe_new(self): - self._do_test(maybe_test_new) + def test_absolute_imports(self): + self._do_test(absolute_import_test) - def test_absolute_imports(self): - self._do_test(absolute_import_test) + def test_relative_imports(self): + self._do_test(relative_import_test) - def test_relative_imports(self): - self._do_test(relative_import_test) + def test_relative_imports_2(self): + self._do_test(relative_import_test_2) - def test_relative_imports_2(self): - self._do_test(relative_import_test_2) + def test_relative_imports_3(self): + self._do_test(relative_import_test_3) - def test_relative_imports_3(self): - self._do_test(relative_import_test_3) def test_main(): - distutils.log.set_threshold(distutils.log.WARN) support.run_unittest(ModuleFinderTest) if __name__ == "__main__": diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 86c68dcd9a..44480723ba 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -260,7 +260,8 @@ class TestStateful(unittest.TestCase): text = '\u4E16\u4E16' encoding = 'iso-2022-jp' expected = b'\x1b$B@$@$' - expected_reset = b'\x1b$B@$@$\x1b(B' + reset = b'\x1b(B' + expected_reset = expected + reset def test_encode(self): self.assertEqual(self.text.encode(self.encoding), self.expected_reset) @@ -271,6 +272,8 @@ class TestStateful(unittest.TestCase): encoder.encode(char) for char in self.text) self.assertEqual(output, self.expected) + self.assertEqual(encoder.encode('', final=True), self.reset) + self.assertEqual(encoder.encode('', final=True), b'') def test_incrementalencoder_final(self): encoder = codecs.getincrementalencoder(self.encoding)() @@ -279,12 +282,14 @@ class TestStateful(unittest.TestCase): encoder.encode(char, index == last_index) for index, char in enumerate(self.text)) self.assertEqual(output, self.expected_reset) + self.assertEqual(encoder.encode('', final=True), b'') class TestHZStateful(TestStateful): text = '\u804a\u804a' encoding = 'hz' expected = b'~{ADAD' - expected_reset = b'~{ADAD~}' + reset = b'~}' + expected_reset = expected + reset def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index 6f1c6c4497..7a7805d59f 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -76,6 +76,23 @@ HAVE_GETVALUE = not getattr(_multiprocessing, 'HAVE_BROKEN_SEM_GETVALUE', False) WIN32 = (sys.platform == "win32") +if WIN32: + from _subprocess import WaitForSingleObject, INFINITE, WAIT_OBJECT_0 + + def wait_for_handle(handle, timeout): + if timeout is None or timeout < 0.0: + timeout = INFINITE + else: + timeout = int(1000 * timeout) + return WaitForSingleObject(handle, timeout) == WAIT_OBJECT_0 +else: + from select import select + _select = util._eintr_retry(select) + + def wait_for_handle(handle, timeout): + if timeout is not None and timeout < 0.0: + timeout = None + return handle in _select([handle], [], [], timeout)[0] try: MAXFD = os.sysconf("SC_OPEN_MAX") @@ -174,6 +191,18 @@ class _TestProcess(BaseTestCase): self.assertEqual(current.ident, os.getpid()) self.assertEqual(current.exitcode, None) + def test_daemon_argument(self): + if self.TYPE == "threads": + return + + # By default uses the current process's daemon flag. + proc0 = self.Process(target=self._test) + self.assertEqual(proc0.daemon, self.current_process().daemon) + proc1 = self.Process(target=self._test, daemon=True) + self.assertTrue(proc1.daemon) + proc2 = self.Process(target=self._test, daemon=False) + self.assertFalse(proc2.daemon) + @classmethod def _test(cls, q, *args, **kwds): current = cls.current_process() @@ -305,6 +334,26 @@ class _TestProcess(BaseTestCase): ] self.assertEqual(result, expected) + @classmethod + def _test_sentinel(cls, event): + event.wait(10.0) + + def test_sentinel(self): + if self.TYPE == "threads": + return + event = self.Event() + p = self.Process(target=self._test_sentinel, args=(event,)) + with self.assertRaises(ValueError): + p.sentinel + p.start() + self.addCleanup(p.join) + sentinel = p.sentinel + self.assertIsInstance(sentinel, int) + self.assertFalse(wait_for_handle(sentinel, timeout=0.0)) + event.set() + p.join() + self.assertTrue(wait_for_handle(sentinel, timeout=DELTA)) + # # # @@ -2010,9 +2059,15 @@ class TestInvalidHandle(unittest.TestCase): @unittest.skipIf(WIN32, "skipped on Windows") def test_invalid_handles(self): - conn = _multiprocessing.Connection(44977608) - self.assertRaises(IOError, conn.poll) - self.assertRaises(IOError, _multiprocessing.Connection, -1) + conn = multiprocessing.connection.Connection(44977608) + try: + self.assertRaises((ValueError, IOError), conn.poll) + finally: + # Hack private attribute _handle to avoid printing an error + # in conn.__del__ + conn._handle = None + self.assertRaises((ValueError, IOError), + multiprocessing.connection.Connection, -1) # # Functions used to create test cases from the base ones in this module diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index a387f61f1e..ec790ada52 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -1,10 +1,11 @@ import io +import socket import datetime import textwrap import unittest import functools import contextlib -import collections +import collections.abc from test import support from nntplib import NNTP, GroupInfo, _have_ssl import nntplib @@ -246,12 +247,32 @@ class NetworkedNNTPTestsMixin: if not name.startswith('test_'): continue meth = getattr(cls, name) - if not isinstance(meth, collections.Callable): + if not isinstance(meth, collections.abc.Callable): continue # Need to use a closure so that meth remains bound to its current # value setattr(cls, name, wrap_meth(meth)) + def test_with_statement(self): + def is_connected(): + if not hasattr(server, 'file'): + return False + try: + server.help() + except (socket.error, EOFError): + return False + return True + + with self.NNTP_CLASS(self.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) as server: + self.assertTrue(is_connected()) + self.assertTrue(server.help()) + self.assertFalse(is_connected()) + + with self.NNTP_CLASS(self.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) as server: + server.quit() + self.assertFalse(is_connected()) + + NetworkedNNTPTestsMixin.wrap_methods() @@ -813,7 +834,7 @@ class NNTPv1v2TestsMixin: def _check_article_body(self, lines): self.assertEqual(len(lines), 4) - self.assertEqual(lines[-1].decode('utf8'), "-- Signed by André.") + self.assertEqual(lines[-1].decode('utf-8'), "-- Signed by André.") self.assertEqual(lines[-2], b"") self.assertEqual(lines[-3], b".Here is a dot-starting line.") self.assertEqual(lines[-4], b"This is just a test article.") diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 7b95612024..d1ae7574b7 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -631,7 +631,7 @@ class TestStandard(BaseTest): option_list=options) def test_required_value(self): - self.assertParseFail(["-a"], "-a option requires an argument") + self.assertParseFail(["-a"], "-a option requires 1 argument") def test_invalid_integer(self): self.assertParseFail(["-b", "5x"], @@ -1023,7 +1023,7 @@ class TestExtendAddTypes(BaseTest): TYPE_CHECKER["file"] = check_file def test_filetype_ok(self): - open(support.TESTFN, "w").close() + support.create_empty_file(support.TESTFN) self.assertParseOK(["--file", support.TESTFN, "-afoo"], {'file': support.TESTFN, 'a': 'foo'}, []) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index efa28ea952..569b2184a9 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -14,18 +14,25 @@ import shutil from test import support import contextlib import mmap +import platform +import re import uuid +import asyncore +import asynchat +import socket +try: + import threading +except ImportError: + threading = None # Detect whether we're on a Linux system that uses the (now outdated # and unmaintained) linuxthreads threading library. There's an issue # when combining linuxthreads with a failed execv call: see # http://bugs.python.org/issue4970. -if (hasattr(os, "confstr_names") and - "CS_GNU_LIBPTHREAD_VERSION" in os.confstr_names): - libpthread = os.confstr("CS_GNU_LIBPTHREAD_VERSION") - USING_LINUXTHREADS= libpthread.startswith("linuxthreads") +if hasattr(sys, 'thread_info') and sys.thread_info.version: + USING_LINUXTHREADS = sys.thread_info.version.startswith("linuxthreads") else: - USING_LINUXTHREADS= False + USING_LINUXTHREADS = False # Tests creating TESTFN class FileTests(unittest.TestCase): @@ -919,8 +926,7 @@ if sys.platform != 'win32': os.mkdir(self.dir) try: for fn in bytesfn: - f = open(os.path.join(self.bdir, fn), "w") - f.close() + support.create_empty_file(os.path.join(self.bdir, fn)) fn = os.fsdecode(fn) if fn in self.unicodefn: raise ValueError("duplicate filename") @@ -1220,6 +1226,380 @@ class LoginTests(unittest.TestCase): self.assertNotEqual(len(user_name), 0) +@unittest.skipUnless(hasattr(os, 'getpriority') and hasattr(os, 'setpriority'), + "needs os.getpriority and os.setpriority") +class ProgramPriorityTests(unittest.TestCase): + """Tests for os.getpriority() and os.setpriority().""" + + def test_set_get_priority(self): + + base = os.getpriority(os.PRIO_PROCESS, os.getpid()) + os.setpriority(os.PRIO_PROCESS, os.getpid(), base + 1) + try: + new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid()) + if base >= 19 and new_prio <= 19: + raise unittest.SkipTest( + "unable to reliably test setpriority at current nice level of %s" % base) + else: + self.assertEqual(new_prio, base + 1) + finally: + try: + os.setpriority(os.PRIO_PROCESS, os.getpid(), base) + except OSError as err: + if err.errno != errno.EACCES: + raise + + +if threading is not None: + class SendfileTestServer(asyncore.dispatcher, threading.Thread): + + class Handler(asynchat.async_chat): + + def __init__(self, conn): + asynchat.async_chat.__init__(self, conn) + self.in_buffer = [] + self.closed = False + self.push(b"220 ready\r\n") + + def handle_read(self): + data = self.recv(4096) + self.in_buffer.append(data) + + def get_data(self): + return b''.join(self.in_buffer) + + def handle_close(self): + self.close() + self.closed = True + + def handle_error(self): + raise + + def __init__(self, address): + threading.Thread.__init__(self) + asyncore.dispatcher.__init__(self) + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.bind(address) + self.listen(5) + self.host, self.port = self.socket.getsockname()[:2] + self.handler_instance = None + self._active = False + self._active_lock = threading.Lock() + + # --- public API + + @property + def running(self): + return self._active + + def start(self): + assert not self.running + self.__flag = threading.Event() + threading.Thread.start(self) + self.__flag.wait() + + def stop(self): + assert self.running + self._active = False + self.join() + + def wait(self): + # wait for handler connection to be closed, then stop the server + while not getattr(self.handler_instance, "closed", False): + time.sleep(0.001) + self.stop() + + # --- internals + + def run(self): + self._active = True + self.__flag.set() + while self._active and asyncore.socket_map: + self._active_lock.acquire() + asyncore.loop(timeout=0.001, count=1) + self._active_lock.release() + asyncore.close_all() + + def handle_accept(self): + conn, addr = self.accept() + self.handler_instance = self.Handler(conn) + + def handle_connect(self): + self.close() + handle_read = handle_connect + + def writable(self): + return 0 + + def handle_error(self): + raise + + +@unittest.skipUnless(threading is not None, "test needs threading module") +@unittest.skipUnless(hasattr(os, 'sendfile'), "test needs os.sendfile()") +class TestSendfile(unittest.TestCase): + + DATA = b"12345abcde" * 16 * 1024 # 160 KB + SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \ + not sys.platform.startswith("solaris") and \ + not sys.platform.startswith("sunos") + + @classmethod + def setUpClass(cls): + with open(support.TESTFN, "wb") as f: + f.write(cls.DATA) + + @classmethod + def tearDownClass(cls): + support.unlink(support.TESTFN) + + def setUp(self): + self.server = SendfileTestServer((support.HOST, 0)) + self.server.start() + self.client = socket.socket() + self.client.connect((self.server.host, self.server.port)) + self.client.settimeout(1) + # synchronize by waiting for "220 ready" response + self.client.recv(1024) + self.sockno = self.client.fileno() + self.file = open(support.TESTFN, 'rb') + self.fileno = self.file.fileno() + + def tearDown(self): + self.file.close() + self.client.close() + if self.server.running: + self.server.stop() + + def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]): + """A higher level wrapper representing how an application is + supposed to use sendfile(). + """ + while 1: + try: + if self.SUPPORT_HEADERS_TRAILERS: + return os.sendfile(sock, file, offset, nbytes, headers, + trailers) + else: + return os.sendfile(sock, file, offset, nbytes) + except OSError as err: + if err.errno == errno.ECONNRESET: + # disconnected + raise + elif err.errno in (errno.EAGAIN, errno.EBUSY): + # we have to retry send data + continue + else: + raise + + def test_send_whole_file(self): + # normal send + total_sent = 0 + offset = 0 + nbytes = 4096 + while total_sent < len(self.DATA): + sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) + if sent == 0: + break + offset += sent + total_sent += sent + self.assertTrue(sent <= nbytes) + self.assertEqual(offset, total_sent) + + self.assertEqual(total_sent, len(self.DATA)) + self.client.shutdown(socket.SHUT_RDWR) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(len(data), len(self.DATA)) + self.assertEqual(data, self.DATA) + + def test_send_at_certain_offset(self): + # start sending a file at a certain offset + total_sent = 0 + offset = len(self.DATA) // 2 + must_send = len(self.DATA) - offset + nbytes = 4096 + while total_sent < must_send: + sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) + if sent == 0: + break + offset += sent + total_sent += sent + self.assertTrue(sent <= nbytes) + + self.client.shutdown(socket.SHUT_RDWR) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + expected = self.DATA[len(self.DATA) // 2:] + self.assertEqual(total_sent, len(expected)) + self.assertEqual(len(data), len(expected)) + self.assertEqual(data, expected) + + def test_offset_overflow(self): + # specify an offset > file size + offset = len(self.DATA) + 4096 + try: + sent = os.sendfile(self.sockno, self.fileno, offset, 4096) + except OSError as e: + # Solaris can raise EINVAL if offset >= file length, ignore. + if e.errno != errno.EINVAL: + raise + else: + self.assertEqual(sent, 0) + self.client.shutdown(socket.SHUT_RDWR) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(data, b'') + + def test_invalid_offset(self): + with self.assertRaises(OSError) as cm: + os.sendfile(self.sockno, self.fileno, -1, 4096) + self.assertEqual(cm.exception.errno, errno.EINVAL) + + # --- headers / trailers tests + + if SUPPORT_HEADERS_TRAILERS: + + def test_headers(self): + total_sent = 0 + sent = os.sendfile(self.sockno, self.fileno, 0, 4096, + headers=[b"x" * 512]) + total_sent += sent + offset = 4096 + nbytes = 4096 + while 1: + sent = self.sendfile_wrapper(self.sockno, self.fileno, + offset, nbytes) + if sent == 0: + break + total_sent += sent + offset += sent + + expected_data = b"x" * 512 + self.DATA + self.assertEqual(total_sent, len(expected_data)) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(hash(data), hash(expected_data)) + + def test_trailers(self): + TESTFN2 = support.TESTFN + "2" + with open(TESTFN2, 'wb') as f: + f.write(b"abcde") + with open(TESTFN2, 'rb')as f: + self.addCleanup(os.remove, TESTFN2) + os.sendfile(self.sockno, f.fileno(), 0, 4096, + trailers=[b"12345"]) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(data, b"abcde12345") + + if hasattr(os, "SF_NODISKIO"): + def test_flags(self): + try: + os.sendfile(self.sockno, self.fileno, 0, 4096, + flags=os.SF_NODISKIO) + except OSError as err: + if err.errno not in (errno.EBUSY, errno.EAGAIN): + raise + + +def supports_extended_attributes(): + if not hasattr(os, "setxattr"): + return False + try: + with open(support.TESTFN, "wb") as fp: + try: + os.fsetxattr(fp.fileno(), b"user.test", b"") + except OSError as e: + if e.errno != errno.ENOTSUP: + raise + return False + finally: + support.unlink(support.TESTFN) + # Kernels < 2.6.39 don't respect setxattr flags. + kernel_version = platform.release() + m = re.match("2.6.(\d{1,2})", kernel_version) + return m is None or int(m.group(1)) >= 39 + + +@unittest.skipUnless(supports_extended_attributes(), + "no non-broken extended attribute support") +class ExtendedAttributeTests(unittest.TestCase): + + def tearDown(self): + support.unlink(support.TESTFN) + + def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr): + fn = support.TESTFN + open(fn, "wb").close() + with self.assertRaises(OSError) as cm: + getxattr(fn, s("user.test")) + self.assertEqual(cm.exception.errno, errno.ENODATA) + self.assertEqual(listxattr(fn), []) + setxattr(fn, s("user.test"), b"") + self.assertEqual(listxattr(fn), ["user.test"]) + self.assertEqual(getxattr(fn, b"user.test"), b"") + setxattr(fn, s("user.test"), b"hello", os.XATTR_REPLACE) + self.assertEqual(getxattr(fn, b"user.test"), b"hello") + with self.assertRaises(OSError) as cm: + setxattr(fn, s("user.test"), b"bye", os.XATTR_CREATE) + self.assertEqual(cm.exception.errno, errno.EEXIST) + with self.assertRaises(OSError) as cm: + setxattr(fn, s("user.test2"), b"bye", os.XATTR_REPLACE) + self.assertEqual(cm.exception.errno, errno.ENODATA) + setxattr(fn, s("user.test2"), b"foo", os.XATTR_CREATE) + self.assertEqual(sorted(listxattr(fn)), ["user.test", "user.test2"]) + removexattr(fn, s("user.test")) + with self.assertRaises(OSError) as cm: + getxattr(fn, s("user.test")) + self.assertEqual(cm.exception.errno, errno.ENODATA) + self.assertEqual(listxattr(fn), ["user.test2"]) + self.assertEqual(getxattr(fn, s("user.test2")), b"foo") + setxattr(fn, s("user.test"), b"a"*1024) + self.assertEqual(getxattr(fn, s("user.test")), b"a"*1024) + removexattr(fn, s("user.test")) + many = sorted("user.test{}".format(i) for i in range(100)) + for thing in many: + setxattr(fn, thing, b"x") + self.assertEqual(sorted(listxattr(fn)), many) + + def _check_xattrs(self, *args): + def make_bytes(s): + return bytes(s, "ascii") + self._check_xattrs_str(str, *args) + support.unlink(support.TESTFN) + self._check_xattrs_str(make_bytes, *args) + + def test_simple(self): + self._check_xattrs(os.getxattr, os.setxattr, os.removexattr, + os.listxattr) + + def test_lpath(self): + self._check_xattrs(os.lgetxattr, os.lsetxattr, os.lremovexattr, + os.llistxattr) + + def test_fds(self): + def getxattr(path, *args): + with open(path, "rb") as fp: + return os.fgetxattr(fp.fileno(), *args) + def setxattr(path, *args): + with open(path, "wb") as fp: + os.fsetxattr(fp.fileno(), *args) + def removexattr(path, *args): + with open(path, "wb") as fp: + os.fremovexattr(fp.fileno(), *args) + def listxattr(path, *args): + with open(path, "rb") as fp: + return os.flistxattr(fp.fileno(), *args) + self._check_xattrs(getxattr, setxattr, removexattr, listxattr) + + +@support.reap_threads def test_main(): support.run_unittest( FileTests, @@ -1240,6 +1620,9 @@ def test_main(): PidTests, LoginTests, LinkTests, + TestSendfile, + ProgramPriorityTests, + ExtendedAttributeTests, ) if __name__ == "__main__": diff --git a/Lib/test/test_ossaudiodev.py b/Lib/test/test_ossaudiodev.py index 9cb89d69a6..3908a0506e 100644 --- a/Lib/test/test_ossaudiodev.py +++ b/Lib/test/test_ossaudiodev.py @@ -170,6 +170,22 @@ class OSSAudioDevTests(unittest.TestCase): pass self.assertTrue(dsp.closed) + def test_on_closed(self): + dsp = ossaudiodev.open('w') + dsp.close() + self.assertRaises(ValueError, dsp.fileno) + self.assertRaises(ValueError, dsp.read, 1) + self.assertRaises(ValueError, dsp.write, b'x') + self.assertRaises(ValueError, dsp.writeall, b'x') + self.assertRaises(ValueError, dsp.bufsize) + self.assertRaises(ValueError, dsp.obufcount) + self.assertRaises(ValueError, dsp.obufcount) + self.assertRaises(ValueError, dsp.obuffree) + self.assertRaises(ValueError, dsp.getptr) + + mixer = ossaudiodev.openmixer() + mixer.close() + self.assertRaises(ValueError, mixer.fileno) def test_main(): try: diff --git a/Lib/test/test_osx_env.py b/Lib/test/test_osx_env.py index 8b3df3783b..24ec2b4403 100644 --- a/Lib/test/test_osx_env.py +++ b/Lib/test/test_osx_env.py @@ -5,6 +5,7 @@ Test suite for OS X interpreter environment variables. from test.support import EnvironmentVarGuard, run_unittest import subprocess import sys +import sysconfig import unittest class OSXEnvironmentVariableTestCase(unittest.TestCase): @@ -27,8 +28,6 @@ class OSXEnvironmentVariableTestCase(unittest.TestCase): self._check_sys('PYTHONEXECUTABLE', '==', 'sys.executable') def test_main(): - from distutils import sysconfig - if sys.platform == 'darwin' and sysconfig.get_config_var('WITH_NEXT_FRAMEWORK'): run_unittest(OSXEnvironmentVariableTestCase) diff --git a/Lib/test/test_packaging.py b/Lib/test/test_packaging.py new file mode 100644 index 0000000000..250d661b74 --- /dev/null +++ b/Lib/test/test_packaging.py @@ -0,0 +1,5 @@ +import sys +from packaging.tests.__main__ import test_main + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 020acd500f..2b50fca818 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -614,6 +614,13 @@ class STObjectTestCase(unittest.TestCase): # XXX tests for pickling and unpickling of ST objects should go here +class OtherParserCase(unittest.TestCase): + + def test_two_args_to_expr(self): + # See bug #12264 + with self.assertRaises(TypeError): + parser.expr("a", "b") + def test_main(): support.run_unittest( @@ -622,6 +629,7 @@ def test_main(): CompileTestCase, ParserStackLimitTestCase, STObjectTestCase, + OtherParserCase, ) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index d861df52bb..c197aff318 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -20,9 +20,12 @@ class PdbTestInput(object): def __enter__(self): self.real_stdin = sys.stdin sys.stdin = _FakeInput(self.input) + self.orig_trace = sys.gettrace() if hasattr(sys, 'gettrace') else None def __exit__(self, *exc): sys.stdin = self.real_stdin + if self.orig_trace: + sys.settrace(self.orig_trace) def test_pdb_displayhook(): diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 53719d3a82..e0e3f630a7 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -3,13 +3,16 @@ import re import sys from io import StringIO import unittest +from math import copysign def disassemble(func): f = StringIO() tmp = sys.stdout sys.stdout = f - dis.dis(func) - sys.stdout = tmp + try: + dis.dis(func) + finally: + sys.stdout = tmp result = f.getvalue() f.close() return result @@ -17,6 +20,7 @@ def disassemble(func): def dis_single(line): return disassemble(compile(line, '', 'single')) + class TestTranforms(unittest.TestCase): def test_unot(self): @@ -99,6 +103,12 @@ class TestTranforms(unittest.TestCase): self.assertIn(elem, asm) self.assertNotIn('BUILD_TUPLE', asm) + # Long tuples should be folded too. + asm = dis_single(repr(tuple(range(10000)))) + # One LOAD_CONST for the tuple, one for the None return value + self.assertEqual(asm.count('LOAD_CONST'), 2) + self.assertNotIn('BUILD_TUPLE', asm) + # Bug 1053819: Tuple of constants misidentified when presented with: # . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . . # The following would segfault upon compilation @@ -216,6 +226,9 @@ class TestTranforms(unittest.TestCase): def test_folding_of_unaryops_on_constants(self): for line, elem in ( ('-0.5', '(-0.5)'), # unary negative + ('-0.0', '(-0.0)'), # -0.0 + ('-(1.0-1.0)','(-0.0)'), # -0.0 after folding + ('-0', '(0)'), # -0 ('~-2', '(1)'), # unary invert ('+1', '(1)'), # unary positive ): @@ -223,6 +236,13 @@ class TestTranforms(unittest.TestCase): self.assertIn(elem, asm, asm) self.assertNotIn('UNARY_', asm) + # Check that -0.0 works after marshaling + def negzero(): + return -(1.0-1.0) + + self.assertNotIn('UNARY_', disassemble(negzero)) + self.assertTrue(copysign(1.0, negzero()) < 0) + # Verify that unfoldables are skipped for line, elem in ( ('-"abc"', "('abc')"), # unary negative @@ -285,6 +305,25 @@ class TestTranforms(unittest.TestCase): asm = disassemble(f) self.assertNotIn('BINARY_ADD', asm) + def test_constant_folding(self): + # Issue #11244: aggressive constant folding. + exprs = [ + "3 * -5", + "-3 * 5", + "2 * (3 * 4)", + "(2 * 3) * 4", + "(-1, 2, 3)", + "(1, -2, 3)", + "(1, 2, -3)", + "(1, 2, -3) * 6", + "lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}", + ] + for e in exprs: + asm = dis_single(e) + self.assertNotIn('UNARY_', asm, e) + self.assertNotIn('BINARY_', asm, e) + self.assertNotIn('BUILD_', asm, e) + class TestBuglets(unittest.TestCase): def test_bug_11510(self): diff --git a/Lib/test/test_pep292.py b/Lib/test/test_pep292.py index 119c7ea8a1..6da8d2e3e1 100644 --- a/Lib/test/test_pep292.py +++ b/Lib/test/test_pep292.py @@ -42,19 +42,6 @@ class TestTemplate(unittest.TestCase): s = Template('$who likes $$') eq(s.substitute(dict(who='tim', what='ham')), 'tim likes $') - def test_invalid(self): - class MyPattern(Template): - pattern = r""" - (?: - (?P<invalid>) | - (?P<escaped>%(delim)s) | - @(?P<named>%(id)s) | - @{(?P<braced>%(id)s)} - ) - """ - s = MyPattern('$') - self.assertRaises(ValueError, s.substitute, dict()) - def test_percents(self): eq = self.assertEqual s = Template('%(foo)s $foo ${foo}') @@ -172,6 +159,26 @@ class TestTemplate(unittest.TestCase): val = t.safe_substitute({'location': 'Cleveland'}) self.assertEqual(val, 'PyCon in Cleveland') + def test_invalid_with_no_lines(self): + # The error formatting for invalid templates + # has a special case for no data that the default + # pattern can't trigger (always has at least '$') + # So we craft a pattern that is always invalid + # with no leading data. + class MyTemplate(Template): + pattern = r""" + (?P<invalid>) | + unreachable( + (?P<named>) | + (?P<braced>) | + (?P<escaped>) + ) + """ + s = MyTemplate('') + with self.assertRaises(ValueError) as err: + s.substitute({}) + self.assertIn('line 1, col 1', str(err.exception)) + def test_unicode_values(self): s = Template('$who likes $what') d = dict(who='t\xffm', what='f\xfe\fed') diff --git a/Lib/test/test_pep3120.py b/Lib/test/test_pep3120.py index 09fedf0d54..496f8da971 100644 --- a/Lib/test/test_pep3120.py +++ b/Lib/test/test_pep3120.py @@ -19,8 +19,8 @@ class PEP3120Test(unittest.TestCase): try: import test.badsyntax_pep3120 except SyntaxError as msg: - msg = str(msg) - self.assertTrue('UTF-8' in msg or 'utf8' in msg) + msg = str(msg).lower() + self.assertTrue('utf-8' in msg or 'utf8' in msg) else: self.fail("expected exception didn't occur") diff --git a/Lib/test/test_pipes.py b/Lib/test/test_pipes.py index d5b886f87e..6a7b45fb46 100644 --- a/Lib/test/test_pipes.py +++ b/Lib/test/test_pipes.py @@ -79,21 +79,6 @@ class SimplePipeTests(unittest.TestCase): with open(TESTFN) as f: self.assertEqual(f.read(), d) - def testQuoting(self): - safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./' - unicode_sample = '\xe9\xe0\xdf' # e + acute accent, a + grave, sharp s - unsafe = '"`$\\!' + unicode_sample - - self.assertEqual(pipes.quote(''), "''") - self.assertEqual(pipes.quote(safeunquoted), safeunquoted) - self.assertEqual(pipes.quote('test file name'), "'test file name'") - for u in unsafe: - self.assertEqual(pipes.quote('test%sname' % u), - "'test%sname'" % u) - for u in unsafe: - self.assertEqual(pipes.quote("test%s'name'" % u), - "'test%s'\"'\"'name'\"'\"''" % u) - def testRepr(self): t = pipes.Template() self.assertEqual(repr(t), "<Template instance, steps=[]>") diff --git a/Lib/test/test_pkgimport.py b/Lib/test/test_pkgimport.py index c37e9362b0..a8426b5c9a 100644 --- a/Lib/test/test_pkgimport.py +++ b/Lib/test/test_pkgimport.py @@ -7,7 +7,7 @@ import tempfile import unittest from imp import cache_from_source -from test.support import run_unittest +from test.support import run_unittest, create_empty_file class TestImport(unittest.TestCase): @@ -29,7 +29,7 @@ class TestImport(unittest.TestCase): self.package_dir = os.path.join(self.test_dir, self.package_name) os.mkdir(self.package_dir) - open(os.path.join(self.package_dir, '__init__.py'), 'w').close() + create_empty_file(os.path.join(self.package_dir, '__init__.py')) self.module_path = os.path.join(self.package_dir, 'foo.py') def tearDown(self): diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 8751aa86ba..cfe623a867 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -1,8 +1,9 @@ -import sys import os -import unittest import platform import subprocess +import sys +import unittest +import warnings from test import support @@ -56,13 +57,11 @@ class PlatformTest(unittest.TestCase): def setUp(self): self.save_version = sys.version - self.save_subversion = sys.subversion self.save_mercurial = sys._mercurial self.save_platform = sys.platform def tearDown(self): sys.version = self.save_version - sys.subversion = self.save_subversion sys._mercurial = self.save_mercurial sys.platform = self.save_platform @@ -77,7 +76,7 @@ class PlatformTest(unittest.TestCase): ('IronPython', '1.0.0', '', '', '', '', '.NET 2.0.50727.42')), ): # branch and revision are not "parsed", but fetched - # from sys.subversion. Ignore them + # from sys._mercurial. Ignore them (name, version, branch, revision, buildno, builddate, compiler) \ = platform._sys_version(input) self.assertEqual( @@ -113,8 +112,6 @@ class PlatformTest(unittest.TestCase): if subversion is None: if hasattr(sys, "_mercurial"): del sys._mercurial - if hasattr(sys, "subversion"): - del sys.subversion else: sys._mercurial = subversion if sys_platform is not None: @@ -247,6 +244,38 @@ class PlatformTest(unittest.TestCase): ): self.assertEqual(platform._parse_release_file(input), output) + def test_popen(self): + mswindows = (sys.platform == "win32") + + if mswindows: + command = '"{}" -c "print(\'Hello\')"'.format(sys.executable) + else: + command = "'{}' -c 'print(\"Hello\")'".format(sys.executable) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + with platform.popen(command) as stdout: + hello = stdout.read().strip() + stdout.close() + self.assertEqual(hello, "Hello") + + data = 'plop' + if mswindows: + command = '"{}" -c "import sys; data=sys.stdin.read(); exit(len(data))"' + else: + command = "'{}' -c 'import sys; data=sys.stdin.read(); exit(len(data))'" + command = command.format(sys.executable) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + with platform.popen(command, 'w') as stdin: + stdout = stdin.write(data) + ret = stdin.close() + self.assertIsNotNone(ret) + if os.name == 'nt': + returncode = ret + else: + returncode = ret >> 8 + self.assertEqual(returncode, len(data)) + def test_main(): support.run_unittest( diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 81af56989f..0a3adccb65 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -108,6 +108,10 @@ class DummyPOP3Handler(asynchat.async_chat): def cmd_apop(self, arg): self.push('+OK done nothing.') + def cmd_quit(self, arg): + self.push('+OK closing.') + self.close_when_done() + class DummyPOP3Server(asyncore.dispatcher, threading.Thread): @@ -165,10 +169,10 @@ class TestPOP3Class(TestCase): def setUp(self): self.server = DummyPOP3Server((HOST, PORT)) self.server.start() - self.client = poplib.POP3(self.server.host, self.server.port) + self.client = poplib.POP3(self.server.host, self.server.port, timeout=3) def tearDown(self): - self.client.quit() + self.client.close() self.server.stop() def test_getwelcome(self): @@ -228,6 +232,12 @@ class TestPOP3Class(TestCase): self.client.uidl() self.client.uidl('foo') + def test_quit(self): + resp = self.client.quit() + self.assertTrue(resp) + self.assertIsNone(self.client.sock) + self.assertIsNone(self.client.file) + SUPPORTS_SSL = False if hasattr(poplib, 'POP3_SSL'): @@ -274,6 +284,7 @@ if hasattr(poplib, 'POP3_SSL'): else: DummyPOP3Handler.handle_read(self) + class TestPOP3_SSLClass(TestPOP3Class): # repeat previous tests by using poplib.POP3_SSL diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 09f04ec348..a098fc088b 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -9,6 +9,7 @@ import errno import sys import time import os +import fcntl import pwd import shutil import stat @@ -42,7 +43,7 @@ class PosixTester(unittest.TestCase): NO_ARG_FUNCTIONS = [ "ctermid", "getcwd", "getcwdb", "uname", "times", "getloadavg", "getegid", "geteuid", "getgid", "getgroups", - "getpid", "getpgrp", "getppid", "getuid", + "getpid", "getpgrp", "getppid", "getuid", "sync", ] for name in NO_ARG_FUNCTIONS: @@ -137,6 +138,156 @@ class PosixTester(unittest.TestCase): finally: fp.close() + @unittest.skipUnless(hasattr(posix, 'truncate'), "test needs posix.truncate()") + def test_truncate(self): + with open(support.TESTFN, 'w') as fp: + fp.write('test') + fp.flush() + posix.truncate(support.TESTFN, 0) + + @unittest.skipUnless(hasattr(posix, 'fexecve'), "test needs posix.fexecve()") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_fexecve(self): + fp = os.open(sys.executable, os.O_RDONLY) + try: + pid = os.fork() + if pid == 0: + os.chdir(os.path.split(sys.executable)[0]) + posix.fexecve(fp, [sys.executable, '-c', 'pass'], os.environ) + else: + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + finally: + os.close(fp) + + @unittest.skipUnless(hasattr(posix, 'waitid'), "test needs posix.waitid()") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_waitid(self): + pid = os.fork() + if pid == 0: + os.chdir(os.path.split(sys.executable)[0]) + posix.execve(sys.executable, [sys.executable, '-c', 'pass'], os.environ) + else: + res = posix.waitid(posix.P_PID, pid, posix.WEXITED) + self.assertEqual(pid, res.si_pid) + + @unittest.skipUnless(hasattr(posix, 'lockf'), "test needs posix.lockf()") + def test_lockf(self): + fd = os.open(support.TESTFN, os.O_WRONLY | os.O_CREAT) + try: + os.write(fd, b'test') + os.lseek(fd, 0, os.SEEK_SET) + posix.lockf(fd, posix.F_LOCK, 4) + # section is locked + posix.lockf(fd, posix.F_ULOCK, 4) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'pread'), "test needs posix.pread()") + def test_pread(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test') + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(b'es', posix.pread(fd, 2, 1)) + # the first pread() shoudn't disturb the file offset + self.assertEqual(b'te', posix.read(fd, 2)) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'pwrite'), "test needs posix.pwrite()") + def test_pwrite(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test') + os.lseek(fd, 0, os.SEEK_SET) + posix.pwrite(fd, b'xx', 1) + self.assertEqual(b'txxt', posix.read(fd, 4)) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'posix_fallocate'), + "test needs posix.posix_fallocate()") + def test_posix_fallocate(self): + fd = os.open(support.TESTFN, os.O_WRONLY | os.O_CREAT) + try: + posix.posix_fallocate(fd, 0, 10) + except OSError as inst: + # issue10812, ZFS doesn't appear to support posix_fallocate, + # so skip Solaris-based since they are likely to have ZFS. + if inst.errno != errno.EINVAL or not sys.platform.startswith("sunos"): + raise + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'posix_fadvise'), + "test needs posix.posix_fadvise()") + def test_posix_fadvise(self): + fd = os.open(support.TESTFN, os.O_RDONLY) + try: + posix.posix_fadvise(fd, 0, 0, posix.POSIX_FADV_WILLNEED) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'futimes'), "test needs posix.futimes()") + def test_futimes(self): + now = time.time() + fd = os.open(support.TESTFN, os.O_RDONLY) + try: + posix.futimes(fd, None) + self.assertRaises(TypeError, posix.futimes, fd, (None, None)) + self.assertRaises(TypeError, posix.futimes, fd, (now, None)) + self.assertRaises(TypeError, posix.futimes, fd, (None, now)) + posix.futimes(fd, (int(now), int(now))) + posix.futimes(fd, (now, now)) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'lutimes'), "test needs posix.lutimes()") + def test_lutimes(self): + now = time.time() + posix.lutimes(support.TESTFN, None) + self.assertRaises(TypeError, posix.lutimes, support.TESTFN, (None, None)) + self.assertRaises(TypeError, posix.lutimes, support.TESTFN, (now, None)) + self.assertRaises(TypeError, posix.lutimes, support.TESTFN, (None, now)) + posix.lutimes(support.TESTFN, (int(now), int(now))) + posix.lutimes(support.TESTFN, (now, now)) + + @unittest.skipUnless(hasattr(posix, 'futimens'), "test needs posix.futimens()") + def test_futimens(self): + now = time.time() + fd = os.open(support.TESTFN, os.O_RDONLY) + try: + self.assertRaises(TypeError, posix.futimens, fd, (None, None), (None, None)) + self.assertRaises(TypeError, posix.futimens, fd, (now, 0), None) + self.assertRaises(TypeError, posix.futimens, fd, None, (now, 0)) + posix.futimens(fd, (int(now), int((now - int(now)) * 1e9)), + (int(now), int((now - int(now)) * 1e9))) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'writev'), "test needs posix.writev()") + def test_writev(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + os.writev(fd, (b'test1', b'tt2', b't3')) + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(b'test1tt2t3', posix.read(fd, 10)) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'readv'), "test needs posix.readv()") + def test_readv(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test1tt2t3') + os.lseek(fd, 0, os.SEEK_SET) + buf = [bytearray(i) for i in [5, 3, 2]] + self.assertEqual(posix.readv(fd, buf), 10) + self.assertEqual([b'test1', b'tt2', b't3'], [bytes(i) for i in buf]) + finally: + os.close(fd) + def test_dup(self): if hasattr(posix, 'dup'): fp = open(support.TESTFN) @@ -162,6 +313,13 @@ class PosixTester(unittest.TestCase): fp1.close() fp2.close() + @unittest.skipUnless(hasattr(os, 'O_CLOEXEC'), "needs os.O_CLOEXEC") + @support.requires_linux_version(2, 6, 23) + def test_oscloexec(self): + fd = os.open(support.TESTFN, os.O_RDONLY|os.O_CLOEXEC) + self.addCleanup(os.close, fd) + self.assertTrue(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + def test_osexlock(self): if hasattr(posix, "O_EXLOCK"): fd = os.open(support.TESTFN, @@ -254,7 +412,7 @@ class PosixTester(unittest.TestCase): self.assertRaises(OSError, posix.chown, support.TESTFN, -1, -1) # re-create the file - open(support.TESTFN, 'w').close() + support.create_empty_file(support.TESTFN) self._test_all_chown_common(posix.chown, support.TESTFN) @unittest.skipUnless(hasattr(posix, 'fchown'), "test needs os.fchown()") @@ -290,6 +448,18 @@ class PosixTester(unittest.TestCase): if hasattr(posix, 'listdir'): self.assertTrue(support.TESTFN in posix.listdir()) + @unittest.skipUnless(hasattr(posix, 'fdlistdir'), "test needs posix.fdlistdir()") + def test_fdlistdir(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + self.assertEqual( + sorted(posix.listdir('.')), + sorted(posix.fdlistdir(f)) + ) + # Check the fd was closed by fdlistdir + with self.assertRaises(OSError) as ctx: + posix.close(f) + self.assertEqual(ctx.exception.errno, errno.EBADF) + def test_access(self): if hasattr(posix, 'access'): self.assertTrue(posix.access(support.TESTFN, os.R_OK)) @@ -310,6 +480,32 @@ class PosixTester(unittest.TestCase): os.close(reader) os.close(writer) + @unittest.skipUnless(hasattr(os, 'pipe2'), "test needs os.pipe2()") + @support.requires_linux_version(2, 6, 27) + def test_pipe2(self): + self.assertRaises(TypeError, os.pipe2, 'DEADBEEF') + self.assertRaises(TypeError, os.pipe2, 0, 0) + + # try calling with flags = 0, like os.pipe() + r, w = os.pipe2(0) + os.close(r) + os.close(w) + + # test flags + r, w = os.pipe2(os.O_CLOEXEC|os.O_NONBLOCK) + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + self.assertTrue(fcntl.fcntl(r, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + self.assertTrue(fcntl.fcntl(w, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + # try reading from an empty pipe: this should fail, not block + self.assertRaises(OSError, os.read, r, 1) + # try a write big enough to fill-up the pipe: this should either + # fail or perform a partial write, not block + try: + os.write(w, b'x' * support.PIPE_MAX_SIZE) + except OSError: + pass + def test_utime(self): if hasattr(posix, 'utime'): now = time.time() @@ -410,6 +606,21 @@ class PosixTester(unittest.TestCase): os.chdir(curdir) support.rmtree(base_path) + @unittest.skipUnless(hasattr(posix, 'getgrouplist'), "test needs posix.getgrouplist()") + @unittest.skipUnless(hasattr(pwd, 'getpwuid'), "test needs pwd.getpwuid()") + @unittest.skipUnless(hasattr(os, 'getuid'), "test needs os.getuid()") + def test_getgrouplist(self): + with os.popen('id -G') as idg: + groups = idg.read().strip() + + if not groups: + raise unittest.SkipTest("need working 'id -G'") + + self.assertEqual( + set([int(x) for x in groups.split()]), + set(posix.getgrouplist(pwd.getpwuid(os.getuid())[0], + pwd.getpwuid(os.getuid())[3]))) + @unittest.skipUnless(hasattr(os, 'getegid'), "test needs os.getegid()") def test_getgroups(self): with os.popen('id -G') as idg: @@ -426,6 +637,351 @@ class PosixTester(unittest.TestCase): set([int(x) for x in groups.split()]), set(posix.getgroups() + [posix.getegid()])) + # tests for the posix *at functions follow + + @unittest.skipUnless(hasattr(posix, 'faccessat'), "test needs posix.faccessat()") + def test_faccessat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + self.assertTrue(posix.faccessat(f, support.TESTFN, os.R_OK)) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'fchmodat'), "test needs posix.fchmodat()") + def test_fchmodat(self): + os.chmod(support.TESTFN, stat.S_IRUSR) + + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.fchmodat(f, support.TESTFN, stat.S_IRUSR | stat.S_IWUSR) + + s = posix.stat(support.TESTFN) + self.assertEqual(s[0] & stat.S_IRWXU, stat.S_IRUSR | stat.S_IWUSR) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'fchownat'), "test needs posix.fchownat()") + def test_fchownat(self): + support.unlink(support.TESTFN) + support.create_empty_file(support.TESTFN) + + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.fchownat(f, support.TESTFN, os.getuid(), os.getgid()) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'fstatat'), "test needs posix.fstatat()") + def test_fstatat(self): + support.unlink(support.TESTFN) + with open(support.TESTFN, 'w') as outfile: + outfile.write("testline\n") + + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + s1 = posix.stat(support.TESTFN) + s2 = posix.fstatat(f, support.TESTFN) + self.assertEqual(s1, s2) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'futimesat'), "test needs posix.futimesat()") + def test_futimesat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + now = time.time() + posix.futimesat(f, support.TESTFN, None) + self.assertRaises(TypeError, posix.futimesat, f, support.TESTFN, (None, None)) + self.assertRaises(TypeError, posix.futimesat, f, support.TESTFN, (now, None)) + self.assertRaises(TypeError, posix.futimesat, f, support.TESTFN, (None, now)) + posix.futimesat(f, support.TESTFN, (int(now), int(now))) + posix.futimesat(f, support.TESTFN, (now, now)) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'linkat'), "test needs posix.linkat()") + def test_linkat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.linkat(f, support.TESTFN, f, support.TESTFN + 'link') + # should have same inodes + self.assertEqual(posix.stat(support.TESTFN)[1], + posix.stat(support.TESTFN + 'link')[1]) + finally: + posix.close(f) + support.unlink(support.TESTFN + 'link') + + @unittest.skipUnless(hasattr(posix, 'mkdirat'), "test needs posix.mkdirat()") + def test_mkdirat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.mkdirat(f, support.TESTFN + 'dir') + posix.stat(support.TESTFN + 'dir') # should not raise exception + finally: + posix.close(f) + support.rmtree(support.TESTFN + 'dir') + + @unittest.skipUnless(hasattr(posix, 'mknodat') and hasattr(stat, 'S_IFIFO'), + "don't have mknodat()/S_IFIFO") + def test_mknodat(self): + # Test using mknodat() to create a FIFO (the only use specified + # by POSIX). + support.unlink(support.TESTFN) + mode = stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.mknodat(f, support.TESTFN, mode, 0) + except OSError as e: + # Some old systems don't allow unprivileged users to use + # mknod(), or only support creating device nodes. + self.assertIn(e.errno, (errno.EPERM, errno.EINVAL)) + else: + self.assertTrue(stat.S_ISFIFO(posix.stat(support.TESTFN).st_mode)) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'openat'), "test needs posix.openat()") + def test_openat(self): + support.unlink(support.TESTFN) + with open(support.TESTFN, 'w') as outfile: + outfile.write("testline\n") + a = posix.open(posix.getcwd(), posix.O_RDONLY) + b = posix.openat(a, support.TESTFN, posix.O_RDONLY) + try: + res = posix.read(b, 9).decode(encoding="utf-8") + self.assertEqual("testline\n", res) + finally: + posix.close(a) + posix.close(b) + + @unittest.skipUnless(hasattr(posix, 'readlinkat'), "test needs posix.readlinkat()") + def test_readlinkat(self): + os.symlink(support.TESTFN, support.TESTFN + 'link') + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + self.assertEqual(posix.readlink(support.TESTFN + 'link'), + posix.readlinkat(f, support.TESTFN + 'link')) + finally: + support.unlink(support.TESTFN + 'link') + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'renameat'), "test needs posix.renameat()") + def test_renameat(self): + support.unlink(support.TESTFN) + support.create_empty_file(support.TESTFN + 'ren') + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.renameat(f, support.TESTFN + 'ren', f, support.TESTFN) + except: + posix.rename(support.TESTFN + 'ren', support.TESTFN) + raise + else: + posix.stat(support.TESTFN) # should not throw exception + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'symlinkat'), "test needs posix.symlinkat()") + def test_symlinkat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.symlinkat(support.TESTFN, f, support.TESTFN + 'link') + self.assertEqual(posix.readlink(support.TESTFN + 'link'), support.TESTFN) + finally: + posix.close(f) + support.unlink(support.TESTFN + 'link') + + @unittest.skipUnless(hasattr(posix, 'unlinkat'), "test needs posix.unlinkat()") + def test_unlinkat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + support.create_empty_file(support.TESTFN + 'del') + posix.stat(support.TESTFN + 'del') # should not throw exception + try: + posix.unlinkat(f, support.TESTFN + 'del') + except: + support.unlink(support.TESTFN + 'del') + raise + else: + self.assertRaises(OSError, posix.stat, support.TESTFN + 'link') + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'utimensat'), "test needs posix.utimensat()") + def test_utimensat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + now = time.time() + posix.utimensat(f, support.TESTFN, None, None) + self.assertRaises(TypeError, posix.utimensat, f, support.TESTFN, (None, None), (None, None)) + self.assertRaises(TypeError, posix.utimensat, f, support.TESTFN, (now, 0), None) + self.assertRaises(TypeError, posix.utimensat, f, support.TESTFN, None, (now, 0)) + posix.utimensat(f, support.TESTFN, (int(now), int((now - int(now)) * 1e9)), + (int(now), int((now - int(now)) * 1e9))) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'mkfifoat'), "don't have mkfifoat()") + def test_mkfifoat(self): + support.unlink(support.TESTFN) + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.mkfifoat(f, support.TESTFN, stat.S_IRUSR | stat.S_IWUSR) + self.assertTrue(stat.S_ISFIFO(posix.stat(support.TESTFN).st_mode)) + finally: + posix.close(f) + + requires_sched_h = unittest.skipUnless(hasattr(posix, 'sched_yield'), + "don't have scheduling support") + requires_sched_affinity = unittest.skipUnless(hasattr(posix, 'cpu_set'), + "don't have sched affinity support") + + @requires_sched_h + def test_sched_yield(self): + # This has no error conditions (at least on Linux). + posix.sched_yield() + + @requires_sched_h + def test_sched_priority(self): + # Round-robin usually has interesting priorities. + pol = posix.SCHED_RR + lo = posix.sched_get_priority_min(pol) + hi = posix.sched_get_priority_max(pol) + self.assertIsInstance(lo, int) + self.assertIsInstance(hi, int) + self.assertGreaterEqual(hi, lo) + # OSX evidently just returns 15 without checking the argument. + if sys.platform != "darwin": + self.assertRaises(OSError, posix.sched_get_priority_min, -23) + self.assertRaises(OSError, posix.sched_get_priority_max, -23) + + @unittest.skipUnless(hasattr(posix, 'sched_setscheduler'), "can't change scheduler") + def test_get_and_set_scheduler_and_param(self): + possible_schedulers = [sched for name, sched in posix.__dict__.items() + if name.startswith("SCHED_")] + mine = posix.sched_getscheduler(0) + self.assertIn(mine, possible_schedulers) + try: + init = posix.sched_getscheduler(1) + except OSError as e: + if e.errno != errno.EPERM and e.errno != errno.ESRCH: + raise + else: + self.assertIn(init, possible_schedulers) + self.assertRaises(OSError, posix.sched_getscheduler, -1) + self.assertRaises(OSError, posix.sched_getparam, -1) + param = posix.sched_getparam(0) + self.assertIsInstance(param.sched_priority, int) + try: + posix.sched_setscheduler(0, mine, param) + except OSError as e: + if e.errno != errno.EPERM: + raise + + # POSIX states that calling sched_setparam() on a process with a + # scheduling policy other than SCHED_FIFO or SCHED_RR is + # implementation-defined: FreeBSD returns EINVAL. + if not sys.platform.startswith('freebsd'): + posix.sched_setparam(0, param) + self.assertRaises(OSError, posix.sched_setparam, -1, param) + + self.assertRaises(OSError, posix.sched_setscheduler, -1, mine, param) + self.assertRaises(TypeError, posix.sched_setscheduler, 0, mine, None) + self.assertRaises(TypeError, posix.sched_setparam, 0, 43) + param = posix.sched_param(None) + self.assertRaises(TypeError, posix.sched_setparam, 0, param) + large = 214748364700 + param = posix.sched_param(large) + self.assertRaises(OverflowError, posix.sched_setparam, 0, param) + param = posix.sched_param(sched_priority=-large) + self.assertRaises(OverflowError, posix.sched_setparam, 0, param) + + @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function") + def test_sched_rr_get_interval(self): + try: + interval = posix.sched_rr_get_interval(0) + except OSError as e: + # This likely means that sched_rr_get_interval is only valid for + # processes with the SCHED_RR scheduler in effect. + if e.errno != errno.EINVAL: + raise + self.skipTest("only works on SCHED_RR processes") + self.assertIsInstance(interval, float) + # Reasonable constraints, I think. + self.assertGreaterEqual(interval, 0.) + self.assertLess(interval, 1.) + + @requires_sched_affinity + def test_sched_affinity(self): + mask = posix.sched_getaffinity(0, 1024) + self.assertGreaterEqual(mask.count(), 1) + self.assertIsInstance(mask, posix.cpu_set) + self.assertRaises(OSError, posix.sched_getaffinity, -1, 1024) + empty = posix.cpu_set(10) + posix.sched_setaffinity(0, mask) + self.assertRaises(OSError, posix.sched_setaffinity, 0, empty) + self.assertRaises(OSError, posix.sched_setaffinity, -1, mask) + + @requires_sched_affinity + def test_cpu_set_basic(self): + s = posix.cpu_set(10) + self.assertEqual(len(s), 10) + self.assertEqual(s.count(), 0) + s.set(0) + s.set(9) + self.assertTrue(s.isset(0)) + self.assertTrue(s.isset(9)) + self.assertFalse(s.isset(5)) + self.assertEqual(s.count(), 2) + s.clear(0) + self.assertFalse(s.isset(0)) + self.assertEqual(s.count(), 1) + s.zero() + self.assertFalse(s.isset(0)) + self.assertFalse(s.isset(9)) + self.assertEqual(s.count(), 0) + self.assertRaises(ValueError, s.set, -1) + self.assertRaises(ValueError, s.set, 10) + self.assertRaises(ValueError, s.clear, -1) + self.assertRaises(ValueError, s.clear, 10) + self.assertRaises(ValueError, s.isset, -1) + self.assertRaises(ValueError, s.isset, 10) + + @requires_sched_affinity + def test_cpu_set_cmp(self): + self.assertNotEqual(posix.cpu_set(11), posix.cpu_set(12)) + l = posix.cpu_set(10) + r = posix.cpu_set(10) + self.assertEqual(l, r) + l.set(1) + self.assertNotEqual(l, r) + r.set(1) + self.assertEqual(l, r) + + @requires_sched_affinity + def test_cpu_set_bitwise(self): + l = posix.cpu_set(5) + l.set(0) + l.set(1) + r = posix.cpu_set(5) + r.set(1) + r.set(2) + b = l & r + self.assertEqual(b.count(), 1) + self.assertTrue(b.isset(1)) + b = l | r + self.assertEqual(b.count(), 3) + self.assertTrue(b.isset(0)) + self.assertTrue(b.isset(1)) + self.assertTrue(b.isset(2)) + b = l ^ r + self.assertEqual(b.count(), 2) + self.assertTrue(b.isset(0)) + self.assertFalse(b.isset(1)) + self.assertTrue(b.isset(2)) + b = l + b |= r + self.assertIs(b, l) + self.assertEqual(l.count(), 3) + class PosixGroupsTester(unittest.TestCase): def setUp(self): @@ -461,9 +1017,11 @@ class PosixGroupsTester(unittest.TestCase): posix.setgroups(groups) self.assertListEqual(groups, posix.getgroups()) - def test_main(): - support.run_unittest(PosixTester, PosixGroupsTester) + try: + support.run_unittest(PosixTester, PosixGroupsTester) + finally: + support.reap_children() if __name__ == '__main__': test_main() diff --git a/Lib/test/test_pulldom.py b/Lib/test/test_pulldom.py new file mode 100644 index 0000000000..b81a595f69 --- /dev/null +++ b/Lib/test/test_pulldom.py @@ -0,0 +1,347 @@ +import io +import unittest +import sys +import xml.sax + +from xml.sax.xmlreader import AttributesImpl +from xml.dom import pulldom + +from test.support import run_unittest, findfile + + +tstfile = findfile("test.xml", subdir="xmltestdata") + +# A handy XML snippet, containing attributes, a namespace prefix, and a +# self-closing tag: +SMALL_SAMPLE = """<?xml version="1.0"?> +<html xmlns="http://www.w3.org/1999/xhtml" xmlns:xdc="http://www.xml.com/books"> +<!-- A comment --> +<title>Introduction to XSL</title> +<hr/> +<p><xdc:author xdc:attrib="prefixed attribute" attrib="other attrib">A. Namespace</xdc:author></p> +</html>""" + + +class PullDOMTestCase(unittest.TestCase): + + def test_parse(self): + """Minimal test of DOMEventStream.parse()""" + + # This just tests that parsing from a stream works. Actual parser + # semantics are tested using parseString with a more focused XML + # fragment. + + # Test with a filename: + handler = pulldom.parse(tstfile) + self.addCleanup(handler.stream.close) + list(handler) + + # Test with a file object: + with open(tstfile, "rb") as fin: + list(pulldom.parse(fin)) + + def test_parse_semantics(self): + """Test DOMEventStream parsing semantics.""" + + items = pulldom.parseString(SMALL_SAMPLE) + evt, node = next(items) + # Just check the node is a Document: + self.assertTrue(hasattr(node, "createElement")) + self.assertEqual(pulldom.START_DOCUMENT, evt) + evt, node = next(items) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("html", node.tagName) + self.assertEqual(2, len(node.attributes)) + self.assertEqual(node.attributes.getNamedItem("xmlns:xdc").value, + "http://www.xml.com/books") + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) # Line break + evt, node = next(items) + # XXX - A comment should be reported here! + # self.assertEqual(pulldom.COMMENT, evt) + # Line break after swallowed comment: + self.assertEqual(pulldom.CHARACTERS, evt) + evt, node = next(items) + self.assertEqual("title", node.tagName) + title_node = node + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) + self.assertEqual("Introduction to XSL", node.data) + evt, node = next(items) + self.assertEqual(pulldom.END_ELEMENT, evt) + self.assertEqual("title", node.tagName) + self.assertTrue(title_node is node) + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) + evt, node = next(items) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("hr", node.tagName) + evt, node = next(items) + self.assertEqual(pulldom.END_ELEMENT, evt) + self.assertEqual("hr", node.tagName) + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) + evt, node = next(items) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("p", node.tagName) + evt, node = next(items) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("xdc:author", node.tagName) + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) + evt, node = next(items) + self.assertEqual(pulldom.END_ELEMENT, evt) + self.assertEqual("xdc:author", node.tagName) + evt, node = next(items) + self.assertEqual(pulldom.END_ELEMENT, evt) + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) + evt, node = next(items) + self.assertEqual(pulldom.END_ELEMENT, evt) + # XXX No END_DOCUMENT item is ever obtained: + #evt, node = next(items) + #self.assertEqual(pulldom.END_DOCUMENT, evt) + + def test_expandItem(self): + """Ensure expandItem works as expected.""" + items = pulldom.parseString(SMALL_SAMPLE) + # Loop through the nodes until we get to a "title" start tag: + for evt, item in items: + if evt == pulldom.START_ELEMENT and item.tagName == "title": + items.expandNode(item) + self.assertEqual(1, len(item.childNodes)) + break + else: + self.fail("No \"title\" element detected in SMALL_SAMPLE!") + # Loop until we get to the next start-element: + for evt, node in items: + if evt == pulldom.START_ELEMENT: + break + self.assertEqual("hr", node.tagName, + "expandNode did not leave DOMEventStream in the correct state.") + # Attempt to expand a standalone element: + items.expandNode(node) + self.assertEqual(next(items)[0], pulldom.CHARACTERS) + evt, node = next(items) + self.assertEqual(node.tagName, "p") + items.expandNode(node) + next(items) # Skip character data + evt, node = next(items) + self.assertEqual(node.tagName, "html") + with self.assertRaises(StopIteration): + next(items) + items.clear() + self.assertIsNone(items.parser) + self.assertIsNone(items.stream) + + @unittest.expectedFailure + def test_comment(self): + """PullDOM does not receive "comment" events.""" + items = pulldom.parseString(SMALL_SAMPLE) + for evt, _ in items: + if evt == pulldom.COMMENT: + break + else: + self.fail("No comment was encountered") + + @unittest.expectedFailure + def test_end_document(self): + """PullDOM does not receive "end-document" events.""" + items = pulldom.parseString(SMALL_SAMPLE) + # Read all of the nodes up to and including </html>: + for evt, node in items: + if evt == pulldom.END_ELEMENT and node.tagName == "html": + break + try: + # Assert that the next node is END_DOCUMENT: + evt, node = next(items) + self.assertEqual(pulldom.END_DOCUMENT, evt) + except StopIteration: + self.fail( + "Ran out of events, but should have received END_DOCUMENT") + + +class ThoroughTestCase(unittest.TestCase): + """Test the hard-to-reach parts of pulldom.""" + + def test_thorough_parse(self): + """Test some of the hard-to-reach parts of PullDOM.""" + self._test_thorough(pulldom.parse(None, parser=SAXExerciser())) + + @unittest.expectedFailure + def test_sax2dom_fail(self): + """SAX2DOM can"t handle a PI before the root element.""" + pd = SAX2DOMTestHelper(None, SAXExerciser(), 12) + self._test_thorough(pd) + + def test_thorough_sax2dom(self): + """Test some of the hard-to-reach parts of SAX2DOM.""" + pd = SAX2DOMTestHelper(None, SAX2DOMExerciser(), 12) + self._test_thorough(pd, False) + + def _test_thorough(self, pd, before_root=True): + """Test some of the hard-to-reach parts of the parser, using a mock + parser.""" + + evt, node = next(pd) + self.assertEqual(pulldom.START_DOCUMENT, evt) + # Just check the node is a Document: + self.assertTrue(hasattr(node, "createElement")) + + if before_root: + evt, node = next(pd) + self.assertEqual(pulldom.COMMENT, evt) + self.assertEqual("a comment", node.data) + evt, node = next(pd) + self.assertEqual(pulldom.PROCESSING_INSTRUCTION, evt) + self.assertEqual("target", node.target) + self.assertEqual("data", node.data) + + evt, node = next(pd) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("html", node.tagName) + + evt, node = next(pd) + self.assertEqual(pulldom.COMMENT, evt) + self.assertEqual("a comment", node.data) + evt, node = next(pd) + self.assertEqual(pulldom.PROCESSING_INSTRUCTION, evt) + self.assertEqual("target", node.target) + self.assertEqual("data", node.data) + + evt, node = next(pd) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("p", node.tagName) + + evt, node = next(pd) + self.assertEqual(pulldom.CHARACTERS, evt) + self.assertEqual("text", node.data) + evt, node = next(pd) + self.assertEqual(pulldom.END_ELEMENT, evt) + self.assertEqual("p", node.tagName) + evt, node = next(pd) + self.assertEqual(pulldom.END_ELEMENT, evt) + self.assertEqual("html", node.tagName) + evt, node = next(pd) + self.assertEqual(pulldom.END_DOCUMENT, evt) + + +class SAXExerciser(object): + """A fake sax parser that calls some of the harder-to-reach sax methods to + ensure it emits the correct events""" + + def setContentHandler(self, handler): + self._handler = handler + + def parse(self, _): + h = self._handler + h.startDocument() + + # The next two items ensure that items preceding the first + # start_element are properly stored and emitted: + h.comment("a comment") + h.processingInstruction("target", "data") + + h.startElement("html", AttributesImpl({})) + + h.comment("a comment") + h.processingInstruction("target", "data") + + h.startElement("p", AttributesImpl({"class": "paraclass"})) + h.characters("text") + h.endElement("p") + h.endElement("html") + h.endDocument() + + def stub(self, *args, **kwargs): + """Stub method. Does nothing.""" + pass + setProperty = stub + setFeature = stub + + +class SAX2DOMExerciser(SAXExerciser): + """The same as SAXExerciser, but without the processing instruction and + comment before the root element, because S2D can"t handle it""" + + def parse(self, _): + h = self._handler + h.startDocument() + h.startElement("html", AttributesImpl({})) + h.comment("a comment") + h.processingInstruction("target", "data") + h.startElement("p", AttributesImpl({"class": "paraclass"})) + h.characters("text") + h.endElement("p") + h.endElement("html") + h.endDocument() + + +class SAX2DOMTestHelper(pulldom.DOMEventStream): + """Allows us to drive SAX2DOM from a DOMEventStream.""" + + def reset(self): + self.pulldom = pulldom.SAX2DOM() + # This content handler relies on namespace support + self.parser.setFeature(xml.sax.handler.feature_namespaces, 1) + self.parser.setContentHandler(self.pulldom) + + +class SAX2DOMTestCase(unittest.TestCase): + + def confirm(self, test, testname="Test"): + self.assertTrue(test, testname) + + def test_basic(self): + """Ensure SAX2DOM can parse from a stream.""" + with io.StringIO(SMALL_SAMPLE) as fin: + sd = SAX2DOMTestHelper(fin, xml.sax.make_parser(), + len(SMALL_SAMPLE)) + for evt, node in sd: + if evt == pulldom.START_ELEMENT and node.tagName == "html": + break + # Because the buffer is the same length as the XML, all the + # nodes should have been parsed and added: + self.assertGreater(len(node.childNodes), 0) + + def testSAX2DOM(self): + """Ensure SAX2DOM expands nodes as expected.""" + sax2dom = pulldom.SAX2DOM() + sax2dom.startDocument() + sax2dom.startElement("doc", {}) + sax2dom.characters("text") + sax2dom.startElement("subelm", {}) + sax2dom.characters("text") + sax2dom.endElement("subelm") + sax2dom.characters("text") + sax2dom.endElement("doc") + sax2dom.endDocument() + + doc = sax2dom.document + root = doc.documentElement + (text1, elm1, text2) = root.childNodes + text3 = elm1.childNodes[0] + + self.assertIsNone(text1.previousSibling) + self.assertIs(text1.nextSibling, elm1) + self.assertIs(elm1.previousSibling, text1) + self.assertIs(elm1.nextSibling, text2) + self.assertIs(text2.previousSibling, elm1) + self.assertIsNone(text2.nextSibling) + self.assertIsNone(text3.previousSibling) + self.assertIsNone(text3.nextSibling) + + self.assertIs(root.parentNode, doc) + self.assertIs(text1.parentNode, root) + self.assertIs(elm1.parentNode, root) + self.assertIs(text2.parentNode, root) + self.assertIs(text3.parentNode, elm1) + doc.unlink() + + +def test_main(): + run_unittest(PullDOMTestCase, ThoroughTestCase, SAX2DOMTestCase) + + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 72f5882cf5..ae56996e88 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -201,7 +201,7 @@ war</tt></dd></dl> missing_pattern = "no Python documentation found for '%s'" # output pattern for module with bad imports -badimport_pattern = "problem in %s - ImportError: No module named %s" +badimport_pattern = "problem in %s - ImportError: No module named %r" def run_pydoc(module_name, *args, **env): """ @@ -256,6 +256,8 @@ class PydocDocTest(unittest.TestCase): @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') def test_html_doc(self): result, doc_loc = get_pydoc_html(pydoc_mod) mod_file = inspect.getabsfile(pydoc_mod) @@ -271,6 +273,8 @@ class PydocDocTest(unittest.TestCase): @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') def test_text_doc(self): result, doc_loc = get_pydoc_text(pydoc_mod) expected_text = expected_text_pattern % \ @@ -349,6 +353,8 @@ class PydocDocTest(unittest.TestCase): @unittest.skipIf(sys.flags.optimize >= 2, 'Docstrings are omitted with -O2 and above') + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') def test_help_output_redirect(self): # issue 940286, if output is set in Helper, then all output from # Helper.help should be redirected diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index fe8bc34039..e3d10f7352 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -428,7 +428,7 @@ class ReTests(unittest.TestCase): self.assertEqual(m.span(), span) def test_re_escape(self): - alnum_chars = string.ascii_letters + string.digits + alnum_chars = string.ascii_letters + string.digits + '_' p = ''.join(chr(i) for i in range(256)) for c in p: if c in alnum_chars: @@ -441,7 +441,7 @@ class ReTests(unittest.TestCase): self.assertMatch(re.escape(p), p) def test_re_escape_byte(self): - alnum_chars = (string.ascii_letters + string.digits).encode('ascii') + alnum_chars = (string.ascii_letters + string.digits + '_').encode('ascii') p = bytes(range(256)) for i in p: b = bytes([i]) diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index b0dc4d7813..439fa33794 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -8,7 +8,7 @@ import os import shutil import unittest -from test.support import run_unittest +from test.support import run_unittest, create_empty_file from reprlib import repr as r # Don't shadow builtin repr from reprlib import Repr from reprlib import recursive_repr @@ -193,10 +193,9 @@ class ReprTests(unittest.TestCase): r(y) r(z) -def touch(path, text=''): - fp = open(path, 'w') - fp.write(text) - fp.close() +def write_file(path, text): + with open(path, 'w', encoding='ASCII') as fp: + fp.write(text) class LongReprTest(unittest.TestCase): def setUp(self): @@ -206,10 +205,10 @@ class LongReprTest(unittest.TestCase): # Make the package and subpackage shutil.rmtree(self.pkgname, ignore_errors=True) os.mkdir(self.pkgname) - touch(os.path.join(self.pkgname, '__init__.py')) + create_empty_file(os.path.join(self.pkgname, '__init__.py')) shutil.rmtree(self.subpkgname, ignore_errors=True) os.mkdir(self.subpkgname) - touch(os.path.join(self.subpkgname, '__init__.py')) + create_empty_file(os.path.join(self.subpkgname, '__init__.py')) # Remember where we are self.here = os.getcwd() sys.path.insert(0, self.here) @@ -231,15 +230,15 @@ class LongReprTest(unittest.TestCase): def test_module(self): eq = self.assertEqual - touch(os.path.join(self.subpkgname, self.pkgname + '.py')) + create_empty_file(os.path.join(self.subpkgname, self.pkgname + '.py')) from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import areallylongpackageandmodulenametotestreprtruncation eq(repr(areallylongpackageandmodulenametotestreprtruncation), - "<module '%s' from '%s'>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__)) + "<module %r from %r>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__)) eq(repr(sys), "<module 'sys' (built-in)>") def test_type(self): eq = self.assertEqual - touch(os.path.join(self.subpkgname, 'foo.py'), '''\ + write_file(os.path.join(self.subpkgname, 'foo.py'), '''\ class foo(object): pass ''') @@ -253,7 +252,7 @@ class foo(object): pass def test_class(self): - touch(os.path.join(self.subpkgname, 'bar.py'), '''\ + write_file(os.path.join(self.subpkgname, 'bar.py'), '''\ class bar: pass ''') @@ -262,7 +261,7 @@ class bar: self.assertEqual(repr(bar.bar), "<class '%s.bar'>" % bar.__name__) def test_instance(self): - touch(os.path.join(self.subpkgname, 'baz.py'), '''\ + write_file(os.path.join(self.subpkgname, 'baz.py'), '''\ class baz: pass ''') @@ -273,7 +272,7 @@ class baz: def test_method(self): eq = self.assertEqual - touch(os.path.join(self.subpkgname, 'qux.py'), '''\ + write_file(os.path.join(self.subpkgname, 'qux.py'), '''\ class aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: def amethod(self): pass ''') diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index f8f3717b64..0b629dc5bc 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -220,6 +220,7 @@ class MiscTest(unittest.TestCase): for func in (do, operator.not_): self.assertRaises(Exc, func, Bad()) + @support.no_tracing def test_recursion(self): # Check that comparison for recursive objects fails gracefully from collections import UserList diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 7ffb6af634..2cede19b40 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -6,7 +6,9 @@ import sys import re import tempfile import py_compile -from test.support import forget, make_legacy_pyc, run_unittest, unload, verbose +from test.support import ( + forget, make_legacy_pyc, run_unittest, unload, verbose, no_tracing, + create_empty_file) from test.script_helper import ( make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir) @@ -112,8 +114,7 @@ class RunModuleTest(unittest.TestCase): def _add_pkg_dir(self, pkg_dir): os.mkdir(pkg_dir) pkg_fname = os.path.join(pkg_dir, "__init__.py") - pkg_file = open(pkg_fname, "w") - pkg_file.close() + create_empty_file(pkg_fname) return pkg_fname def _make_pkg(self, source, depth, mod_base="runpy_test"): @@ -218,8 +219,7 @@ class RunModuleTest(unittest.TestCase): module_dir = os.path.join(module_dir, pkg_name) # Add sibling module sibling_fname = os.path.join(module_dir, "sibling.py") - sibling_file = open(sibling_fname, "w") - sibling_file.close() + create_empty_file(sibling_fname) if verbose: print(" Added sibling module:", sibling_fname) # Add nephew module uncle_dir = os.path.join(parent_dir, "uncle") @@ -229,8 +229,7 @@ class RunModuleTest(unittest.TestCase): self._add_pkg_dir(cousin_dir) if verbose: print(" Added cousin package:", cousin_dir) nephew_fname = os.path.join(cousin_dir, "nephew.py") - nephew_file = open(nephew_fname, "w") - nephew_file.close() + create_empty_file(nephew_fname) if verbose: print(" Added nephew module:", nephew_fname) def _check_relative_imports(self, depth, run_name=None): @@ -395,6 +394,7 @@ argv0 = sys.argv[0] msg = "can't find '__main__' module in %r" % zip_name self._check_import_error(zip_name, msg) + @no_tracing def test_main_recursion_error(self): with temp_dir() as script_dir, temp_dir() as dummy_dir: mod_name = '__main__' diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 1225d6e769..8e00889ef5 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -20,8 +20,8 @@ import unittest TEST_XMLFILE = findfile("test.xml", subdir="xmltestdata") TEST_XMLFILE_OUT = findfile("test.xml.out", subdir="xmltestdata") try: - TEST_XMLFILE.encode("utf8") - TEST_XMLFILE_OUT.encode("utf8") + TEST_XMLFILE.encode("utf-8") + TEST_XMLFILE_OUT.encode("utf-8") except UnicodeEncodeError: raise unittest.SkipTest("filename is not encodable to utf8") diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index fbc87aa80c..129a18aade 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -1,5 +1,5 @@ import unittest -from test.support import check_syntax_error, run_unittest +from test.support import check_syntax_error, cpython_only, run_unittest class ScopeTests(unittest.TestCase): @@ -496,23 +496,22 @@ class ScopeTests(unittest.TestCase): self.assertNotIn("x", varnames) self.assertIn("y", varnames) + @cpython_only def testLocalsClass_WithTrace(self): # Issue23728: after the trace function returns, the locals() # dictionary is used to update all variables, this used to # include free variables. But in class statements, free # variables are not inserted... import sys + self.addCleanup(sys.settrace, sys.gettrace()) sys.settrace(lambda a,b,c:None) - try: - x = 12 + x = 12 - class C: - def f(self): - return x + class C: + def f(self): + return x - self.assertEqual(x, 12) # Used to raise UnboundLocalError - finally: - sys.settrace(None) + self.assertEqual(x, 12) # Used to raise UnboundLocalError def testBoundAndFree(self): # var is bound and free in class @@ -527,6 +526,7 @@ class ScopeTests(unittest.TestCase): inst = f(3)() self.assertEqual(inst.a, inst.m()) + @cpython_only def testInteractionWithTraceFunc(self): import sys @@ -543,6 +543,7 @@ class ScopeTests(unittest.TestCase): class TestClass: pass + self.addCleanup(sys.settrace, sys.gettrace()) sys.settrace(tracer) adaptgetter("foo", TestClass, (1, "")) sys.settrace(None) diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py index fe92f45145..4a13adebfd 100644 --- a/Lib/test/test_select.py +++ b/Lib/test/test_select.py @@ -20,6 +20,7 @@ class SelectTestCase(unittest.TestCase): self.assertRaises(TypeError, select.select, [self.Nope()], [], []) self.assertRaises(TypeError, select.select, [self.Almost()], [], []) self.assertRaises(TypeError, select.select, [], [], [], "not a number") + self.assertRaises(ValueError, select.select, [], [], [], -1) def test_returned_list_identity(self): # See issue #8329 diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 3e73f52b87..13c126566d 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -2,7 +2,7 @@ import unittest import shelve import glob from test import support -from collections import MutableMapping +from collections.abc import MutableMapping from test.test_dbm import dbm_iterator def L1(s): @@ -129,8 +129,8 @@ class TestCase(unittest.TestCase): shelve.Shelf(d)[key] = [1] self.assertIn(key.encode('utf-8'), d) # but a different one can be given - shelve.Shelf(d, keyencoding='latin1')[key] = [1] - self.assertIn(key.encode('latin1'), d) + shelve.Shelf(d, keyencoding='latin-1')[key] = [1] + self.assertIn(key.encode('latin-1'), d) # with all consequences s = shelve.Shelf(d, keyencoding='ascii') self.assertRaises(UnicodeEncodeError, s.__setitem__, key, [1]) diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 25e4b6df6c..d4463f3004 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -1,6 +1,7 @@ -import unittest -import os, sys, io +import io import shlex +import string +import unittest from test import support @@ -173,6 +174,22 @@ class ShlexTest(unittest.TestCase): "%s: %s != %s" % (self.data[i][0], l, self.data[i][1:])) + def testQuote(self): + safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./' + unicode_sample = '\xe9\xe0\xdf' # e + acute accent, a + grave, sharp s + unsafe = '"`$\\!' + unicode_sample + + self.assertEqual(shlex.quote(''), "''") + self.assertEqual(shlex.quote(safeunquoted), safeunquoted) + self.assertEqual(shlex.quote('test file name'), "'test file name'") + for u in unsafe: + self.assertEqual(shlex.quote('test%sname' % u), + "'test%sname'" % u) + for u in unsafe: + self.assertEqual(shlex.quote("test%s'name'" % u), + "'test%s'\"'\"'name'\"'\"''" % u) + + # Allow this test to be used with old shlex.py if not getattr(shlex, "split", None): for methname in dir(ShlexTest): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index e2310e26f3..b16980393c 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -21,7 +21,7 @@ import tarfile import warnings from test import support -from test.support import TESTFN, check_warnings, captured_stdout +from test.support import TESTFN, check_warnings, captured_stdout, requires_zlib try: import bz2 @@ -39,11 +39,6 @@ except ImportError: UID_GID_SUPPORT = False try: - import zlib -except ImportError: - zlib = None - -try: import zipfile ZIP_SUPPORT = True except ImportError: @@ -64,6 +59,31 @@ def mock_rename(func): os.rename = builtin_rename return wrap +def write_file(path, content, binary=False): + """Write *content* to a file located at *path*. + + If *path* is a tuple instead of a string, os.path.join will be used to + make a path. If *binary* is true, the file will be opened in binary + mode. + """ + if isinstance(path, tuple): + path = os.path.join(*path) + with open(path, 'wb' if binary else 'w') as fp: + fp.write(content) + +def read_file(path, binary=False): + """Return contents from a file located at *path*. + + If *path* is a tuple instead of a string, os.path.join will be used to + make a path. If *binary* is true, the file will be opened in binary + mode. + """ + if isinstance(path, tuple): + path = os.path.join(*path) + with open(path, 'rb' if binary else 'r') as fp: + return fp.read() + + class TestShutil(unittest.TestCase): def setUp(self): @@ -76,19 +96,6 @@ class TestShutil(unittest.TestCase): d = self.tempdirs.pop() shutil.rmtree(d, os.name in ('nt', 'cygwin')) - def write_file(self, path, content='xxx'): - """Writes a file in the given path. - - - path can be a string or a sequence. - """ - if isinstance(path, (list, tuple)): - path = os.path.join(*path) - f = open(path, 'w') - try: - f.write(content) - finally: - f.close() def mkdtemp(self): """Create a temporary directory that will be cleaned up. @@ -112,8 +119,7 @@ class TestShutil(unittest.TestCase): self.errorState = 0 os.mkdir(TESTFN) self.childpath = os.path.join(TESTFN, 'a') - f = open(self.childpath, 'w') - f.close() + support.create_empty_file(self.childpath) old_dir_mode = os.stat(TESTFN).st_mode old_child_mode = os.stat(self.childpath).st_mode # Make unwritable. @@ -161,81 +167,46 @@ class TestShutil(unittest.TestCase): def test_rmtree_dont_delete_file(self): # When called on a file instead of a directory, don't delete it. handle, path = tempfile.mkstemp() - os.fdopen(handle).close() + os.close(handle) self.assertRaises(OSError, shutil.rmtree, path) os.remove(path) - def _write_data(self, path, data): - f = open(path, "w") - f.write(data) - f.close() - def test_copytree_simple(self): - - def read_data(path): - f = open(path) - data = f.read() - f.close() - return data - src_dir = tempfile.mkdtemp() dst_dir = os.path.join(tempfile.mkdtemp(), 'destination') - self._write_data(os.path.join(src_dir, 'test.txt'), '123') + self.addCleanup(shutil.rmtree, src_dir) + self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) + write_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') - - try: - shutil.copytree(src_dir, dst_dir) - self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) - self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir'))) - self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir', - 'test.txt'))) - actual = read_data(os.path.join(dst_dir, 'test.txt')) - self.assertEqual(actual, '123') - actual = read_data(os.path.join(dst_dir, 'test_dir', 'test.txt')) - self.assertEqual(actual, '456') - finally: - for path in ( - os.path.join(src_dir, 'test.txt'), - os.path.join(dst_dir, 'test.txt'), - os.path.join(src_dir, 'test_dir', 'test.txt'), - os.path.join(dst_dir, 'test_dir', 'test.txt'), - ): - if os.path.exists(path): - os.remove(path) - for path in (src_dir, - os.path.dirname(dst_dir) - ): - if os.path.exists(path): - shutil.rmtree(path) + write_file((src_dir, 'test_dir', 'test.txt'), '456') + + shutil.copytree(src_dir, dst_dir) + self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) + self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir'))) + self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir', + 'test.txt'))) + actual = read_file((dst_dir, 'test.txt')) + self.assertEqual(actual, '123') + actual = read_file((dst_dir, 'test_dir', 'test.txt')) + self.assertEqual(actual, '456') def test_copytree_with_exclude(self): - - def read_data(path): - f = open(path) - data = f.read() - f.close() - return data - # creating data join = os.path.join exists = os.path.exists src_dir = tempfile.mkdtemp() try: dst_dir = join(tempfile.mkdtemp(), 'destination') - self._write_data(join(src_dir, 'test.txt'), '123') - self._write_data(join(src_dir, 'test.tmp'), '123') + write_file((src_dir, 'test.txt'), '123') + write_file((src_dir, 'test.tmp'), '123') os.mkdir(join(src_dir, 'test_dir')) - self._write_data(join(src_dir, 'test_dir', 'test.txt'), '456') + write_file((src_dir, 'test_dir', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2')) - self._write_data(join(src_dir, 'test_dir2', 'test.txt'), '456') + write_file((src_dir, 'test_dir2', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2', 'subdir')) os.mkdir(join(src_dir, 'test_dir2', 'subdir2')) - self._write_data(join(src_dir, 'test_dir2', 'subdir', 'test.txt'), - '456') - self._write_data(join(src_dir, 'test_dir2', 'subdir2', 'test.py'), - '456') - + write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') + write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') # testing glob-like patterns try: @@ -243,21 +214,19 @@ class TestShutil(unittest.TestCase): shutil.copytree(src_dir, dst_dir, ignore=patterns) # checking the result: some elements should not be copied self.assertTrue(exists(join(dst_dir, 'test.txt'))) - self.assertTrue(not exists(join(dst_dir, 'test.tmp'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2'))) + self.assertFalse(exists(join(dst_dir, 'test.tmp'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2'))) finally: - if os.path.exists(dst_dir): - shutil.rmtree(dst_dir) + shutil.rmtree(dst_dir) try: patterns = shutil.ignore_patterns('*.tmp', 'subdir*') shutil.copytree(src_dir, dst_dir, ignore=patterns) # checking the result: some elements should not be copied - self.assertTrue(not exists(join(dst_dir, 'test.tmp'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir2'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir'))) + self.assertFalse(exists(join(dst_dir, 'test.tmp'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir'))) finally: - if os.path.exists(dst_dir): - shutil.rmtree(dst_dir) + shutil.rmtree(dst_dir) # testing callable-style try: @@ -276,13 +245,12 @@ class TestShutil(unittest.TestCase): shutil.copytree(src_dir, dst_dir, ignore=_filter) # checking the result: some elements should not be copied - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir2', - 'test.py'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2', + 'test.py'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir'))) finally: - if os.path.exists(dst_dir): - shutil.rmtree(dst_dir) + shutil.rmtree(dst_dir) finally: shutil.rmtree(src_dir) shutil.rmtree(os.path.dirname(dst_dir)) @@ -377,9 +345,9 @@ class TestShutil(unittest.TestCase): src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') - self._write_data(os.path.join(src_dir, 'test.txt'), '123') + write_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') + write_file((src_dir, 'test_dir', 'test.txt'), '456') copied = [] def _copy(src, dst): @@ -396,7 +364,7 @@ class TestShutil(unittest.TestCase): dst_dir = os.path.join(self.mkdtemp(), 'destination') os.symlink('IDONTEXIST', os.path.join(src_dir, 'test.txt')) os.mkdir(os.path.join(src_dir, 'test_dir')) - self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') + write_file((src_dir, 'test_dir', 'test.txt'), '456') self.assertRaises(Error, shutil.copytree, src_dir, dst_dir) # a dangling symlink is ignored with the proper flag @@ -412,7 +380,7 @@ class TestShutil(unittest.TestCase): def _copy_file(self, method): fname = 'test.txt' tmpdir = self.mkdtemp() - self.write_file([tmpdir, fname]) + write_file((tmpdir, fname), 'xxx') file1 = os.path.join(tmpdir, fname) tmpdir2 = self.mkdtemp() method(file1, tmpdir2) @@ -444,14 +412,14 @@ class TestShutil(unittest.TestCase): self.assertEqual(getattr(file1_stat, 'st_flags'), getattr(file2_stat, 'st_flags')) - @unittest.skipUnless(zlib, "requires zlib") + @requires_zlib def test_make_tarball(self): # creating something to tar tmpdir = self.mkdtemp() - self.write_file([tmpdir, 'file1'], 'xxx') - self.write_file([tmpdir, 'file2'], 'xxx') + write_file((tmpdir, 'file1'), 'xxx') + write_file((tmpdir, 'file2'), 'xxx') os.mkdir(os.path.join(tmpdir, 'sub')) - self.write_file([tmpdir, 'sub', 'file3'], 'xxx') + write_file((tmpdir, 'sub', 'file3'), 'xxx') tmpdir2 = self.mkdtemp() # force shutil to create the directory @@ -498,16 +466,16 @@ class TestShutil(unittest.TestCase): tmpdir = self.mkdtemp() dist = os.path.join(tmpdir, 'dist') os.mkdir(dist) - self.write_file([dist, 'file1'], 'xxx') - self.write_file([dist, 'file2'], 'xxx') + write_file((dist, 'file1'), 'xxx') + write_file((dist, 'file2'), 'xxx') os.mkdir(os.path.join(dist, 'sub')) - self.write_file([dist, 'sub', 'file3'], 'xxx') + write_file((dist, 'sub', 'file3'), 'xxx') os.mkdir(os.path.join(dist, 'sub2')) tmpdir2 = self.mkdtemp() base_name = os.path.join(tmpdir2, 'archive') return tmpdir, tmpdir2, base_name - @unittest.skipUnless(zlib, "Requires zlib") + @requires_zlib @unittest.skipUnless(find_executable('tar') and find_executable('gzip'), 'Need the tar command to run') def test_tarfile_vs_tar(self): @@ -562,13 +530,13 @@ class TestShutil(unittest.TestCase): tarball = base_name + '.tar' self.assertTrue(os.path.exists(tarball)) - @unittest.skipUnless(zlib, "Requires zlib") + @requires_zlib @unittest.skipUnless(ZIP_SUPPORT, 'Need zip support to run') def test_make_zipfile(self): # creating something to tar tmpdir = self.mkdtemp() - self.write_file([tmpdir, 'file1'], 'xxx') - self.write_file([tmpdir, 'file2'], 'xxx') + write_file((tmpdir, 'file1'), 'xxx') + write_file((tmpdir, 'file2'), 'xxx') tmpdir2 = self.mkdtemp() # force shutil to create the directory @@ -586,7 +554,7 @@ class TestShutil(unittest.TestCase): base_name = os.path.join(tmpdir, 'archive') self.assertRaises(ValueError, make_archive, base_name, 'xxx') - @unittest.skipUnless(zlib, "Requires zlib") + @requires_zlib def test_make_archive_owner_group(self): # testing make_archive with owner and group, with various combinations # this works even if there's not gid/uid support @@ -614,7 +582,7 @@ class TestShutil(unittest.TestCase): self.assertTrue(os.path.exists(res)) - @unittest.skipUnless(zlib, "Requires zlib") + @requires_zlib @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") def test_tarfile_root_owner(self): tmpdir, tmpdir2, base_name = self._create_files() @@ -683,7 +651,7 @@ class TestShutil(unittest.TestCase): diff.append(file_) return diff - @unittest.skipUnless(zlib, "Requires zlib") + @requires_zlib def test_unpack_archive(self): formats = ['tar', 'gztar', 'zip'] if BZ2_SUPPORTED: @@ -734,6 +702,75 @@ class TestShutil(unittest.TestCase): unregister_unpack_format('Boo2') self.assertEqual(get_unpack_formats(), formats) + @unittest.skipUnless(hasattr(shutil, 'disk_usage'), + "disk_usage not available on this platform") + def test_disk_usage(self): + usage = shutil.disk_usage(os.getcwd()) + self.assertGreater(usage.total, 0) + self.assertGreater(usage.used, 0) + self.assertGreaterEqual(usage.free, 0) + self.assertGreaterEqual(usage.total, usage.used) + self.assertGreater(usage.total, usage.free) + + @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") + @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown') + def test_chown(self): + + # cleaned-up automatically by TestShutil.tearDown method + dirname = self.mkdtemp() + filename = tempfile.mktemp(dir=dirname) + write_file(filename, 'testing chown function') + + with self.assertRaises(ValueError): + shutil.chown(filename) + + with self.assertRaises(LookupError): + shutil.chown(filename, user='non-exising username') + + with self.assertRaises(LookupError): + shutil.chown(filename, group='non-exising groupname') + + with self.assertRaises(TypeError): + shutil.chown(filename, b'spam') + + with self.assertRaises(TypeError): + shutil.chown(filename, 3.14) + + uid = os.getuid() + gid = os.getgid() + + def check_chown(path, uid=None, gid=None): + s = os.stat(filename) + if uid is not None: + self.assertEqual(uid, s.st_uid) + if gid is not None: + self.assertEqual(gid, s.st_gid) + + shutil.chown(filename, uid, gid) + check_chown(filename, uid, gid) + shutil.chown(filename, uid) + check_chown(filename, uid) + shutil.chown(filename, user=uid) + check_chown(filename, uid) + shutil.chown(filename, group=gid) + check_chown(filename, gid=gid) + + shutil.chown(dirname, uid, gid) + check_chown(dirname, uid, gid) + shutil.chown(dirname, uid) + check_chown(dirname, uid) + shutil.chown(dirname, user=uid) + check_chown(dirname, uid) + shutil.chown(dirname, group=gid) + check_chown(dirname, gid=gid) + + user = pwd.getpwuid(uid)[0] + group = grp.getgrgid(gid)[0] + shutil.chown(filename, user, group) + check_chown(filename, uid, gid) + shutil.chown(dirname, user, group) + check_chown(dirname, uid, gid) + class TestMove(unittest.TestCase): @@ -965,8 +1002,7 @@ class TestCopyFile(unittest.TestCase): shutil.move(self.src_dir, dst_dir) self.assertTrue(os.path.isdir(dst_dir)) finally: - if os.path.exists(dst_dir): - os.rmdir(dst_dir) + os.rmdir(dst_dir) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 8df1bf0a19..c349252dbc 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1,17 +1,19 @@ -import errno +import unittest +from test import support +from contextlib import closing import gc -import os import pickle import select import signal +import struct import subprocess -import sys -import time import traceback -import unittest -from test import support -from contextlib import closing +import sys, os, time, errno from test.script_helper import assert_python_ok, spawn_python +try: + import threading +except ImportError: + threading = None if sys.platform in ('os2', 'riscos'): raise unittest.SkipTest("Can't test signal on %s" % sys.platform) @@ -57,15 +59,9 @@ class InterProcessSignalTests(unittest.TestCase): def handlerA(self, signum, frame): self.a_called = True - if support.verbose: - print("handlerA invoked from signal %s at:\n%s" % ( - signum, self.format_frame(frame, limit=1))) def handlerB(self, signum, frame): self.b_called = True - if support.verbose: - print ("handlerB invoked from signal %s at:\n%s" % ( - signum, self.format_frame(frame, limit=1))) raise HandlerBCalled(signum, self.format_frame(frame)) def wait(self, child): @@ -92,8 +88,6 @@ class InterProcessSignalTests(unittest.TestCase): # Let the sub-processes know who to send signals to. pid = os.getpid() - if support.verbose: - print("test runner's pid is", pid) child = ignoring_eintr(subprocess.Popen, ['kill', '-HUP', str(pid)]) if child: @@ -117,8 +111,6 @@ class InterProcessSignalTests(unittest.TestCase): except HandlerBCalled: self.assertTrue(self.b_called) self.assertFalse(self.a_called) - if support.verbose: - print("HandlerBCalled exception caught") child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)]) if child: @@ -134,8 +126,7 @@ class InterProcessSignalTests(unittest.TestCase): # may return early. time.sleep(1) except KeyboardInterrupt: - if support.verbose: - print("KeyboardInterrupt (the alarm() went off)") + pass except: self.fail("Some other exception woke us from pause: %s" % traceback.format_exc()) @@ -191,7 +182,7 @@ class InterProcessSignalTests(unittest.TestCase): @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") -class BasicSignalTests(unittest.TestCase): +class PosixTests(unittest.TestCase): def trivial_signal_handler(self, *args): pass @@ -233,31 +224,41 @@ class WindowsSignalTests(unittest.TestCase): @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") class WakeupSignalTests(unittest.TestCase): - def check_wakeup(self, test_body): - # use a subprocess to have only one thread and to not change signal - # handling of the parent process + def check_wakeup(self, test_body, *signals): + # use a subprocess to have only one thread code = """if 1: import fcntl import os import signal + import struct + + signals = {!r} def handler(signum, frame): pass + def check_signum(signals): + data = os.read(read, len(signals)+1) + raised = struct.unpack('%uB' % len(data), data) + if raised != signals: + raise Exception("%r != %r" % (raised, signals)) + {} signal.signal(signal.SIGALRM, handler) read, write = os.pipe() - flags = fcntl.fcntl(write, fcntl.F_GETFL, 0) - flags = flags | os.O_NONBLOCK - fcntl.fcntl(write, fcntl.F_SETFL, flags) + for fd in (read, write): + flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0) + flags = flags | os.O_NONBLOCK + fcntl.fcntl(fd, fcntl.F_SETFL, flags) signal.set_wakeup_fd(write) test() + check_signum(signals) os.close(read) os.close(write) - """.format(test_body) + """.format(signals, test_body) assert_python_ok('-c', code) @@ -283,7 +284,7 @@ class WakeupSignalTests(unittest.TestCase): dt = after_time - mid_time if dt >= TIMEOUT_HALF: raise Exception("%s >= %s" % (dt, TIMEOUT_HALF)) - """) + """, signal.SIGALRM) def test_wakeup_fd_during(self): self.check_wakeup("""def test(): @@ -306,7 +307,37 @@ class WakeupSignalTests(unittest.TestCase): dt = after_time - before_time if dt >= TIMEOUT_HALF: raise Exception("%s >= %s" % (dt, TIMEOUT_HALF)) - """) + """, signal.SIGALRM) + + def test_signum(self): + self.check_wakeup("""def test(): + signal.signal(signal.SIGUSR1, handler) + os.kill(os.getpid(), signal.SIGUSR1) + os.kill(os.getpid(), signal.SIGALRM) + """, signal.SIGUSR1, signal.SIGALRM) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def test_pending(self): + signals = (signal.SIGUSR1, signal.SIGUSR2) + # when signals are unblocked, pending signal ared delivered in the + # reverse order of their number + signals = tuple(sorted(signals, reverse=True)) + + self.check_wakeup("""def test(): + signum1 = signal.SIGUSR1 + signum2 = signal.SIGUSR2 + + signal.signal(signum1, handler) + signal.signal(signum2, handler) + + signal.pthread_sigmask(signal.SIG_BLOCK, (signum1, signum2)) + os.kill(os.getpid(), signum1) + os.kill(os.getpid(), signum2) + # Unblocking the 2 signals calls the C signal handler twice + signal.pthread_sigmask(signal.SIG_UNBLOCK, (signum1, signum2)) + """, *signals) + @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") class SiginterruptTest(unittest.TestCase): @@ -316,9 +347,6 @@ class SiginterruptTest(unittest.TestCase): read is interrupted by the signal and raises an exception. Return False if it returns normally. """ - class Timeout(Exception): - pass - # use a subprocess to have only one thread, to have a timeout on the # blocking read and to not touch signal handling in this process code = """if 1: @@ -359,18 +387,8 @@ class SiginterruptTest(unittest.TestCase): # wait until the child process is loaded and has started first_line = process.stdout.readline() - # Wait the process with a timeout of 5 seconds - timeout = time.time() + 5.0 - while True: - if timeout < time.time(): - raise Timeout() - status = process.poll() - if status is not None: - break - time.sleep(0.1) - - stdout, stderr = process.communicate() - except Timeout: + stdout, stderr = process.communicate(timeout=5.0) + except subprocess.TimeoutExpired: process.kill() return False else: @@ -419,8 +437,6 @@ class ItimerTest(unittest.TestCase): def sig_alrm(self, *args): self.hndl_called = True - if support.verbose: - print("SIGALRM handler invoked", args) def sig_vtalrm(self, *args): self.hndl_called = True @@ -432,21 +448,13 @@ class ItimerTest(unittest.TestCase): elif self.hndl_count == 3: # disable ITIMER_VIRTUAL, this function shouldn't be called anymore signal.setitimer(signal.ITIMER_VIRTUAL, 0) - if support.verbose: - print("last SIGVTALRM handler call") self.hndl_count += 1 - if support.verbose: - print("SIGVTALRM handler invoked", args) - def sig_prof(self, *args): self.hndl_called = True signal.setitimer(signal.ITIMER_PROF, 0) - if support.verbose: - print("SIGPROF handler invoked", args) - def test_itimer_exc(self): # XXX I'm assuming -1 is an invalid itimer, but maybe some platform # defines it ? @@ -459,10 +467,7 @@ class ItimerTest(unittest.TestCase): def test_itimer_real(self): self.itimer = signal.ITIMER_REAL signal.setitimer(self.itimer, 1.0) - if support.verbose: - print("\ncall pause()...") signal.pause() - self.assertEqual(self.hndl_called, True) # Issue 3864, unknown if this affects earlier versions of freebsd also @@ -511,11 +516,361 @@ class ItimerTest(unittest.TestCase): # and the handler should have been called self.assertEqual(self.hndl_called, True) + +class PendingSignalsTests(unittest.TestCase): + """ + Test pthread_sigmask(), pthread_kill(), sigpending() and sigwait() + functions. + """ + @unittest.skipUnless(hasattr(signal, 'sigpending'), + 'need signal.sigpending()') + def test_sigpending_empty(self): + self.assertEqual(signal.sigpending(), set()) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + @unittest.skipUnless(hasattr(signal, 'sigpending'), + 'need signal.sigpending()') + def test_sigpending(self): + code = """if 1: + import os + import signal + + def handler(signum, frame): + 1/0 + + signum = signal.SIGUSR1 + signal.signal(signum, handler) + + signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + os.kill(os.getpid(), signum) + pending = signal.sigpending() + if pending != {signum}: + raise Exception('%s != {%s}' % (pending, signum)) + try: + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + """ + assert_python_ok('-c', code) + + @unittest.skipUnless(hasattr(signal, 'pthread_kill'), + 'need signal.pthread_kill()') + def test_pthread_kill(self): + code = """if 1: + import signal + import threading + import sys + + signum = signal.SIGUSR1 + + def handler(signum, frame): + 1/0 + + signal.signal(signum, handler) + + if sys.platform == 'freebsd6': + # Issue #12392 and #12469: send a signal to the main thread + # doesn't work before the creation of the first thread on + # FreeBSD 6 + def noop(): + pass + thread = threading.Thread(target=noop) + thread.start() + thread.join() + + tid = threading.get_ident() + try: + signal.pthread_kill(tid, signum) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + """ + assert_python_ok('-c', code) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def wait_helper(self, blocked, test): + """ + test: body of the "def test(signum):" function. + blocked: number of the blocked signal + """ + code = '''if 1: + import signal + import sys + + def handler(signum, frame): + 1/0 + + %s + + blocked = %s + signum = signal.SIGALRM + + # child: block and wait the signal + try: + signal.signal(signum, handler) + signal.pthread_sigmask(signal.SIG_BLOCK, [blocked]) + + # Do the tests + test(signum) + + # The handler must not be called on unblock + try: + signal.pthread_sigmask(signal.SIG_UNBLOCK, [blocked]) + except ZeroDivisionError: + print("the signal handler has been called", + file=sys.stderr) + sys.exit(1) + except BaseException as err: + print("error: {}".format(err), file=sys.stderr) + sys.stderr.flush() + sys.exit(1) + ''' % (test.strip(), blocked) + + # sig*wait* must be called with the signal blocked: since the current + # process might have several threads running, use a subprocess to have + # a single thread. + assert_python_ok('-c', code) + + @unittest.skipUnless(hasattr(signal, 'sigwait'), + 'need signal.sigwait()') + def test_sigwait(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + signal.alarm(1) + received = signal.sigwait([signum]) + if received != signum: + raise Exception('received %s, not %s' % (received, signum)) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), + 'need signal.sigwaitinfo()') + def test_sigwaitinfo(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + signal.alarm(1) + info = signal.sigwaitinfo([signum]) + if info.si_signo != signum: + raise Exception("info.si_signo != %s" % signum) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + signal.alarm(1) + info = signal.sigtimedwait([signum], (10, 1000)) + if info.si_signo != signum: + raise Exception('info.si_signo != %s' % signum) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait_poll(self): + # check that polling with sigtimedwait works + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + import os + os.kill(os.getpid(), signum) + info = signal.sigtimedwait([signum], (0, 0)) + if info.si_signo != signum: + raise Exception('info.si_signo != %s' % signum) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait_timeout(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + received = signal.sigtimedwait([signum], (1, 0)) + if received is not None: + raise Exception("received=%r" % (received,)) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait_negative_timeout(self): + signum = signal.SIGALRM + self.assertRaises(ValueError, signal.sigtimedwait, [signum], (-1, -1)) + self.assertRaises(ValueError, signal.sigtimedwait, [signum], (0, -1)) + self.assertRaises(ValueError, signal.sigtimedwait, [signum], (-1, 0)) + + @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), + 'need signal.sigwaitinfo()') + def test_sigwaitinfo_interrupted(self): + self.wait_helper(signal.SIGUSR1, ''' + def test(signum): + import errno + + hndl_called = True + def alarm_handler(signum, frame): + hndl_called = False + + signal.signal(signal.SIGALRM, alarm_handler) + signal.alarm(1) + try: + signal.sigwaitinfo([signal.SIGUSR1]) + except OSError as e: + if e.errno == errno.EINTR: + if not hndl_called: + raise Exception("SIGALRM handler not called") + else: + raise Exception("Expected EINTR to be raised by sigwaitinfo") + else: + raise Exception("Expected EINTR to be raised by sigwaitinfo") + ''') + + @unittest.skipUnless(hasattr(signal, 'sigwait'), + 'need signal.sigwait()') + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + @unittest.skipIf(threading is None, "test needs threading module") + def test_sigwait_thread(self): + # Check that calling sigwait() from a thread doesn't suspend the whole + # process. A new interpreter is spawned to avoid problems when mixing + # threads and fork(): only async-safe functions are allowed between + # fork() and exec(). + assert_python_ok("-c", """if True: + import os, threading, sys, time, signal + + # the default handler terminates the process + signum = signal.SIGUSR1 + + def kill_later(): + # wait until the main thread is waiting in sigwait() + time.sleep(1) + os.kill(os.getpid(), signum) + + # the signal must be blocked by all the threads + signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + killer = threading.Thread(target=kill_later) + killer.start() + received = signal.sigwait([signum]) + if received != signum: + print("sigwait() received %s, not %s" % (received, signum), + file=sys.stderr) + sys.exit(1) + killer.join() + # unblock the signal, which should have been cleared by sigwait() + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + """) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def test_pthread_sigmask_arguments(self): + self.assertRaises(TypeError, signal.pthread_sigmask) + self.assertRaises(TypeError, signal.pthread_sigmask, 1) + self.assertRaises(TypeError, signal.pthread_sigmask, 1, 2, 3) + self.assertRaises(OSError, signal.pthread_sigmask, 1700, []) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def test_pthread_sigmask(self): + code = """if 1: + import signal + import os; import threading + + def handler(signum, frame): + 1/0 + + def kill(signum): + os.kill(os.getpid(), signum) + + def read_sigmask(): + return signal.pthread_sigmask(signal.SIG_BLOCK, []) + + signum = signal.SIGUSR1 + + # Install our signal handler + old_handler = signal.signal(signum, handler) + + # Unblock SIGUSR1 (and copy the old mask) to test our signal handler + old_mask = signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + try: + kill(signum) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + + # Block and then raise SIGUSR1. The signal is blocked: the signal + # handler is not called, and the signal is now pending + signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + kill(signum) + + # Check the new mask + blocked = read_sigmask() + if signum not in blocked: + raise Exception("%s not in %s" % (signum, blocked)) + if old_mask ^ blocked != {signum}: + raise Exception("%s ^ %s != {%s}" % (old_mask, blocked, signum)) + + # Unblock SIGUSR1 + try: + # unblock the pending signal calls immediatly the signal handler + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + try: + kill(signum) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + + # Check the new mask + unblocked = read_sigmask() + if signum in unblocked: + raise Exception("%s in %s" % (signum, unblocked)) + if blocked ^ unblocked != {signum}: + raise Exception("%s ^ %s != {%s}" % (blocked, unblocked, signum)) + if old_mask != unblocked: + raise Exception("%s != %s" % (old_mask, unblocked)) + """ + assert_python_ok('-c', code) + + @unittest.skipIf(sys.platform == 'freebsd6', + "issue #12392: send a signal to the main thread doesn't work " + "before the creation of the first thread on FreeBSD 6") + @unittest.skipUnless(hasattr(signal, 'pthread_kill'), + 'need signal.pthread_kill()') + def test_pthread_kill_main_thread(self): + # Test that a signal can be sent to the main thread with pthread_kill() + # before any other thread has been created (see issue #12392). + code = """if True: + import threading + import signal + import sys + + def handler(signum, frame): + sys.exit(3) + + signal.signal(signal.SIGUSR1, handler) + signal.pthread_kill(threading.get_ident(), signal.SIGUSR1) + sys.exit(2) + """ + + with spawn_python('-c', code) as process: + stdout, stderr = process.communicate() + exitcode = process.wait() + if exitcode != 3: + raise Exception("Child error (exit code %s): %s" % + (exitcode, stdout)) + + def test_main(): try: - support.run_unittest(BasicSignalTests, InterProcessSignalTests, + support.run_unittest(PosixTests, InterProcessSignalTests, WakeupSignalTests, SiginterruptTest, - ItimerTest, WindowsSignalTests) + ItimerTest, WindowsSignalTests, + PendingSignalsTests) finally: support.reap_children() diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 2cb0d1ab52..05d97ef4b7 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -72,6 +72,14 @@ class GeneralTests(unittest.TestCase): smtp = smtplib.SMTP(HOST, self.port) smtp.close() + def testSourceAddress(self): + mock_socket.reply_with(b"220 Hola mundo") + # connects + smtp = smtplib.SMTP(HOST, self.port, + source_address=('127.0.0.1',19876)) + self.assertEqual(smtp.source_address, ('127.0.0.1', 19876)) + smtp.close() + def testBasic2(self): mock_socket.reply_with(b"220 Hola mundo") # connects, include port in host name @@ -204,6 +212,20 @@ class DebuggingServerTests(unittest.TestCase): smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) smtp.quit() + def testSourceAddress(self): + # connect + port = support.find_unused_port() + try: + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=3, source_address=('127.0.0.1', port)) + self.assertEqual(smtp.source_address, ('127.0.0.1', port)) + self.assertEqual(smtp.local_hostname, 'localhost') + smtp.quit() + except IOError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to port %d" % port) + raise + def testNOOP(self): smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) expected = (250, b'Ok') @@ -560,6 +582,9 @@ sim_lists = {'list-1':['Mr.A@somewhere.com','Mrs.C@somewhereesle.com'], # Simulated SMTP channel & server class SimSMTPChannel(smtpd.SMTPChannel): + # For testing failures in QUIT when using the context manager API. + quit_response = None + def __init__(self, extra_features, *args, **kw): self._extrafeatures = ''.join( [ "250-{0}\r\n".format(x) for x in extra_features ]) @@ -610,19 +635,31 @@ class SimSMTPChannel(smtpd.SMTPChannel): else: self.push('550 No access for you!') + def smtp_QUIT(self, arg): + # args is ignored + if self.quit_response is None: + super(SimSMTPChannel, self).smtp_QUIT(arg) + else: + self.push(self.quit_response) + self.close_when_done() + def handle_error(self): raise class SimSMTPServer(smtpd.SMTPServer): + # For testing failures in QUIT when using the context manager API. + quit_response = None + def __init__(self, *args, **kw): self._extra_features = [] smtpd.SMTPServer.__init__(self, *args, **kw) def handle_accepted(self, conn, addr): - self._SMTPchannel = SimSMTPChannel(self._extra_features, - self, conn, addr) + self._SMTPchannel = SimSMTPChannel( + self._extra_features, self, conn, addr) + self._SMTPchannel.quit_response = self.quit_response def process_message(self, peer, mailfrom, rcpttos, data): pass @@ -752,6 +789,25 @@ class SMTPSimTests(unittest.TestCase): self.assertIn(sim_auth_credentials['cram-md5'], str(err)) smtp.close() + def test_with_statement(self): + with smtplib.SMTP(HOST, self.port) as smtp: + code, message = smtp.noop() + self.assertEqual(code, 250) + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo') + with smtplib.SMTP(HOST, self.port) as smtp: + smtp.close() + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo') + + def test_with_statement_QUIT_failure(self): + self.serv.quit_response = '421 QUIT FAILED' + with self.assertRaises(smtplib.SMTPResponseException) as error: + with smtplib.SMTP(HOST, self.port) as smtp: + smtp.noop() + self.assertEqual(error.exception.smtp_code, 421) + self.assertEqual(error.exception.smtp_error, b'QUIT FAILED') + # We don't need to clean up self.serv.quit_response because a new + # server is always instantiated in the setUp(). + #TODO: add tests for correct AUTH method fallback now that the #test infrastructure can support it. diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py index 0198ab669e..86224ef2e4 100644 --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -4,28 +4,60 @@ import unittest from test import support import smtplib +ssl = support.import_module("ssl") + support.requires("network") + +class SmtpTest(unittest.TestCase): + testServer = 'smtp.gmail.com' + remotePort = 25 + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + + def test_connect_starttls(self): + support.get_attribute(smtplib, 'SMTP_SSL') + with support.transient_internet(self.testServer): + server = smtplib.SMTP(self.testServer, self.remotePort) + try: + server.starttls(context=self.context) + except smtplib.SMTPException as e: + if e.args[0] == 'STARTTLS extension not supported by server.': + unittest.skip(e.args[0]) + else: + raise + server.ehlo() + server.quit() + + class SmtpSSLTest(unittest.TestCase): testServer = 'smtp.gmail.com' remotePort = 465 + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) def test_connect(self): support.get_attribute(smtplib, 'SMTP_SSL') with support.transient_internet(self.testServer): server = smtplib.SMTP_SSL(self.testServer, self.remotePort) - server.ehlo() - server.quit() + server.ehlo() + server.quit() def test_connect_default_port(self): support.get_attribute(smtplib, 'SMTP_SSL') with support.transient_internet(self.testServer): server = smtplib.SMTP_SSL(self.testServer) - server.ehlo() - server.quit() + server.ehlo() + server.quit() + + def test_connect_using_sslcontext(self): + support.get_attribute(smtplib, 'SMTP_SSL') + with support.transient_internet(self.testServer): + server = smtplib.SMTP_SSL(self.testServer, self.remotePort, context=self.context) + server.ehlo() + server.quit() + def test_main(): - support.run_unittest(SmtpSSLTest) + support.run_unittest(SmtpTest, SmtpSSLTest) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index baca4c12c7..83351b9b98 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -7,6 +7,8 @@ import errno import io import socket import select +import tempfile +import _testcapi import time import traceback import queue @@ -18,34 +20,14 @@ import contextlib from weakref import proxy import signal import math +import pickle try: import fcntl except ImportError: fcntl = False -def try_address(host, port=0, family=socket.AF_INET): - """Try to bind a socket on the given host:port and return True - if that has been possible.""" - try: - sock = socket.socket(family, socket.SOCK_STREAM) - sock.bind((host, port)) - except (socket.error, socket.gaierror): - return False - else: - sock.close() - return True - -def linux_version(): - try: - # platform.release() is something like '2.6.33.7-desktop-2mnb' - version_string = platform.release().split('-')[0] - return tuple(map(int, version_string.split('.'))) - except ValueError: - return 0, 0, 0 - HOST = support.HOST -MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf8') ## test unicode string and carriage return -SUPPORTS_IPV6 = socket.has_ipv6 and try_address('::1', family=socket.AF_INET6) +MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return try: import _thread as thread @@ -54,6 +36,9 @@ except ImportError: thread = None threading = None +# Size in bytes of the int type +SIZEOF_INT = array.array("i").itemsize + class SocketTCPTest(unittest.TestCase): def setUp(self): @@ -75,6 +60,26 @@ class SocketUDPTest(unittest.TestCase): self.serv.close() self.serv = None +class ThreadSafeCleanupTestCase(unittest.TestCase): + """Subclass of unittest.TestCase with thread-safe cleanup methods. + + This subclass protects the addCleanup() and doCleanups() methods + with a recursive lock. + """ + + if threading: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cleanup_lock = threading.RLock() + + def addCleanup(self, *args, **kwargs): + with self._cleanup_lock: + return super().addCleanup(*args, **kwargs) + + def doCleanups(self, *args, **kwargs): + with self._cleanup_lock: + return super().doCleanups(*args, **kwargs) + class ThreadableTest: """Threadable Test class @@ -257,6 +262,243 @@ class SocketPairTest(unittest.TestCase, ThreadableTest): ThreadableTest.clientTearDown(self) +# The following classes are used by the sendmsg()/recvmsg() tests. +# Combining, for instance, ConnectedStreamTestMixin and TCPTestBase +# gives a drop-in replacement for SocketConnectedTest, but different +# address families can be used, and the attributes serv_addr and +# cli_addr will be set to the addresses of the endpoints. + +class SocketTestBase(unittest.TestCase): + """A base class for socket tests. + + Subclasses must provide methods newSocket() to return a new socket + and bindSock(sock) to bind it to an unused address. + + Creates a socket self.serv and sets self.serv_addr to its address. + """ + + def setUp(self): + self.serv = self.newSocket() + self.bindServer() + + def bindServer(self): + """Bind server socket and set self.serv_addr to its address.""" + self.bindSock(self.serv) + self.serv_addr = self.serv.getsockname() + + def tearDown(self): + self.serv.close() + self.serv = None + + +class SocketListeningTestMixin(SocketTestBase): + """Mixin to listen on the server socket.""" + + def setUp(self): + super().setUp() + self.serv.listen(1) + + +class ThreadedSocketTestMixin(ThreadSafeCleanupTestCase, SocketTestBase, + ThreadableTest): + """Mixin to add client socket and allow client/server tests. + + Client socket is self.cli and its address is self.cli_addr. See + ThreadableTest for usage information. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = self.newClientSocket() + self.bindClient() + + def newClientSocket(self): + """Return a new socket for use as client.""" + return self.newSocket() + + def bindClient(self): + """Bind client socket and set self.cli_addr to its address.""" + self.bindSock(self.cli) + self.cli_addr = self.cli.getsockname() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +class ConnectedStreamTestMixin(SocketListeningTestMixin, + ThreadedSocketTestMixin): + """Mixin to allow client/server stream tests with connected client. + + Server's socket representing connection to client is self.cli_conn + and client's connection to server is self.serv_conn. (Based on + SocketConnectedTest.) + """ + + def setUp(self): + super().setUp() + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + super().tearDown() + + def clientSetUp(self): + super().clientSetUp() + self.cli.connect(self.serv_addr) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + super().clientTearDown() + + +class UnixSocketTestBase(SocketTestBase): + """Base class for Unix-domain socket tests.""" + + # This class is used for file descriptor passing tests, so we + # create the sockets in a private directory so that other users + # can't send anything that might be problematic for a privileged + # user running the tests. + + def setUp(self): + self.dir_path = tempfile.mkdtemp() + self.addCleanup(os.rmdir, self.dir_path) + super().setUp() + + def bindSock(self, sock): + path = tempfile.mktemp(dir=self.dir_path) + sock.bind(path) + self.addCleanup(support.unlink, path) + +class UnixStreamBase(UnixSocketTestBase): + """Base class for Unix-domain SOCK_STREAM tests.""" + + def newSocket(self): + return socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + +class InetTestBase(SocketTestBase): + """Base class for IPv4 socket tests.""" + + host = HOST + + def setUp(self): + super().setUp() + self.port = self.serv_addr[1] + + def bindSock(self, sock): + support.bind_port(sock, host=self.host) + +class TCPTestBase(InetTestBase): + """Base class for TCP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + +class UDPTestBase(InetTestBase): + """Base class for UDP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + +class SCTPStreamBase(InetTestBase): + """Base class for SCTP tests in one-to-one (SOCK_STREAM) mode.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM, + socket.IPPROTO_SCTP) + + +class Inet6TestBase(InetTestBase): + """Base class for IPv6 socket tests.""" + + # Don't use "localhost" here - it may not have an IPv6 address + # assigned to it by default (e.g. in /etc/hosts), and if someone + # has assigned it an IPv4-mapped address, then it's unlikely to + # work with the full IPv6 API. + host = "::1" + +class UDP6TestBase(Inet6TestBase): + """Base class for UDP-over-IPv6 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + + +# Test-skipping decorators for use with ThreadableTest. + +def skipWithClientIf(condition, reason): + """Skip decorated test if condition is true, add client_skip decorator. + + If the decorated object is not a class, sets its attribute + "client_skip" to a decorator which will return an empty function + if the test is to be skipped, or the original function if it is + not. This can be used to avoid running the client part of a + skipped test when using ThreadableTest. + """ + def client_pass(*args, **kwargs): + pass + def skipdec(obj): + retval = unittest.skip(reason)(obj) + if not isinstance(obj, type): + retval.client_skip = lambda f: client_pass + return retval + def noskipdec(obj): + if not (isinstance(obj, type) or hasattr(obj, "client_skip")): + obj.client_skip = lambda f: f + return obj + return skipdec if condition else noskipdec + + +def requireAttrs(obj, *attributes): + """Skip decorated test if obj is missing any of the given attributes. + + Sets client_skip attribute as skipWithClientIf() does. + """ + missing = [name for name in attributes if not hasattr(obj, name)] + return skipWithClientIf( + missing, "don't have " + ", ".join(name for name in missing)) + + +def requireSocket(*args): + """Skip decorated test if a socket cannot be created with given arguments. + + When an argument is given as a string, will use the value of that + attribute of the socket module, or skip the test if it doesn't + exist. Sets client_skip attribute as skipWithClientIf() does. + """ + err = None + missing = [obj for obj in args if + isinstance(obj, str) and not hasattr(socket, obj)] + if missing: + err = "don't have " + ", ".join(name for name in missing) + else: + callargs = [getattr(socket, obj) if isinstance(obj, str) else obj + for obj in args] + try: + s = socket.socket(*callargs) + except socket.error as e: + # XXX: check errno? + err = str(e) + else: + s.close() + return skipWithClientIf( + err is not None, + "can't create socket({0}): {1}".format( + ", ".join(str(o) for o in args), err)) + + ####################################################################### ## Begin Tests @@ -282,18 +524,13 @@ class GeneralModuleTests(unittest.TestCase): def testSocketError(self): # Testing socket module exceptions - def raise_error(*args, **kwargs): + msg = "Error raising socket exception (%s)." + with self.assertRaises(socket.error, msg=msg % 'socket.error'): raise socket.error - def raise_herror(*args, **kwargs): + with self.assertRaises(socket.error, msg=msg % 'socket.herror'): raise socket.herror - def raise_gaierror(*args, **kwargs): + with self.assertRaises(socket.error, msg=msg % 'socket.gaierror'): raise socket.gaierror - self.assertRaises(socket.error, raise_error, - "Error raising socket exception.") - self.assertRaises(socket.error, raise_herror, - "Error raising socket exception.") - self.assertRaises(socket.error, raise_gaierror, - "Error raising socket exception.") def testSendtoErrors(self): # Testing that sendto doens't masks failures. See #10169. @@ -369,6 +606,52 @@ class GeneralModuleTests(unittest.TestCase): if not fqhn in all_host_names: self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") + @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") + def test_sethostname(self): + oldhn = socket.gethostname() + try: + socket.sethostname('new') + except socket.error as e: + if e.errno == errno.EPERM: + self.skipTest("test should be run as root") + else: + raise + try: + # running test as root! + self.assertEqual(socket.gethostname(), 'new') + # Should work with bytes objects too + socket.sethostname(b'bar') + self.assertEqual(socket.gethostname(), 'bar') + finally: + socket.sethostname(oldhn) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInterfaceNameIndex(self): + interfaces = socket.if_nameindex() + for index, name in interfaces: + self.assertIsInstance(index, int) + self.assertIsInstance(name, str) + # interface indices are non-zero integers + self.assertGreater(index, 0) + _index = socket.if_nametoindex(name) + self.assertIsInstance(_index, int) + self.assertEqual(index, _index) + _name = socket.if_indextoname(index) + self.assertIsInstance(_name, str) + self.assertEqual(name, _name) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInvalidInterfaceNameIndex(self): + # test nonexistent interface index/name + self.assertRaises(socket.error, socket.if_indextoname, 0) + self.assertRaises(socket.error, socket.if_nametoindex, '_DEADBEEF') + # test with invalid values + self.assertRaises(TypeError, socket.if_nametoindex, 0) + self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') + def testRefCountGetNameInfo(self): # Testing reference count for getnameinfo if hasattr(sys, "getrefcount"): @@ -421,10 +704,8 @@ class GeneralModuleTests(unittest.TestCase): # Find one service that exists, then check all the related interfaces. # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. - if (sys.platform.startswith('linux') or - sys.platform.startswith('freebsd') or - sys.platform.startswith('netbsd') or - sys.platform == 'darwin'): + if (sys.platform.startswith(('freebsd', 'netbsd')) + or sys.platform in ('linux', 'darwin')): # avoid the 'echo' service on this platform, as there is an # assumption breaking non-standard port/protocol entry services = ('daytime', 'qotd', 'domain') @@ -654,7 +935,7 @@ class GeneralModuleTests(unittest.TestCase): socket.getaddrinfo('localhost', 80) socket.getaddrinfo('127.0.0.1', 80) socket.getaddrinfo(None, 80) - if SUPPORTS_IPV6: + if support.IPV6_ENABLED: socket.getaddrinfo('::1', 80) # port can be a string service name such as "http", a numeric # port number or None @@ -774,7 +1055,13 @@ class GeneralModuleTests(unittest.TestCase): fp.close() self.assertEqual(repr(fp), "<_io.BufferedReader name=-1>") - def testListenBacklog0(self): + def test_pickle(self): + sock = socket.socket() + with sock: + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(TypeError, pickle.dumps, sock, protocol) + + def test_listen_backlog0(self): srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) srv.bind((HOST, 0)) # backlog = 0 @@ -920,6 +1207,1839 @@ class BasicUDPTest(ThreadedUDPSocketTest): def _testRecvFromNegative(self): self.cli.sendto(MSG, 0, (HOST, self.port)) + +# Tests for the sendmsg()/recvmsg() interface. Where possible, the +# same test code is used with different families and types of socket +# (e.g. stream, datagram), and tests using recvmsg() are repeated +# using recvmsg_into(). +# +# The generic test classes such as SendmsgTests and +# RecvmsgGenericTests inherit from SendrecvmsgBase and expect to be +# supplied with sockets cli_sock and serv_sock representing the +# client's and the server's end of the connection respectively, and +# attributes cli_addr and serv_addr holding their (numeric where +# appropriate) addresses. +# +# The final concrete test classes combine these with subclasses of +# SocketTestBase which set up client and server sockets of a specific +# type, and with subclasses of SendrecvmsgBase such as +# SendrecvmsgDgramBase and SendrecvmsgConnectedBase which map these +# sockets to cli_sock and serv_sock and override the methods and +# attributes of SendrecvmsgBase to fill in destination addresses if +# needed when sending, check for specific flags in msg_flags, etc. +# +# RecvmsgIntoMixin provides a version of doRecvmsg() implemented using +# recvmsg_into(). + +# XXX: like the other datagram (UDP) tests in this module, the code +# here assumes that datagram delivery on the local machine will be +# reliable. + +class SendrecvmsgBase(ThreadSafeCleanupTestCase): + # Base class for sendmsg()/recvmsg() tests. + + # Time in seconds to wait before considering a test failed, or + # None for no timeout. Not all tests actually set a timeout. + fail_timeout = 3.0 + + def setUp(self): + self.misc_event = threading.Event() + super().setUp() + + def sendToServer(self, msg): + # Send msg to the server. + return self.cli_sock.send(msg) + + # Tuple of alternative default arguments for sendmsg() when called + # via sendmsgToServer() (e.g. to include a destination address). + sendmsg_to_server_defaults = () + + def sendmsgToServer(self, *args): + # Call sendmsg() on self.cli_sock with the given arguments, + # filling in any arguments which are not supplied with the + # corresponding items of self.sendmsg_to_server_defaults, if + # any. + return self.cli_sock.sendmsg( + *(args + self.sendmsg_to_server_defaults[len(args):])) + + def doRecvmsg(self, sock, bufsize, *args): + # Call recvmsg() on sock with given arguments and return its + # result. Should be used for tests which can use either + # recvmsg() or recvmsg_into() - RecvmsgIntoMixin overrides + # this method with one which emulates it using recvmsg_into(), + # thus allowing the same test to be used for both methods. + result = sock.recvmsg(bufsize, *args) + self.registerRecvmsgResult(result) + return result + + def registerRecvmsgResult(self, result): + # Called by doRecvmsg() with the return value of recvmsg() or + # recvmsg_into(). Can be overridden to arrange cleanup based + # on the returned ancillary data, for instance. + pass + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer. + self.assertEqual(addr1, addr2) + + # Flags that are normally unset in msg_flags + msg_flags_common_unset = 0 + for name in ("MSG_CTRUNC", "MSG_OOB"): + msg_flags_common_unset |= getattr(socket, name, 0) + + # Flags that are normally set + msg_flags_common_set = 0 + + # Flags set when a complete record has been received (e.g. MSG_EOR + # for SCTP) + msg_flags_eor_indicator = 0 + + # Flags set when a complete record has not been received + # (e.g. MSG_TRUNC for datagram sockets) + msg_flags_non_eor_indicator = 0 + + def checkFlags(self, flags, eor=None, checkset=0, checkunset=0, ignore=0): + # Method to check the value of msg_flags returned by recvmsg[_into](). + # + # Checks that all bits in msg_flags_common_set attribute are + # set in "flags" and all bits in msg_flags_common_unset are + # unset. + # + # The "eor" argument specifies whether the flags should + # indicate that a full record (or datagram) has been received. + # If "eor" is None, no checks are done; otherwise, checks + # that: + # + # * if "eor" is true, all bits in msg_flags_eor_indicator are + # set and all bits in msg_flags_non_eor_indicator are unset + # + # * if "eor" is false, all bits in msg_flags_non_eor_indicator + # are set and all bits in msg_flags_eor_indicator are unset + # + # If "checkset" and/or "checkunset" are supplied, they require + # the given bits to be set or unset respectively, overriding + # what the attributes require for those bits. + # + # If any bits are set in "ignore", they will not be checked, + # regardless of the other inputs. + # + # Will raise Exception if the inputs require a bit to be both + # set and unset, and it is not ignored. + + defaultset = self.msg_flags_common_set + defaultunset = self.msg_flags_common_unset + + if eor: + defaultset |= self.msg_flags_eor_indicator + defaultunset |= self.msg_flags_non_eor_indicator + elif eor is not None: + defaultset |= self.msg_flags_non_eor_indicator + defaultunset |= self.msg_flags_eor_indicator + + # Function arguments override defaults + defaultset &= ~checkunset + defaultunset &= ~checkset + + # Merge arguments with remaining defaults, and check for conflicts + checkset |= defaultset + checkunset |= defaultunset + inboth = checkset & checkunset & ~ignore + if inboth: + raise Exception("contradictory set, unset requirements for flags " + "{0:#x}".format(inboth)) + + # Compare with given msg_flags value + mask = (checkset | checkunset) & ~ignore + self.assertEqual(flags & mask, checkset & mask) + + +class RecvmsgIntoMixin(SendrecvmsgBase): + # Mixin to implement doRecvmsg() using recvmsg_into(). + + def doRecvmsg(self, sock, bufsize, *args): + buf = bytearray(bufsize) + result = sock.recvmsg_into([buf], *args) + self.registerRecvmsgResult(result) + self.assertGreaterEqual(result[0], 0) + self.assertLessEqual(result[0], bufsize) + return (bytes(buf[:result[0]]),) + result[1:] + + +class SendrecvmsgDgramFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for datagram sockets. + + @property + def msg_flags_non_eor_indicator(self): + return super().msg_flags_non_eor_indicator | socket.MSG_TRUNC + + +class SendrecvmsgSCTPFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for SCTP sockets. + + @property + def msg_flags_eor_indicator(self): + return super().msg_flags_eor_indicator | socket.MSG_EOR + + +class SendrecvmsgConnectionlessBase(SendrecvmsgBase): + # Base class for tests on connectionless-mode sockets. Users must + # supply sockets on attributes cli and serv to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.serv + + @property + def cli_sock(self): + return self.cli + + @property + def sendmsg_to_server_defaults(self): + return ([], [], 0, self.serv_addr) + + def sendToServer(self, msg): + return self.cli_sock.sendto(msg, self.serv_addr) + + +class SendrecvmsgConnectedBase(SendrecvmsgBase): + # Base class for tests on connected sockets. Users must supply + # sockets on attributes serv_conn and cli_conn (representing the + # connections *to* the server and the client), to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.cli_conn + + @property + def cli_sock(self): + return self.serv_conn + + def checkRecvmsgAddress(self, addr1, addr2): + # Address is currently "unspecified" for a connected socket, + # so we don't examine it + pass + + +class SendrecvmsgServerTimeoutBase(SendrecvmsgBase): + # Base class to set a timeout on server's socket. + + def setUp(self): + super().setUp() + self.serv_sock.settimeout(self.fail_timeout) + + +class SendmsgTests(SendrecvmsgServerTimeoutBase): + # Tests for sendmsg() which can use any socket type and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsg(self): + # Send a simple message with sendmsg(). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG]), len(MSG)) + + def testSendmsgDataGenerator(self): + # Send from buffer obtained from a generator (not a sequence). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgDataGenerator(self): + self.assertEqual(self.sendmsgToServer((o for o in [MSG])), + len(MSG)) + + def testSendmsgAncillaryGenerator(self): + # Gather (empty) ancillary data from a generator. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgAncillaryGenerator(self): + self.assertEqual(self.sendmsgToServer([MSG], (o for o in [])), + len(MSG)) + + def testSendmsgArray(self): + # Send data from an array instead of the usual bytes object. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgArray(self): + self.assertEqual(self.sendmsgToServer([array.array("B", MSG)]), + len(MSG)) + + def testSendmsgGather(self): + # Send message data from more than one buffer (gather write). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgGather(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + def testSendmsgBadArgs(self): + # Check that sendmsg() rejects invalid arguments. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadArgs(self): + self.assertRaises(TypeError, self.cli_sock.sendmsg) + self.assertRaises(TypeError, self.sendmsgToServer, + b"not in an iterable") + self.assertRaises(TypeError, self.sendmsgToServer, + object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG, object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], 0, object()) + self.sendToServer(b"done") + + def testSendmsgBadCmsg(self): + # Check that invalid ancillary data items are rejected. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(object(), 0, b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, object(), b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, object())]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0)]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b"data", 42)]) + self.sendToServer(b"done") + + @requireAttrs(socket, "CMSG_SPACE") + def testSendmsgBadMultiCmsg(self): + # Check that invalid ancillary data items are rejected when + # more than one item is present. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + @testSendmsgBadMultiCmsg.client_skip + def _testSendmsgBadMultiCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [0, 0, b""]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b""), object()]) + self.sendToServer(b"done") + + def testSendmsgExcessCmsgReject(self): + # Check that sendmsg() rejects excess ancillary data items + # when the number that can be sent is limited. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgExcessCmsgReject(self): + if not hasattr(socket, "CMSG_SPACE"): + # Can only send one item + with self.assertRaises(socket.error) as cm: + self.sendmsgToServer([MSG], [(0, 0, b""), (0, 0, b"")]) + self.assertIsNone(cm.exception.errno) + self.sendToServer(b"done") + + def testSendmsgAfterClose(self): + # Check that sendmsg() fails on a closed socket. + pass + + def _testSendmsgAfterClose(self): + self.cli_sock.close() + self.assertRaises(socket.error, self.sendmsgToServer, [MSG]) + + +class SendmsgStreamTests(SendmsgTests): + # Tests for sendmsg() which require a stream socket and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsgExplicitNoneAddr(self): + # Check that peer address can be specified as None. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgExplicitNoneAddr(self): + self.assertEqual(self.sendmsgToServer([MSG], [], 0, None), len(MSG)) + + def testSendmsgTimeout(self): + # Check that timeout works with sendmsg(). + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + def _testSendmsgTimeout(self): + try: + self.cli_sock.settimeout(0.03) + with self.assertRaises(socket.timeout): + while True: + self.sendmsgToServer([b"a"*512]) + finally: + self.misc_event.set() + + # XXX: would be nice to have more tests for sendmsg flags argument. + + # Linux supports MSG_DONTWAIT when sending, but in general, it + # only works when receiving. Could add other platforms if they + # support it too. + @skipWithClientIf(sys.platform not in {"linux2"}, + "MSG_DONTWAIT not known to work on this platform when " + "sending") + def testSendmsgDontWait(self): + # Check that MSG_DONTWAIT in flags causes non-blocking behaviour. + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @testSendmsgDontWait.client_skip + def _testSendmsgDontWait(self): + try: + with self.assertRaises(socket.error) as cm: + while True: + self.sendmsgToServer([b"a"*512], [], socket.MSG_DONTWAIT) + self.assertIn(cm.exception.errno, + (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + self.misc_event.set() + + +class SendmsgConnectionlessTests(SendmsgTests): + # Tests for sendmsg() which require a connectionless-mode + # (e.g. datagram) socket, and do not involve recvmsg() or + # recvmsg_into(). + + def testSendmsgNoDestAddr(self): + # Check that sendmsg() fails when no destination address is + # given for unconnected socket. + pass + + def _testSendmsgNoDestAddr(self): + self.assertRaises(socket.error, self.cli_sock.sendmsg, + [MSG]) + self.assertRaises(socket.error, self.cli_sock.sendmsg, + [MSG], [], 0, None) + + +class RecvmsgGenericTests(SendrecvmsgBase): + # Tests for recvmsg() which can also be emulated using + # recvmsg_into(), and can use any socket type. + + def testRecvmsg(self): + # Receive a simple message with recvmsg[_into](). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsg(self): + self.sendToServer(MSG) + + def testRecvmsgExplicitDefaults(self): + # Test recvmsg[_into]() with default arguments provided explicitly. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgExplicitDefaults(self): + self.sendToServer(MSG) + + def testRecvmsgShorter(self): + # Receive a message smaller than buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) + 42) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShorter(self): + self.sendToServer(MSG) + + def testRecvmsgTrunc(self): + # Receive part of message, check for truncation indicators. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + def _testRecvmsgTrunc(self): + self.sendToServer(MSG) + + def testRecvmsgShortAncillaryBuf(self): + # Test ancillary data buffer too small to hold any ancillary data. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 1) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShortAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgLongAncillaryBuf(self): + # Test large ancillary data buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgLongAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgAfterClose(self): + # Check that recvmsg[_into]() fails on a closed socket. + self.serv_sock.close() + self.assertRaises(socket.error, self.doRecvmsg, self.serv_sock, 1024) + + def _testRecvmsgAfterClose(self): + pass + + def testRecvmsgTimeout(self): + # Check that timeout works. + try: + self.serv_sock.settimeout(0.03) + self.assertRaises(socket.timeout, + self.doRecvmsg, self.serv_sock, len(MSG)) + finally: + self.misc_event.set() + + def _testRecvmsgTimeout(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @requireAttrs(socket, "MSG_PEEK") + def testRecvmsgPeek(self): + # Check that MSG_PEEK in flags enables examination of pending + # data without consuming it. + + # Receive part of data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3, 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + # Ignoring MSG_TRUNC here (so this test is the same for stream + # and datagram sockets). Some wording in POSIX seems to + # suggest that it needn't be set when peeking, but that may + # just be a slip. + self.checkFlags(flags, eor=False, + ignore=getattr(socket, "MSG_TRUNC", 0)) + + # Receive all data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + # Check that the same data can still be received normally. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgPeek.client_skip + def _testRecvmsgPeek(self): + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + def testRecvmsgFromSendmsg(self): + # Test receiving with recvmsg[_into]() when message is sent + # using sendmsg(). + self.serv_sock.settimeout(self.fail_timeout) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgFromSendmsg.client_skip + def _testRecvmsgFromSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + +class RecvmsgGenericStreamTests(RecvmsgGenericTests): + # Tests which require a stream socket and can use either recvmsg() + # or recvmsg_into(). + + def testRecvmsgEOF(self): + # Receive end-of-stream indicator (b"", peer socket closed). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.assertEqual(msg, b"") + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=None) # Might not have end-of-record marker + + def _testRecvmsgEOF(self): + self.cli_sock.close() + + def testRecvmsgOverflow(self): + # Receive a message in more than one chunk. + seg1, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + seg2, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testRecvmsgOverflow(self): + self.sendToServer(MSG) + + +class RecvmsgTests(RecvmsgGenericTests): + # Tests for recvmsg() which can use any socket type. + + def testRecvmsgBadArgs(self): + # Check that recvmsg() rejects invalid arguments. + self.assertRaises(TypeError, self.serv_sock.recvmsg) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + -1, 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + len(MSG), -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + [bytearray(10)], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + object(), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), 0, object()) + + msg, ancdata, flags, addr = self.serv_sock.recvmsg(len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgBadArgs(self): + self.sendToServer(MSG) + + +class RecvmsgIntoTests(RecvmsgIntoMixin, RecvmsgGenericTests): + # Tests for recvmsg_into() which can use any socket type. + + def testRecvmsgIntoBadArgs(self): + # Check that recvmsg_into() rejects invalid arguments. + buf = bytearray(len(MSG)) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + len(MSG), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + buf, 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [object()], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [b"I'm not writable"], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf, object()], 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg_into, + [buf], -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], 0, object()) + + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf], 0, 0) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoBadArgs(self): + self.sendToServer(MSG) + + def testRecvmsgIntoGenerator(self): + # Receive into buffer obtained from a generator (not a sequence). + buf = bytearray(len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + (o for o in [buf])) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoGenerator(self): + self.sendToServer(MSG) + + def testRecvmsgIntoArray(self): + # Receive into an array rather than the usual bytearray. + buf = array.array("B", [0] * len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf]) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf.tobytes(), MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoArray(self): + self.sendToServer(MSG) + + def testRecvmsgIntoScatter(self): + # Receive into multiple buffers (scatter write). + b1 = bytearray(b"----") + b2 = bytearray(b"0123456789") + b3 = bytearray(b"--------------") + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + [b1, memoryview(b2)[2:9], b3]) + self.assertEqual(nbytes, len(b"Mary had a little lamb")) + self.assertEqual(b1, bytearray(b"Mary")) + self.assertEqual(b2, bytearray(b"01 had a 9")) + self.assertEqual(b3, bytearray(b"little lamb---")) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoScatter(self): + self.sendToServer(b"Mary had a little lamb") + + +class CmsgMacroTests(unittest.TestCase): + # Test the functions CMSG_LEN() and CMSG_SPACE(). Tests + # assumptions used by sendmsg() and recvmsg[_into](), which share + # code with these functions. + + # Match the definition in socketmodule.c + socklen_t_limit = min(0x7fffffff, _testcapi.INT_MAX) + + @requireAttrs(socket, "CMSG_LEN") + def testCMSG_LEN(self): + # Test CMSG_LEN() with various valid and invalid values, + # checking the assumptions used by recvmsg() and sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_LEN(0) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(socket.CMSG_LEN(0), array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_LEN(n) + # This is how recvmsg() calculates the data size + self.assertEqual(ret - socket.CMSG_LEN(0), n) + self.assertLessEqual(ret, self.socklen_t_limit) + + self.assertRaises(OverflowError, socket.CMSG_LEN, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_LEN, toobig) + self.assertRaises(OverflowError, socket.CMSG_LEN, sys.maxsize) + + @requireAttrs(socket, "CMSG_SPACE") + def testCMSG_SPACE(self): + # Test CMSG_SPACE() with various valid and invalid values, + # checking the assumptions used by sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + last = socket.CMSG_SPACE(0) + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(last, array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_SPACE(n) + self.assertGreaterEqual(ret, last) + self.assertGreaterEqual(ret, socket.CMSG_LEN(n)) + self.assertGreaterEqual(ret, n + socket.CMSG_LEN(0)) + self.assertLessEqual(ret, self.socklen_t_limit) + last = ret + + self.assertRaises(OverflowError, socket.CMSG_SPACE, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_SPACE, toobig) + self.assertRaises(OverflowError, socket.CMSG_SPACE, sys.maxsize) + + +class SCMRightsTest(SendrecvmsgServerTimeoutBase): + # Tests for file descriptor passing on Unix-domain sockets. + + # Invalid file descriptor value that's unlikely to evaluate to a + # real FD even if one of its bytes is replaced with a different + # value (which shouldn't actually happen). + badfd = -0x5555 + + def newFDs(self, n): + # Return a list of n file descriptors for newly-created files + # containing their list indices as ASCII numbers. + fds = [] + for i in range(n): + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + self.addCleanup(os.close, fd) + os.write(fd, str(i).encode()) + fds.append(fd) + return fds + + def checkFDs(self, fds): + # Check that the file descriptors in the given list contain + # their correct list indices as ASCII numbers. + for n, fd in enumerate(fds): + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(os.read(fd, 1024), str(n).encode()) + + def registerRecvmsgResult(self, result): + self.addCleanup(self.closeRecvmsgFDs, result) + + def closeRecvmsgFDs(self, recvmsg_result): + # Close all file descriptors specified in the ancillary data + # of the given return value from recvmsg() or recvmsg_into(). + for cmsg_level, cmsg_type, cmsg_data in recvmsg_result[1]: + if (cmsg_level == socket.SOL_SOCKET and + cmsg_type == socket.SCM_RIGHTS): + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + for fd in fds: + os.close(fd) + + def createAndSendFDs(self, n): + # Send n new file descriptors created by newFDs() to the + # server, with the constant MSG as the non-ancillary data. + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(n)))]), + len(MSG)) + + def checkRecvmsgFDs(self, numfds, result, maxcmsgs=1, ignoreflags=0): + # Check that constant MSG was received with numfds file + # descriptors in a maximum of maxcmsgs control messages (which + # must contain only complete integers). By default, check + # that MSG_CTRUNC is unset, but ignore any flags in + # ignoreflags. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertIsInstance(ancdata, list) + self.assertLessEqual(len(ancdata), maxcmsgs) + fds = array.array("i") + for item in ancdata: + self.assertIsInstance(item, tuple) + cmsg_level, cmsg_type, cmsg_data = item + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data) % SIZEOF_INT, 0) + fds.frombytes(cmsg_data) + + self.assertEqual(len(fds), numfds) + self.checkFDs(fds) + + def testFDPassSimple(self): + # Pass a single FD (array read from bytes object). + self.checkRecvmsgFDs(1, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testFDPassSimple(self): + self.assertEqual( + self.sendmsgToServer( + [MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(1)).tobytes())]), + len(MSG)) + + def testMultipleFDPass(self): + # Pass multiple FDs in a single array. + self.checkRecvmsgFDs(4, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testMultipleFDPass(self): + self.createAndSendFDs(4) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassCMSG_SPACE(self): + # Test using CMSG_SPACE() to calculate ancillary buffer size. + self.checkRecvmsgFDs( + 4, self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(4 * SIZEOF_INT))) + + @testFDPassCMSG_SPACE.client_skip + def _testFDPassCMSG_SPACE(self): + self.createAndSendFDs(4) + + def testFDPassCMSG_LEN(self): + # Test using CMSG_LEN() to calculate ancillary buffer size. + self.checkRecvmsgFDs(1, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(4 * SIZEOF_INT)), + # RFC 3542 says implementations may set + # MSG_CTRUNC if there isn't enough space + # for trailing padding. + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassCMSG_LEN(self): + self.createAndSendFDs(1) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparate(self): + # Pass two FDs in two separate arrays. Arrays may be combined + # into a single control message by the OS. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), 10240), + maxcmsgs=2) + + @testFDPassSeparate.client_skip + def _testFDPassSeparate(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparateMinSpace(self): + # Pass two FDs in two separate arrays, receiving them into the + # minimum space for two arrays. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(SIZEOF_INT)), + maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) + + @testFDPassSeparateMinSpace.client_skip + def _testFDPassSeparateMinSpace(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + def sendAncillaryIfPossible(self, msg, ancdata): + # Try to send msg and ancdata to server, but if the system + # call fails, just send msg with no ancillary data. + try: + nbytes = self.sendmsgToServer([msg], ancdata) + except socket.error as e: + # Check that it was the system call that failed + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer([msg]) + self.assertEqual(nbytes, len(msg)) + + def testFDPassEmpty(self): + # Try to pass an empty FD array. Can receive either no array + # or an empty array. + self.checkRecvmsgFDs(0, self.doRecvmsg(self.serv_sock, + len(MSG), 10240), + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassEmpty(self): + self.sendAncillaryIfPossible(MSG, [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + b"")]) + + def testFDPassPartialInt(self): + # Try to pass a truncated FD array. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 1) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + def _testFDPassPartialInt(self): + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [self.badfd]).tobytes()[:-1])]) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassPartialIntInMiddle(self): + # Try to pass two FD arrays, the first of which is truncated. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 2) + fds = array.array("i") + # Arrays may have been combined in a single control message + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.assertLessEqual(len(fds), 2) + self.checkFDs(fds) + + @testFDPassPartialIntInMiddle.client_skip + def _testFDPassPartialIntInMiddle(self): + fd0, fd1 = self.newFDs(2) + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0, self.badfd]).tobytes()[:-1]), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]) + + def checkTruncatedHeader(self, result, ignoreflags=0): + # Check that no ancillary data items are returned when data is + # truncated inside the cmsghdr structure. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no buffer size + # is specified. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG)), + # BSD seems to set MSG_CTRUNC only + # if an item has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTruncNoBufSize(self): + self.createAndSendFDs(1) + + def testCmsgTrunc0(self): + # Check that no ancillary data is received when buffer size is 0. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0), + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTrunc0(self): + self.createAndSendFDs(1) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + def testCmsgTrunc1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) + + def _testCmsgTrunc1(self): + self.createAndSendFDs(1) + + def testCmsgTrunc2Int(self): + # The cmsghdr structure has at least three members, two of + # which are ints, so we still shouldn't see any ancillary + # data. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + SIZEOF_INT * 2)) + + def _testCmsgTrunc2Int(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Minus1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(0) - 1)) + + def _testCmsgTruncLen0Minus1(self): + self.createAndSendFDs(1) + + # The following tests try to truncate the control message in the + # middle of the FD array. + + def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): + # Check that file descriptor data is truncated to between + # mindata and maxdata bytes when received with buffer size + # ancbuf, and that any complete file descriptor numbers are + # valid. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbuf) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + if mindata == 0 and ancdata == []: + return + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertGreaterEqual(len(cmsg_data), mindata) + self.assertLessEqual(len(cmsg_data), maxdata) + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.checkFDs(fds) + + def testCmsgTruncLen0(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) + + def _testCmsgTruncLen0(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Plus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) + + def _testCmsgTruncLen0Plus1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), + maxdata=SIZEOF_INT) + + def _testCmsgTruncLen1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen2Minus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, + maxdata=(2 * SIZEOF_INT) - 1) + + def _testCmsgTruncLen2Minus1(self): + self.createAndSendFDs(2) + + +class RFC3542AncillaryTest(SendrecvmsgServerTimeoutBase): + # Test sendmsg() and recvmsg[_into]() using the ancillary data + # features of the RFC 3542 Advanced Sockets API for IPv6. + # Currently we can only handle certain data items (e.g. traffic + # class, hop limit, MTU discovery and fragmentation settings) + # without resorting to unportable means such as the struct module, + # but the tests here are aimed at testing the ancillary data + # handling in sendmsg() and recvmsg() rather than the IPv6 API + # itself. + + # Test value to use when setting hop limit of packet + hop_limit = 2 + + # Test value to use when setting traffic class of packet. + # -1 means "use kernel default". + traffic_class = -1 + + def ancillaryMapping(self, ancdata): + # Given ancillary data list ancdata, return a mapping from + # pairs (cmsg_level, cmsg_type) to corresponding cmsg_data. + # Check that no (level, type) pair appears more than once. + d = {} + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertNotIn((cmsg_level, cmsg_type), d) + d[(cmsg_level, cmsg_type)] = cmsg_data + return d + + def checkHopLimit(self, ancbufsize, maxhop=255, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space. Check that data is MSG, ancillary data is not + # truncated (but ignore any flags in ignoreflags), and hop + # limit is between 0 and maxhop inclusive. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + self.assertIsInstance(ancdata[0], tuple) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimit(self): + # Test receiving the packet hop limit as ancillary data. + self.checkHopLimit(ancbufsize=10240) + + @testRecvHopLimit.client_skip + def _testRecvHopLimit(self): + # Need to wait until server has asked to receive ancillary + # data, as implementations are not required to buffer it + # otherwise. + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimitCMSG_SPACE(self): + # Test receiving hop limit, using CMSG_SPACE to calculate buffer size. + self.checkHopLimit(ancbufsize=socket.CMSG_SPACE(SIZEOF_INT)) + + @testRecvHopLimitCMSG_SPACE.client_skip + def _testRecvHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Could test receiving into buffer sized using CMSG_LEN, but RFC + # 3542 says portable applications must provide space for trailing + # padding. Implementations may set MSG_CTRUNC if there isn't + # enough space for the padding. + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSetHopLimit(self): + # Test setting hop limit on outgoing packet and receiving it + # at the other end. + self.checkHopLimit(ancbufsize=10240, maxhop=self.hop_limit) + + @testSetHopLimit.client_skip + def _testSetHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + def checkTrafficClassAndHopLimit(self, ancbufsize, maxhop=255, + ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space. Check that data is MSG, ancillary + # data is not truncated (but ignore any flags in ignoreflags), + # and traffic class and hop limit are in range (hop limit no + # more than maxhop). + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + self.assertEqual(len(ancdata), 2) + ancmap = self.ancillaryMapping(ancdata) + + tcdata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_TCLASS)] + self.assertEqual(len(tcdata), SIZEOF_INT) + a = array.array("i") + a.frombytes(tcdata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + hldata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT)] + self.assertEqual(len(hldata), SIZEOF_INT) + a = array.array("i") + a.frombytes(hldata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimit(self): + # Test receiving traffic class and hop limit as ancillary data. + self.checkTrafficClassAndHopLimit(ancbufsize=10240) + + @testRecvTrafficClassAndHopLimit.client_skip + def _testRecvTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + # Test receiving traffic class and hop limit, using + # CMSG_SPACE() to calculate buffer size. + self.checkTrafficClassAndHopLimit( + ancbufsize=socket.CMSG_SPACE(SIZEOF_INT) * 2) + + @testRecvTrafficClassAndHopLimitCMSG_SPACE.client_skip + def _testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSetTrafficClassAndHopLimit(self): + # Test setting traffic class and hop limit on outgoing packet, + # and receiving them at the other end. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testSetTrafficClassAndHopLimit.client_skip + def _testSetTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testOddCmsgSize(self): + # Try to send ancillary data with first item one byte too + # long. Fall back to sending with correct size if this fails, + # and check that second item was handled correctly. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testOddCmsgSize.client_skip + def _testOddCmsgSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + try: + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class]).tobytes() + b"\x00"), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + except socket.error as e: + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + self.assertEqual(nbytes, len(MSG)) + + # Tests for proper handling of truncated ancillary data + + def checkHopLimitTruncatedHeader(self, ancbufsize, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space, which should be too small to contain the ancillary + # data header (if ancbufsize is None, pass no second argument + # to recvmsg()). Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and no ancillary data is + # returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + args = () if ancbufsize is None else (ancbufsize,) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), *args) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no ancillary + # buffer size is provided. + self.checkHopLimitTruncatedHeader(ancbufsize=None, + # BSD seems to set + # MSG_CTRUNC only if an item + # has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + @testCmsgTruncNoBufSize.client_skip + def _testCmsgTruncNoBufSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc0(self): + # Check that no ancillary data is received when ancillary + # buffer size is zero. + self.checkHopLimitTruncatedHeader(ancbufsize=0, + ignoreflags=socket.MSG_CTRUNC) + + @testSingleCmsgTrunc0.client_skip + def _testSingleCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=1) + + @testSingleCmsgTrunc1.client_skip + def _testSingleCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc2Int(self): + self.checkHopLimitTruncatedHeader(ancbufsize=2 * SIZEOF_INT) + + @testSingleCmsgTrunc2Int.client_skip + def _testSingleCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncLen0Minus1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=socket.CMSG_LEN(0) - 1) + + @testSingleCmsgTruncLen0Minus1.client_skip + def _testSingleCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncInData(self): + # Test truncation of a control message inside its associated + # data. The message may be returned with its data truncated, + # or not returned at all. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + self.assertLessEqual(len(ancdata), 1) + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + @testSingleCmsgTruncInData.client_skip + def _testSingleCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + def checkTruncatedSecondHeader(self, ancbufsize, ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space, which should be large enough to + # contain the first item, but too small to contain the header + # of the second. Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and only one ancillary + # data item is returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertIn(cmsg_type, {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT}) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + # Try the above test with various buffer sizes. + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc0(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT), + ignoreflags=socket.MSG_CTRUNC) + + @testSecondCmsgTrunc0.client_skip + def _testSecondCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 1) + + @testSecondCmsgTrunc1.client_skip + def _testSecondCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc2Int(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + 2 * SIZEOF_INT) + + @testSecondCmsgTrunc2Int.client_skip + def _testSecondCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTruncLen0Minus1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(0) - 1) + + @testSecondCmsgTruncLen0Minus1.client_skip + def _testSecondCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecomdCmsgTruncInData(self): + # Test truncation of the second of two control messages inside + # its associated data. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + cmsg_types = {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT} + + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + self.assertEqual(ancdata, []) + + @testSecomdCmsgTruncInData.client_skip + def _testSecomdCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + +# Derive concrete test classes for different socket types. + +class SendrecvmsgUDPTestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDPTest(SendmsgConnectionlessTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDPTest(RecvmsgTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDPTest(RecvmsgIntoTests, SendrecvmsgUDPTestBase): + pass + + +class SendrecvmsgUDP6TestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDP6TestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(socket.has_ipv6, "Python not built with IPv6 support") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDP6Test(SendmsgConnectionlessTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(socket.has_ipv6, "Python not built with IPv6 support") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDP6Test(RecvmsgTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(socket.has_ipv6, "Python not built with IPv6 support") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDP6Test(RecvmsgIntoTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(socket.has_ipv6, "Python not built with IPv6 support") +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgRFC3542AncillaryUDP6Test(RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(socket.has_ipv6, "Python not built with IPv6 support") +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoRFC3542AncillaryUDP6Test(RecvmsgIntoMixin, + RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + + +class SendrecvmsgTCPTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, TCPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgTCPTest(SendmsgStreamTests, SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgTCPTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoTCPTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + + +class SendrecvmsgSCTPStreamTestBase(SendrecvmsgSCTPFlagsBase, + SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, SCTPStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgSCTPStreamTest(SendmsgStreamTests, SendrecvmsgSCTPStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCTPStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCTPStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + pass + + +class SendrecvmsgUnixStreamTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, UnixStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUnixStreamTest(SendmsgStreamTests, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUnixStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUnixStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCMRightsStreamTest(SCMRightsTest, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg_into") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCMRightsStreamTest(RecvmsgIntoMixin, SCMRightsTest, + SendrecvmsgUnixStreamTestBase): + pass + + +# Test interrupting the interruptible send/receive methods with a +# signal when a timeout is set. These tests avoid having multiple +# threads alive during the test so that the OS cannot deliver the +# signal to the wrong one. + +class InterruptedTimeoutBase(unittest.TestCase): + # Base class for interrupted send/receive tests. Installs an + # empty handler for SIGALRM and removes it on teardown, along with + # any scheduled alarms. + + def setUp(self): + super().setUp() + orig_alrm_handler = signal.signal(signal.SIGALRM, + lambda signum, frame: None) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(self.setAlarm, 0) + + # Timeout for socket operations + timeout = 4.0 + + # Provide setAlarm() method to schedule delivery of SIGALRM after + # given number of seconds, or cancel it if zero, and an + # appropriate time value to use. Use setitimer() if available. + if hasattr(signal, "setitimer"): + alarm_time = 0.05 + + def setAlarm(self, seconds): + signal.setitimer(signal.ITIMER_REAL, seconds) + else: + # Old systems may deliver the alarm up to one second early + alarm_time = 2 + + def setAlarm(self, seconds): + signal.alarm(seconds) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +class InterruptedRecvTimeoutTest(InterruptedTimeoutBase, UDPTestBase): + # Test interrupting the recv*() methods with signals when a + # timeout is set. + + def setUp(self): + super().setUp() + self.serv.settimeout(self.timeout) + + def checkInterruptedRecv(self, func, *args, **kwargs): + # Check that func(*args, **kwargs) raises socket.error with an + # errno of EINTR when interrupted by a signal. + self.setAlarm(self.alarm_time) + with self.assertRaises(socket.error) as cm: + func(*args, **kwargs) + self.assertNotIsInstance(cm.exception, socket.timeout) + self.assertEqual(cm.exception.errno, errno.EINTR) + + def testInterruptedRecvTimeout(self): + self.checkInterruptedRecv(self.serv.recv, 1024) + + def testInterruptedRecvIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recv_into, bytearray(1024)) + + def testInterruptedRecvfromTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom, 1024) + + def testInterruptedRecvfromIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom_into, bytearray(1024)) + + @requireAttrs(socket.socket, "recvmsg") + def testInterruptedRecvmsgTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg, 1024) + + @requireAttrs(socket.socket, "recvmsg_into") + def testInterruptedRecvmsgIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg_into, [bytearray(1024)]) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +@unittest.skipUnless(thread, 'Threading required for this test.') +class InterruptedSendTimeoutTest(InterruptedTimeoutBase, + ThreadSafeCleanupTestCase, + SocketListeningTestMixin, TCPTestBase): + # Test interrupting the interruptible send*() methods with signals + # when a timeout is set. + + def setUp(self): + super().setUp() + self.serv_conn = self.newSocket() + self.addCleanup(self.serv_conn.close) + # Use a thread to complete the connection, but wait for it to + # terminate before running the test, so that there is only one + # thread to accept the signal. + cli_thread = threading.Thread(target=self.doConnect) + cli_thread.start() + self.cli_conn, addr = self.serv.accept() + self.addCleanup(self.cli_conn.close) + cli_thread.join() + self.serv_conn.settimeout(self.timeout) + + def doConnect(self): + self.serv_conn.connect(self.serv_addr) + + def checkInterruptedSend(self, func, *args, **kwargs): + # Check that func(*args, **kwargs), run in a loop, raises + # socket.error with an errno of EINTR when interrupted by a + # signal. + with self.assertRaises(socket.error) as cm: + while True: + self.setAlarm(self.alarm_time) + func(*args, **kwargs) + self.assertNotIsInstance(cm.exception, socket.timeout) + self.assertEqual(cm.exception.errno, errno.EINTR) + + def testInterruptedSendTimeout(self): + self.checkInterruptedSend(self.serv_conn.send, b"a"*512) + + def testInterruptedSendtoTimeout(self): + # Passing an actual address here as Python's wrapper for + # sendto() doesn't allow passing a zero-length one; POSIX + # requires that the address is ignored since the socket is + # connection-mode, however. + self.checkInterruptedSend(self.serv_conn.sendto, b"a"*512, + self.serv_addr) + + @requireAttrs(socket.socket, "sendmsg") + def testInterruptedSendmsgTimeout(self): + self.checkInterruptedSend(self.serv_conn.sendmsg, [b"a"*512]) + + @unittest.skipUnless(thread, 'Threading required for this test.') class TCPCloserTest(ThreadedTCPSocketTest): @@ -996,11 +3116,8 @@ class NonBlockingTCPTests(ThreadedTCPSocketTest): pass if hasattr(socket, "SOCK_NONBLOCK"): + @support.requires_linux_version(2, 6, 28) def testInitNonBlocking(self): - v = linux_version() - if v < (2, 6, 28): - self.skipTest("Linux kernel 2.6.28 or higher required, not %s" - % ".".join(map(str, v))) # reinit server socket self.serv.close() self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM | @@ -1102,7 +3219,7 @@ class FileObjectClassTestCase(SocketConnectedTest): """ bufsize = -1 # Use default buffer size - encoding = 'utf8' + encoding = 'utf-8' errors = 'strict' newline = None @@ -1323,7 +3440,7 @@ class FileObjectInterruptedTestCase(unittest.TestCase): data = b'' else: data = '' - expecting = expecting.decode('utf8') + expecting = expecting.decode('utf-8') while len(data) != len(expecting): part = fo.read(size) if not part: @@ -1485,7 +3602,7 @@ class UnicodeReadFileObjectClassTestCase(FileObjectClassTestCase): """Tests for socket.makefile() in text mode (rather than binary)""" read_mode = 'r' - read_msg = MSG.decode('utf8') + read_msg = MSG.decode('utf-8') write_mode = 'wb' write_msg = MSG newline = '' @@ -1497,7 +3614,7 @@ class UnicodeWriteFileObjectClassTestCase(FileObjectClassTestCase): read_mode = 'rb' read_msg = MSG write_mode = 'w' - write_msg = MSG.decode('utf8') + write_msg = MSG.decode('utf-8') newline = '' @@ -1505,9 +3622,9 @@ class UnicodeReadWriteFileObjectClassTestCase(FileObjectClassTestCase): """Tests for socket.makefile() in text mode (rather than binary)""" read_mode = 'r' - read_msg = MSG.decode('utf8') + read_msg = MSG.decode('utf-8') write_mode = 'w' - write_msg = MSG.decode('utf8') + write_msg = MSG.decode('utf-8') newline = '' @@ -1974,11 +4091,8 @@ class ContextManagersTest(ThreadedTCPSocketTest): "SOCK_CLOEXEC not defined") @unittest.skipUnless(fcntl, "module fcntl not available") class CloexecConstantTest(unittest.TestCase): + @support.requires_linux_version(2, 6, 28) def test_SOCK_CLOEXEC(self): - v = linux_version() - if v < (2, 6, 28): - self.skipTest("Linux kernel 2.6.28 or higher required, not %s" - % ".".join(map(str, v))) with socket.socket(socket.AF_INET, socket.SOCK_STREAM | socket.SOCK_CLOEXEC) as s: self.assertTrue(s.type & socket.SOCK_CLOEXEC) @@ -1996,11 +4110,8 @@ class NonblockConstantTest(unittest.TestCase): self.assertFalse(s.type & socket.SOCK_NONBLOCK) self.assertEqual(s.gettimeout(), None) + @support.requires_linux_version(2, 6, 28) def test_SOCK_NONBLOCK(self): - v = linux_version() - if v < (2, 6, 28): - self.skipTest("Linux kernel 2.6.28 or higher required, not %s" - % ".".join(map(str, v))) # a lot of it seems silly and redundant, but I wanted to test that # changing back and forth worked ok with socket.socket(socket.AF_INET, @@ -2056,11 +4167,36 @@ def test_main(): ]) if hasattr(socket, "socketpair"): tests.append(BasicSocketPairTest) - if sys.platform == 'linux2': + if sys.platform == 'linux': tests.append(TestLinuxAbstractNamespace) if isTipcAvailable(): tests.append(TIPCTest) tests.append(TIPCThreadableTest) + tests.extend([ + CmsgMacroTests, + SendmsgUDPTest, + RecvmsgUDPTest, + RecvmsgIntoUDPTest, + SendmsgUDP6Test, + RecvmsgUDP6Test, + RecvmsgRFC3542AncillaryUDP6Test, + RecvmsgIntoRFC3542AncillaryUDP6Test, + RecvmsgIntoUDP6Test, + SendmsgTCPTest, + RecvmsgTCPTest, + RecvmsgIntoTCPTest, + SendmsgSCTPStreamTest, + RecvmsgSCTPStreamTest, + RecvmsgIntoSCTPStreamTest, + SendmsgUnixStreamTest, + RecvmsgUnixStreamTest, + RecvmsgIntoUnixStreamTest, + RecvmsgSCMRightsStreamTest, + RecvmsgIntoSCMRightsStreamTest, + # These are slow when setitimer() is not available + InterruptedRecvTimeoutTest, + InterruptedSendTimeoutTest, + ]) thread_info = support.threading_setup() support.run_unittest(*tests) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 869381a501..e386325341 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -42,6 +42,9 @@ ONLYCERT = data_file("ssl_cert.pem") ONLYKEY = data_file("ssl_key.pem") BYTES_ONLYCERT = os.fsencode(ONLYCERT) BYTES_ONLYKEY = os.fsencode(ONLYKEY) +CERTFILE_PROTECTED = data_file("keycert.passwd.pem") +ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem") +KEY_PASSWORD = "somepass" CAPATH = data_file("capath") BYTES_CAPATH = os.fsencode(CAPATH) @@ -102,6 +105,16 @@ class BasicSocketTests(unittest.TestCase): sys.stdout.write("\n RAND_status is %d (%s)\n" % (v, (v and "sufficient randomness") or "insufficient randomness")) + + data, is_cryptographic = ssl.RAND_pseudo_bytes(16) + self.assertEqual(len(data), 16) + self.assertEqual(is_cryptographic, v == 1) + if v: + data = ssl.RAND_bytes(16) + self.assertEqual(len(data), 16) + else: + self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) + try: ssl.RAND_egd(1) except TypeError: @@ -311,6 +324,25 @@ class BasicSocketTests(unittest.TestCase): self.assertRaises(ValueError, ctx.wrap_socket, sock, True, server_hostname="some.hostname") + def test_unknown_channel_binding(self): + # should raise ValueError for unknown type + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s) + with self.assertRaises(ValueError): + ss.get_channel_binding("unknown-type") + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + # unconnected should return None for known type + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s) + self.assertIsNone(ss.get_channel_binding("tls-unique")) + # the same for server-side + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s, server_side=True, certfile=CERTFILE) + self.assertIsNone(ss.get_channel_binding("tls-unique")) + class ContextTests(unittest.TestCase): @skip_if_broken_ubuntu_ssl @@ -401,6 +433,60 @@ class ContextTests(unittest.TestCase): ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): ctx.load_cert_chain(SVN_PYTHON_ORG_ROOT_CERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=bytearray(KEY_PASSWORD.encode())) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode()) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, + bytearray(KEY_PASSWORD.encode())) + with self.assertRaisesRegex(TypeError, "should be a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=True) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass") + with self.assertRaisesRegex(ValueError, "cannot be longer"): + # openssl has a fixed limit on the password buffer. + # PEM_BUFSIZE is generally set to 1kb. + # Return a string larger than this. + ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400) + # Password callback + def getpass_unicode(): + return KEY_PASSWORD + def getpass_bytes(): + return KEY_PASSWORD.encode() + def getpass_bytearray(): + return bytearray(KEY_PASSWORD.encode()) + def getpass_badpass(): + return "badpass" + def getpass_huge(): + return b'a' * (1024 * 1024) + def getpass_bad_type(): + return 9 + def getpass_exception(): + raise Exception('getpass error') + class GetPassCallable: + def __call__(self): + return KEY_PASSWORD + def getpass(self): + return KEY_PASSWORD + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=GetPassCallable().getpass) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass) + with self.assertRaisesRegex(ValueError, "cannot be longer"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge) + with self.assertRaisesRegex(TypeError, "must return a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type) + with self.assertRaisesRegex(Exception, "getpass error"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception) + # Make sure the password function isn't called if it isn't needed + ctx.load_cert_chain(CERTFILE, password=getpass_exception) def test_load_verify_locations(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) @@ -645,25 +731,30 @@ class NetworkedTests(unittest.TestCase): sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) def test_get_server_certificate(self): - with support.transient_internet("svn.python.org"): - pem = ssl.get_server_certificate(("svn.python.org", 443)) - if not pem: - self.fail("No server certificate on svn.python.org:443!") + def _test_get_server_certificate(host, port, cert=None): + with support.transient_internet(host): + pem = ssl.get_server_certificate((host, port)) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) - try: - pem = ssl.get_server_certificate(("svn.python.org", 443), ca_certs=CERTFILE) - except ssl.SSLError as x: - #should fail + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + self.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + + pem = ssl.get_server_certificate((host, port), ca_certs=cert) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) if support.verbose: - sys.stdout.write("%s\n" % x) - else: - self.fail("Got server certificate %s for svn.python.org!" % pem) + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) - pem = ssl.get_server_certificate(("svn.python.org", 443), ca_certs=SVN_PYTHON_ORG_ROOT_CERT) - if not pem: - self.fail("No server certificate on svn.python.org:443!") - if support.verbose: - sys.stdout.write("\nVerified certificate for svn.python.org:443 is\n%s\n" % pem) + _test_get_server_certificate('svn.python.org', 443, SVN_PYTHON_ORG_ROOT_CERT) + if support.IPV6_ENABLED: + _test_get_server_certificate('ipv6.google.com', 443) def test_ciphers(self): remote = ("svn.python.org", 443) @@ -811,6 +902,11 @@ else: self.sslconn = None if support.verbose and self.server.connectionchatty: sys.stdout.write(" server: connection is now unencrypted...\n") + elif stripped == b'CB tls-unique': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n") + data = self.sslconn.get_channel_binding("tls-unique") + self.write(repr(data).encode("us-ascii") + b"\n") else: if (support.verbose and self.server.connectionchatty): @@ -1222,7 +1318,8 @@ else: t.join() @skip_if_broken_ubuntu_ssl - @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), "need SSLv2") + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), + "OpenSSL is compiled without SSLv2 support") def test_protocol_sslv2(self): """Connecting to an SSLv2 server with various client options""" if support.verbose: @@ -1554,6 +1651,14 @@ else: # consume data s.read() + # Make sure sendmsg et al are disallowed to avoid + # inadvertent disclosure of data and/or corruption + # of the encrypted data stream + self.assertRaises(NotImplementedError, s.sendmsg, [b"data"]) + self.assertRaises(NotImplementedError, s.recvmsg, 100) + self.assertRaises(NotImplementedError, + s.recvmsg_into, bytearray(100)) + s.write(b"over\n") s.close() finally: @@ -1609,6 +1714,73 @@ else: t.join() server.close() + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + """Test tls-unique channel binding.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + flag = threading.Event() + server.start(flag) + # wait for it to start + flag.wait() + # try to connect + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + try: + # get the data + cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got channel binding data: {0!r}\n" + .format(cb_data)) + + # check if it is sane + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + + # and compare with the peers version + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(cb_data).encode("us-ascii")) + s.close() + + # now, again + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + new_cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got another channel binding data: {0!r}\n" + .format(new_cb_data)) + # is it really unique + self.assertNotEqual(cb_data, new_cb_data) + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(new_cb_data).encode("us-ascii")) + s.close() + finally: + server.stop() + server.join() def test_main(verbose=False): if support.verbose: diff --git a/Lib/test/test_string.py b/Lib/test/test_string.py index a352ee3560..161573200b 100644 --- a/Lib/test/test_string.py +++ b/Lib/test/test_string.py @@ -26,15 +26,38 @@ class ModuleTest(unittest.TestCase): self.assertEqual(string.capwords('\taBc\tDeF\t'), 'Abc Def') self.assertEqual(string.capwords('\taBc\tDeF\t', '\t'), '\tAbc\tDef\t') - def test_formatter(self): + def test_basic_formatter(self): fmt = string.Formatter() self.assertEqual(fmt.format("foo"), "foo") - self.assertEqual(fmt.format("foo{0}", "bar"), "foobar") self.assertEqual(fmt.format("foo{1}{0}-{1}", "bar", 6), "foo6bar-6") + + def test_conversion_specifiers(self): + fmt = string.Formatter() self.assertEqual(fmt.format("-{arg!r}-", arg='test'), "-'test'-") + self.assertEqual(fmt.format("{0!s}", 'test'), 'test') + self.assertRaises(ValueError, fmt.format, "{0!h}", 'test') + + def test_name_lookup(self): + fmt = string.Formatter() + class AnyAttr: + def __getattr__(self, attr): + return attr + x = AnyAttr() + self.assertEqual(fmt.format("{0.lumber}{0.jack}", x), 'lumberjack') + with self.assertRaises(AttributeError): + fmt.format("{0.lumber}{0.jack}", '') + + def test_index_lookup(self): + fmt = string.Formatter() + lookup = ["eggs", "and", "spam"] + self.assertEqual(fmt.format("{0[2]}{0[0]}", lookup), 'spameggs') + with self.assertRaises(IndexError): + fmt.format("{0[2]}{0[0]}", []) + with self.assertRaises(KeyError): + fmt.format("{0[2]}{0[0]}", {}) - # override get_value ############################################ + def test_override_get_value(self): class NamespaceFormatter(string.Formatter): def __init__(self, namespace={}): string.Formatter.__init__(self) @@ -54,7 +77,7 @@ class ModuleTest(unittest.TestCase): self.assertEqual(fmt.format("{greeting}, world!"), 'hello, world!') - # override format_field ######################################### + def test_override_format_field(self): class CallFormatter(string.Formatter): def format_field(self, value, format_spec): return format(value(), format_spec) @@ -63,18 +86,18 @@ class ModuleTest(unittest.TestCase): self.assertEqual(fmt.format('*{0}*', lambda : 'result'), '*result*') - # override convert_field ######################################## + def test_override_convert_field(self): class XFormatter(string.Formatter): def convert_field(self, value, conversion): if conversion == 'x': return None - return super(XFormatter, self).convert_field(value, conversion) + return super().convert_field(value, conversion) fmt = XFormatter() self.assertEqual(fmt.format("{0!r}:{0!x}", 'foo', 'foo'), "'foo':None") - # override parse ################################################ + def test_override_parse(self): class BarFormatter(string.Formatter): # returns an iterable that contains tuples of the form: # (literal_text, field_name, format_spec, conversion) @@ -90,7 +113,7 @@ class ModuleTest(unittest.TestCase): fmt = BarFormatter() self.assertEqual(fmt.format('*|+0:^10s|*', 'foo'), '* foo *') - # test all parameters used + def test_check_unused_args(self): class CheckAllUsedFormatter(string.Formatter): def check_unused_args(self, used_args, args, kwargs): # Track which arguments actually got used @@ -112,28 +135,13 @@ class ModuleTest(unittest.TestCase): self.assertRaises(ValueError, fmt.format, "{0}", 10, 20, i=100) self.assertRaises(ValueError, fmt.format, "{i}", 10, 20, i=100) - def test_vformat_assert(self): - cls = string.Formatter() - kwargs = { - "i": 100 - } - self.assertRaises(ValueError, cls._vformat, - cls.format, "{0}", kwargs, set(), -2) - - def test_convert_field(self): - cls = string.Formatter() - self.assertEqual(cls.format("{0!s}", 'foo'), 'foo') - self.assertRaises(ValueError, cls.format, "{0!h}", 'foo') - - def test_get_field(self): - cls = string.Formatter() - class MyClass: - name = 'lumberjack' - x = MyClass() - self.assertEqual(cls.format("{0.name}", x), 'lumberjack') - - lookup = ["eggs", "and", "spam"] - self.assertEqual(cls.format("{0[2]}", lookup), 'spam') + def test_vformat_recursion_limit(self): + fmt = string.Formatter() + args = () + kwargs = dict(i=100) + with self.assertRaises(ValueError) as err: + fmt._vformat("{i}", args, kwargs, set(), -1) + self.assertIn("recursion", str(err.exception)) def test_main(): diff --git a/Lib/test/test_strlit.py b/Lib/test/test_strlit.py index 9eb30e9c54..fb9cdbbffa 100644 --- a/Lib/test/test_strlit.py +++ b/Lib/test/test_strlit.py @@ -130,7 +130,7 @@ class TestLiterals(unittest.TestCase): self.assertRaises(SyntaxError, self.check_encoding, "utf-8", extra) def test_file_utf8(self): - self.check_encoding("utf8") + self.check_encoding("utf-8") def test_file_iso_8859_1(self): self.check_encoding("iso-8859-1") diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index ff76ec445c..432d324749 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -58,6 +58,8 @@ class BaseTestCase(unittest.TestCase): # shutdown time. That frustrates tests trying to check stderr produced # from a spawned Python process. actual = support.strip_python_stderr(stderr) + # strip_python_stderr also strips whitespace, so we do too. + expected = expected.strip() self.assertEqual(actual, expected, msg) @@ -69,6 +71,15 @@ class ProcessTestCase(BaseTestCase): "import sys; sys.exit(47)"]) self.assertEqual(rc, 47) + def test_call_timeout(self): + # call() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.call waits for the + # child. + self.assertRaises(subprocess.TimeoutExpired, subprocess.call, + [sys.executable, "-c", "while True: pass"], + timeout=0.1) + def test_check_call_zero(self): # check_call() function with zero return code rc = subprocess.check_call([sys.executable, "-c", @@ -111,6 +122,21 @@ class ProcessTestCase(BaseTestCase): self.fail("Expected ValueError when stdout arg supplied.") self.assertIn('stdout', c.exception.args[0]) + def test_check_output_timeout(self): + # check_output() function with timeout arg + with self.assertRaises(subprocess.TimeoutExpired) as c: + output = subprocess.check_output( + [sys.executable, "-c", + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"], + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3) + self.fail("Expected TimeoutExpired.") + self.assertEqual(c.exception.output, b'BDFL') + def test_call_kwargs(self): # call() function with keyword args newenv = os.environ.copy() @@ -310,6 +336,31 @@ class ProcessTestCase(BaseTestCase): rc = subprocess.call([sys.executable, "-c", cmd], stdout=1) self.assertEqual(rc, 2) + def test_stdout_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'for i in range(10240):' + 'print("x" * 1024)'], + stdout=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdout, None) + + def test_stderr_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys\n' + 'for i in range(10240):' + 'sys.stderr.write("x" * 1024)'], + stderr=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stderr, None) + + def test_stdin_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdin.read(1)'], + stdin=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdin, None) + def test_cwd(self): tmpdir = tempfile.gettempdir() # We cannot use os.path.realpath to canonicalize the path, @@ -398,6 +449,41 @@ class ProcessTestCase(BaseTestCase): self.assertEqual(stdout, b"banana") self.assertStderrEqual(stderr, b"pineapple") + def test_communicate_timeout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stderr.write("pineapple\\n");' + 'time.sleep(1);' + 'sys.stderr.write("pear\\n");' + 'sys.stdout.write(sys.stdin.read())'], + universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, "banana", + timeout=0.3) + # Make sure we can keep waiting for it, and that we get the whole output + # after it completes. + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "banana") + self.assertStderrEqual(stderr.encode(), b"pineapple\npear\n") + + def test_communicate_timeout_large_ouput(self): + # Test a expring timeout while the child is outputting lots of data. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));'], + stdout=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, timeout=0.4) + (stdout, _) = p.communicate() + self.assertEqual(len(stdout), 4 * 64 * 1024) + # Test for the fd leak reported in http://bugs.python.org/issue2791. def test_communicate_pipe_fd_leak(self): for stdin_pipe in (False, True): @@ -434,24 +520,21 @@ class ProcessTestCase(BaseTestCase): # This test will probably deadlock rather than fail, if # communicate() does not work properly. x, y = os.pipe() - if mswindows: - pipe_buf = 512 - else: - pipe_buf = os.fpathconf(x, "PC_PIPE_BUF") os.close(x) os.close(y) p = subprocess.Popen([sys.executable, "-c", 'import sys,os;' 'sys.stdout.write(sys.stdin.read(47));' - 'sys.stderr.write("xyz"*%d);' - 'sys.stdout.write(sys.stdin.read())' % pipe_buf], + 'sys.stderr.write("x" * %d);' + 'sys.stdout.write(sys.stdin.read())' % + support.PIPE_MAX_SIZE], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.addCleanup(p.stdout.close) self.addCleanup(p.stderr.close) self.addCleanup(p.stdin.close) - string_to_write = b"abc"*pipe_buf + string_to_write = b"a" * support.PIPE_MAX_SIZE (stdout, stderr) = p.communicate(string_to_write) self.assertEqual(stdout, string_to_write) @@ -625,6 +708,15 @@ class ProcessTestCase(BaseTestCase): # Subsequent invocations should just return the returncode self.assertEqual(p.wait(), 0) + def test_wait_timeout(self): + p = subprocess.Popen([sys.executable, + "-c", "import time; time.sleep(0.1)"]) + with self.assertRaises(subprocess.TimeoutExpired) as c: + p.wait(timeout=0.01) + self.assertIn("0.01", str(c.exception)) # For coverage of __str__. + # Some heavily loaded buildbots (sparc Debian 3.x) require this much + # time to start. + self.assertEqual(p.wait(timeout=3), 0) def test_invalid_bufsize(self): # an invalid type of the bufsize argument should raise @@ -1220,6 +1312,11 @@ class POSIXProcessTestCase(BaseTestCase): exitcode = subprocess.call([abs_program, "-c", "pass"]) self.assertEqual(exitcode, 0) + # absolute bytes path as a string + cmd = b"'" + abs_program + b"' -c pass" + exitcode = subprocess.call(cmd, shell=True) + self.assertEqual(exitcode, 0) + # bytes program, unicode PATH env = os.environ.copy() env["PATH"] = path @@ -1375,7 +1472,7 @@ class POSIXProcessTestCase(BaseTestCase): stdout, stderr = p.communicate() self.assertEqual(0, p.returncode, "sigchild_ignore.py exited" " non-zero with this error:\n%s" % - stderr.decode('utf8')) + stderr.decode('utf-8')) def test_select_unbuffered(self): # Issue #11459: bufsize=0 should really set the pipes as @@ -1588,28 +1685,6 @@ class ProcessTestCaseNoPoll(ProcessTestCase): ProcessTestCase.tearDown(self) -@unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False), - "_posixsubprocess extension module not found.") -class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase): - @classmethod - def setUpClass(cls): - global subprocess - assert subprocess._posixsubprocess - # Reimport subprocess while forcing _posixsubprocess to not exist. - with support.check_warnings(('.*_posixsubprocess .* not being used.*', - RuntimeWarning)): - subprocess = support.import_fresh_module( - 'subprocess', blocked=['_posixsubprocess']) - assert not subprocess._posixsubprocess - - @classmethod - def tearDownClass(cls): - global subprocess - # Reimport subprocess as it should be, restoring order to the universe. - subprocess = support.import_fresh_module('subprocess') - assert subprocess._posixsubprocess - - class HelperFunctionTests(unittest.TestCase): @unittest.skipIf(mswindows, "errno and EINTR make no sense on windows") def test_eintr_retry_call(self): @@ -1718,7 +1793,6 @@ def test_main(): unit_tests = (ProcessTestCase, POSIXProcessTestCase, Win32ProcessTestCase, - ProcessTestCasePOSIXPurePython, CommandTests, ProcessTestCaseNoPoll, HelperFunctionTests, diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 914216d2da..298cae0011 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -81,6 +81,16 @@ class TestSuper(unittest.TestCase): self.assertEqual(E().f(), 'AE') + def test___class___set(self): + # See issue #12370 + class X(A): + def f(self): + return super().f() + __class__ = 413 + x = X() + self.assertEqual(x.f(), 'A') + self.assertEqual(x.__class__, 413) + def test_main(): support.run_unittest(TestSuper) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py new file mode 100644 index 0000000000..394e210337 --- /dev/null +++ b/Lib/test/test_support.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python + +import sys +import os +import unittest +import socket +import tempfile +import errno +from test import support + +TESTFN = support.TESTFN +TESTDIRN = os.path.basename(tempfile.mkdtemp(dir='.')) + + +class TestSupport(unittest.TestCase): + def setUp(self): + support.unlink(TESTFN) + support.rmtree(TESTDIRN) + tearDown = setUp + + def test_import_module(self): + support.import_module("ftplib") + self.assertRaises(unittest.SkipTest, support.import_module, "foo") + + def test_import_fresh_module(self): + support.import_fresh_module("ftplib") + + def test_get_attribute(self): + self.assertEqual(support.get_attribute(self, "test_get_attribute"), + self.test_get_attribute) + self.assertRaises(unittest.SkipTest, support.get_attribute, self, "foo") + + @unittest.skip("failing buildbots") + def test_get_original_stdout(self): + self.assertEqual(support.get_original_stdout(), sys.stdout) + + def test_unload(self): + import sched + self.assertIn("sched", sys.modules) + support.unload("sched") + self.assertNotIn("sched", sys.modules) + + def test_unlink(self): + with open(TESTFN, "w") as f: + pass + support.unlink(TESTFN) + self.assertFalse(os.path.exists(TESTFN)) + support.unlink(TESTFN) + + def test_rmtree(self): + os.mkdir(TESTDIRN) + os.mkdir(os.path.join(TESTDIRN, TESTDIRN)) + support.rmtree(TESTDIRN) + self.assertFalse(os.path.exists(TESTDIRN)) + support.rmtree(TESTDIRN) + + def test_forget(self): + mod_filename = TESTFN + '.py' + with open(mod_filename, 'w') as f: + print('foo = 1', file=f) + sys.path.insert(0, os.curdir) + try: + mod = __import__(TESTFN) + self.assertIn(TESTFN, sys.modules) + + support.forget(TESTFN) + self.assertNotIn(TESTFN, sys.modules) + finally: + del sys.path[0] + support.unlink(mod_filename) + + def test_HOST(self): + s = socket.socket() + s.bind((support.HOST, 0)) + s.close() + + def test_find_unused_port(self): + port = support.find_unused_port() + s = socket.socket() + s.bind((support.HOST, port)) + s.close() + + def test_bind_port(self): + s = socket.socket() + support.bind_port(s) + s.listen(1) + s.close() + + def test_temp_cwd(self): + here = os.getcwd() + with support.temp_cwd(name=TESTFN): + self.assertEqual(os.path.basename(os.getcwd()), TESTFN) + self.assertFalse(os.path.exists(TESTFN)) + self.assertTrue(os.path.basename(os.getcwd()), here) + + def test_sortdict(self): + self.assertEqual(support.sortdict({3:3, 2:2, 1:1}), "{1: 1, 2: 2, 3: 3}") + + def test_make_bad_fd(self): + fd = support.make_bad_fd() + with self.assertRaises(OSError) as cm: + os.write(fd, b"foo") + self.assertEqual(cm.exception.errno, errno.EBADF) + + def test_check_syntax_error(self): + support.check_syntax_error(self, "def class") + self.assertRaises(AssertionError, support.check_syntax_error, self, "1") + + def test_CleanImport(self): + import importlib + with support.CleanImport("asyncore"): + importlib.import_module("asyncore") + + def test_DirsOnSysPath(self): + with support.DirsOnSysPath('foo', 'bar'): + self.assertIn("foo", sys.path) + self.assertIn("bar", sys.path) + self.assertNotIn("foo", sys.path) + self.assertNotIn("bar", sys.path) + + def test_captured_stdout(self): + with support.captured_stdout() as s: + print("hello") + self.assertEqual(s.getvalue(), "hello\n") + + def test_captured_stderr(self): + with support.captured_stderr() as s: + print("hello", file=sys.stderr) + self.assertEqual(s.getvalue(), "hello\n") + + def test_captured_stdin(self): + with support.captured_stdin() as s: + print("hello", file=sys.stdin) + self.assertEqual(s.getvalue(), "hello\n") + + def test_gc_collect(self): + support.gc_collect() + + def test_python_is_optimized(self): + self.assertIsInstance(support.python_is_optimized(), bool) + + def test_swap_attr(self): + class Obj: + x = 1 + obj = Obj() + with support.swap_attr(obj, "x", 5): + self.assertEqual(obj.x, 5) + self.assertEqual(obj.x, 1) + + def test_swap_item(self): + D = {"item":1} + with support.swap_item(D, "item", 5): + self.assertEqual(D["item"], 5) + self.assertEqual(D["item"], 1) + + # XXX -follows a list of untested API + # make_legacy_pyc + # is_resource_enabled + # requires + # fcmp + # umaks + # findfile + # check_warnings + # EnvironmentVarGuard + # TransientResource + # transient_internet + # run_with_locale + # set_memlimit + # bigmemtest + # precisionbigmemtest + # bigaddrspacetest + # requires_resource + # run_doctest + # threading_cleanup + # reap_threads + # reap_children + # strip_python_stderr + # args_from_interpreter_flags + # can_symlink + # skip_unless_symlink + + +def test_main(): + tests = [TestSupport] + support.run_unittest(*tests) + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index db27cdf347..4355ef5bc8 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -303,6 +303,7 @@ class SysModuleTest(unittest.TestCase): self.assertEqual(sys.getdlopenflags(), oldflags+1) sys.setdlopenflags(oldflags) + @test.support.refcount_test def test_refcount(self): # n here must be a global in order for this test to pass while # tracing with a python function. Tracing calls PyFrame_FastToLocals @@ -342,7 +343,7 @@ class SysModuleTest(unittest.TestCase): # Test sys._current_frames() in a WITH_THREADS build. @test.support.reap_threads def current_frames_with_threads(self): - import threading, _thread + import threading import traceback # Spawn a thread that blocks at a known place. Then the main @@ -356,7 +357,7 @@ class SysModuleTest(unittest.TestCase): g456() def g456(): - thread_info.append(_thread.get_ident()) + thread_info.append(threading.get_ident()) entered_g.set() leave_g.wait() @@ -372,7 +373,7 @@ class SysModuleTest(unittest.TestCase): d = sys._current_frames() - main_id = _thread.get_ident() + main_id = threading.get_ident() self.assertIn(main_id, d) self.assertIn(thread_id, d) @@ -473,6 +474,14 @@ class SysModuleTest(unittest.TestCase): if not sys.platform.startswith('win'): self.assertIsInstance(sys.abiflags, str) + @unittest.skipUnless(hasattr(sys, 'thread_info'), + 'Threading required for this test.') + def test_thread_info(self): + info = sys.thread_info + self.assertEqual(len(info), 3) + self.assertIn(info.name, ('nt', 'os2', 'pthread', 'solaris', None)) + self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) + def test_43581(self): # Can't use sys.stdout, as this is a StringIO object when # the test runs under regrtest. @@ -500,7 +509,7 @@ class SysModuleTest(unittest.TestCase): def test_sys_flags(self): self.assertTrue(sys.flags) - attrs = ("debug", "division_warning", + attrs = ("debug", "inspect", "interactive", "optimize", "dont_write_bytecode", "no_user_site", "no_site", "ignore_environment", "verbose", "bytes_warning", "quiet") @@ -656,7 +665,7 @@ class SizeofTest(unittest.TestCase): return inner check(get_cell().__closure__[0], size(h + 'P')) # code - check(get_cell().__code__, size(h + '5i8Pi3P')) + check(get_cell().__code__, size(h + '5i9Pi3P')) # complex check(complex(0,1), size(h + '2d')) # method_descriptor (descriptor object) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 43134e98bd..acb8e29f19 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -251,6 +251,7 @@ class TraceTestCase(unittest.TestCase): def setUp(self): self.using_gc = gc.isenabled() gc.disable() + self.addCleanup(sys.settrace, sys.gettrace()) def tearDown(self): if self.using_gc: @@ -389,6 +390,9 @@ class TraceTestCase(unittest.TestCase): class RaisingTraceFuncTestCase(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + def trace(self, frame, event, arg): """A trace function that raises an exception in response to a specific trace event.""" @@ -688,6 +692,10 @@ def no_jump_without_trace_function(): class JumpTestCase(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + sys.settrace(None) + def compare_jump_output(self, expected, received): if received != expected: self.fail( "Outputs don't match:\n" + @@ -739,6 +747,8 @@ class JumpTestCase(unittest.TestCase): def test_18_no_jump_to_non_integers(self): self.run_test(no_jump_to_non_integers) def test_19_no_jump_without_trace_function(self): + # Must set sys.settrace(None) in setUp(), else condition is not + # triggered. no_jump_without_trace_function() def test_20_large_function(self): diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index b2a3224b1a..773d5231ac 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,9 +1,3 @@ -"""Tests for 'site'. - -Tests assume the initial paths in sys.path once the interpreter has begun -executing have not been removed. - -""" import unittest import sys import os @@ -16,14 +10,14 @@ from test.support import (run_unittest, TESTFN, unlink, get_attribute, import sysconfig from sysconfig import (get_paths, get_platform, get_config_vars, - get_path, get_path_names, _INSTALL_SCHEMES, + get_path, get_path_names, _SCHEMES, _get_default_scheme, _expand_vars, get_scheme_names, get_config_var, _main) + class TestSysConfig(unittest.TestCase): def setUp(self): - """Make a copy of sys.path""" super(TestSysConfig, self).setUp() self.sys_path = sys.path[:] # patching os.uname @@ -43,10 +37,15 @@ class TestSysConfig(unittest.TestCase): self.isabs = os.path.isabs self.splitdrive = os.path.splitdrive self._config_vars = copy(sysconfig._CONFIG_VARS) - self.old_environ = deepcopy(os.environ) + self._added_envvars = [] + self._changed_envvars = [] + for var in ('MACOSX_DEPLOYMENT_TARGET', 'Path'): + if var in os.environ: + self._changed_envvars.append((var, os.environ[var])) + else: + self._added_envvars.append(var) def tearDown(self): - """Restore sys.path""" sys.path[:] = self.sys_path self._cleanup_testfn() if self.uname is not None: @@ -61,13 +60,10 @@ class TestSysConfig(unittest.TestCase): os.path.isabs = self.isabs os.path.splitdrive = self.splitdrive sysconfig._CONFIG_VARS = copy(self._config_vars) - for key, value in self.old_environ.items(): - if os.environ.get(key) != value: - os.environ[key] = value - - for key in list(os.environ.keys()): - if key not in self.old_environ: - del os.environ[key] + for var, value in self._changed_envvars: + os.environ[var] = value + for var in self._added_envvars: + os.environ.pop(var, None) super(TestSysConfig, self).tearDown() @@ -85,7 +81,7 @@ class TestSysConfig(unittest.TestCase): shutil.rmtree(path) def test_get_path_names(self): - self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS) + self.assertEqual(get_path_names(), _SCHEMES.options('posix_prefix')) def test_get_paths(self): scheme = get_paths() @@ -99,8 +95,8 @@ class TestSysConfig(unittest.TestCase): def test_get_path(self): # xxx make real tests here - for scheme in _INSTALL_SCHEMES: - for name in _INSTALL_SCHEMES[scheme]: + for scheme in _SCHEMES: + for name in _SCHEMES[scheme]: res = get_path(name, scheme) def test_get_config_vars(self): @@ -139,8 +135,6 @@ class TestSysConfig(unittest.TestCase): ('Darwin Kernel Version 8.11.1: ' 'Wed Oct 10 18:23:28 PDT 2007; ' 'root:xnu-792.25.20~1/RELEASE_I386'), 'PowerPC')) - - get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.3' get_config_vars()['CFLAGS'] = ('-fno-strict-aliasing -DNDEBUG -g ' @@ -155,7 +149,6 @@ class TestSysConfig(unittest.TestCase): finally: sys.maxsize = maxint - self._set_uname(('Darwin', 'macziade', '8.11.1', ('Darwin Kernel Version 8.11.1: ' 'Wed Oct 10 18:23:28 PDT 2007; ' @@ -213,9 +206,9 @@ class TestSysConfig(unittest.TestCase): get_config_vars()['CFLAGS'] = ('-arch %s -isysroot ' '/Developer/SDKs/MacOSX10.4u.sdk ' '-fno-strict-aliasing -fno-common ' - '-dynamic -DNDEBUG -g -O3'%(arch,)) + '-dynamic -DNDEBUG -g -O3' % arch) - self.assertEqual(get_platform(), 'macosx-10.4-%s'%(arch,)) + self.assertEqual(get_platform(), 'macosx-10.4-%s' % arch) # linux debian sarge os.name = 'posix' @@ -283,7 +276,6 @@ class TestSysConfig(unittest.TestCase): self.assertIn(ldflags, ldshared) - @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") def test_platform_in_subprocess(self): my_platform = sysconfig.get_platform() @@ -309,28 +301,29 @@ class TestSysConfig(unittest.TestCase): self.assertEqual(status, 0) self.assertEqual(my_platform, test_platform) - # Test with MACOSX_DEPLOYMENT_TARGET in the environment, and # using a value that is unlikely to be the default one. env = os.environ.copy() env['MACOSX_DEPLOYMENT_TARGET'] = '10.1' - p = subprocess.Popen([ - sys.executable, '-c', - 'import sysconfig; print(sysconfig.get_platform())', - ], - stdout=subprocess.PIPE, - stderr=open('/dev/null'), - env=env) - test_platform = p.communicate()[0].strip() - test_platform = test_platform.decode('utf-8') - status = p.wait() + with open('/dev/null') as dev_null: + p = subprocess.Popen([ + sys.executable, '-c', + 'import sysconfig; print(sysconfig.get_platform())', + ], + stdout=subprocess.PIPE, + stderr=dev_null, + env=env) + test_platform = p.communicate()[0].strip() + test_platform = test_platform.decode('utf-8') + status = p.wait() - self.assertEqual(status, 0) - self.assertEqual(my_platform, test_platform) + self.assertEqual(status, 0) + self.assertEqual(my_platform, test_platform) class MakefileTests(unittest.TestCase): + @unittest.skipIf(sys.platform.startswith('win'), 'Test is not Windows compatible') def test_get_makefile_filename(self): diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index b45f168990..034765c5e7 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -82,7 +82,7 @@ class UstarReadTest(ReadTest): def test_fileobj_iter(self): self.tar.extract("ustar/regtype", TEMPDIR) tarinfo = self.tar.getmember("ustar/regtype") - with open(os.path.join(TEMPDIR, "ustar/regtype"), "rU") as fobj1: + with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1: lines1 = fobj1.readlines() fobj2 = self.tar.extractfile(tarinfo) try: @@ -703,7 +703,7 @@ class GNUReadTest(LongnameTest): # Return True if the platform knows the st_blocks stat attribute and # uses st_blocks units of 512 bytes, and if the filesystem is able to # store holes in files. - if sys.platform == "linux2": + if sys.platform.startswith("linux"): # Linux evidentially has 512 byte st_blocks units. name = os.path.join(TEMPDIR, "sparse-test") with open(name, "wb") as fobj: @@ -893,7 +893,7 @@ class WriteTest(WriteTestBase): try: for name in ("foo", "bar", "baz"): name = os.path.join(tempdir, name) - open(name, "wb").close() + support.create_empty_file(name) exclude = os.path.isfile @@ -920,7 +920,7 @@ class WriteTest(WriteTestBase): try: for name in ("foo", "bar", "baz"): name = os.path.join(tempdir, name) - open(name, "wb").close() + support.create_empty_file(name) def filter(tarinfo): if os.path.basename(tarinfo.name) == "bar": @@ -959,7 +959,7 @@ class WriteTest(WriteTestBase): # and compare the stored name with the original. foo = os.path.join(TEMPDIR, "foo") if not dir: - open(foo, "w").close() + support.create_empty_file(foo) else: os.mkdir(foo) @@ -1319,7 +1319,7 @@ class UstarUnicodeTest(unittest.TestCase): self._test_unicode_filename("utf7") def test_utf8_filename(self): - self._test_unicode_filename("utf8") + self._test_unicode_filename("utf-8") def _test_unicode_filename(self, encoding): tar = tarfile.open(tmpname, "w", format=self.format, encoding=encoding, errors="strict") @@ -1398,7 +1398,7 @@ class GNUUnicodeTest(UstarUnicodeTest): def test_bad_pax_header(self): # Test for issue #8633. GNU tar <= 1.23 creates raw binary fields # without a hdrcharset=BINARY header. - for encoding, name in (("utf8", "pax/bad-pax-\udce4\udcf6\udcfc"), + for encoding, name in (("utf-8", "pax/bad-pax-\udce4\udcf6\udcfc"), ("iso8859-1", "pax/bad-pax-\xe4\xf6\xfc"),): with tarfile.open(tarname, encoding=encoding, errors="surrogateescape") as tar: try: @@ -1413,7 +1413,7 @@ class PAXUnicodeTest(UstarUnicodeTest): def test_binary_header(self): # Test a POSIX.1-2008 compatible header with a hdrcharset=BINARY field. - for encoding, name in (("utf8", "pax/hdrcharset-\udce4\udcf6\udcfc"), + for encoding, name in (("utf-8", "pax/hdrcharset-\udce4\udcf6\udcfc"), ("iso8859-1", "pax/hdrcharset-\xe4\xf6\xfc"),): with tarfile.open(tarname, encoding=encoding, errors="surrogateescape") as tar: try: diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py index a78b938b86..efd91c7ed2 100644 --- a/Lib/test/test_telnetlib.py +++ b/Lib/test/test_telnetlib.py @@ -39,6 +39,7 @@ class GeneralTests(TestCase): def tearDown(self): self.evt.wait() self.thread.join() + del self.thread # Clear out any dangling Thread objects. def testBasic(self): # connects diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index f7f5bdadbd..014fca01a9 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -20,7 +20,7 @@ has_spawnl = hasattr(os, 'spawnl') # TEST_FILES may need to be tweaked for systems depending on the maximum # number of files that can be opened at one time (see ulimit -n) -if sys.platform in ('openbsd3', 'openbsd4'): +if sys.platform.startswith('openbsd'): TEST_FILES = 48 else: TEST_FILES = 100 diff --git a/Lib/test/test_threaded_import.py b/Lib/test/test_threaded_import.py index 7791935fdf..789920b7fa 100644 --- a/Lib/test/test_threaded_import.py +++ b/Lib/test/test_threaded_import.py @@ -11,8 +11,8 @@ import sys import time import shutil import unittest -from test.support import verbose, import_module, run_unittest, TESTFN -thread = import_module('_thread') +from test.support import ( + verbose, import_module, run_unittest, TESTFN, reap_threads) threading = import_module('threading') def task(N, done, done_tasks, errors): @@ -30,7 +30,7 @@ def task(N, done, done_tasks, errors): except Exception as e: errors.append(e.with_traceback(None)) finally: - done_tasks.append(thread.get_ident()) + done_tasks.append(threading.get_ident()) finished = len(done_tasks) == N if finished: done.set() @@ -62,7 +62,7 @@ class Finder: def __init__(self): self.numcalls = 0 self.x = 0 - self.lock = thread.allocate_lock() + self.lock = threading.Lock() def find_module(self, name, path=None): # Simulate some thread-unsafe behaviour. If calls to find_module() @@ -113,7 +113,9 @@ class ThreadedImportTests(unittest.TestCase): done_tasks = [] done.clear() for i in range(N): - thread.start_new_thread(task, (N, done, done_tasks, errors,)) + t = threading.Thread(target=task, + args=(N, done, done_tasks, errors,)) + t.start() done.wait(60) self.assertFalse(errors) if verbose: @@ -203,6 +205,7 @@ class ThreadedImportTests(unittest.TestCase): self.assertEqual(set(results), {'a', 'b'}) +@reap_threads def test_main(): run_unittest(ThreadedImportTests) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index ef15d10dbd..da90e17eab 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -173,7 +173,7 @@ class ThreadTests(BaseTestCase): exception = ctypes.py_object(AsyncExc) # First check it works when setting the exception from the same thread. - tid = _thread.get_ident() + tid = threading.get_ident() try: result = set_async_exc(ctypes.c_long(tid), exception) @@ -202,7 +202,7 @@ class ThreadTests(BaseTestCase): class Worker(threading.Thread): def run(self): - self.id = _thread.get_ident() + self.id = threading.get_ident() self.finished = False try: @@ -407,6 +407,14 @@ class ThreadTests(BaseTestCase): t.daemon = True self.assertTrue('daemon' in repr(t)) + def test_deamon_param(self): + t = threading.Thread() + self.assertFalse(t.daemon) + t = threading.Thread(daemon=False) + self.assertFalse(t.daemon) + t = threading.Thread(daemon=True) + self.assertTrue(t.daemon) + class ThreadJoinOnShutdown(BaseTestCase): @@ -688,6 +696,10 @@ class ThreadingExceptionTests(BaseTestCase): thread.start() self.assertRaises(RuntimeError, setattr, thread, "daemon", True) + def test_releasing_unacquired_lock(self): + lock = threading.Lock() + self.assertRaises(RuntimeError, lock.release) + @unittest.skipUnless(sys.platform == 'darwin', 'test macosx problem') def test_recursion_limit(self): # Issue 9670 @@ -748,6 +760,7 @@ class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): class BarrierTests(lock_tests.BarrierTests): barriertype = staticmethod(threading.Barrier) + def test_main(): test.support.run_unittest(LockTests, PyRLockTests, CRLockTests, EventTests, ConditionAsRLockTests, ConditionTests, @@ -755,7 +768,7 @@ def test_main(): ThreadTests, ThreadJoinOnShutdown, ThreadingExceptionTests, - BarrierTests + BarrierTests, ) if __name__ == "__main__": diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py index e0af31dae4..f975a75e85 100644 --- a/Lib/test/test_threadsignals.py +++ b/Lib/test/test_threadsignals.py @@ -14,10 +14,8 @@ if sys.platform[:3] in ('win', 'os2') or sys.platform=='riscos': process_pid = os.getpid() signalled_all=thread.allocate_lock() -# Issue #11223: Locks are implemented using a mutex and a condition variable of -# the pthread library on FreeBSD6. POSIX condition variables cannot be -# interrupted by signals (see pthread_cond_wait manual page). -USING_PTHREAD_COND = (sys.platform == 'freebsd6') +USING_PTHREAD_COND = (sys.thread_info.name == 'pthread' + and sys.thread_info.lock == 'mutex+cond') def registerSignals(for_usr1, for_usr2, for_alrm): usr1 = signal.signal(signal.SIGUSR1, for_usr1) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 8499af8157..94de098911 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -27,6 +27,8 @@ class TimeTestCase(unittest.TestCase): int(self.t)) def test_sleep(self): + self.assertRaises(ValueError, time.sleep, -2) + self.assertRaises(ValueError, time.sleep, -1) time.sleep(1.2) def test_strftime(self): @@ -96,12 +98,13 @@ class TimeTestCase(unittest.TestCase): self._bounds_checking(lambda tup: time.strftime('', tup)) def test_default_values_for_zero(self): - # Make sure that using all zeros uses the proper default values. - # No test for daylight savings since strftime() does not change output - # based on its value. + # Make sure that using all zeros uses the proper default + # values. No test for daylight savings since strftime() does + # not change output based on its value and no test for year + # because systems vary in their support for year 0. expected = "2000 01 01 00 00 00 1 001" with support.check_warnings(): - result = time.strftime("%Y %m %d %H %M %S %w %j", (0,)*9) + result = time.strftime("%Y %m %d %H %M %S %w %j", (2000,)+(0,)*8) self.assertEqual(expected, result) def test_strptime(self): @@ -271,15 +274,6 @@ class TestLocale(unittest.TestCase): class _BaseYearTest(unittest.TestCase): - accept2dyear = None - - def setUp(self): - self.saved_accept2dyear = time.accept2dyear - time.accept2dyear = self.accept2dyear - - def tearDown(self): - time.accept2dyear = self.saved_accept2dyear - def yearstr(self, y): raise NotImplementedError() @@ -306,23 +300,7 @@ class _TestStrftimeYear: self.assertEqual(text, '12345') self.assertEqual(self.yearstr(123456789), '123456789') -class _Test2dYear(_BaseYearTest): - accept2dyear = 1 - - def test_year(self): - with support.check_warnings(): - self.assertEqual(self.yearstr(0), '2000') - self.assertEqual(self.yearstr(69), '1969') - self.assertEqual(self.yearstr(68), '2068') - self.assertEqual(self.yearstr(99), '1999') - - def test_invalid(self): - self.assertRaises(ValueError, self.yearstr, -1) - self.assertRaises(ValueError, self.yearstr, 100) - self.assertRaises(ValueError, self.yearstr, 999) - class _Test4dYear(_BaseYearTest): - accept2dyear = 0 def test_year(self): self.assertIn(self.yearstr(1), ('1', '0001')) @@ -361,46 +339,19 @@ class _Test4dYear(_BaseYearTest): except OverflowError: pass -class TestAsctimeAccept2dYear(_TestAsctimeYear, _Test2dYear): - pass - -class TestStrftimeAccept2dYear(_TestStrftimeYear, _Test2dYear): - pass - class TestAsctime4dyear(_TestAsctimeYear, _Test4dYear): pass class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear): pass -class Test2dyearBool(_TestAsctimeYear, _Test2dYear): - accept2dyear = True - -class Test4dyearBool(_TestAsctimeYear, _Test4dYear): - accept2dyear = False - -class TestAccept2YearBad(_TestAsctimeYear, _BaseYearTest): - class X: - def __bool__(self): - raise RuntimeError('boo') - accept2dyear = X() - def test_2dyear(self): - pass - def test_invalid(self): - self.assertRaises(RuntimeError, self.yearstr, 200) - def test_main(): support.run_unittest( TimeTestCase, TestLocale, - TestAsctimeAccept2dYear, - TestStrftimeAccept2dYear, TestAsctime4dyear, - TestStrftime4dyear, - Test2dyearBool, - Test4dyearBool, - TestAccept2YearBad) + TestStrftime4dyear) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 461d1d8e6a..d9bef389de 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -102,6 +102,7 @@ class TracedClass(object): class TestLineCounts(unittest.TestCase): """White-box testing of line-counting, via runfunc""" def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0) self.my_py_filename = fix_ext_py(__file__) @@ -192,6 +193,7 @@ class TestRunExecCounts(unittest.TestCase): """A simple sanity test of line-counting, via runctx (exec)""" def setUp(self): self.my_py_filename = fix_ext_py(__file__) + self.addCleanup(sys.settrace, sys.gettrace()) def test_exec_counts(self): self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0) @@ -218,6 +220,7 @@ class TestRunExecCounts(unittest.TestCase): class TestFuncs(unittest.TestCase): """White-box testing of funcs tracing""" def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) self.tracer = Trace(count=0, trace=0, countfuncs=1) self.filemod = my_file_and_modname() @@ -242,6 +245,8 @@ class TestFuncs(unittest.TestCase): } self.assertEqual(self.tracer.results().calledfuncs, expected) + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'pre-existing trace function throws off measurements') def test_inst_method_calling(self): obj = TracedClass(20) self.tracer.runfunc(obj.inst_method_calling, 1) @@ -257,9 +262,12 @@ class TestFuncs(unittest.TestCase): class TestCallers(unittest.TestCase): """White-box testing of callers tracing""" def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) self.tracer = Trace(count=0, trace=0, countcallers=1) self.filemod = my_file_and_modname() + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'pre-existing trace function throws off measurements') def test_loop_caller_importing(self): self.tracer.runfunc(traced_func_importing_caller, 1) @@ -280,6 +288,9 @@ class TestCallers(unittest.TestCase): # Created separately for issue #3821 class TestCoverage(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + def tearDown(self): rmtree(TESTFN) unlink(TESTFN) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 5cd2927f2d..f7424c0b67 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -775,7 +775,7 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('{0!s}'.format(G('data')), 'string is data') msg = 'object.__format__ with a non-empty format string is deprecated' - with support.check_warnings((msg, PendingDeprecationWarning)): + with support.check_warnings((msg, DeprecationWarning)): self.assertEqual('{0:^10}'.format(E('data')), ' E(data) ') self.assertEqual('{0:^10s}'.format(E('data')), ' E(data) ') self.assertEqual('{0:>15s}'.format(G('data')), ' string is data') @@ -1357,11 +1357,14 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('hello'.encode('ascii'), b'hello') self.assertEqual('hello'.encode('utf-7'), b'hello') self.assertEqual('hello'.encode('utf-8'), b'hello') - self.assertEqual('hello'.encode('utf8'), b'hello') + self.assertEqual('hello'.encode('utf-8'), b'hello') self.assertEqual('hello'.encode('utf-16-le'), b'h\000e\000l\000l\000o\000') self.assertEqual('hello'.encode('utf-16-be'), b'\000h\000e\000l\000l\000o') self.assertEqual('hello'.encode('latin-1'), b'hello') + # Default encoding is utf-8 + self.assertEqual('\u2603'.encode(), b'\xe2\x98\x83') + # Roundtrip safety for BMP (just the first 1024 chars) for c in range(1024): u = chr(c) @@ -1602,7 +1605,9 @@ class UnicodeTest(string_tests.CommonTest, # Test PyUnicode_FromFormat() def test_from_format(self): support.import_module('ctypes') - from ctypes import pythonapi, py_object, c_int + from ctypes import (pythonapi, py_object, + c_int, c_long, c_longlong, c_ssize_t, + c_uint, c_ulong, c_ulonglong, c_size_t) if sys.maxunicode == 65535: name = "PyUnicodeUCS2_FromFormat" else: @@ -1627,13 +1632,40 @@ class UnicodeTest(string_tests.CommonTest, 'string, got a non-ASCII byte: 0xe9$', PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') + # test "%c" self.assertEqual(PyUnicode_FromFormat(b'%c', c_int(0xabcd)), '\uabcd') self.assertEqual(PyUnicode_FromFormat(b'%c', c_int(0x10ffff)), '\U0010ffff') - # other tests + # test "%" + self.assertEqual(PyUnicode_FromFormat(b'%'), '%') + self.assertEqual(PyUnicode_FromFormat(b'%%'), '%') + self.assertEqual(PyUnicode_FromFormat(b'%%s'), '%s') + self.assertEqual(PyUnicode_FromFormat(b'[%%]'), '[%]') + self.assertEqual(PyUnicode_FromFormat(b'%%%s', b'abc'), '%abc') + + # test integer formats (%i, %d, %u) + self.assertEqual(PyUnicode_FromFormat(b'%03i', c_int(10)), '010') + self.assertEqual(PyUnicode_FromFormat(b'%0.4i', c_int(10)), '0010') + self.assertEqual(PyUnicode_FromFormat(b'%i', c_int(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%li', c_long(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%lli', c_longlong(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%zi', c_ssize_t(-123)), '-123') + + self.assertEqual(PyUnicode_FromFormat(b'%d', c_int(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%ld', c_long(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%lld', c_longlong(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%zd', c_ssize_t(-123)), '-123') + + self.assertEqual(PyUnicode_FromFormat(b'%u', c_uint(123)), '123') + self.assertEqual(PyUnicode_FromFormat(b'%lu', c_ulong(123)), '123') + self.assertEqual(PyUnicode_FromFormat(b'%llu', c_ulonglong(123)), '123') + self.assertEqual(PyUnicode_FromFormat(b'%zu', c_size_t(123)), '123') + + # test %A text = PyUnicode_FromFormat(b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') self.assertEqual(text, r"%A:'abc\xe9\uabcd\U0010ffff'") + # test %V text = PyUnicode_FromFormat(b'repr=%V', 'abc', b'xyz') self.assertEqual(text, 'repr=abc') @@ -1647,6 +1679,13 @@ class UnicodeTest(string_tests.CommonTest, text = PyUnicode_FromFormat(b'repr=%V', None, b'abc\xff') self.assertEqual(text, 'repr=abc\ufffd') + # not supported: copy the raw format string. these tests are just here + # to check for crashs and should not be considered as specifications + self.assertEqual(PyUnicode_FromFormat(b'%1%s', b'abc'), '%s') + self.assertEqual(PyUnicode_FromFormat(b'%1abc'), '%1abc') + self.assertEqual(PyUnicode_FromFormat(b'%+i', c_int(10)), '%+i') + self.assertEqual(PyUnicode_FromFormat(b'%.%s', b'abc'), '%.%s') + # Test PyUnicode_AsWideChar() def test_aswidechar(self): from _testcapi import unicode_aswidechar diff --git a/Lib/test/test_unicode_file.py b/Lib/test/test_unicode_file.py index 6c2011ace8..68bd658a75 100644 --- a/Lib/test/test_unicode_file.py +++ b/Lib/test/test_unicode_file.py @@ -6,7 +6,7 @@ import unicodedata import unittest from test.support import (run_unittest, rmtree, - TESTFN_ENCODING, TESTFN_UNICODE, TESTFN_UNENCODABLE) + TESTFN_ENCODING, TESTFN_UNICODE, TESTFN_UNENCODABLE, create_empty_file) if not os.path.supports_unicode_filenames: try: @@ -99,8 +99,7 @@ class TestUnicodeFiles(unittest.TestCase): # top-level 'test' functions would be if they could take params def _test_single(self, filename): remove_if_exists(filename) - f = open(filename, "w") - f.close() + create_empty_file(filename) try: self._do_single(filename) finally: diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index ac023740ce..7d35f10b12 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -289,7 +289,7 @@ class urlretrieve_FileTests(unittest.TestCase): def constructLocalFileUrl(self, filePath): filePath = os.path.abspath(filePath) try: - filePath.encode("utf8") + filePath.encode("utf-8") except UnicodeEncodeError: raise unittest.SkipTest("filePath is not encodable to utf8") return "file://%s" % urllib.request.pathname2url(filePath) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 03cd927498..713a97f774 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -5,6 +5,7 @@ import os import io import socket import array +import sys import urllib.request # The proxy bypass method imported below has logic specific to the OSX @@ -605,7 +606,7 @@ class OpenerDirectorTests(unittest.TestCase): def sanepathname2url(path): try: - path.encode("utf8") + path.encode("utf-8") except UnicodeEncodeError: raise unittest.SkipTest("path is not encodable to utf8") urlpath = urllib.request.pathname2url(path) @@ -1166,6 +1167,8 @@ class HandlerTests(unittest.TestCase): self.assertEqual(req.get_host(), "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"),"FooBar") + # TODO: This should be only for OSX + @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { 'exclude_simple': False, @@ -1269,6 +1272,26 @@ class HandlerTests(unittest.TestCase): # _test_basic_auth called .open() twice) self.assertEqual(opener.recorded, ["digest", "basic"]*2) + def test_unsupported_auth_digest_handler(self): + opener = OpenerDirector() + # While using DigestAuthHandler + digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None) + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Kerberos\r\n\r\n') + opener.add_handler(digest_auth_handler) + opener.add_handler(http_handler) + self.assertRaises(ValueError,opener.open,"http://www.example.com") + + def test_unsupported_auth_basic_handler(self): + # While using BasicAuthHandler + opener = OpenerDirector() + basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None) + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: NTLM\r\n\r\n') + opener.add_handler(basic_auth_handler) + opener.add_handler(http_handler) + self.assertRaises(ValueError,opener.open,"http://www.example.com") + def _test_basic_auth(self, opener, auth_handler, auth_header, realm, http_handler, password_manager, request_url, protected_url): @@ -1306,6 +1329,7 @@ class HandlerTests(unittest.TestCase): self.assertEqual(len(http_handler.requests), 1) self.assertFalse(http_handler.requests[0].has_header(auth_header)) + class MiscTests(unittest.TestCase): def test_build_opener(self): diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py index 868ed24ccc..6381070f56 100644 --- a/Lib/test/test_userlist.py +++ b/Lib/test/test_userlist.py @@ -52,6 +52,12 @@ class UserListTest(list_tests.CommonTest): return str(key) + '!!!' self.assertEqual(next(iter(T((1,2)))), "0!!!") + def test_userlist_copy(self): + u = self.type2test([6, 8, 1, 9, 1]) + v = u.copy() + self.assertEqual(u, v) + self.assertEqual(type(u), type(v)) + def test_main(): support.run_unittest(UserListTest) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 43fa656102..7bc59ed9fd 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -471,14 +471,14 @@ class TestUUID(TestCase): if pid == 0: os.close(fds[0]) value = uuid.uuid4() - os.write(fds[1], value.hex.encode('latin1')) + os.write(fds[1], value.hex.encode('latin-1')) os._exit(0) else: os.close(fds[1]) parent_value = uuid.uuid4().hex os.waitpid(pid, 0) - child_value = os.read(fds[0], 100).decode('latin1') + child_value = os.read(fds[0], 100).decode('latin-1') self.assertNotEqual(parent_value, child_value) diff --git a/Lib/test/test_wait3.py b/Lib/test/test_wait3.py index 786e60b327..bd06c8d8bd 100644 --- a/Lib/test/test_wait3.py +++ b/Lib/test/test_wait3.py @@ -19,13 +19,16 @@ except AttributeError: class Wait3Test(ForkWait): def wait_impl(self, cpid): - for i in range(10): + # This many iterations can be required, since some previously run + # tests (e.g. test_ctypes) could have spawned a lot of children + # very quickly. + for i in range(30): # wait3() shouldn't hang, but some of the buildbots seem to hang # in the forking tests. This is an attempt to fix the problem. spid, status, rusage = os.wait3(os.WNOHANG) if spid == cpid: break - time.sleep(1.0) + time.sleep(0.1) self.assertEqual(spid, cpid) self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8)) diff --git a/Lib/test/test_warnings.py b/Lib/test/test_warnings.py index 79be835bbd..953b2827be 100644 --- a/Lib/test/test_warnings.py +++ b/Lib/test/test_warnings.py @@ -512,12 +512,11 @@ class _WarningsTests(BaseTest): def test_showwarning_not_callable(self): with original_warnings.catch_warnings(module=self.module): self.module.filterwarnings("always", category=UserWarning) - old_showwarning = self.module.showwarning + self.module.showwarning = print + with support.captured_output('stdout'): + self.module.warn('Warning!') self.module.showwarning = 23 - try: - self.assertRaises(TypeError, self.module.warn, "Warning!") - finally: - self.module.showwarning = old_showwarning + self.assertRaises(TypeError, self.module.warn, "Warning!") def test_show_warning_output(self): # With showarning() missing, make sure that output is okay. @@ -547,10 +546,13 @@ class _WarningsTests(BaseTest): globals_dict = globals() oldfile = globals_dict['__file__'] try: - with original_warnings.catch_warnings(module=self.module) as w: + catch = original_warnings.catch_warnings(record=True, + module=self.module) + with catch as w: self.module.filterwarnings("always", category=UserWarning) globals_dict['__file__'] = None original_warnings.warn('test', UserWarning) + self.assertTrue(len(w)) finally: globals_dict['__file__'] = oldfile diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 22fafa9a2e..40c22917c0 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -22,7 +22,7 @@ from xml.etree import ElementTree as ET SIMPLE_XMLFILE = findfile("simple.xml", subdir="xmltestdata") try: - SIMPLE_XMLFILE.encode("utf8") + SIMPLE_XMLFILE.encode("utf-8") except UnicodeEncodeError: raise unittest.SkipTest("filename is not encodable to utf8") SIMPLE_NS_XMLFILE = findfile("simple-ns.xml", subdir="xmltestdata") @@ -1255,8 +1255,8 @@ def processinginstruction(): >>> ET.tostring(ET.PI('test', '<testing&>')) b'<?test <testing&>?>' - >>> ET.tostring(ET.PI('test', '<testing&>\xe3'), 'latin1') - b"<?xml version='1.0' encoding='latin1'?>\\n<?test <testing&>\\xe3?>" + >>> ET.tostring(ET.PI('test', '<testing&>\xe3'), 'latin-1') + b"<?xml version='1.0' encoding='latin-1'?>\\n<?test <testing&>\\xe3?>" """ # diff --git a/Lib/test/test_xmlrpc_net.py b/Lib/test/test_xmlrpc_net.py index 5df79f0fb6..b9853ed80b 100644 --- a/Lib/test/test_xmlrpc_net.py +++ b/Lib/test/test_xmlrpc_net.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import collections +import collections.abc import errno import socket import sys @@ -48,7 +48,7 @@ class CurrentTimeTest(unittest.TestCase): # Perform a minimal sanity check on the result, just to be sure # the request means what we think it means. - self.assertIsInstance(builders, collections.Sequence) + self.assertIsInstance(builders, collections.abc.Sequence) self.assertTrue([x for x in builders if "3.x" in x], builders) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 782020c6c8..4dcb69059c 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -1,9 +1,3 @@ -# We can test part of the module without zlib. -try: - import zlib -except ImportError: - zlib = None - import io import os import sys @@ -19,7 +13,7 @@ from tempfile import TemporaryFile from random import randint, random from unittest import skipUnless -from test.support import TESTFN, run_unittest, findfile, unlink +from test.support import TESTFN, run_unittest, findfile, unlink, requires_zlib TESTFN2 = TESTFN + "2" TESTFNDIR = TESTFN + "d" @@ -269,44 +263,44 @@ class TestsWithSourceFile(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_iterlines_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_open_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_open_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_random_open_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_random_open_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readline_read_deflated(self): # Issue #7610: calls to readline() interleaved with calls to read(). for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_readline_read_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readline_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_readline_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readlines_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_readlines_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_iterlines_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_iterlines_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_low_compression(self): """Check for cases where compressed data is larger than original.""" # Create the ZIP archive @@ -377,7 +371,7 @@ class TestsWithSourceFile(unittest.TestCase): with open(TESTFN, "rb") as f: self.assertEqual(zipfp.read(TESTFN), f.read()) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_per_file_compression(self): """Check that files within a Zip archive can have different compression options.""" @@ -446,19 +440,18 @@ class TestsWithSourceFile(unittest.TestCase): # remove the test file subdirectories shutil.rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) - def test_writestr_compression(self): + def test_writestr_compression_stored(self): zipfp = zipfile.ZipFile(TESTFN2, "w") zipfp.writestr("a.txt", "hello world", compress_type=zipfile.ZIP_STORED) - if zlib: - zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED) - info = zipfp.getinfo('a.txt') self.assertEqual(info.compress_type, zipfile.ZIP_STORED) - if zlib: - info = zipfp.getinfo('b.txt') - self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED) - + @requires_zlib + def test_writestr_compression_deflated(self): + zipfp = zipfile.ZipFile(TESTFN2, "w") + zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED) + info = zipfp.getinfo('b.txt') + self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED) def zip_test_writestr_permissions(self, f, compression): # Make sure that writestr creates files with mode 0600, @@ -507,7 +500,7 @@ class TestsWithSourceFile(unittest.TestCase): except zipfile.BadZipFile: self.assertTrue(zipfp2.fp is None, 'zipfp is not closed') - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_unicode_filenames(self): # bug #10801 fname = findfile('zip_cp437_header.zip') @@ -616,7 +609,7 @@ class TestZip64InSmallFiles(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_test(f, zipfile.ZIP_DEFLATED) @@ -976,7 +969,7 @@ class OtherTests(unittest.TestCase): def test_testzip_with_bad_crc_stored(self): self.check_testzip_with_bad_crc(zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_testzip_with_bad_crc_deflated(self): self.check_testzip_with_bad_crc(zipfile.ZIP_DEFLATED) @@ -1004,7 +997,7 @@ class OtherTests(unittest.TestCase): def test_read_with_bad_crc_stored(self): self.check_read_with_bad_crc(zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_read_with_bad_crc_deflated(self): self.check_read_with_bad_crc(zipfile.ZIP_DEFLATED) @@ -1024,7 +1017,7 @@ class OtherTests(unittest.TestCase): def test_read_return_size_stored(self): self.check_read_return_size(zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_read_return_size_deflated(self): self.check_read_return_size(zipfile.ZIP_DEFLATED) @@ -1110,7 +1103,7 @@ class DecryptionTests(unittest.TestCase): self.zip2.setpassword(b"perl") self.assertRaises(RuntimeError, self.zip2.read, "zero") - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_good_password(self): self.zip.setpassword(b"python") self.assertEqual(self.zip.read("test.txt"), self.plain) @@ -1160,7 +1153,7 @@ class TestsWithRandomBinaryFiles(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_test(f, zipfile.ZIP_DEFLATED) @@ -1200,7 +1193,7 @@ class TestsWithRandomBinaryFiles(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_open_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_open_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_open_test(f, zipfile.ZIP_DEFLATED) @@ -1228,13 +1221,13 @@ class TestsWithRandomBinaryFiles(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_random_open_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_random_open_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_random_open_test(f, zipfile.ZIP_DEFLATED) -@skipUnless(zlib, "requires zlib") +@requires_zlib class TestsWithMultipleOpens(unittest.TestCase): def setUp(self): # Create the ZIP archive @@ -1426,28 +1419,28 @@ class UniversalNewlineTests(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.iterlines_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_read_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.read_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readline_read_deflated(self): # Issue #7610: calls to readline() interleaved with calls to read(). for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.readline_read_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readline_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.readline_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readlines_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.readlines_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_iterlines_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.iterlines_test(f, zipfile.ZIP_DEFLATED) diff --git a/Lib/test/test_zipfile64.py b/Lib/test/test_zipfile64.py index 0e7d73f1ff..a8fb7aba54 100644 --- a/Lib/test/test_zipfile64.py +++ b/Lib/test/test_zipfile64.py @@ -11,12 +11,6 @@ support.requires( 'test requires loads of disk-space bytes and a long time to run' ) -# We can test part of the module without zlib. -try: - import zlib -except ImportError: - zlib = None - import zipfile, os, unittest import time import sys @@ -24,7 +18,7 @@ import sys from io import StringIO from tempfile import TemporaryFile -from test.support import TESTFN, run_unittest +from test.support import TESTFN, run_unittest, requires_zlib TESTFN2 = TESTFN + "2" @@ -81,12 +75,12 @@ class TestsWithSourceFile(unittest.TestCase): for f in TemporaryFile(), TESTFN2: self.zipTest(f, zipfile.ZIP_STORED) - if zlib: - def testDeflated(self): - # Try the temp file first. If we do TESTFN2 first, then it hogs - # gigabytes of disk space for the duration of the test. - for f in TemporaryFile(), TESTFN2: - self.zipTest(f, zipfile.ZIP_DEFLATED) + @requires_zlib + def testDeflated(self): + # Try the temp file first. If we do TESTFN2 first, then it hogs + # gigabytes of disk space for the duration of the test. + for f in TemporaryFile(), TESTFN2: + self.zipTest(f, zipfile.ZIP_DEFLATED) def tearDown(self): for fname in TESTFN, TESTFN2: diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index df5ff9d6a6..56141efbfb 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -9,12 +9,6 @@ import unittest from test import support from test.test_importhooks import ImportHooksBaseTestCase, test_src, test_co -# some tests can be ran even without zlib -try: - import zlib -except ImportError: - zlib = None - from zipfile import ZipFile, ZipInfo, ZIP_STORED, ZIP_DEFLATED import zipimport @@ -392,7 +386,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): os.remove(filename) -@unittest.skipUnless(zlib, "requires zlib") +@support.requires_zlib class CompressedZipImportTestCase(UncompressedZipImportTestCase): compression = ZIP_DEFLATED @@ -417,7 +411,7 @@ class BadFileZipImportTestCase(unittest.TestCase): def testEmptyFile(self): support.unlink(TESTMOD) - open(TESTMOD, 'w+').close() + support.create_empty_file(TESTMOD) self.assertZipFailure(TESTMOD) def testFileUnreadable(self): diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index a558d7dd23..0c93a8cc1a 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -156,20 +156,19 @@ class ZipSupportTests(unittest.TestCase): test_zipped_doctest.test_DocTestRunner.verbose_flag, test_zipped_doctest.test_Example, test_zipped_doctest.test_debug, - test_zipped_doctest.test_pdb_set_trace, - test_zipped_doctest.test_pdb_set_trace_nested, test_zipped_doctest.test_testsource, test_zipped_doctest.test_trailing_space_in_test, test_zipped_doctest.test_DocTestSuite, test_zipped_doctest.test_DocTestFinder, ] - # These remaining tests are the ones which need access + # These tests are the ones which need access # to the data files, so we don't run them fail_due_to_missing_data_files = [ test_zipped_doctest.test_DocFileSuite, test_zipped_doctest.test_testfile, test_zipped_doctest.test_unittest_reportflags, ] + for obj in known_good_tests: _run_object_doctest(obj, test_zipped_doctest) finally: diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 68dd3eabe1..dddde47f8c 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -447,6 +447,26 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): y += dco.flush() self.assertEqual(y, b'foo') + def test_decompress_eof(self): + x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' + dco = zlib.decompressobj() + self.assertFalse(dco.eof) + dco.decompress(x[:-5]) + self.assertFalse(dco.eof) + dco.decompress(x[-5:]) + self.assertTrue(dco.eof) + dco.flush() + self.assertTrue(dco.eof) + + def test_decompress_eof_incomplete_stream(self): + x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' + dco = zlib.decompressobj() + self.assertFalse(dco.eof) + dco.decompress(x[:-5]) + self.assertFalse(dco.eof) + dco.flush() + self.assertFalse(dco.eof) + if hasattr(zlib.compressobj(), "copy"): def test_compresscopy(self): # Test copying a compression object diff --git a/Lib/test/threaded_import_hangers.py b/Lib/test/threaded_import_hangers.py index adf03e31ff..5484e60a0d 100644 --- a/Lib/test/threaded_import_hangers.py +++ b/Lib/test/threaded_import_hangers.py @@ -35,8 +35,11 @@ for name, func, args in [ ("os.path.abspath", os.path.abspath, ('.',)), ]: - t = Worker(func, args) - t.start() - t.join(TIMEOUT) - if t.is_alive(): - errors.append("%s appeared to hang" % name) + try: + t = Worker(func, args) + t.start() + t.join(TIMEOUT) + if t.is_alive(): + errors.append("%s appeared to hang" % name) + finally: + del t diff --git a/Lib/textwrap.py b/Lib/textwrap.py index dfb400548b..0aeba3f324 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -5,7 +5,7 @@ # Copyright (C) 2002, 2003 Python Software Foundation. # Written by Greg Ward <gward@python.net> -import string, re +import re __all__ = ['TextWrapper', 'wrap', 'fill', 'dedent'] diff --git a/Lib/threading.py b/Lib/threading.py index d260983452..b6632327ce 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -20,12 +20,12 @@ from _weakrefset import WeakSet __all__ = ['active_count', 'Condition', 'current_thread', 'enumerate', 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Barrier', - 'Timer', 'setprofile', 'settrace', 'local', 'stack_size'] + 'Timer', 'ThreadError', 'setprofile', 'settrace', 'local', 'stack_size'] # Rename some stuff so "from threading import *" is safe _start_new_thread = _thread.start_new_thread _allocate_lock = _thread.allocate_lock -_get_ident = _thread.get_ident +get_ident = _thread.get_ident ThreadError = _thread.error try: _CRLock = _thread.RLock @@ -53,7 +53,7 @@ if __debug__: format = format % args # Issue #4188: calling current_thread() can incur an infinite # recursion if it has to create a DummyThread on the fly. - ident = _get_ident() + ident = get_ident() try: name = _active[ident].name except KeyError: @@ -111,7 +111,7 @@ class _RLock(_Verbose): self.__class__.__name__, owner, self._count) def acquire(self, blocking=True, timeout=-1): - me = _get_ident() + me = get_ident() if self._owner == me: self._count = self._count + 1 if __debug__: @@ -131,7 +131,7 @@ class _RLock(_Verbose): __enter__ = acquire def release(self): - if self._owner != _get_ident(): + if self._owner != get_ident(): raise RuntimeError("cannot release un-acquired lock") self._count = count = self._count - 1 if not count: @@ -157,6 +157,8 @@ class _RLock(_Verbose): def _release_save(self): if __debug__: self._note("%s._release_save()", self) + if self._count == 0: + raise RuntimeError("cannot release un-acquired lock") count = self._count self._count = 0 owner = self._owner @@ -165,15 +167,12 @@ class _RLock(_Verbose): return (count, owner) def _is_owned(self): - return self._owner == _get_ident() + return self._owner == get_ident() _PyRLock = _RLock -def Condition(*args, **kwargs): - return _Condition(*args, **kwargs) - -class _Condition(_Verbose): +class Condition(_Verbose): def __init__(self, lock=None, verbose=None): _Verbose.__init__(self, verbose) @@ -306,10 +305,7 @@ class _Condition(_Verbose): notifyAll = notify_all -def Semaphore(*args, **kwargs): - return _Semaphore(*args, **kwargs) - -class _Semaphore(_Verbose): +class Semaphore(_Verbose): # After Tim Peters' semaphore class, but not quite the same (no maximum) @@ -364,25 +360,19 @@ class _Semaphore(_Verbose): self.release() -def BoundedSemaphore(*args, **kwargs): - return _BoundedSemaphore(*args, **kwargs) - -class _BoundedSemaphore(_Semaphore): +class BoundedSemaphore(Semaphore): """Semaphore that checks that # releases is <= # acquires""" def __init__(self, value=1, verbose=None): - _Semaphore.__init__(self, value, verbose) + Semaphore.__init__(self, value, verbose) self._initial_value = value def release(self): if self._value >= self._initial_value: raise ValueError("Semaphore released too many times") - return _Semaphore.release(self) - + return Semaphore.release(self) -def Event(*args, **kwargs): - return _Event(*args, **kwargs) -class _Event(_Verbose): +class Event(_Verbose): # After Tim Peters' event class (without is_posted()) @@ -625,7 +615,7 @@ class Thread(_Verbose): #XXX __exc_clear = _sys.exc_clear def __init__(self, group=None, target=None, name=None, - args=(), kwargs=None, verbose=None): + args=(), kwargs=None, verbose=None, *, daemon=None): assert group is None, "group argument must be None for now" _Verbose.__init__(self, verbose) if kwargs is None: @@ -634,7 +624,10 @@ class Thread(_Verbose): self._name = str(name or _newname()) self._args = args self._kwargs = kwargs - self._daemonic = self._set_daemon() + if daemon is not None: + self._daemonic = daemon + else: + self._daemonic = current_thread().daemon self._ident = None self._started = Event() self._stopped = False @@ -652,10 +645,6 @@ class Thread(_Verbose): self._block.__init__() self._started._reset_internal_locks() - def _set_daemon(self): - # Overridden in _MainThread and _DummyThread - return current_thread().daemon - def __repr__(self): assert self._initialized, "Thread.__init__() was not called" status = "initial" @@ -717,7 +706,7 @@ class Thread(_Verbose): raise def _set_ident(self): - self._ident = _get_ident() + self._ident = get_ident() def _bootstrap_inner(self): try: @@ -790,7 +779,7 @@ class Thread(_Verbose): try: # We don't call self._delete() because it also # grabs _active_limbo_lock. - del _active[_get_ident()] + del _active[get_ident()] except: pass @@ -826,7 +815,7 @@ class Thread(_Verbose): try: with _active_limbo_lock: - del _active[_get_ident()] + del _active[get_ident()] # There must not be any python code between the previous line # and after the lock is released. Otherwise a tracing function # could try to acquire the lock again in the same thread, (in @@ -917,10 +906,7 @@ class Thread(_Verbose): # The timer class was contributed by Itamar Shtull-Trauring -def Timer(*args, **kwargs): - return _Timer(*args, **kwargs) - -class _Timer(Thread): +class Timer(Thread): """Call a function after a specified number of seconds: t = Timer(30.0, f, args=[], kwargs={}) @@ -952,15 +938,12 @@ class _Timer(Thread): class _MainThread(Thread): def __init__(self): - Thread.__init__(self, name="MainThread") + Thread.__init__(self, name="MainThread", daemon=False) self._started.set() self._set_ident() with _active_limbo_lock: _active[self._ident] = self - def _set_daemon(self): - return False - def _exitfunc(self): self._stop() t = _pickSomeNonDaemonThread() @@ -992,7 +975,7 @@ def _pickSomeNonDaemonThread(): class _DummyThread(Thread): def __init__(self): - Thread.__init__(self, name=_newname("Dummy-%d")) + Thread.__init__(self, name=_newname("Dummy-%d"), daemon=True) # Thread._block consumes an OS-level locking primitive, which # can never be used by a _DummyThread. Since a _DummyThread @@ -1004,9 +987,6 @@ class _DummyThread(Thread): with _active_limbo_lock: _active[self._ident] = self - def _set_daemon(self): - return True - def join(self, timeout=None): assert False, "cannot join a dummy thread" @@ -1015,9 +995,8 @@ class _DummyThread(Thread): def current_thread(): try: - return _active[_get_ident()] + return _active[get_ident()] except KeyError: - ##print "current_thread(): no current thread for", _get_ident() return _DummyThread() currentThread = current_thread @@ -1071,7 +1050,7 @@ def _after_fork(): if thread is current: # There is only one active thread. We reset the ident to # its new value since it can have changed. - ident = _get_ident() + ident = get_ident() thread._ident = ident # Any condition variables hanging off of the active thread may # be in an invalid state, so we reinitialize them. diff --git a/Lib/timeit.py b/Lib/timeit.py index 461c891070..f45168b445 100644 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -79,10 +79,10 @@ else: # being indented 8 spaces. template = """ def inner(_it, _timer): - %(setup)s + {setup} _t0 = _timer() for _i in _it: - %(stmt)s + {stmt} _t1 = _timer() return _t1 - _t0 """ @@ -126,9 +126,9 @@ class Timer: stmt = reindent(stmt, 8) if isinstance(setup, str): setup = reindent(setup, 4) - src = template % {'stmt': stmt, 'setup': setup} + src = template.format(stmt=stmt, setup=setup) elif hasattr(setup, '__call__'): - src = template % {'stmt': stmt, 'setup': '_setup()'} + src = template.format(stmt=stmt, setup='_setup()') ns['_setup'] = setup else: raise ValueError("setup is neither a string nor callable") diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index e54c53f18e..ee87147a7d 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -30,8 +30,6 @@ button.pack(side=BOTTOM) tk.mainloop() """ -__version__ = "$Revision$" - import sys if sys.platform == "win32": # Attempt to configure Tcl/Tk without requiring PATH diff --git a/Lib/turtle.py b/Lib/turtle.py index 8fb366a7eb..ad7518336f 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -108,7 +108,6 @@ import tkinter as TK import types import math import time -import os import inspect from os.path import isfile, split, join diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 3133907e4a..d82fa20927 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -104,9 +104,9 @@ def expectedFailure(func): class _AssertRaisesBaseContext(object): def __init__(self, expected, test_case, callable_obj=None, - expected_regex=None): + expected_regex=None): self.expected = expected - self.failureException = test_case.failureException + self.test_case = test_case if callable_obj is not None: try: self.obj_name = callable_obj.__name__ @@ -117,6 +117,24 @@ class _AssertRaisesBaseContext(object): if isinstance(expected_regex, (bytes, str)): expected_regex = re.compile(expected_regex) self.expected_regex = expected_regex + self.msg = None + + def _raiseFailure(self, standardMsg): + msg = self.test_case._formatMessage(self.msg, standardMsg) + raise self.test_case.failureException(msg) + + def handle(self, name, callable_obj, args, kwargs): + """ + If callable_obj is None, assertRaises/Warns is being used as a + context manager, so check for a 'msg' kwarg and return self. + If callable_obj is not None, call it passing args and kwargs. + """ + if callable_obj is None: + self.msg = kwargs.pop('msg', None) + return self + with self: + callable_obj(*args, **kwargs) + class _AssertRaisesContext(_AssertRaisesBaseContext): @@ -132,11 +150,10 @@ class _AssertRaisesContext(_AssertRaisesBaseContext): except AttributeError: exc_name = str(self.expected) if self.obj_name: - raise self.failureException("{0} not raised by {1}" - .format(exc_name, self.obj_name)) + self._raiseFailure("{} not raised by {}".format(exc_name, + self.obj_name)) else: - raise self.failureException("{0} not raised" - .format(exc_name)) + self._raiseFailure("{} not raised".format(exc_name)) if not issubclass(exc_type, self.expected): # let unexpected exceptions pass through return False @@ -147,8 +164,8 @@ class _AssertRaisesContext(_AssertRaisesBaseContext): expected_regex = self.expected_regex if not expected_regex.search(str(exc_value)): - raise self.failureException('"%s" does not match "%s"' % - (expected_regex.pattern, str(exc_value))) + self._raiseFailure('"{}" does not match "{}"'.format( + expected_regex.pattern, str(exc_value))) return True @@ -192,14 +209,13 @@ class _AssertWarnsContext(_AssertRaisesBaseContext): return # Now we simply try to choose a helpful failure message if first_matching is not None: - raise self.failureException('"%s" does not match "%s"' % - (self.expected_regex.pattern, str(first_matching))) + self._raiseFailure('"{}" does not match "{}"'.format( + self.expected_regex.pattern, str(first_matching))) if self.obj_name: - raise self.failureException("{0} not triggered by {1}" - .format(exc_name, self.obj_name)) + self._raiseFailure("{} not triggered by {}".format(exc_name, + self.obj_name)) else: - raise self.failureException("{0} not triggered" - .format(exc_name)) + self._raiseFailure("{} not triggered".format(exc_name)) class TestCase(object): @@ -452,7 +468,7 @@ class TestCase(object): warnings.warn("TestResult has no addExpectedFailure method, reporting as passes", RuntimeWarning) result.addSuccess(self) - + return result finally: result.stopTest(self) if orig_result is None: @@ -526,7 +542,6 @@ class TestCase(object): except UnicodeDecodeError: return '%s : %s' % (safe_repr(standardMsg), safe_repr(msg)) - def assertRaises(self, excClass, callableObj=None, *args, **kwargs): """Fail unless an exception of class excClass is thrown by callableObj when invoked with arguments args and keyword @@ -541,6 +556,9 @@ class TestCase(object): with self.assertRaises(SomeException): do_something() + An optional keyword argument 'msg' can be provided when assertRaises + is used as a context object. + The context manager keeps a reference to the exception as the 'exception' attribute. This allows you to inspect the exception after the assertion:: @@ -551,25 +569,25 @@ class TestCase(object): self.assertEqual(the_exception.error_code, 3) """ context = _AssertRaisesContext(excClass, self, callableObj) - if callableObj is None: - return context - with context: - callableObj(*args, **kwargs) + return context.handle('assertRaises', callableObj, args, kwargs) def assertWarns(self, expected_warning, callable_obj=None, *args, **kwargs): """Fail unless a warning of class warnClass is triggered - by callableObj when invoked with arguments args and keyword + by callable_obj when invoked with arguments args and keyword arguments kwargs. If a different type of warning is triggered, it will not be handled: depending on the other warning filtering rules in effect, it might be silenced, printed out, or raised as an exception. - If called with callableObj omitted or None, will return a + If called with callable_obj omitted or None, will return a context object used like this:: with self.assertWarns(SomeWarning): do_something() + An optional keyword argument 'msg' can be provided when assertWarns + is used as a context object. + The context manager keeps a reference to the first matching warning as the 'warning' attribute; similarly, the 'filename' and 'lineno' attributes give you information about the line @@ -582,10 +600,7 @@ class TestCase(object): self.assertEqual(the_warning.some_attribute, 147) """ context = _AssertWarnsContext(expected_warning, self, callable_obj) - if callable_obj is None: - return context - with context: - callable_obj(*args, **kwargs) + return context.handle('assertWarns', callable_obj, args, kwargs) def _getAssertEqualityFunc(self, first, second): """Get a detailed comparison function for the types of the two args. @@ -951,48 +966,6 @@ class TestCase(object): self.fail(self._formatMessage(msg, standardMsg)) - def assertSameElements(self, expected_seq, actual_seq, msg=None): - """An unordered sequence specific comparison. - - Raises with an error message listing which elements of expected_seq - are missing from actual_seq and vice versa if any. - - Duplicate elements are ignored when comparing *expected_seq* and - *actual_seq*. It is the equivalent of ``assertEqual(set(expected), - set(actual))`` but it works with sequences of unhashable objects as - well. - """ - warnings.warn('assertSameElements is deprecated', - DeprecationWarning) - try: - expected = set(expected_seq) - actual = set(actual_seq) - missing = sorted(expected.difference(actual)) - unexpected = sorted(actual.difference(expected)) - except TypeError: - # Fall back to slower list-compare if any of the objects are - # not hashable. - expected = list(expected_seq) - actual = list(actual_seq) - try: - expected.sort() - actual.sort() - except TypeError: - missing, unexpected = unorderable_list_difference(expected, - actual) - else: - missing, unexpected = sorted_list_difference(expected, actual) - errors = [] - if missing: - errors.append('Expected, but missing:\n %s' % - safe_repr(missing)) - if unexpected: - errors.append('Unexpected, but present:\n %s' % - safe_repr(unexpected)) - if errors: - standardMsg = '\n'.join(errors) - self.fail(self._formatMessage(msg, standardMsg)) - def assertCountEqual(self, first, second, msg=None): """An unordered sequence comparison asserting that the same elements, @@ -1106,15 +1079,15 @@ class TestCase(object): expected_regex: Regex (re pattern object or string) expected to be found in error message. callable_obj: Function to be called. + msg: Optional message used in case of failure. Can only be used + when assertRaisesRegex is used as a context manager. args: Extra args. kwargs: Extra kwargs. """ context = _AssertRaisesContext(expected_exception, self, callable_obj, expected_regex) - if callable_obj is None: - return context - with context: - callable_obj(*args, **kwargs) + + return context.handle('assertRaisesRegex', callable_obj, args, kwargs) def assertWarnsRegex(self, expected_warning, expected_regex, callable_obj=None, *args, **kwargs): @@ -1128,15 +1101,14 @@ class TestCase(object): expected_regex: Regex (re pattern object or string) expected to be found in error message. callable_obj: Function to be called. + msg: Optional message used in case of failure. Can only be used + when assertWarnsRegex is used as a context manager. args: Extra args. kwargs: Extra kwargs. """ context = _AssertWarnsContext(expected_warning, self, callable_obj, expected_regex) - if callable_obj is None: - return context - with context: - callable_obj(*args, **kwargs) + return context.handle('assertWarnsRegex', callable_obj, args, kwargs) def assertRegex(self, text, expected_regex, msg=None): """Fail the test unless the text matches the regular expression.""" diff --git a/Lib/unittest/test/test_assertions.py b/Lib/unittest/test/test_assertions.py index a1d20ebf2d..d43fe5a601 100644 --- a/Lib/unittest/test/test_assertions.py +++ b/Lib/unittest/test/test_assertions.py @@ -1,6 +1,7 @@ import datetime import warnings import unittest +from itertools import product class Test_Assertions(unittest.TestCase): @@ -145,6 +146,14 @@ class TestLongMessage(unittest.TestCase): self.testableTrue._formatMessage(one, '\uFFFD') def assertMessages(self, methodName, args, errors): + """ + Check that methodName(*args) raises the correct error messages. + errors should be a list of 4 regex that match the error when: + 1) longMessage = False and no msg passed; + 2) longMessage = False and msg passed; + 3) longMessage = True and no msg passed; + 4) longMessage = True and msg passed; + """ def getMethod(i): useTestableFalse = i < 2 if useTestableFalse: @@ -284,3 +293,67 @@ class TestLongMessage(unittest.TestCase): ["^unexpectedly identical: None$", "^oops$", "^unexpectedly identical: None$", "^unexpectedly identical: None : oops$"]) + + + def assertMessagesCM(self, methodName, args, func, errors): + """ + Check that the correct error messages are raised while executing: + with method(*args): + func() + *errors* should be a list of 4 regex that match the error when: + 1) longMessage = False and no msg passed; + 2) longMessage = False and msg passed; + 3) longMessage = True and no msg passed; + 4) longMessage = True and msg passed; + """ + p = product((self.testableFalse, self.testableTrue), + ({}, {"msg": "oops"})) + for (cls, kwargs), err in zip(p, errors): + method = getattr(cls, methodName) + with self.assertRaisesRegex(cls.failureException, err): + with method(*args, **kwargs) as cm: + func() + + def testAssertRaises(self): + self.assertMessagesCM('assertRaises', (TypeError,), lambda: None, + ['^TypeError not raised$', '^oops$', + '^TypeError not raised$', + '^TypeError not raised : oops$']) + + def testAssertRaisesRegex(self): + # test error not raised + self.assertMessagesCM('assertRaisesRegex', (TypeError, 'unused regex'), + lambda: None, + ['^TypeError not raised$', '^oops$', + '^TypeError not raised$', + '^TypeError not raised : oops$']) + # test error raised but with wrong message + def raise_wrong_message(): + raise TypeError('foo') + self.assertMessagesCM('assertRaisesRegex', (TypeError, 'regex'), + raise_wrong_message, + ['^"regex" does not match "foo"$', '^oops$', + '^"regex" does not match "foo"$', + '^"regex" does not match "foo" : oops$']) + + def testAssertWarns(self): + self.assertMessagesCM('assertWarns', (UserWarning,), lambda: None, + ['^UserWarning not triggered$', '^oops$', + '^UserWarning not triggered$', + '^UserWarning not triggered : oops$']) + + def testAssertWarnsRegex(self): + # test error not raised + self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'unused regex'), + lambda: None, + ['^UserWarning not triggered$', '^oops$', + '^UserWarning not triggered$', + '^UserWarning not triggered : oops$']) + # test warning raised but with wrong message + def raise_wrong_message(): + warnings.warn('foo') + self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'regex'), + raise_wrong_message, + ['^"regex" does not match "foo"$', '^oops$', + '^"regex" does not match "foo"$', + '^"regex" does not match "foo" : oops$']) diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py index c74a539515..fdb2e78462 100644 --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -4,6 +4,7 @@ import pickle import re import sys import warnings +import weakref import inspect from copy import deepcopy @@ -386,27 +387,62 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing): self.assertIsInstance(Foo().id(), str) - # "If result is omitted or None, a temporary result object is created - # and used, but is not made available to the caller. As TestCase owns the + # "If result is omitted or None, a temporary result object is created, + # used, and is made available to the caller. As TestCase owns the # temporary result startTestRun and stopTestRun are called. def test_run__uses_defaultTestResult(self): events = [] + defaultResult = LoggingResult(events) class Foo(unittest.TestCase): def test(self): events.append('test') def defaultTestResult(self): - return LoggingResult(events) + return defaultResult # Make run() find a result object on its own - Foo('test').run() + result = Foo('test').run() + self.assertIs(result, defaultResult) expected = ['startTestRun', 'startTest', 'test', 'addSuccess', 'stopTest', 'stopTestRun'] self.assertEqual(events, expected) + + # "The result object is returned to run's caller" + def test_run__returns_given_result(self): + + class Foo(unittest.TestCase): + def test(self): + pass + + result = unittest.TestResult() + + retval = Foo('test').run(result) + self.assertIs(retval, result) + + + # "The same effect [as method run] may be had by simply calling the + # TestCase instance." + def test_call__invoking_an_instance_delegates_to_run(self): + resultIn = unittest.TestResult() + resultOut = unittest.TestResult() + + class Foo(unittest.TestCase): + def test(self): + pass + + def run(self, result): + self.assertIs(result, resultIn) + return resultOut + + retval = Foo('test')(resultIn) + + self.assertIs(retval, resultOut) + + def testShortDescriptionWithoutDocstring(self): self.assertIsNone(self.shortDescription()) @@ -1140,7 +1176,6 @@ test case (self.assert_, (True,)), (self.failUnlessRaises, (TypeError, lambda _: 3.14 + 'spam')), (self.failIf, (False,)), - (self.assertSameElements, ([1, 1, 2, 3], [1, 2, 3])), (self.assertDictContainsSubset, (dict(a=1, b=2), dict(a=1, b=2, c=3))), (self.assertRaisesRegexp, (KeyError, 'foo', lambda: {}['foo'])), (self.assertRegexpMatches, ('bar', 'bar')), @@ -1149,18 +1184,20 @@ test case with self.assertWarns(DeprecationWarning): meth(*args) - def testDeprecatedFailMethods(self): - """Test that the deprecated fail* methods get removed in 3.3""" + # disable this test for now. When the version where the fail* methods will + # be removed is decided, re-enable it and update the version + def _testDeprecatedFailMethods(self): + """Test that the deprecated fail* methods get removed in 3.x""" if sys.version_info[:2] < (3, 3): return deprecated_names = [ 'failIfEqual', 'failUnlessEqual', 'failUnlessAlmostEqual', 'failIfAlmostEqual', 'failUnless', 'failUnlessRaises', 'failIf', - 'assertSameElements', 'assertDictContainsSubset', + 'assertDictContainsSubset', ] for deprecated_name in deprecated_names: with self.assertRaises(AttributeError): - getattr(self, deprecated_name) # remove these in 3.3 + getattr(self, deprecated_name) # remove these in 3.x def testDeepcopy(self): # Issue: 5660 @@ -1268,3 +1305,11 @@ test case klass('test_something').run(result) self.assertEqual(len(result.errors), 1) self.assertEqual(result.testsRun, 1) + + @support.cpython_only + def testNoCycles(self): + case = unittest.TestCase() + wr = weakref.ref(case) + with support.disable_gc(): + del case + self.assertFalse(wr()) diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py index f7e31a57f1..d1b9ef5a0a 100644 --- a/Lib/unittest/test/test_loader.py +++ b/Lib/unittest/test/test_loader.py @@ -239,7 +239,7 @@ class Test_TestLoader(unittest.TestCase): try: loader.loadTestsFromName('sdasfasfasdf') except ImportError as e: - self.assertEqual(str(e), "No module named sdasfasfasdf") + self.assertEqual(str(e), "No module named 'sdasfasfasdf'") else: self.fail("TestLoader.loadTestsFromName failed to raise ImportError") @@ -619,7 +619,7 @@ class Test_TestLoader(unittest.TestCase): try: loader.loadTestsFromNames(['sdasfasfasdf']) except ImportError as e: - self.assertEqual(str(e), "No module named sdasfasfasdf") + self.assertEqual(str(e), "No module named 'sdasfasfasdf'") else: self.fail("TestLoader.loadTestsFromNames failed to raise ImportError") diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 773025a09b..4a571e8b9f 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -825,14 +825,20 @@ class AbstractBasicAuthHandler: self.retried += 1 if authreq: - mo = AbstractBasicAuthHandler.rx.search(authreq) - if mo: - scheme, quote, realm = mo.groups() - if scheme.lower() == 'basic': - response = self.retry_http_basic_auth(host, req, realm) - if response and response.code != 401: - self.retried = 0 - return response + scheme = authreq.split()[0] + if not scheme.lower() == 'basic': + raise ValueError("AbstractBasicAuthHandler does not" + " support the following scheme: '%s'" % + scheme) + else: + mo = AbstractBasicAuthHandler.rx.search(authreq) + if mo: + scheme, quote, realm = mo.groups() + if scheme.lower() == 'basic': + response = self.retry_http_basic_auth(host, req, realm) + if response and response.code != 401: + self.retried = 0 + return response def retry_http_basic_auth(self, host, req, realm): user, pw = self.passwd.find_user_password(realm, host) @@ -918,6 +924,9 @@ class AbstractDigestAuthHandler: scheme = authreq.split()[0] if scheme.lower() == 'digest': return self.retry_http_digest_auth(req, authreq) + elif not scheme.lower() == 'basic': + raise ValueError("AbstractDigestAuthHandler does not support" + " the following scheme: '%s'" % scheme) def retry_http_digest_auth(self, req, auth): token, challenge = auth.split(' ', 1) @@ -1875,7 +1884,7 @@ class URLopener: if encoding == 'base64': import base64 # XXX is this encoding/decoding ok? - data = base64.decodebytes(data.encode('ascii')).decode('latin1') + data = base64.decodebytes(data.encode('ascii')).decode('latin-1') else: data = unquote(data) msg.append('Content-Length: %d' % len(data)) diff --git a/Lib/urllib/response.py b/Lib/urllib/response.py index 8c6dcca5aa..b43e57595b 100644 --- a/Lib/urllib/response.py +++ b/Lib/urllib/response.py @@ -37,12 +37,15 @@ class addbase(object): id(self), self.fp) def close(self): + if self.fp: + self.fp.close() + self.fp = None self.read = None self.readline = None self.readlines = None self.fileno = None - if self.fp: self.fp.close() - self.fp = None + self.__iter__ = None + self.__next__ = None def __enter__(self): if self.fp is None: diff --git a/Lib/wsgiref.egg-info b/Lib/wsgiref.egg-info deleted file mode 100644 index c0b7893c34..0000000000 --- a/Lib/wsgiref.egg-info +++ /dev/null @@ -1,8 +0,0 @@ -Metadata-Version: 1.0 -Name: wsgiref -Version: 0.1.2 -Summary: WSGI (PEP 333) Reference Library -Author: Phillip J. Eby -Author-email: web-sig@python.org -License: PSF or ZPL -Platform: UNKNOWN diff --git a/Lib/xml/parsers/expat.py b/Lib/xml/parsers/expat.py index a805b828d8..bcbe9fb1f8 100644 --- a/Lib/xml/parsers/expat.py +++ b/Lib/xml/parsers/expat.py @@ -1,6 +1,4 @@ """Interface to the Expat non-validating XML parser.""" -__version__ = '$Revision$' - import sys from pyexpat import * |
