diff options
| -rw-r--r-- | Doc/ACKS.txt | 1 | ||||
| -rw-r--r-- | Doc/Makefile | 5 | ||||
| -rw-r--r-- | Doc/README.txt | 2 | ||||
| -rw-r--r-- | Doc/library/_ast.rst | 83 | ||||
| -rw-r--r-- | Doc/library/abc.rst | 6 | ||||
| -rw-r--r-- | Doc/library/ast.rst | 257 | ||||
| -rw-r--r-- | Doc/library/cmd.rst | 7 | ||||
| -rw-r--r-- | Doc/library/inspect.rst | 15 | ||||
| -rw-r--r-- | Doc/library/language.rst | 2 | ||||
| -rw-r--r-- | Doc/library/logging.rst | 10 | ||||
| -rw-r--r-- | Doc/library/parser.rst | 167 | ||||
| -rw-r--r-- | Doc/library/threading.rst | 11 | ||||
| -rw-r--r-- | Doc/whatsnew/2.5.rst | 4 | ||||
| -rw-r--r-- | Doc/whatsnew/2.6.rst | 96 | ||||
| -rw-r--r-- | Include/bytearrayobject.h | 8 | ||||
| -rw-r--r-- | Include/bytesobject.h | 8 | ||||
| -rw-r--r-- | Lib/ast.py | 300 | ||||
| -rw-r--r-- | Lib/inspect.py | 7 | ||||
| -rw-r--r-- | Lib/test/test_ast.py | 109 | ||||
| -rw-r--r-- | Lib/test/test_complex.py | 8 | ||||
| -rw-r--r-- | Lib/test/test_inspect.py | 4 | ||||
| -rw-r--r-- | Lib/test/test_threading.py | 5 | ||||
| -rw-r--r-- | Lib/threading.py | 14 | ||||
| -rw-r--r-- | Misc/ACKS | 2 | ||||
| -rw-r--r-- | Misc/developers.txt | 3 | ||||
| -rw-r--r-- | Modules/itertoolsmodule.c | 6 | ||||
| -rw-r--r-- | Objects/complexobject.c | 3 |
27 files changed, 931 insertions, 212 deletions
diff --git a/Doc/ACKS.txt b/Doc/ACKS.txt index 764a65b507..1130762873 100644 --- a/Doc/ACKS.txt +++ b/Doc/ACKS.txt @@ -157,6 +157,7 @@ docs@python.org), and we'll be glad to correct the problem. * Bernhard Reiter * Armin Rigo * Wes Rishel + * Armin Ronacher * Jim Roskind * Guido van Rossum * Donald Wallace Rouse II diff --git a/Doc/Makefile b/Doc/Makefile index df55289789..c1abe119e2 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -21,6 +21,7 @@ help: @echo " web to make file usable by Sphinx.web" @echo " htmlhelp to make HTML files and a HTML help project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " text to make plain text files" @echo " changes to make an overview over all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " coverage to check documentation coverage for library and C API" @@ -75,6 +76,10 @@ latex: build @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." +text: BUILDER = text +text: build + @echo "Build finished; the text files are in build/text." + changes: BUILDER = changes changes: build @echo "The overview file is in build/changes." diff --git a/Doc/README.txt b/Doc/README.txt index b70d23f6ed..fc263f0e0d 100644 --- a/Doc/README.txt +++ b/Doc/README.txt @@ -56,6 +56,8 @@ Available make targets are: * "latex", which builds LaTeX source files that can be run with "pdflatex" to produce PDF documents. + * "text", which builds a plain text file for each source file. + * "linkcheck", which checks all external references to see whether they are broken, redirected or malformed, and outputs this information to stdout as well as a plain-text (.txt) file. diff --git a/Doc/library/_ast.rst b/Doc/library/_ast.rst deleted file mode 100644 index cb7e44ecf5..0000000000 --- a/Doc/library/_ast.rst +++ /dev/null @@ -1,83 +0,0 @@ -.. _ast: - -Abstract Syntax Trees -===================== - -.. module:: _ast - :synopsis: Abstract Syntax Tree classes. - -.. sectionauthor:: Martin v. Löwis <martin@v.loewis.de> - - -The ``_ast`` module helps Python applications to process trees of the Python -abstract syntax grammar. The abstract syntax itself might change with each -Python release; this module helps to find out programmatically what the current -grammar looks like. - -An abstract syntax tree can be generated by passing :data:`_ast.PyCF_ONLY_AST` -as a flag to the :func:`compile` builtin function. The result will be a tree of -objects whose classes all inherit from :class:`_ast.AST`. - -A modified abstract syntax tree can be compiled into a Python code object using -the built-in :func:`compile` function. - -The actual classes are derived from the ``Parser/Python.asdl`` file, which is -reproduced below. There is one class defined for each left-hand side symbol in -the abstract grammar (for example, ``_ast.stmt`` or ``_ast.expr``). In addition, -there is one class defined for each constructor on the right-hand side; these -classes inherit from the classes for the left-hand side trees. For example, -``_ast.BinOp`` inherits from ``_ast.expr``. For production rules with -alternatives (aka "sums"), the left-hand side class is abstract: only instances -of specific constructor nodes are ever created. - -Each concrete class has an attribute ``_fields`` which gives the names of all -child nodes. - -Each instance of a concrete class has one attribute for each child node, of the -type as defined in the grammar. For example, ``_ast.BinOp`` instances have an -attribute ``left`` of type ``_ast.expr``. Instances of ``_ast.expr`` and -``_ast.stmt`` subclasses also have lineno and col_offset attributes. The lineno -is the line number of source text (1 indexed so the first line is line 1) and -the col_offset is the utf8 byte offset of the first token that generated the -node. The utf8 offset is recorded because the parser uses utf8 internally. - -If these attributes are marked as optional in the grammar (using a question -mark), the value might be ``None``. If the attributes can have zero-or-more -values (marked with an asterisk), the values are represented as Python lists. -All possible attributes must be present and have valid values when compiling an -AST with :func:`compile`. - -The constructor of a class ``_ast.T`` parses their arguments as follows: - -* If there are positional arguments, there must be as many as there are items in - ``T._fields``; they will be assigned as attributes of these names. -* If there are keyword arguments, they will set the attributes of the same names - to the given values. - -For example, to create and populate a ``UnaryOp`` node, you could use :: - - node = _ast.UnaryOp() - node.op = _ast.USub() - node.operand = _ast.Num() - node.operand.n = 5 - node.operand.lineno = 0 - node.operand.col_offset = 0 - node.lineno = 0 - node.col_offset = 0 - -or the more compact :: - - node = _ast.UnaryOp(_ast.USub(), _ast.Num(5, lineno=0, col_offset=0), - lineno=0, col_offset=0) - - - -Abstract Grammar ----------------- - -The module defines a string constant ``__version__`` which is the decimal -subversion revision number of the file shown below. - -The abstract grammar is currently defined as follows: - -.. literalinclude:: ../../Parser/Python.asdl diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index aa92913285..f1a6c06471 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -9,9 +9,9 @@ .. much of the content adapted from docstrings This module provides the infrastructure for defining abstract base classes -(ABCs) in Python, as outlined in :pep:`3119`; see the PEP for why this -was added to Python. (See also, :pep:`3141` regarding a type hierarchy -for numbers based on ABCs.) +(ABCs) in Python, as outlined in :pep:`3119`; see the PEP for why this was added +to Python. (See also :pep:`3141` and the :mod:`numbers` module regarding a type +hierarchy for numbers based on ABCs.) The :mod:`collections` module has some concrete classes that derive from ABCs; these can, of course, be further derived. In addition the diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst new file mode 100644 index 0000000000..70840da779 --- /dev/null +++ b/Doc/library/ast.rst @@ -0,0 +1,257 @@ +.. _ast: + +Abstract Syntax Trees +===================== + +.. module:: ast + :synopsis: Abstract Syntax Tree classes and manipulation. + +.. sectionauthor:: Martin v. Löwis <martin@v.loewis.de> +.. sectionauthor:: Georg Brandl <georg@python.org> + +.. versionadded:: 2.5 + The low-level ``_ast`` module containing only the node classes. + +.. versionadded:: 2.6 + The high-level ``ast`` module containing all helpers. + + +The :mod:`ast` module helps Python applications to process trees of the Python +abstract syntax grammar. The abstract syntax itself might change with each +Python release; this module helps to find out programmatically what the current +grammar looks like. + +An abstract syntax tree can be generated by passing :data:`_ast.PyCF_ONLY_AST` +as a flag to the :func:`compile` builtin function, or using the :func:`parse` +helper provided in this module. The result will be a tree of objects whose +classes all inherit from :class:`ast.AST`. + +A modified abstract syntax tree can be compiled into a Python code object using +the built-in :func:`compile` function. + +Node classes +------------ + +.. class:: AST + + This is the base of all AST node classes. The actual node classes are + derived from the :file:`Parser/Python.asdl` file, which is reproduced + :ref:`below <abstract-grammar>`. They are defined in the :mod:`_ast` C + module and re-exported in :mod:`ast`. + + There is one class defined for each left-hand side symbol in the abstract + grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, + there is one class defined for each constructor on the right-hand side; these + classes inherit from the classes for the left-hand side trees. For example, + :class:`ast.BinOp` inherits from :class:`ast.expr`. For production rules + with alternatives (aka "sums"), the left-hand side class is abstract: only + instances of specific constructor nodes are ever created. + + .. attribute:: _fields + + Each concrete class has an attribute :attr:`_fields` which gives the names + of all child nodes. + + Each instance of a concrete class has one attribute for each child node, + of the type as defined in the grammar. For example, :class:`ast.BinOp` + instances have an attribute :attr:`left` of type :class:`ast.expr`. + + If these attributes are marked as optional in the grammar (using a + question mark), the value might be ``None``. If the attributes can have + zero-or-more values (marked with an asterisk), the values are represented + as Python lists. All possible attributes must be present and have valid + values when compiling an AST with :func:`compile`. + + .. attribute:: lineno + col_offset + + Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have + :attr:`lineno` and :attr:`col_offset` attributes. The :attr:`lineno` is + the line number of source text (1-indexed so the first line is line 1) and + the :attr:`col_offset` is the UTF-8 byte offset of the first token that + generated the node. The UTF-8 offset is recorded because the parser uses + UTF-8 internally. + + The constructor of a class :class:`ast.T` parses its arguments as follows: + + * If there are positional arguments, there must be as many as there are items + in :attr:`T._fields`; they will be assigned as attributes of these names. + * If there are keyword arguments, they will set the attributes of the same + names to the given values. + + For example, to create and populate an :class:`ast.UnaryOp` node, you could + use :: + + node = ast.UnaryOp() + node.op = ast.USub() + node.operand = ast.Num() + node.operand.n = 5 + node.operand.lineno = 0 + node.operand.col_offset = 0 + node.lineno = 0 + node.col_offset = 0 + + or the more compact :: + + node = ast.UnaryOp(ast.USub(), ast.Num(5, lineno=0, col_offset=0), + lineno=0, col_offset=0) + + +.. _abstract-grammar: + +Abstract Grammar +---------------- + +The module defines a string constant ``__version__`` which is the decimal +Subversion revision number of the file shown below. + +The abstract grammar is currently defined as follows: + +.. literalinclude:: ../../Parser/Python.asdl + + +:mod:`ast` Helpers +------------------ + +.. versionadded:: 2.6 + +Apart from the node classes, :mod:`ast` module defines these utility functions +and classes for traversing abstract syntax trees: + +.. function:: parse(expr, filename='<unknown>', mode='exec') + + Parse an expression into an AST node. Equivalent to ``compile(expr, + filename, mode, PyCF_ONLY_AST)``. + + +.. function:: literal_eval(node_or_string) + + Safely evaluate an expression node or a string containing a Python + expression. The string or node provided may only consist of the following + Python literal structures: strings, numbers, tuples, lists, dicts, booleans, + and ``None``. + + This can be used for safely evaluating strings containing Python expressions + from untrusted sources without the need to parse the values oneself. + + +.. function:: get_docstring(node, clean=True): + + Return the docstring of the given *node* (which must be a + :class:`FunctionDef`, :class:`ClassDef` or :class:`Module` node), or ``None`` + if it has no docstring. If *clean* is true, clean up the docstring's + indentation with :func:`inspect.cleandoc`. + + +.. function:: fix_missing_locations(node) + + When you compile a node tree with :func:`compile`, the compiler expects + :attr:`lineno` and :attr:`col_offset` attributes for every node that supports + them. This is rather tedious to fill in for generated nodes, so this helper + adds these attributes recursively where not already set, by setting them to + the values of the parent node. It works recursively starting at *node*. + + +.. function:: increment_lineno(node, n=1) + + Increment the line number of each node in the tree starting at *node* by *n*. + This is useful to "move code" to a different location in a file. + + +.. function:: copy_location(new_node, old_node) + + Copy source location (:attr:`lineno` and :attr:`col_offset`) from *old_node* + to *new_node* if possible, and return *new_node*. + + +.. function:: iter_fields(node) + + Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` + that is present on *node*. + + +.. function:: iter_child_nodes(node) + + Yield all direct child nodes of *node*, that is, all fields that are nodes + and all items of fields that are lists of nodes. + + +.. function:: walk(node) + + Recursively yield all child nodes of *node*, in no specified order. This is + useful if you only want to modify nodes in place and don't care about the + context. + + +.. class:: NodeVisitor() + + A node visitor base class that walks the abstract syntax tree and calls a + visitor function for every node found. This function may return a value + which is forwarded by the `visit` method. + + This class is meant to be subclassed, with the subclass adding visitor + methods. + + .. method:: visit(node) + + Visit a node. The default implementation calls the method called + :samp:`self.visit_{classname}` where *classname* is the name of the node + class, or :meth:`generic_visit` if that method doesn't exist. + + .. method:: generic_visit(node) + + This visitor calls :meth:`visit` on all children of the node. + + Note that child nodes of nodes that have a custom visitor method won't be + visited unless the visitor calls :meth:`generic_visit` or visits them + itself. + + Don't use the :class:`NodeVisitor` if you want to apply changes to nodes + during traversal. For this a special visitor exists + (:class:`NodeTransformer`) that allows modifications. + + +.. class:: NodeTransformer() + + A :class:`NodeVisitor` subclass that walks the abstract syntax tree and + allows modification of nodes. + + The `NodeTransformer` will walk the AST and use the return value of the + visitor methods to replace or remove the old node. If the return value of + the visitor method is ``None``, the node will be removed from its location, + otherwise it is replaced with the return value. The return value may be the + original node in which case no replacement takes place. + + Here is an example transformer that rewrites all occurrences of name lookups + (``foo``) to ``data['foo']``:: + + class RewriteName(NodeTransformer): + + def visit_Name(self, node): + return copy_location(Subscript( + value=Name(id='data', ctx=Load()), + slice=Index(value=Str(s=node.id)), + ctx=node.ctx + ), node) + + Keep in mind that if the node you're operating on has child nodes you must + either transform the child nodes yourself or call the :meth:`generic_visit` + method for the node first. + + For nodes that were part of a collection of statements (that applies to all + statement nodes), the visitor may also return a list of nodes rather than + just a single node. + + Usually you use the transformer like this:: + + node = YourTransformer().visit(node) + + +.. function:: dump(node, annotate_fields=True, include_attributes=False) + + Return a formatted dump of the tree in *node*. This is mainly useful for + debugging purposes. The returned string will show the names and the values + for fields. This makes the code impossible to evaluate, so if evaluation is + wanted *annotate_fields* must be set to False. Attributes such as line + numbers and column offsets are dumped by default. If this is wanted, + *include_attributes* can be set to ``True``. diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index 716db6741a..c31c9a20b7 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -26,7 +26,12 @@ interface. The optional arguments *stdin* and *stdout* specify the input and output file objects that the Cmd instance or subclass instance will use for input and - output. If not specified, they will default to *sys.stdin* and *sys.stdout*. + output. If not specified, they will default to :data:`sys.stdin` and + :data:`sys.stdout`. + + If you want a given *stdin* to be used, make sure to set the instance's + :attr:`use_rawinput` attribute to ``False``, otherwise *stdin* will be + ignored. .. _cmd-objects: diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 7ef5d3f181..a87651f4a2 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -320,13 +320,9 @@ attributes: Retrieving source code ---------------------- - .. function:: getdoc(object) - Get the documentation string for an object. All tabs are expanded to spaces. To - clean up docstrings that are indented to line up with blocks of code, any - whitespace than can be uniformly removed from the second line onwards is - removed. + Get the documentation string for an object, cleaned up with :func:`cleandoc`. .. function:: getcomments(object) @@ -373,6 +369,15 @@ Retrieving source code cannot be retrieved. +.. function:: cleandoc(doc) + + Clean up indentation from docstrings that are indented to line up with blocks + of code. Any whitespace that can be uniformly removed from the second line + onwards is removed. Also, all tabs are expanded to spaces. + + .. versionadded:: 2.6 + + .. _inspect-classes-functions: Classes and functions diff --git a/Doc/library/language.rst b/Doc/library/language.rst index 7d6af7deff..bcf9ac0e6e 100644 --- a/Doc/library/language.rst +++ b/Doc/library/language.rst @@ -15,7 +15,7 @@ These modules include: .. toctree:: parser.rst - _ast.rst + ast.rst symbol.rst token.rst keyword.rst diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index cac29e9997..3f6b86036f 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1618,7 +1618,7 @@ The :class:`TimedRotatingFileHandler` class, located in the timed intervals. -.. class:: TimedRotatingFileHandler(filename [,when [,interval [,backupCount[, encoding[, delay]]]]]) +.. class:: TimedRotatingFileHandler(filename [,when [,interval [,backupCount[, encoding[, delay[, utc]]]]]]) Returns a new instance of the :class:`TimedRotatingFileHandler` class. The specified file is opened and used as the stream for logging. On rotating it also @@ -1626,7 +1626,7 @@ timed intervals. *interval*. You can use the *when* to specify the type of *interval*. The list of possible - values is, note that they are not case sensitive: + values is below. Note that they are not case sensitive. +----------------+-----------------------+ | Value | Type of interval | @@ -1647,7 +1647,11 @@ timed intervals. The system will save old log files by appending extensions to the filename. The extensions are date-and-time based, using the strftime format ``%Y-%m-%d_%H-%M-%S`` or a leading portion thereof, depending on the - rollover interval. If *backupCount* is nonzero, at most *backupCount* files + rollover interval. + If the *utc* argument is true, times in UTC will be used; otherwise + local time is used. + + If *backupCount* is nonzero, at most *backupCount* files will be kept, and if more would be created when rollover occurs, the oldest one is deleted. The deletion logic uses the interval to determine which files to delete, so changing the interval may leave old files lying around. diff --git a/Doc/library/parser.rst b/Doc/library/parser.rst index b0e509a137..5df8feb4f1 100644 --- a/Doc/library/parser.rst +++ b/Doc/library/parser.rst @@ -24,6 +24,17 @@ from this. This is better than trying to parse and modify an arbitrary Python code fragment as a string because parsing is performed in a manner identical to the code forming the application. It is also faster. +.. note:: + + From Python 2.5 onward, it's much more convenient to cut in at the Abstract + Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` + module. + + The :mod:`parser` module exports the names documented here also with "st" + replaced by "ast"; this is a legacy from the time when there was no other + AST and has nothing to do with the AST found in Python 2.5. This is also the + reason for the functions' keyword arguments being called *ast*, not *st*. + There are a few things to note about this module which are important to making use of the data structures created. This is not a tutorial on editing the parse trees for Python code, but some examples of using the :mod:`parser` module are @@ -34,9 +45,9 @@ internal parser is required. For full information on the language syntax, refer to :ref:`reference-index`. The parser itself is created from a grammar specification defined in the file :file:`Grammar/Grammar` in the standard Python distribution. The parse trees -stored in the AST objects created by this module are the actual output from the +stored in the ST objects created by this module are the actual output from the internal parser when created by the :func:`expr` or :func:`suite` functions, -described below. The AST objects created by :func:`sequence2ast` faithfully +described below. The ST objects created by :func:`sequence2st` faithfully simulate those structures. Be aware that the values of the sequences which are considered "correct" will vary from one version of Python to another as the formal grammar for the language is revised. However, transporting code from one @@ -46,7 +57,7 @@ migrating to an older version of the interpreter will not support more recent language constructs. The parse trees are not typically compatible from one version to another, whereas source code has always been forward-compatible. -Each element of the sequences returned by :func:`ast2list` or :func:`ast2tuple` +Each element of the sequences returned by :func:`st2list` or :func:`st2tuple` has a simple form. Sequences representing non-terminal elements in the grammar always have a length greater than one. The first element is an integer which identifies a production in the grammar. These integers are given symbolic names @@ -69,19 +80,19 @@ of the :keyword:`if` keyword above is representative. The various types of terminal symbols are defined in the C header file :file:`Include/token.h` and the Python module :mod:`token`. -The AST objects are not required to support the functionality of this module, +The ST objects are not required to support the functionality of this module, but are provided for three purposes: to allow an application to amortize the cost of processing complex parse trees, to provide a parse tree representation which conserves memory space when compared to the Python list or tuple representation, and to ease the creation of additional modules in C which manipulate parse trees. A simple "wrapper" class may be created in Python to -hide the use of AST objects. +hide the use of ST objects. The :mod:`parser` module defines functions for a few distinct purposes. The -most important purposes are to create AST objects and to convert AST objects to +most important purposes are to create ST objects and to convert ST objects to other representations such as parse trees and compiled code objects, but there are also functions which serve to query the type of parse tree represented by an -AST object. +ST object. .. seealso:: @@ -94,20 +105,20 @@ AST object. testing node values. -.. _creating-asts: +.. _creating-sts: -Creating AST Objects --------------------- +Creating ST Objects +------------------- -AST objects may be created from source code or from a parse tree. When creating -an AST object from source, different functions are used to create the ``'eval'`` +ST objects may be created from source code or from a parse tree. When creating +an ST object from source, different functions are used to create the ``'eval'`` and ``'exec'`` forms. .. function:: expr(source) The :func:`expr` function parses the parameter *source* as if it were an input - to ``compile(source, 'file.py', 'eval')``. If the parse succeeds, an AST object + to ``compile(source, 'file.py', 'eval')``. If the parse succeeds, an ST object is created to hold the internal parse tree representation, otherwise an appropriate exception is thrown. @@ -115,22 +126,22 @@ and ``'exec'`` forms. .. function:: suite(source) The :func:`suite` function parses the parameter *source* as if it were an input - to ``compile(source, 'file.py', 'exec')``. If the parse succeeds, an AST object + to ``compile(source, 'file.py', 'exec')``. If the parse succeeds, an ST object is created to hold the internal parse tree representation, otherwise an appropriate exception is thrown. -.. function:: sequence2ast(sequence) +.. function:: sequence2st(sequence) This function accepts a parse tree represented as a sequence and builds an internal representation if possible. If it can validate that the tree conforms to the Python grammar and all nodes are valid node types in the host version of - Python, an AST object is created from the internal representation and returned + Python, an ST object is created from the internal representation and returned to the called. If there is a problem creating the internal representation, or if the tree cannot be validated, a :exc:`ParserError` exception is thrown. An - AST object created this way should not be assumed to compile correctly; normal - exceptions thrown by compilation may still be initiated when the AST object is - passed to :func:`compileast`. This may indicate problems not related to syntax + ST object created this way should not be assumed to compile correctly; normal + exceptions thrown by compilation may still be initiated when the ST object is + passed to :func:`compilest`. This may indicate problems not related to syntax (such as a :exc:`MemoryError` exception), but may also be due to constructs such as the result of parsing ``del f(0)``, which escapes the Python parser but is checked by the bytecode compiler. @@ -142,31 +153,31 @@ and ``'exec'`` forms. symbols in the input tree. -.. function:: tuple2ast(sequence) +.. function:: tuple2st(sequence) - This is the same function as :func:`sequence2ast`. This entry point is + This is the same function as :func:`sequence2st`. This entry point is maintained for backward compatibility. -.. _converting-asts: +.. _converting-sts: -Converting AST Objects ----------------------- +Converting ST Objects +--------------------- -AST objects, regardless of the input used to create them, may be converted to +ST objects, regardless of the input used to create them, may be converted to parse trees represented as list- or tuple- trees, or may be compiled into executable code objects. Parse trees may be extracted with or without line numbering information. -.. function:: ast2list(ast[, line_info]) +.. function:: st2list(ast[, line_info]) - This function accepts an AST object from the caller in *ast* and returns a + This function accepts an ST object from the caller in *ast* and returns a Python list representing the equivalent parse tree. The resulting list representation can be used for inspection or the creation of a new parse tree in list form. This function does not fail so long as memory is available to build the list representation. If the parse tree will only be used for inspection, - :func:`ast2tuple` should be used instead to reduce memory consumption and + :func:`st2tuple` should be used instead to reduce memory consumption and fragmentation. When the list representation is required, this function is significantly faster than retrieving a tuple representation and converting that to nested lists. @@ -177,31 +188,31 @@ numbering information. This information is omitted if the flag is false or omitted. -.. function:: ast2tuple(ast[, line_info]) +.. function:: st2tuple(ast[, line_info]) - This function accepts an AST object from the caller in *ast* and returns a + This function accepts an ST object from the caller in *ast* and returns a Python tuple representing the equivalent parse tree. Other than returning a - tuple instead of a list, this function is identical to :func:`ast2list`. + tuple instead of a list, this function is identical to :func:`st2list`. If *line_info* is true, line number information will be included for all terminal tokens as a third element of the list representing the token. This information is omitted if the flag is false or omitted. -.. function:: compileast(ast[, filename='<ast>']) +.. function:: compilest(ast[, filename='<syntax-tree>']) .. index:: builtin: exec builtin: eval - The Python byte compiler can be invoked on an AST object to produce code objects + The Python byte compiler can be invoked on an ST object to produce code objects which can be used as part of a call to the built-in :func:`exec` or :func:`eval` functions. This function provides the interface to the compiler, passing the internal parse tree from *ast* to the parser, using the source file name specified by the *filename* parameter. The default value supplied for *filename* - indicates that the source was an AST object. + indicates that the source was an ST object. - Compiling an AST object may result in exceptions related to compilation; an + Compiling an ST object may result in exceptions related to compilation; an example would be a :exc:`SyntaxError` caused by the parse tree for ``del f(0)``: this statement is considered legal within the formal grammar for Python but is not a legal language construct. The :exc:`SyntaxError` raised for this @@ -211,15 +222,15 @@ numbering information. tree. -.. _querying-asts: +.. _querying-sts: -Queries on AST Objects ----------------------- +Queries on ST Objects +--------------------- -Two functions are provided which allow an application to determine if an AST was +Two functions are provided which allow an application to determine if an ST was created as an expression or a suite. Neither of these functions can be used to -determine if an AST was created from source code via :func:`expr` or -:func:`suite` or from a parse tree via :func:`sequence2ast`. +determine if an ST was created from source code via :func:`expr` or +:func:`suite` or from a parse tree via :func:`sequence2st`. .. function:: isexpr(ast) @@ -229,19 +240,19 @@ determine if an AST was created from source code via :func:`expr` or When *ast* represents an ``'eval'`` form, this function returns true, otherwise it returns false. This is useful, since code objects normally cannot be queried for this information using existing built-in functions. Note that the code - objects created by :func:`compileast` cannot be queried like this either, and + objects created by :func:`compilest` cannot be queried like this either, and are identical to those created by the built-in :func:`compile` function. .. function:: issuite(ast) - This function mirrors :func:`isexpr` in that it reports whether an AST object + This function mirrors :func:`isexpr` in that it reports whether an ST object represents an ``'exec'`` form, commonly known as a "suite." It is not safe to assume that this function is equivalent to ``not isexpr(ast)``, as additional syntactic fragments may be supported in the future. -.. _ast-errors: +.. _st-errors: Exceptions and Error Handling ----------------------------- @@ -257,12 +268,12 @@ function for information about the exceptions it can raise. generally produced for validation failures rather than the built in :exc:`SyntaxError` thrown during normal parsing. The exception argument is either a string describing the reason of the failure or a tuple containing a - sequence causing the failure from a parse tree passed to :func:`sequence2ast` - and an explanatory string. Calls to :func:`sequence2ast` need to be able to + sequence causing the failure from a parse tree passed to :func:`sequence2st` + and an explanatory string. Calls to :func:`sequence2st` need to be able to handle either type of exception, while calls to other functions in the module will only need to be aware of the simple string values. -Note that the functions :func:`compileast`, :func:`expr`, and :func:`suite` may +Note that the functions :func:`compilest`, :func:`expr`, and :func:`suite` may throw exceptions which are normally thrown by the parsing and compilation process. These include the built in exceptions :exc:`MemoryError`, :exc:`OverflowError`, :exc:`SyntaxError`, and :exc:`SystemError`. In these @@ -270,49 +281,49 @@ cases, these exceptions carry all the meaning normally associated with them. Refer to the descriptions of each function for detailed information. -.. _ast-objects: +.. _st-objects: -AST Objects ------------ +ST Objects +---------- -Ordered and equality comparisons are supported between AST objects. Pickling of -AST objects (using the :mod:`pickle` module) is also supported. +Ordered and equality comparisons are supported between ST objects. Pickling of +ST objects (using the :mod:`pickle` module) is also supported. -.. data:: ASTType +.. data:: STType The type of the objects returned by :func:`expr`, :func:`suite` and - :func:`sequence2ast`. + :func:`sequence2st`. -AST objects have the following methods: +ST objects have the following methods: -.. method:: AST.compile([filename]) +.. method:: ST.compile([filename]) - Same as ``compileast(ast, filename)``. + Same as ``compilest(st, filename)``. -.. method:: AST.isexpr() +.. method:: ST.isexpr() - Same as ``isexpr(ast)``. + Same as ``isexpr(st)``. -.. method:: AST.issuite() +.. method:: ST.issuite() - Same as ``issuite(ast)``. + Same as ``issuite(st)``. -.. method:: AST.tolist([line_info]) +.. method:: ST.tolist([line_info]) - Same as ``ast2list(ast, line_info)``. + Same as ``st2list(st, line_info)``. -.. method:: AST.totuple([line_info]) +.. method:: ST.totuple([line_info]) - Same as ``ast2tuple(ast, line_info)``. + Same as ``st2tuple(st, line_info)``. -.. _ast-examples: +.. _st-examples: Examples -------- @@ -340,27 +351,27 @@ to the code :: 10 The equivalent operation using the :mod:`parser` module is somewhat longer, and -allows the intermediate internal parse tree to be retained as an AST object:: +allows the intermediate internal parse tree to be retained as an ST object:: >>> import parser - >>> ast = parser.expr('a + 5') - >>> code = ast.compile('file.py') + >>> st = parser.expr('a + 5') + >>> code = st.compile('file.py') >>> a = 5 >>> eval(code) 10 -An application which needs both AST and code objects can package this code into +An application which needs both ST and code objects can package this code into readily available functions:: import parser def load_suite(source_string): - ast = parser.suite(source_string) - return ast, ast.compile() + st = parser.suite(source_string) + return st, st.compile() def load_expression(source_string): - ast = parser.expr(source_string) - return ast, ast.compile() + st = parser.expr(source_string) + return st, st.compile() Information Discovery @@ -414,8 +425,8 @@ tuples. :: >>> import parser >>> import pprint - >>> ast = parser.suite(open('docstring.py').read()) - >>> tup = ast.totuple() + >>> st = parser.suite(open('docstring.py').read()) + >>> tup = st.totuple() >>> pprint.pprint(tup) (257, (264, @@ -670,8 +681,8 @@ file :file:`example.py`.) :: source = open(fileName).read() basename = os.path.basename(os.path.splitext(fileName)[0]) - ast = parser.suite(source) - return ModuleInfo(ast.totuple(), basename) + st = parser.suite(source) + return ModuleInfo(st.totuple(), basename) This provides an easy-to-use interface to the documentation of a module. If information is required which is not extracted by the code of this example, the diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 2381f24838..22c174b2f1 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -643,6 +643,17 @@ impossible to detect the termination of alien threads. constructor. +.. method:: Thread.getIdent() + + Return the 'thread identifier' of this thread or None if the thread has not + been started. This is a nonzero integer. See the :mod:`thread` module's + :func:`get_ident()` function. Thread identifiers may be recycled when a + thread exits and another thread is created. The identifier is returned + even after the thread has exited. + + .. versionadded:: 2.6 + + .. method:: Thread.isAlive() Return whether the thread is alive. diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index a5169f34bc..925fd0fd31 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -2205,10 +2205,10 @@ Port-Specific Changes * MacOS X (10.3 and higher): dynamic loading of modules now uses the :cfunc:`dlopen` function instead of MacOS-specific functions. -* MacOS X: a :option:`--enable-universalsdk` switch was added to the +* MacOS X: an :option:`--enable-universalsdk` switch was added to the :program:`configure` script that compiles the interpreter as a universal binary able to run on both PowerPC and Intel processors. (Contributed by Ronald - Oussoren.) + Oussoren; :issue:`2573`.) * Windows: :file:`.dll` is no longer supported as a filename extension for extension modules. :file:`.pyd` is now the only filename extension that will be diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 54ff968826..c4c909379a 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -648,6 +648,7 @@ the type's :meth:`__format__` method with the provided specifier:: >>> format(75.6564, '.2f') '75.66' + .. seealso:: :ref:`formatstrings` @@ -1252,6 +1253,11 @@ Here are all of the changes that Python 2.6 makes to the core Python language. (Contributed by Alexander Belopolsky; :issue:`1686487`.) +* A new built-in, ``next(*iterator*, [*default*])`` returns the next item + from the specified iterator. If the *default* argument is supplied, + it will be returned if *iterator* has been exhausted; otherwise, + the :exc:`StopIteration` exception will be raised. (:issue:`2719`) + * Tuples now have an :meth:`index` method matching the list type's :meth:`index` method:: @@ -1553,6 +1559,7 @@ details. :mod:`terminalcommand`. A number of old IRIX-specific modules were deprecated: + :mod:`al` and :mod:`AL`, :mod:`cd`, :mod:`cddb`, :mod:`cdplayer`, @@ -1664,7 +1671,11 @@ details. (Contributed by Raymond Hettinger.) -* The :mod:`ctypes` module now supports a :class:`c_bool` datatype +* XXX Describe the new ctypes calling convention that allows safe + access to errno. + (Implemented by Thomas Heller; :issue:`1798`.) + +* The :mod:`ctypes` module now supports a :class:`c_bool` datatype that represents the C99 ``bool`` type. (Contributed by David Remahl; :issue:`1649190`.) @@ -1733,7 +1744,14 @@ details. to drop the built-in in the 2.x series. (Patched by Christian Heimes; :issue:`1739906`.) -* The :func:`glob.glob` function can now return Unicode filenames if +* When possible, the :mod:`getpass` module will now use + :file:`/dev/tty` (when available) to print + a prompting message and read the password, falling back to using + standard error and standard input. If the password may be echoed to + the terminal, a warning is printed before the prompt is displayed. + (Contributed by Gregory P. Smith.) + +* The :func:`glob.glob` function can now return Unicode filenames if a Unicode path was used and Unicode filenames are matched within the directory. (:issue:`1001604`) @@ -1752,6 +1770,10 @@ details. This is more efficient than making a call to :func:`heappush` and then :func:`heappop`. + :mod:`heapq` is now implemented to only use less-than comparison, + instead of the less-than-or-equal comparison it previously used. + This makes :mod:`heapq`'s usage of a type match that of the + :meth:`list.sort` method. (Contributed by Raymond Hettinger.) * An optional ``timeout`` parameter was added to the @@ -1846,6 +1868,11 @@ details. is true, opening of the log file is deferred until the first :meth:`emit` call is made. (Contributed by Vinay Sajip.) + :class:`TimedRotatingFileHandler` also has a *utc* constructor + parameter. If the argument is true, UTC time will be used + in determining when midnight occurs and in generating filenames; + otherwise local time will be used. + * The :mod:`macfs` module has been removed. This in turn required the :func:`macostools.touched` function to be removed because it depended on the :mod:`macfs` module. (:issue:`1490190`) @@ -2112,13 +2139,21 @@ details. are written or not. (Contributed by Neal Norwitz and Georg Brandl.) - Information about the command-line arguments supplied to the Python - interpreter are available as attributes of a ``sys.flags`` named - tuple. For example, the :attr:`verbose` attribute is true if Python + Information about the command-line arguments supplied to the Python + interpreter is available by reading attributes of a named + tuple available as ``sys.flags``. For example, the :attr:`verbose` + attribute is true if Python was executed in verbose mode, :attr:`debug` is true in debugging mode, etc. These attributes are all read-only. (Contributed by Christian Heimes.) + A new function, :func:`getsizeof`, takes a Python object and returns + the amount of memory used by the object, measured in bytes. Built-in + objects return correct results; third-party extensions may not, + but can define a :meth:`__sizeof__` method to return the + object's size. + (Contributed by Robert Schuppenies; :issue:`2898`.) + It's now possible to determine the current profiler and tracer functions by calling :func:`sys.getprofile` and :func:`sys.gettrace`. (Contributed by Georg Brandl; :issue:`1648`.) @@ -2204,7 +2239,11 @@ details. (Contributed by Dwayne Bailey; :issue:`1581073`.) -* The :mod:`timeit` module now accepts callables as well as strings +* The :mod:`threading` module's :class:`Thread` objects + gained a :meth:`getIdent` method that returns the thread's + identifier, a nonzero integer. (Contributed by XXX; :issue:`2871`.) + +* The :mod:`timeit` module now accepts callables as well as strings for the statement being timed and for the setup code. Two convenience functions were added for creating :class:`Timer` instances: @@ -2213,6 +2252,24 @@ details. the corresponding method. (Contributed by Erik Demaine; :issue:`1533909`.) +* The :mod:`turtle` module for turtle graphics was greatly enhanced by + Gregor Lingl. New features in the module include: + + * Better animation of turtle movement and rotation. + * Control over turtle movement using the new delay(), + tracer(), and speed() methods. + * The ability to set new shapes for the turtle, and to + define a new coordinate system. + * Turtles now have an undo() method that can roll back actions. + * Simple support for reacting to input events such as mouse and keyboard + activity, making it possible to write simple games. + * A :file:`turtle.cfg` file can be used to customize the starting appearance + of the turtle's screen. + * The module's docstrings can be replaced by new docstrings that have been + translated into another language. + + (:issue:`1513695`) + * An optional ``timeout`` parameter was added to the :func:`urllib.urlopen` function and the :class:`urllib.ftpwrapper` class constructor, as well as the @@ -2255,8 +2312,10 @@ details. not necessarily correct for all applications. Code using :mod:`xmlrpclib` should convert :class:`date` and :class:`time` instances. (:issue:`1330538`) The code can also handle - dates before 1900. (Contributed by Ralf Schmitt; :issue:`2014`.) - + dates before 1900 (contributed by Ralf Schmitt; :issue:`2014`) + and 64-bit integers represented by using ``<i8>`` in XML-RPC responses + (contributed by XXX; :issue:`2985`). + * The :mod:`zipfile` module's :class:`ZipFile` class now has :meth:`extract` and :meth:`extractall` methods that will unpack a single file or all the files in the archive to the current directory, or @@ -2272,9 +2331,14 @@ details. (Contributed by Alan McIntyre; :issue:`467924`.) - Also, :mod:`zipfile` now supports using Unicode filenames - for archived files. (Contributed by Alexey Borzenkov; :issue:`1734346`.) + The :meth:`open`, :meth:`read` and :meth:`extract` methods can now + take either a filename or a :class:`ZipInfo` object. This is useful when an + archive accidentally contains a duplicated filename. + (Contributed by Graham Horler; :issue:`1775025`.) + Finally, :mod:`zipfile` now supports using Unicode filenames + for archived files. (Contributed by Alexey Borzenkov; :issue:`1734346`.) + .. ====================================================================== .. whole new modules get described in subsections here @@ -2469,10 +2533,8 @@ Changes to Python's build process and to the C API include: results, and then compiles using these results for optimization. (Contributed by Gregory P. Smith.) - .. ====================================================================== - Port-Specific Changes: Windows ----------------------------------- @@ -2517,6 +2579,16 @@ Port-Specific Changes: Windows .. ====================================================================== +Port-Specific Changes: MacOS X +----------------------------------- + +* When compiling a framework build of Python, you can now specify the + framework name to be used by providing the + :option:`--with-framework-name=` option to the + :program:`configure` script. + +.. ====================================================================== + .. _section-other: diff --git a/Include/bytearrayobject.h b/Include/bytearrayobject.h index 2c0b7344c7..265b4bbdd0 100644 --- a/Include/bytearrayobject.h +++ b/Include/bytearrayobject.h @@ -1,7 +1,7 @@ -/* Bytes object interface */ +/* ByteArray object interface */ -#ifndef Py_BYTESOBJECT_H -#define Py_BYTESOBJECT_H +#ifndef Py_BYTEARRAYOBJECT_H +#define Py_BYTEARRAYOBJECT_H #ifdef __cplusplus extern "C" { #endif @@ -50,4 +50,4 @@ PyAPI_FUNC(int) PyByteArray_Resize(PyObject *, Py_ssize_t); #ifdef __cplusplus } #endif -#endif /* !Py_BYTESOBJECT_H */ +#endif /* !Py_BYTEARRAYOBJECT_H */ diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 529b028c82..b2d451a536 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -1,8 +1,8 @@ -/* String object interface */ +/* Bytes (String) object interface */ -#ifndef Py_STRINGOBJECT_H -#define Py_STRINGOBJECT_H +#ifndef Py_BYTESOBJECT_H +#define Py_BYTESOBJECT_H #ifdef __cplusplus extern "C" { #endif @@ -107,4 +107,4 @@ PyAPI_FUNC(int) _PyBytes_InsertThousandsGrouping(char *buffer, #ifdef __cplusplus } #endif -#endif /* !Py_STRINGOBJECT_H */ +#endif /* !Py_BYTESOBJECT_H */ diff --git a/Lib/ast.py b/Lib/ast.py new file mode 100644 index 0000000000..2aac2dd0fb --- /dev/null +++ b/Lib/ast.py @@ -0,0 +1,300 @@ +# -*- coding: utf-8 -*- +""" + ast + ~~~ + + The `ast` module helps Python applications to process trees of the Python + abstract syntax grammar. The abstract syntax itself might change with + each Python release; this module helps to find out programmatically what + the current grammar looks like and allows modifications of it. + + An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as + a flag to the `compile()` builtin function or by using the `parse()` + function from this module. The result will be a tree of objects whose + classes all inherit from `ast.AST`. + + A modified abstract syntax tree can be compiled into a Python code object + using the built-in `compile()` function. + + Additionally various helper functions are provided that make working with + the trees simpler. The main intention of the helper functions and this + module in general is to provide an easy to use interface for libraries + that work tightly with the python syntax (template engines for example). + + + :copyright: Copyright 2008 by Armin Ronacher. + :license: Python License. +""" +from _ast import * + + +def parse(expr, filename='<unknown>', mode='exec'): + """ + Parse an expression into an AST node. + Equivalent to compile(expr, filename, mode, PyCF_ONLY_AST). + """ + return compile(expr, filename, mode, PyCF_ONLY_AST) + + +def literal_eval(node_or_string): + """ + Safely evaluate an expression node or a string containing a Python + expression. The string or node provided may only consist of the following + Python literal structures: strings, numbers, tuples, lists, dicts, booleans, + and None. + """ + _safe_names = {'None': None, 'True': True, 'False': False} + if isinstance(node_or_string, str): + node_or_string = parse(node_or_string, mode='eval') + if isinstance(node_or_string, Expression): + node_or_string = node_or_string.body + def _convert(node): + if isinstance(node, Str): + return node.s + elif isinstance(node, Num): + return node.n + elif isinstance(node, Tuple): + return tuple(map(_convert, node.elts)) + elif isinstance(node, List): + return list(map(_convert, node.elts)) + elif isinstance(node, Dict): + return dict((_convert(k), _convert(v)) for k, v + in zip(node.keys, node.values)) + elif isinstance(node, Name): + if node.id in _safe_names: + return _safe_names[node.id] + raise ValueError('malformed string') + return _convert(node_or_string) + + +def dump(node, annotate_fields=True, include_attributes=False): + """ + Return a formatted dump of the tree in *node*. This is mainly useful for + debugging purposes. The returned string will show the names and the values + for fields. This makes the code impossible to evaluate, so if evaluation is + wanted *annotate_fields* must be set to False. Attributes such as line + numbers and column offsets are dumped by default. If this is wanted, + *include_attributes* can be set to True. + """ + def _format(node): + if isinstance(node, AST): + fields = [(a, _format(b)) for a, b in iter_fields(node)] + rv = '%s(%s' % (node.__class__.__name__, ', '.join( + ('%s=%s' % field for field in fields) + if annotate_fields else + (b for a, b in fields) + )) + if include_attributes and node._attributes: + rv += fields and ', ' or ' ' + rv += ', '.join('%s=%s' % (a, _format(getattr(node, a))) + for a in node._attributes) + return rv + ')' + elif isinstance(node, list): + return '[%s]' % ', '.join(_format(x) for x in node) + return repr(node) + if not isinstance(node, AST): + raise TypeError('expected AST, got %r' % node.__class__.__name__) + return _format(node) + + +def copy_location(new_node, old_node): + """ + Copy source location (`lineno` and `col_offset` attributes) from + *old_node* to *new_node* if possible, and return *new_node*. + """ + for attr in 'lineno', 'col_offset': + if attr in old_node._attributes and attr in new_node._attributes \ + and hasattr(old_node, attr): + setattr(new_node, attr, getattr(old_node, attr)) + return new_node + + +def fix_missing_locations(node): + """ + When you compile a node tree with compile(), the compiler expects lineno and + col_offset attributes for every node that supports them. This is rather + tedious to fill in for generated nodes, so this helper adds these attributes + recursively where not already set, by setting them to the values of the + parent node. It works recursively starting at *node*. + """ + def _fix(node, lineno, col_offset): + if 'lineno' in node._attributes: + if not hasattr(node, 'lineno'): + node.lineno = lineno + else: + lineno = node.lineno + if 'col_offset' in node._attributes: + if not hasattr(node, 'col_offset'): + node.col_offset = col_offset + else: + col_offset = node.col_offset + for child in iter_child_nodes(node): + _fix(child, lineno, col_offset) + _fix(node, 1, 0) + return node + + +def increment_lineno(node, n=1): + """ + Increment the line number of each node in the tree starting at *node* by *n*. + This is useful to "move code" to a different location in a file. + """ + if 'lineno' in node._attributes: + node.lineno = getattr(node, 'lineno', 0) + n + for child in walk(node): + if 'lineno' in child._attributes: + child.lineno = getattr(child, 'lineno', 0) + n + return node + + +def iter_fields(node): + """ + Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` + that is present on *node*. + """ + for field in node._fields: + try: + yield field, getattr(node, field) + except AttributeError: + pass + + +def iter_child_nodes(node): + """ + Yield all direct child nodes of *node*, that is, all fields that are nodes + and all items of fields that are lists of nodes. + """ + for name, field in iter_fields(node): + if isinstance(field, AST): + yield field + elif isinstance(field, list): + for item in field: + if isinstance(item, AST): + yield item + + +def get_docstring(node, clean=True): + """ + Return the docstring for the given node or None if no docstring can + be found. If the node provided does not have docstrings a TypeError + will be raised. + """ + if not isinstance(node, (FunctionDef, ClassDef, Module)): + raise TypeError("%r can't have docstrings" % node.__class__.__name__) + if node.body and isinstance(node.body[0], Expr) and \ + isinstance(node.body[0].value, Str): + if clean: + import inspect + return inspect.cleandoc(node.body[0].value.s) + return node.body[0].value.s + + +def walk(node): + """ + Recursively yield all child nodes of *node*, in no specified order. This is + useful if you only want to modify nodes in place and don't care about the + context. + """ + from collections import deque + todo = deque([node]) + while todo: + node = todo.popleft() + todo.extend(iter_child_nodes(node)) + yield node + + +class NodeVisitor(object): + """ + A node visitor base class that walks the abstract syntax tree and calls a + visitor function for every node found. This function may return a value + which is forwarded by the `visit` method. + + This class is meant to be subclassed, with the subclass adding visitor + methods. + + Per default the visitor functions for the nodes are ``'visit_'`` + + class name of the node. So a `TryFinally` node visit function would + be `visit_TryFinally`. This behavior can be changed by overriding + the `visit` method. If no visitor function exists for a node + (return value `None`) the `generic_visit` visitor is used instead. + + Don't use the `NodeVisitor` if you want to apply changes to nodes during + traversing. For this a special visitor exists (`NodeTransformer`) that + allows modifications. + """ + + def visit(self, node): + """Visit a node.""" + method = 'visit_' + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + return visitor(node) + + def generic_visit(self, node): + """Called if no explicit visitor function exists for a node.""" + for field, value in iter_fields(node): + if isinstance(value, list): + for item in value: + if isinstance(item, AST): + self.visit(item) + elif isinstance(value, AST): + self.visit(value) + + +class NodeTransformer(NodeVisitor): + """ + A :class:`NodeVisitor` subclass that walks the abstract syntax tree and + allows modification of nodes. + + The `NodeTransformer` will walk the AST and use the return value of the + visitor methods to replace or remove the old node. If the return value of + the visitor method is ``None``, the node will be removed from its location, + otherwise it is replaced with the return value. The return value may be the + original node in which case no replacement takes place. + + Here is an example transformer that rewrites all occurrences of name lookups + (``foo``) to ``data['foo']``:: + + class RewriteName(NodeTransformer): + + def visit_Name(self, node): + return copy_location(Subscript( + value=Name(id='data', ctx=Load()), + slice=Index(value=Str(s=node.id)), + ctx=node.ctx + ), node) + + Keep in mind that if the node you're operating on has child nodes you must + either transform the child nodes yourself or call the :meth:`generic_visit` + method for the node first. + + For nodes that were part of a collection of statements (that applies to all + statement nodes), the visitor may also return a list of nodes rather than + just a single node. + + Usually you use the transformer like this:: + + node = YourTransformer().visit(node) + """ + + def generic_visit(self, node): + for field, old_value in iter_fields(node): + old_value = getattr(node, field, None) + if isinstance(old_value, list): + new_values = [] + for value in old_value: + if isinstance(value, AST): + value = self.visit(value) + if value is None: + continue + elif not isinstance(value, AST): + new_values.extend(value) + continue + new_values.append(value) + old_value[:] = new_values + elif isinstance(old_value, AST): + new_node = self.visit(old_value) + if new_node is None: + delattr(node, field) + else: + setattr(node, field, new_node) + return node diff --git a/Lib/inspect.py b/Lib/inspect.py index 6039d4e47f..5758abd3d8 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -368,6 +368,13 @@ def getdoc(object): return None if not isinstance(doc, str): return None + return cleandoc(doc) + +def cleandoc(doc): + """Clean up indentation from docstrings. + + Any whitespace that can be uniformly removed from the second line + onwards is removed.""" try: lines = doc.expandtabs().split('\n') except UnicodeError: diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 4c74cf7dbf..2aa3b8fb00 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1,6 +1,6 @@ import sys, unittest from test import support -import _ast +import ast def to_tuple(t): if t is None or isinstance(t, (str, int, complex)): @@ -117,9 +117,9 @@ eval_tests = [ class AST_Tests(unittest.TestCase): def _assert_order(self, ast_node, parent_pos): - if not isinstance(ast_node, _ast.AST) or ast_node._fields is None: + if not isinstance(ast_node, ast.AST) or ast_node._fields is None: return - if isinstance(ast_node, (_ast.expr, _ast.stmt, _ast.excepthandler)): + if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)): node_pos = (ast_node.lineno, ast_node.col_offset) self.assert_(node_pos >= parent_pos) parent_pos = (ast_node.lineno, ast_node.col_offset) @@ -136,29 +136,29 @@ class AST_Tests(unittest.TestCase): (single_tests, single_results, "single"), (eval_tests, eval_results, "eval")): for i, o in zip(input, output): - ast_tree = compile(i, "?", kind, _ast.PyCF_ONLY_AST) + ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) self.assertEquals(to_tuple(ast_tree), o) self._assert_order(ast_tree, (0, 0)) def test_nodeclasses(self): - x = _ast.BinOp(1, 2, 3, lineno=0) + x = ast.BinOp(1, 2, 3, lineno=0) self.assertEquals(x.left, 1) self.assertEquals(x.op, 2) self.assertEquals(x.right, 3) self.assertEquals(x.lineno, 0) # node raises exception when not given enough arguments - self.assertRaises(TypeError, _ast.BinOp, 1, 2) + self.assertRaises(TypeError, ast.BinOp, 1, 2) # can set attributes through kwargs too - x = _ast.BinOp(left=1, op=2, right=3, lineno=0) + x = ast.BinOp(left=1, op=2, right=3, lineno=0) self.assertEquals(x.left, 1) self.assertEquals(x.op, 2) self.assertEquals(x.right, 3) self.assertEquals(x.lineno, 0) # this used to fail because Sub._fields was None - x = _ast.Sub() + x = ast.Sub() def test_pickling(self): import pickle @@ -175,8 +175,99 @@ class AST_Tests(unittest.TestCase): ast2 = mod.loads(mod.dumps(ast, protocol)) self.assertEquals(to_tuple(ast2), to_tuple(ast)) + +class ASTHelpers_Test(unittest.TestCase): + + def test_parse(self): + a = ast.parse('foo(1 + 1)') + b = compile('foo(1 + 1)', '<unknown>', 'exec', ast.PyCF_ONLY_AST) + self.assertEqual(ast.dump(a), ast.dump(b)) + + def test_dump(self): + node = ast.parse('spam(eggs, "and cheese")') + self.assertEqual(ast.dump(node), + "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " + "args=[Name(id='eggs', ctx=Load()), Str(s='and cheese')], " + "keywords=[], starargs=None, kwargs=None))])" + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " + "Str('and cheese')], [], None, None))])" + ) + self.assertEqual(ast.dump(node, include_attributes=True), + "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), " + "lineno=1, col_offset=5), Str(s='and cheese', lineno=1, " + "col_offset=11)], keywords=[], starargs=None, kwargs=None, " + "lineno=1, col_offset=0), lineno=1, col_offset=0)])" + ) + + def test_copy_location(self): + src = ast.parse('1 + 1', mode='eval') + src.body.right = ast.copy_location(ast.Num(2), src.body.right) + self.assertEqual(ast.dump(src, include_attributes=True), + 'Expression(body=BinOp(left=Num(n=1, lineno=1, col_offset=0), ' + 'op=Add(), right=Num(n=2, lineno=1, col_offset=4), lineno=1, ' + 'col_offset=0))' + ) + + def test_fix_missing_locations(self): + src = ast.parse('write("spam")') + src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), + [ast.Str('eggs')], [], None, None))) + self.assertEqual(src, ast.fix_missing_locations(src)) + self.assertEqual(ast.dump(src, include_attributes=True), + "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " + "lineno=1, col_offset=0), args=[Str(s='spam', lineno=1, " + "col_offset=6)], keywords=[], starargs=None, kwargs=None, " + "lineno=1, col_offset=0), lineno=1, col_offset=0), " + "Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, " + "col_offset=0), args=[Str(s='eggs', lineno=1, col_offset=0)], " + "keywords=[], starargs=None, kwargs=None, lineno=1, " + "col_offset=0), lineno=1, col_offset=0)])" + ) + + def test_increment_lineno(self): + src = ast.parse('1 + 1', mode='eval') + self.assertEqual(ast.increment_lineno(src, n=3), src) + self.assertEqual(ast.dump(src, include_attributes=True), + 'Expression(body=BinOp(left=Num(n=1, lineno=4, col_offset=0), ' + 'op=Add(), right=Num(n=1, lineno=4, col_offset=4), lineno=4, ' + 'col_offset=0))' + ) + + def test_iter_fields(self): + node = ast.parse('foo()', mode='eval') + d = dict(ast.iter_fields(node.body)) + self.assertEqual(d.pop('func').id, 'foo') + self.assertEqual(d, {'keywords': [], 'kwargs': None, + 'args': [], 'starargs': None}) + + def test_iter_child_nodes(self): + node = ast.parse("spam(23, 42, eggs='leek')", mode='eval') + self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4) + iterator = ast.iter_child_nodes(node.body) + self.assertEqual(next(iterator).id, 'spam') + self.assertEqual(next(iterator).n, 23) + self.assertEqual(next(iterator).n, 42) + self.assertEqual(ast.dump(next(iterator)), + "keyword(arg='eggs', value=Str(s='leek'))" + ) + + def test_get_docstring(self): + node = ast.parse('def foo():\n """line one\n line two"""') + self.assertEqual(ast.get_docstring(node.body[0]), + 'line one\nline two') + + def test_literal_eval(self): + self.assertEqual(ast.literal_eval('[1, 2, 3]'), [1, 2, 3]) + self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42}) + self.assertEqual(ast.literal_eval('(True, False, None)'), (True, False, None)) + self.assertRaises(ValueError, ast.literal_eval, 'foo()') + + def test_main(): - support.run_unittest(AST_Tests) + support.run_unittest(AST_Tests, ASTHelpers_Test) def main(): if __name__ != '__main__': diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 44c8999b6f..ab3c1d1d2d 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -352,6 +352,14 @@ class ComplexTest(unittest.TestCase): except (OSError, IOError): pass + def test_getnewargs(self): + self.assertEqual((1+2j).__getnewargs__(), (1.0, 2.0)) + self.assertEqual((1-2j).__getnewargs__(), (1.0, -2.0)) + self.assertEqual((2j).__getnewargs__(), (0.0, 2.0)) + self.assertEqual((-0j).__getnewargs__(), (0.0, -0.0)) + self.assertEqual(complex(0, INF).__getnewargs__(), (0.0, INF)) + self.assertEqual(complex(INF, 0).__getnewargs__(), (INF, 0.0)) + if float.__getformat__("double").startswith("IEEE"): def test_plus_minus_0j(self): # test that -0j and 0j literals are not identified diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 55fae20734..7b85616205 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -195,6 +195,10 @@ class TestRetrievingSourceCode(GetSourceBase): self.assertEqual(inspect.getdoc(git.abuse), 'Another\n\ndocstring\n\ncontaining\n\ntabs') + def test_cleandoc(self): + self.assertEqual(inspect.cleandoc('An\n indented\n docstring.'), + 'An\nindented\ndocstring.') + def test_getcomments(self): self.assertEqual(inspect.getcomments(mod), '# line 1\n') self.assertEqual(inspect.getcomments(mod.StupidGit), '# line 20\n') diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 3c09b0bd27..15edb1b141 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -3,6 +3,7 @@ import test.support from test.support import verbose import random +import re import sys import threading import _thread @@ -71,6 +72,8 @@ class ThreadTests(unittest.TestCase): for i in range(NUMTASKS): t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning) threads.append(t) + self.failUnlessEqual(t.getIdent(), None) + self.assert_(re.match('<TestThread\(.*, initial\)>', repr(t))) t.start() if verbose: @@ -78,6 +81,8 @@ class ThreadTests(unittest.TestCase): for t in threads: t.join(NUMTASKS) self.assert_(not t.isAlive()) + self.failIfEqual(t.getIdent(), 0) + self.assert_(re.match('<TestThread\(.*, \w+ -?\d+\)>', repr(t))) if verbose: print('all tasks done') self.assertEqual(numrunning.get(), 0) diff --git a/Lib/threading.py b/Lib/threading.py index 673d88e8ef..bce291ed31 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -401,6 +401,7 @@ class Thread(_Verbose): self._args = args self._kwargs = kwargs self._daemonic = self._set_daemon() + self._ident = None self._started = Event() self._stopped = False self._block = Condition(Lock()) @@ -421,7 +422,9 @@ class Thread(_Verbose): if self._stopped: status = "stopped" if self._daemonic: - status = status + " daemon" + status += " daemon" + if self._ident is not None: + status += " %s" % self._ident return "<%s(%s, %s)>" % (self.__class__.__name__, self._name, status) def start(self): @@ -469,9 +472,10 @@ class Thread(_Verbose): def _bootstrap_inner(self): try: + self._ident = _get_ident() self._started.set() _active_limbo_lock.acquire() - _active[_get_ident()] = self + _active[self._ident] = self del _limbo[self] _active_limbo_lock.release() if __debug__: @@ -536,7 +540,7 @@ class Thread(_Verbose): with _active_limbo_lock: self._stop() try: - # We don't call self.__delete() because it also + # We don't call self._delete() because it also # grabs _active_limbo_lock. del _active[_get_ident()] except: @@ -625,6 +629,10 @@ class Thread(_Verbose): assert self._initialized, "Thread.__init__() not called" self._name = str(name) + def getIdent(self): + assert self._initialized, "Thread.__init__() not called" + return self._ident + def isAlive(self): assert self._initialized, "Thread.__init__() not called" return self._started.isSet() and not self._stopped @@ -155,6 +155,7 @@ Ben Darnell Jonathan Dasteel John DeGood Vincent Delft +Arnaud Delobelle Erik Demaine Roger Dev Raghuram Devarakonda @@ -573,6 +574,7 @@ Andy Robinson Kevin Rodgers Giampaolo Rodola Mike Romberg +Armin Ronacher Case Roole Timothy Roscoe Jim Roskind diff --git a/Misc/developers.txt b/Misc/developers.txt index c97900b997..2904a6840a 100644 --- a/Misc/developers.txt +++ b/Misc/developers.txt @@ -17,6 +17,9 @@ the format to accommodate documentation needs as they arise. Permissions History ------------------- +- Gregor Lingl was given SVN access on 10 June 2008 by MvL, + for work on the turtle module. + - Robert Schuppenies was given SVN access on 21 May 2008 by MvL, for GSoC contributions. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index a8063c5e1a..fb613021f1 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -2019,7 +2019,7 @@ empty: } PyDoc_STRVAR(combinations_doc, -"combinations(iterables) --> combinations object\n\ +"combinations(iterable[, r]) --> combinations object\n\ \n\ Return successive r-length combinations of elements in the iterable.\n\n\ combinations(range(4), 3) --> (0,1,2), (0,1,3), (0,2,3), (1,2,3)"); @@ -2294,10 +2294,10 @@ empty: } PyDoc_STRVAR(permutations_doc, -"permutations(iterables[, r]) --> permutations object\n\ +"permutations(iterable[, r]) --> permutations object\n\ \n\ Return successive r-length permutations of elements in the iterable.\n\n\ -permutations(range(4), 3) --> (0,1,2), (0,1,3), (0,2,3), (1,2,3)"); +permutations(range(3), 2) --> (0,1), (0,2), (1,0), (1,2), (2,0), (2,1)"); static PyTypeObject permutations_type = { PyVarObject_HEAD_INIT(NULL, 0) diff --git a/Objects/complexobject.c b/Objects/complexobject.c index acd5a4a1a2..2e501fc0c5 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -693,7 +693,8 @@ PyDoc_STRVAR(complex_conjugate_doc, static PyObject * complex_getnewargs(PyComplexObject *v) { - return Py_BuildValue("(D)", &v->cval); + Py_complex c = v->cval; + return Py_BuildValue("(dd)", c.real, c.imag); } #if 0 |