Issue #17119: Fixed integer overflows when processing large strings and tuples

in the tkinter module.

12 files changed, 131 insertions, 43 deletions

diff --git a/Lib/lib2to3/btm_utils.py b/Lib/lib2to3/btm_utils.py
index 2276dc9e96..339750edba 100644
--- a/Lib/lib2to3/btm_utils.py
+++ b/Lib/lib2to3/btm_utils.py
@@ -96,8 +96,7 @@ class MinNode(object):
     def leaves(self):
         "Generator that returns the leaves of the tree"
         for child in self.children:
-            for x in child.leaves():
-                yield x
+            yield from child.leaves()
         if not self.children:
             yield self
 
@@ -277,7 +276,6 @@ def rec_test(sequence, test_func):
     sub-iterables"""
     for x in sequence:
         if isinstance(x, (list, tuple)):
-            for y in rec_test(x, test_func):
-                yield y
+            yield from rec_test(x, test_func)
         else:
             yield test_func(x)
diff --git a/Lib/lib2to3/fixer_util.py b/Lib/lib2to3/fixer_util.py
index 60d219f577..6e259c54ac 100644
--- a/Lib/lib2to3/fixer_util.py
+++ b/Lib/lib2to3/fixer_util.py
@@ -129,6 +129,29 @@ def FromImport(package_name, name_leafs):
     imp = Node(syms.import_from, children)
     return imp
 
+def ImportAndCall(node, results, names):
+    """Returns an import statement and calls a method
+    of the module:
+
+    import module
+    module.name()"""
+    obj = results["obj"].clone()
+    if obj.type == syms.arglist:
+        newarglist = obj.clone()
+    else:
+        newarglist = Node(syms.arglist, [obj.clone()])
+    after = results["after"]
+    if after:
+        after = [n.clone() for n in after]
+    new = Node(syms.power,
+               Attr(Name(names[0]), Name(names[1])) +
+               [Node(syms.trailer,
+                     [results["lpar"].clone(),
+                      newarglist,
+                      results["rpar"].clone()])] + after)
+    new.prefix = node.prefix
+    return new
+
 
 ###########################################################
 ### Determine whether a node represents a given literal
diff --git a/Lib/lib2to3/fixes/fix_intern.py b/Lib/lib2to3/fixes/fix_intern.py
index 6be11cd39d..fb2973c242 100644
--- a/Lib/lib2to3/fixes/fix_intern.py
+++ b/Lib/lib2to3/fixes/fix_intern.py
@@ -6,9 +6,8 @@
 intern(s) -> sys.intern(s)"""
 
 # Local imports
-from .. import pytree
 from .. import fixer_base
-from ..fixer_util import Name, Attr, touch_import
+from ..fixer_util import ImportAndCall, touch_import
 
 
 class FixIntern(fixer_base.BaseFix):
@@ -26,21 +25,7 @@ class FixIntern(fixer_base.BaseFix):
     """
 
     def transform(self, node, results):
-        syms = self.syms
-        obj = results["obj"].clone()
-        if obj.type == syms.arglist:
-            newarglist = obj.clone()
-        else:
-            newarglist = pytree.Node(syms.arglist, [obj.clone()])
-        after = results["after"]
-        if after:
-            after = [n.clone() for n in after]
-        new = pytree.Node(syms.power,
-                          Attr(Name("sys"), Name("intern")) +
-                          [pytree.Node(syms.trailer,
-                                       [results["lpar"].clone(),
-                                        newarglist,
-                                        results["rpar"].clone()])] + after)
-        new.prefix = node.prefix
+        names = ('sys', 'intern')
+        new = ImportAndCall(node, results, names)
         touch_import(None, 'sys', node)
         return new
diff --git a/Lib/lib2to3/fixes/fix_reload.py b/Lib/lib2to3/fixes/fix_reload.py
new file mode 100644
index 0000000000..1855357588
--- /dev/null
+++ b/Lib/lib2to3/fixes/fix_reload.py
@@ -0,0 +1,28 @@
+"""Fixer for reload().
+
+reload(s) -> imp.reload(s)"""
+
+# Local imports
+from .. import fixer_base
+from ..fixer_util import ImportAndCall, touch_import
+
+
+class FixReload(fixer_base.BaseFix):
+    BM_compatible = True
+    order = "pre"
+
+    PATTERN = """
+    power< 'reload'
+           trailer< lpar='('
+                    ( not(arglist | argument<any '=' any>) obj=any
+                      | obj=arglist<(not argument<any '=' any>) any ','> )
+                    rpar=')' >
+           after=any*
+    >
+    """
+
+    def transform(self, node, results):
+        names = ('imp', 'reload')
+        new = ImportAndCall(node, results, names)
+        touch_import(None, 'imp', node)
+        return new
diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py
index f9cc18b5d3..93bae9021a 100644
--- a/Lib/lib2to3/main.py
+++ b/Lib/lib2to3/main.py
@@ -90,11 +90,11 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
             if os.path.lexists(backup):
                 try:
                     os.remove(backup)
-                except os.error as err:
+                except OSError as err:
                     self.log_message("Can't remove backup %s", backup)
             try:
                 os.rename(filename, backup)
-            except os.error as err:
+            except OSError as err:
                 self.log_message("Can't rename %s to %s", filename, backup)
         # Actually write the new file
         write = super(StdoutRefactoringTool, self).write_file
diff --git a/Lib/lib2to3/pgen2/conv.py b/Lib/lib2to3/pgen2/conv.py
index bf49762ae4..ed0cac532e 100644
--- a/Lib/lib2to3/pgen2/conv.py
+++ b/Lib/lib2to3/pgen2/conv.py
@@ -60,7 +60,7 @@ class Converter(grammar.Grammar):
         """
         try:
             f = open(filename)
-        except IOError as err:
+        except OSError as err:
             print("Can't open %s: %s" % (filename, err))
             return False
         self.symbol2number = {}
@@ -111,7 +111,7 @@ class Converter(grammar.Grammar):
         """
         try:
             f = open(filename)
-        except IOError as err:
+        except OSError as err:
             print("Can't open %s: %s" % (filename, err))
             return False
         # The code below essentially uses f's iterator-ness!
diff --git a/Lib/lib2to3/pgen2/driver.py b/Lib/lib2to3/pgen2/driver.py
index 4c611c6640..3ccc69dcf6 100644
--- a/Lib/lib2to3/pgen2/driver.py
+++ b/Lib/lib2to3/pgen2/driver.py
@@ -123,7 +123,7 @@ def load_grammar(gt="Grammar.txt", gp=None,
             logger.info("Writing grammar tables to %s", gp)
             try:
                 g.dump(gp)
-            except IOError as e:
+            except OSError as e:
                 logger.info("Writing failed:"+str(e))
     else:
         g = grammar.Grammar()
diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py
index 14c5f70f1d..7f1c5648e2 100644
--- a/Lib/lib2to3/pgen2/grammar.py
+++ b/Lib/lib2to3/pgen2/grammar.py
@@ -86,15 +86,13 @@ class Grammar(object):
 
     def dump(self, filename):
         """Dump the grammar tables to a pickle file."""
-        f = open(filename, "wb")
-        pickle.dump(self.__dict__, f, 2)
-        f.close()
+        with open(filename, "wb") as f:
+            pickle.dump(self.__dict__, f, 2)
 
     def load(self, filename):
         """Load the grammar tables from a pickle file."""
-        f = open(filename, "rb")
-        d = pickle.load(f)
-        f.close()
+        with open(filename, "rb") as f:
+            d = pickle.load(f)
         self.__dict__.update(d)
 
     def copy(self):
diff --git a/Lib/lib2to3/pytree.py b/Lib/lib2to3/pytree.py
index 17cbf0a2f9..c4a1be3500 100644
--- a/Lib/lib2to3/pytree.py
+++ b/Lib/lib2to3/pytree.py
@@ -194,8 +194,7 @@ class Base(object):
 
     def leaves(self):
         for child in self.children:
-            for x in child.leaves():
-                yield x
+            yield from child.leaves()
 
     def depth(self):
         if self.parent is None:
@@ -274,16 +273,14 @@ class Node(Base):
     def post_order(self):
         """Return a post-order iterator for the tree."""
         for child in self.children:
-            for node in child.post_order():
-                yield node
+            yield from child.post_order()
         yield self
 
     def pre_order(self):
         """Return a pre-order iterator for the tree."""
         yield self
         for child in self.children:
-            for node in child.pre_order():
-                yield node
+            yield from child.pre_order()
 
     def _prefix_getter(self):
         """
diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py
index 201e193fe2..8100317871 100644
--- a/Lib/lib2to3/refactor.py
+++ b/Lib/lib2to3/refactor.py
@@ -326,7 +326,7 @@ class RefactoringTool(object):
         """
         try:
             f = open(filename, "rb")
-        except IOError as err:
+        except OSError as err:
             self.log_error("Can't open %s: %s", filename, err)
             return None, None
         try:
@@ -534,12 +534,12 @@ class RefactoringTool(object):
         """
         try:
             f = _open_with_encoding(filename, "w", encoding=encoding)
-        except os.error as err:
+        except OSError as err:
             self.log_error("Can't create %s: %s", filename, err)
             return
         try:
             f.write(_to_system_newlines(new_text))
-        except os.error as err:
+        except OSError as err:
             self.log_error("Can't write %s: %s", filename, err)
         finally:
             f.close()
diff --git a/Lib/lib2to3/tests/pytree_idempotency.py b/Lib/lib2to3/tests/pytree_idempotency.py
index a02bbfe201..731c403120 100755
--- a/Lib/lib2to3/tests/pytree_idempotency.py
+++ b/Lib/lib2to3/tests/pytree_idempotency.py
@@ -53,7 +53,7 @@ def main():
     for dir in sys.path:
         try:
             names = os.listdir(dir)
-        except os.error:
+        except OSError:
             continue
         print("Scanning", dir, "...", file=sys.stderr)
         for name in names:
diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py
index 914b3bf928..d7659fa267 100644
--- a/Lib/lib2to3/tests/test_fixers.py
+++ b/Lib/lib2to3/tests/test_fixers.py
@@ -282,6 +282,65 @@ class Test_apply(FixerTestCase):
         b = """f(*args, **kwds)"""
         self.check(a, b)
 
+class Test_reload(FixerTestCase):
+    fixer = "reload"
+
+    def test(self):
+        b = """reload(a)"""
+        a = """import imp\nimp.reload(a)"""
+        self.check(b, a)
+
+    def test_comment(self):
+        b = """reload( a ) # comment"""
+        a = """import imp\nimp.reload( a ) # comment"""
+        self.check(b, a)
+
+        # PEP 8 comments
+        b = """reload( a )  # comment"""
+        a = """import imp\nimp.reload( a )  # comment"""
+        self.check(b, a)
+
+    def test_space(self):
+        b = """reload( a )"""
+        a = """import imp\nimp.reload( a )"""
+        self.check(b, a)
+
+        b = """reload( a)"""
+        a = """import imp\nimp.reload( a)"""
+        self.check(b, a)
+
+        b = """reload(a )"""
+        a = """import imp\nimp.reload(a )"""
+        self.check(b, a)
+
+    def test_unchanged(self):
+        s = """reload(a=1)"""
+        self.unchanged(s)
+
+        s = """reload(f, g)"""
+        self.unchanged(s)
+
+        s = """reload(f, *h)"""
+        self.unchanged(s)
+
+        s = """reload(f, *h, **i)"""
+        self.unchanged(s)
+
+        s = """reload(f, **i)"""
+        self.unchanged(s)
+
+        s = """reload(*h, **i)"""
+        self.unchanged(s)
+
+        s = """reload(*h)"""
+        self.unchanged(s)
+
+        s = """reload(**i)"""
+        self.unchanged(s)
+
+        s = """reload()"""
+        self.unchanged(s)
+
 class Test_intern(FixerTestCase):
     fixer = "intern"