Merge issue-324 fix

11 files changed, 207 insertions, 92 deletions

diff --git a/AUTHORS.txt b/AUTHORS.txt
index fb2f0bc..66e27aa 100644
--- a/AUTHORS.txt
+++ b/AUTHORS.txt
@@ -43,6 +43,7 @@ Marcus Cobden
 Mark van der Wal
 Martin Fuzzey
 Matthew Desmarais
+Mickie Betz
 Noel O'Boyle
 Pablo Carballo
 Patrick Mezard
diff --git a/coverage/parser.py b/coverage/parser.py
index f488367..fc751eb 100644
--- a/coverage/parser.py
+++ b/coverage/parser.py
@@ -460,7 +460,7 @@ class ByteParser(object):
 
         # Walk the byte codes building chunks.
         for bc in bytecodes:
-            # Maybe have to start a new chunk
+            # Maybe have to start a new chunk.
             start_new_chunk = False
             first_chunk = False
             if bc.offset in bytes_lines_map:
@@ -481,9 +481,13 @@ class ByteParser(object):
                 if chunk:
                     chunk.exits.add(bc.offset)
                 chunk = Chunk(bc.offset, chunk_lineno, first_chunk)
+                if not chunks:
+                    # The very first chunk of a code object is always an
+                    # entrance.
+                    chunk.entrance = True
                 chunks.append(chunk)
 
-            # Look at the opcode
+            # Look at the opcode.
             if bc.jump_to >= 0 and bc.op not in OPS_NO_JUMP:
                 if ignore_branch:
                     # Someone earlier wanted us to ignore this branch.
@@ -570,15 +574,15 @@ class ByteParser(object):
         """
         chunks = self._split_into_chunks()
 
-        # A map from byte offsets to chunks jumped into.
+        # A map from byte offsets to the chunk starting at that offset.
         byte_chunks = dict((c.byte, c) for c in chunks)
 
-        # There's always an entrance at the first chunk.
-        yield (-1, byte_chunks[0].line)
-
         # Traverse from the first chunk in each line, and yield arcs where
         # the trace function will be invoked.
         for chunk in chunks:
+            if chunk.entrance:
+                yield (-1, chunk.line)
+
             if not chunk.first:
                 continue
 
@@ -586,7 +590,7 @@ class ByteParser(object):
             chunks_to_consider = [chunk]
             while chunks_to_consider:
                 # Get the chunk we're considering, and make sure we don't
-                # consider it again
+                # consider it again.
                 this_chunk = chunks_to_consider.pop()
                 chunks_considered.add(this_chunk)
 
@@ -649,6 +653,8 @@ class Chunk(object):
 
     .. _basic block: http://en.wikipedia.org/wiki/Basic_block
 
+    `byte` is the offset to the bytecode starting this chunk.
+
     `line` is the source line number containing this chunk.
 
     `first` is true if this is the first chunk in the source line.
@@ -656,19 +662,24 @@ class Chunk(object):
     An exit < 0 means the chunk can leave the code (return).  The exit is
     the negative of the starting line number of the code block.
 
+    The `entrance` attribute is a boolean indicating whether the code object
+    can be entered at this chunk.
+
     """
     def __init__(self, byte, line, first):
         self.byte = byte
         self.line = line
         self.first = first
         self.length = 0
+        self.entrance = False
         self.exits = set()
 
     def __repr__(self):
-        if self.first:
-            bang = "!"
-        else:
-            bang = ""
-        return "<%d+%d @%d%s %r>" % (
-            self.byte, self.length, self.line, bang, list(self.exits)
+        return "<%d+%d @%d%s%s %r>" % (
+            self.byte,
+            self.length,
+            self.line,
+            "!" if self.first else "",
+            "v" if self.entrance else "",
+            list(self.exits),
             )
diff --git a/coverage/pytracer.py b/coverage/pytracer.py
index 0eafbef..3f03aaf 100644
--- a/coverage/pytracer.py
+++ b/coverage/pytracer.py
@@ -1,7 +1,15 @@
 """Raw data collector for Coverage."""
 
+import dis
 import sys
 
+from coverage import env
+
+# We need the YIELD_VALUE opcode below, in a comparison-friendly form.
+YIELD_VALUE = dis.opmap['YIELD_VALUE']
+if env.PY2:
+    YIELD_VALUE = chr(YIELD_VALUE)
+
 
 class PyTracer(object):
     """Python implementation of the raw data tracer."""
@@ -79,9 +87,11 @@ class PyTracer(object):
                 if tracename not in self.data:
                     self.data[tracename] = {}
                 self.cur_file_dict = self.data[tracename]
-            # Set the last_line to -1 because the next arc will be entering a
-            # code block, indicated by (-1, n).
-            self.last_line = -1
+            # The call event is really a "start frame" event, and happens for
+            # function calls and re-entering generators.  The f_lasti field is
+            # -1 for calls, and a real offset for generators.  Use -1 as the
+            # line number for calls, and the real line number for generators.
+            self.last_line = -1 if (frame.f_lasti < 0) else frame.f_lineno
         elif event == 'line':
             # Record an executed line.
             if self.cur_file_dict is not None:
@@ -93,8 +103,12 @@ class PyTracer(object):
                 self.last_line = lineno
         elif event == 'return':
             if self.arcs and self.cur_file_dict:
-                first = frame.f_code.co_firstlineno
-                self.cur_file_dict[(self.last_line, -first)] = None
+                # Record an arc leaving the function, but beware that a
+                # "return" event might just mean yielding from a generator.
+                bytecode = frame.f_code.co_code[frame.f_lasti]
+                if bytecode != YIELD_VALUE:
+                    first = frame.f_code.co_firstlineno
+                    self.cur_file_dict[(self.last_line, -first)] = None
             # Leaving this function, pop the filename stack.
             self.cur_file_dict, self.last_line = self.data_stack.pop()
         elif event == 'exception':
diff --git a/coverage/results.py b/coverage/results.py
index c1718d4..7b621c1 100644
--- a/coverage/results.py
+++ b/coverage/results.py
@@ -101,10 +101,13 @@ class Analysis(object):
         # Exclude arcs here which connect a line to itself.  They can occur
         # in executed data in some cases.  This is where they can cause
         # trouble, and here is where it's the least burden to remove them.
+        # Also, generators can somehow cause arcs from "enter" to "exit", so
+        # make sure we have at least one positive value.
         unpredicted = (
             e for e in executed
                 if e not in possible
                     and e[0] != e[1]
+                    and (e[0] > 0 or e[1] > 0)
         )
         return sorted(unpredicted)
 
diff --git a/coverage/tracer.c b/coverage/tracer.c
index 52543b8..1ce5ed2 100644
--- a/coverage/tracer.c
+++ b/coverage/tracer.c
@@ -3,6 +3,7 @@
 #include "Python.h"
 #include "structmember.h"
 #include "frameobject.h"
+#include "opcode.h"
 
 /* Compile-time debugging helpers */
 #undef WHAT_LOG         /* Define to log the WHAT params in the trace function. */
@@ -613,8 +614,16 @@ CTracer_handle_call(CTracer *self, PyFrameObject *frame)
         SHOWLOG(self->pdata_stack->depth, frame->f_lineno, filename, "skipped");
     }
 
+<<<<<<< local
     self->cur_entry.disposition = disposition;
     self->cur_entry.last_line = -1;
+=======
+    /* A call event is really a "start frame" event, and can happen for
+     * re-entering a generator also.  f_lasti is -1 for a true call, and a
+     * real byte offset for a generator re-entry.
+     */
+    self->cur_entry.last_line = (frame->f_lasti < 0) ? -1 : frame->f_lineno;
+>>>>>>> other
 
 ok:
     ret = RET_OK;
@@ -808,9 +817,13 @@ CTracer_handle_return(CTracer *self, PyFrameObject *frame)
     }
     if (self->pdata_stack->depth >= 0) {
         if (self->tracing_arcs && self->cur_entry.file_data) {
-            int first = frame->f_code->co_firstlineno;
-            if (CTracer_record_pair(self, self->cur_entry.last_line, -first) < 0) {
-                goto error;
+            /* Need to distinguish between RETURN_VALUE and YIELD_VALUE. */
+            int bytecode = MyText_AS_STRING(frame->f_code->co_code)[frame->f_lasti];
+            if (bytecode != YIELD_VALUE) {
+                int first = frame->f_code->co_firstlineno;
+                if (CTracer_record_pair(self, self->cur_entry.last_line, -first) < 0) {
+                    goto error;
+                }
             }
         }
 
diff --git a/lab/parser.py b/lab/parser.py
index 1783468..662183a 100644
--- a/lab/parser.py
+++ b/lab/parser.py
@@ -9,8 +9,8 @@ from optparse import OptionParser
 import disgen
 
 from coverage.misc import CoverageException
-from coverage.files import get_python_source
 from coverage.parser import ByteParser, PythonParser
+from coverage.python import get_python_source
 
 opcode_counts = collections.Counter()
 
@@ -82,7 +82,7 @@ class ParserMain(object):
             self.disassemble(bp, histogram=options.histogram)
 
         arcs = bp._all_arcs()
-        if options.chunks and not options.dis:
+        if options.chunks:# and not options.dis:
             chunks = bp._all_chunks()
             if options.recursive:
                 print("%6d: %s" % (len(chunks), filename))
@@ -116,7 +116,7 @@ class ParserMain(object):
                         m2 = 'C'
                     if lineno in cp.excluded:
                         m3 = 'x'
-                    a = arc_chars.get(lineno, '').ljust(arc_width)
+                    a = arc_chars[lineno].ljust(arc_width)
                     print("%4d %s%s%s%s%s %s" %
                                 (lineno, m0, m1, m2, m3, a, ltext)
                         )
@@ -162,12 +162,12 @@ class ParserMain(object):
         dictionary mapping line numbers to ascii strings to draw for that line.
 
         """
-        arc_chars = {}
+        arc_chars = collections.defaultdict(str)
         for lfrom, lto in sorted(arcs):
             if lfrom < 0:
-                arc_chars[lto] = arc_chars.get(lto, '') + 'v'
+                arc_chars[lto] += 'v'
             elif lto < 0:
-                arc_chars[lfrom] = arc_chars.get(lfrom, '') + '^'
+                arc_chars[lfrom] += '^'
             else:
                 if lfrom == lto - 1:
                     # Don't show obvious arcs.
@@ -176,7 +176,7 @@ class ParserMain(object):
                     l1, l2 = lfrom, lto
                 else:
                     l1, l2 = lto, lfrom
-                w = max([len(arc_chars.get(l, '')) for l in range(l1, l2+1)])
+                w = max(len(arc_chars[l]) for l in range(l1, l2+1))
                 for l in range(l1, l2+1):
                     if l == lfrom:
                         ch = '<'
@@ -184,11 +184,11 @@ class ParserMain(object):
                         ch = '>'
                     else:
                         ch = '|'
-                    arc_chars[l] = arc_chars.get(l, '').ljust(w) + ch
+                    arc_chars[l] = arc_chars[l].ljust(w) + ch
                 arc_width = 0
 
         if arc_chars:
-            arc_width = max([len(a) for a in arc_chars.values()])
+            arc_width = max(len(a) for a in arc_chars.values())
         else:
             arc_width = 0
 
diff --git a/lab/run_trace.py b/lab/run_trace.py
new file mode 100644
index 0000000..3822a80
--- /dev/null
+++ b/lab/run_trace.py
@@ -0,0 +1,32 @@
+"""Run a simple trace function on a file of Python code."""
+
+import os, sys
+
+nest = 0
+
+def trace(frame, event, arg):
+    global nest
+
+    if nest is None:
+        # This can happen when Python is shutting down.
+        return None
+
+    print "%s%s %s %d @%d" % (
+        "    " * nest,
+        event,
+        os.path.basename(frame.f_code.co_filename),
+        frame.f_lineno,
+        frame.f_lasti,
+        )
+
+    if event == 'call':
+        nest += 1
+    if event == 'return':
+        nest -= 1
+
+    return trace
+
+the_program = sys.argv[1]
+
+sys.settrace(trace)
+execfile(the_program)
diff --git a/lab/sample.py b/lab/sample.py
deleted file mode 100644
index bb62848..0000000
--- a/lab/sample.py
+++ /dev/null
@@ -1,5 +0,0 @@
-a, b = 1, 0
-if a or b or fn():
-    # Hey
-    a = 3
-d = 4
diff --git a/lab/trace_sample.py b/lab/trace_sample.py
deleted file mode 100644
index 3f81919..0000000
--- a/lab/trace_sample.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import os, sys
-
-global nest
-nest = 0
-
-def trace(frame, event, arg):
-    #if event == 'line':
-    global nest
-
-    print "%s%s %s %d" % (
-        "   " * nest,
-        event,
-        os.path.basename(frame.f_code.co_filename),
-        frame.f_lineno,
-        )
-
-    if event == 'call':
-        nest += 1
-    if event == 'return':
-        nest -= 1
-
-    return trace
-
-def trace2(frame, event, arg):
-    #if event == 'line':
-    global nest
-
-    print "2: %s%s %s %d" % (
-        "   " * nest,
-        event,
-        os.path.basename(frame.f_code.co_filename),
-        frame.f_lineno,
-        )
-
-    if event == 'call':
-        nest += 1
-    if event == 'return':
-        nest -= 1
-
-    return trace2
-
-sys.settrace(trace)
-
-def bar():
-    print "nar"
-
-a = 26
-def foo(n):
-    a = 28
-    sys.settrace(sys.gettrace())
-    bar()
-    a = 30
-    return 2*n
-
-print foo(a)
-#import sample
-#import littleclass
diff --git a/tests/test_arcs.py b/tests/test_arcs.py
index 81fa7e6..2b7dafd 100644
--- a/tests/test_arcs.py
+++ b/tests/test_arcs.py
@@ -560,6 +560,93 @@ class ExceptionArcTest(CoverageTest):
             arcz_missing="67 7B", arcz_unpredicted="68")
 
 
+class YieldTest(CoverageTest):
+    """Arc tests for generators."""
+
+    def test_yield_in_loop(self):
+        self.check_coverage("""\
+            def gen(inp):
+                for n in inp:
+                    yield n
+
+            list(gen([1,2,3]))
+            """,
+            arcz=".1 .2 23 2. 32 15 5.",
+            arcz_missing="",
+            arcz_unpredicted="")
+
+    def test_padded_yield_in_loop(self):
+        self.check_coverage("""\
+            def gen(inp):
+                i = 2
+                for n in inp:
+                    i = 4
+                    yield n
+                    i = 6
+                i = 7
+
+            list(gen([1,2,3]))
+            """,
+            arcz=".1 19 9.  .2 23 34 45 56 63 37 7.",
+            arcz_missing="",
+            arcz_unpredicted="")
+
+    def test_bug_308(self):
+        self.check_coverage("""\
+            def run():
+                for i in range(10):
+                    yield lambda: i
+
+            for f in run():
+                print(f())
+            """,
+            arcz=".1 15 56 65 5.  .2 23 32 2.  .3 3-3",
+            arcz_missing="",
+            arcz_unpredicted="")
+
+        self.check_coverage("""\
+            def run():
+                yield lambda: 100
+                for i in range(10):
+                    yield lambda: i
+
+            for f in run():
+                print(f())
+            """,
+            arcz=".1 16 67 76 6.  .2 23 34 43 3.   2-2  .4 4-4",
+            arcz_missing="",
+            arcz_unpredicted="")
+
+        self.check_coverage("""\
+            def run():
+                yield lambda: 100  # no branch miss
+
+            for f in run():
+                print(f())
+            """,
+            arcz=".1 14 45 54 4.  .2 2.  2-2",
+            arcz_missing="",
+            arcz_unpredicted="")
+
+    def test_bug_324(self):
+        # This code is tricky: the list() call pulls all the values from gen(),
+        # but each of them is a generator itself that is never iterated.  As a
+        # result, the generator expression on line 3 is never entered or run.
+        self.check_coverage("""\
+            def gen(inp):
+                for n in inp:
+                    yield (i * 2 for i in range(n))
+
+            list(gen([1,2,3]))
+            """,
+            arcz=
+                ".1 15 5. "     # The module level
+                ".2 23 32 2. "  # The gen() function
+                ".3 3-3",       # The generator expression
+            arcz_missing=".3 3-3",
+            arcz_unpredicted="")
+
+
 class MiscArcTest(CoverageTest):
     """Miscellaneous arc-measuring tests."""
 
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 244d4c7..81916a9 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -34,6 +34,22 @@ class PythonParserTest(CoverageTest):
             2:1, 3:1, 4:2, 5:1, 7:1, 9:1, 10:1
             })
 
+    def test_generator_exit_counts(self):
+        # https://bitbucket.org/ned/coveragepy/issue/324/yield-in-loop-confuses-branch-coverage
+        parser = self.parse_source("""\
+            def gen(input):
+                for n in inp:
+                    yield (i * 2 for i in range(n))
+
+            list(gen([1,2,3]))
+            """)
+        self.assertEqual(parser.exit_counts(), {
+            1:1,    # def -> list
+            2:2,    # for -> yield; for -> exit
+            3:2,    # yield -> for;  genexp exit
+            5:1,    # list -> exit
+        })
+
     def test_try_except(self):
         parser = self.parse_source("""\
             try: