refactor line numbering handling

--HG--
branch : _ast_compat

6 files changed, 207 insertions, 188 deletions

diff --git a/_nodes_ast.py b/_nodes_ast.py
index d0e252e4..c0233173 100644
--- a/_nodes_ast.py
+++ b/_nodes_ast.py
@@ -146,6 +146,7 @@ def _init_set_doc(node):
     node.doc = None
     try:
         if isinstance(node.body[0], Discard) and isinstance(node.body[0].value, Str):
+            node.tolineno = node.body[0].lineno
             node.doc = node.body[0].value.s
             node.body = node.body[1:]
     except IndexError:
diff --git a/lookup.py b/lookup.py
index 3d6717bd..f98a5ee4 100644
--- a/lookup.py
+++ b/lookup.py
@@ -136,14 +136,12 @@ def _filter_stmts(self, stmts, frame, offset):
         return stmts
     mystmt = self.statement()
     # line filtering if we are in the same frame
-    if myframe is frame:
-        try:
-            mylineno = mystmt.source_line() + offset
-        except TypeError, ex:
-            # node has no lineno information (this is the case for nodes
-            # inserted for living objects for instance)
-            # disabling lineno filtering
-            mylineno = 0
+    #
+    # take care node may be missing lineno information (this is the case for
+    # nodes inserted for living objects)
+    if myframe is frame and mystmt.fromlineno is not None:
+        assert mystmt.fromlineno is not None, mystmt
+        mylineno = mystmt.fromlineno + offset
     else:
         # disabling lineno filtering
         mylineno = 0
@@ -152,7 +150,7 @@ def _filter_stmts(self, stmts, frame, offset):
     for node in stmts:
         stmt = node.statement()
         # line filtering is on and we have reached our location, break
-        if mylineno > 0 and stmt.source_line() > mylineno:
+        if mylineno > 0 and stmt.fromlineno > mylineno:
             break
         if isinstance(node, nodes.Class) and self in node.bases:
             break
diff --git a/nodes.py b/nodes.py
index 1e363025..4d59f01e 100644
--- a/nodes.py
+++ b/nodes.py
@@ -41,6 +41,8 @@ __docformat__ = "restructuredtext en"
 
 from itertools import imap
 
+from logilab.common.decorators import monkeypatch
+
 try:
     from logilab.astng._nodes_ast import *
     from logilab.astng._nodes_ast import _const_factory
@@ -215,58 +217,54 @@ class NodeNG:
         given list of nodes
         """
         myroot = self.root()
-        mylineno = self.source_line()
+        mylineno = self.fromlineno
         nearest = None, 0
         for node in nodes:
             assert node.root() is myroot, \
                    'not from the same module %s' % (self, node)
-            lineno = node.source_line()
-            if node.source_line() > mylineno:
+            lineno = node.fromlineno
+            if node.fromlineno > mylineno:
                 break
             if lineno > nearest[1]:
                 nearest = node, lineno
         # FIXME: raise an exception if nearest is None ?
         return nearest[0]
 
-    def source_line(self):
+    def set_line_info(self, lastchild):
+        if self.lineno is None:
+            self.fromlineno = self._fixed_source_line()
+        else:
+            self.fromlineno = self.lineno
+        if lastchild is None:
+            self.tolineno = self.fromlineno
+        else:
+            self.tolineno = lastchild.tolineno
+        assert self.fromlineno is not None, self
+        assert self.tolineno is not None, self
+        
+    def _fixed_source_line(self):
         """return the line number where the given node appears
 
         we need this method since not all nodes have the lineno attribute
         correctly set...
         """
         line = self.lineno
-        if line is None:
-            _node = self
-            try:
-                while line is None:
-                    _node = _node.get_children().next()
-                    line = _node.lineno
-            except StopIteration:
-                _node = self.parent
-                while _node and line is None:
-                    line = _node.lineno
-                    _node = _node.parent
-            self.lineno = line
+        _node = self
+        try:
+            while line is None:
+                _node = _node.get_children().next()
+                line = _node.lineno
+        except StopIteration:
+            _node = self.parent
+            while _node and line is None:
+                line = _node.lineno
+                _node = _node.parent
         return line
     
-    def last_source_line(self):
-        """return the last block line number for this node (i.e. including
-        children)
-        """
-        try:
-            return self.__dict__['_cached_last_source_line']
-        except KeyError:
-            line = self.source_line()
-            # XXX the latest children would be enough, no ?
-            for node in self.get_children():
-                line = max(line, node.last_source_line())
-            self._cached_last_source_line = line
-            return line
-
     def block_range(self, lineno):
         """handle block line numbers range for non block opening statements
         """
-        return lineno, self.last_source_line()
+        return lineno, self.tolineno
 
     def set_local(self, name, stmt):
         """delegate to a scoped parent handling a locals dictionary
@@ -383,7 +381,7 @@ def object_block_range(node, lineno):
 
     start from the "def" or "class" position whatever the given lineno
     """
-    return node.source_line(), node.last_source_line()
+    return node.fromlineno, node.tolineno
 
 Function.block_range = object_block_range
 Class.block_range = object_block_range
@@ -392,25 +390,31 @@ Module.block_range = object_block_range
 # XXX only if compiler mode ?
 def if_block_range(node, lineno):
     """handle block line numbers range for if/elif statements"""
-    if lineno == node.body[0].source_line():
+    if lineno == node.body[0].fromlineno:
         return lineno, lineno
-    if lineno <= node.body[-1].last_source_line():
-        return lineno, node.body[-1].last_source_line()
-    return elsed_block_range(node, lineno, node.body[0].source_line() - 1)
+    if lineno <= node.body[-1].tolineno:
+        return lineno, node.body[-1].tolineno
+    return elsed_block_range(node, lineno, node.body[0].fromlineno - 1)
 
 If.block_range = if_block_range
 
+@monkeypatch(If)
+def set_line_info(self, lastchild):
+    self.fromlineno = self.lineno
+    self.tolineno = lastchild.tolineno
+    self.blockstart_tolineno = self.test.tolineno
+
 def try_except_block_range(node, lineno):
     """handle block line numbers range for try/except statements
     """
     last = None
     for exhandler in node.handlers:
-        if exhandler.type and lineno == exhandler.type.source_line():
+        if exhandler.type and lineno == exhandler.type.fromlineno:
             return lineno, lineno
-        if exhandler.body[0].source_line() <= lineno <= exhandler.body[-1].last_source_line():
-            return lineno, exhandler.body[-1].last_source_line()
+        if exhandler.body[0].fromlineno <= lineno <= exhandler.body[-1].tolineno:
+            return lineno, exhandler.body[-1].tolineno
         if last is None:
-            last = exhandler.body[0].source_line() - 1
+            last = exhandler.body[0].fromlineno - 1
     return elsed_block_range(node, lineno, last)
 
 TryExcept.block_range = try_except_block_range
@@ -419,13 +423,13 @@ def elsed_block_range(node, lineno, last=None):
     """handle block line numbers range for try/finally, for and while
     statements
     """
-    if lineno == node.source_line():
+    if lineno == node.fromlineno:
         return lineno, lineno
     if node.orelse:
-        if lineno >= node.orelse[0].source_line():
-            return lineno, node.orelse[-1].last_source_line()
-        return lineno, node.orelse[0].source_line() - 1
-    return lineno, last or node.last_source_line()
+        if lineno >= node.orelse[0].fromlineno:
+            return lineno, node.orelse[-1].tolineno
+        return lineno, node.orelse[0].fromlineno - 1
+    return lineno, last or node.tolineno
 
 TryFinally.block_range = elsed_block_range
 While.block_range = elsed_block_range
diff --git a/rebuilder.py b/rebuilder.py
index 113defa3..38694e14 100644
--- a/rebuilder.py
+++ b/rebuilder.py
@@ -86,12 +86,17 @@ class RebuildVisitor(ASTVisitor):
 
     def _walk(self, node, parent=None):
         """default visit method, handle the parent attribute"""
+        node.fromlineno = node.lineno
         node.parent = parent
         node.accept(self.rebuilder)
         handle_leave = node.accept(self)
+        child = None
         for child in node.get_children():
             self.set_context(node, child)
             self._walk(child, node)
+            if self.asscontext is child:
+                self.asscontext = None
+        node.set_line_info(child)
         if handle_leave:
             leave = getattr(self, "leave_" + node.__class__.__name__.lower() )
             leave(node)
diff --git a/scoped_nodes.py b/scoped_nodes.py
index b912080c..c025b59a 100644
--- a/scoped_nodes.py
+++ b/scoped_nodes.py
@@ -312,9 +312,11 @@ class FunctionNG(object):
 
     # function's type, 'function' | 'method' | 'staticmethod' | 'classmethod'
     type = 'function'
-    # list of argument names. MAY BE NONE on some builtin functions where
-    # arguments are unknown
-    argnames = None
+    
+    def set_line_info(self, lastchild):
+        self.fromlineno = self.lineno
+        self.tolineno = lastchild.tolineno
+        self.blockstart_tolineno = self.args.tolineno
 
     def pytype(self):
         if 'method' in self.type:
@@ -542,7 +544,14 @@ class ClassNG(object):
     newstyle = property(_newstyle_impl,
                         doc="boolean indicating if it's a new style class"
                         "or not")
-
+    
+    def set_line_info(self, lastchild):
+        self.fromlineno = self.lineno
+        self.blockstart_tolineno = self.bases and self.bases[-1].tolineno or self.fromlineno
+        if lastchild is not None:
+            self.tolineno = lastchild.tolineno
+        # else this is a class with only a docstring, then tolineno is (should be) already ok
+        
     def pytype(self):
         if self.newstyle:
             return '__builtin__.type'
diff --git a/test/unittest_builder.py b/test/unittest_builder.py
index d03eb88b..d6235688 100644
--- a/test/unittest_builder.py
+++ b/test/unittest_builder.py
@@ -10,8 +10,7 @@
 # You should have received a copy of the GNU General Public License along with
 # this program; if not, write to the Free Software Foundation, Inc.,
 # 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-"""tests for the astng builder module
-"""
+"""tests for the astng builder and rebuilder module"""
 
 import unittest
 import sys
@@ -28,115 +27,105 @@ from logilab.astng.nodes_as_string import as_string
 import data
 from data import module as test_module
 
-if nodes.AST_MODE == 'compiler':
-
-    class TransformerTC(TestCase):
-
-        def setUp(self):
-            transformer = patchcomptransformer.ASTNGTransformer()
-            self.astng = transformer.parsesuite(open('data/format.py').read())
-
-        def test_callfunc_lineno(self):
-            stmts = self.astng.getChildNodes()[0].nodes
-            # on line 4:
-            #    function('aeozrijz\
-            #    earzer', hop)
-            discard = stmts[0]
-            self.assertIsInstance(discard, nodes.Discard)
-            self.assertEquals(discard.fromlineno, 4)
-            self.assertEquals(discard.tolineno, 5)
-            callfunc = discard.expr
-            self.assertIsInstance(callfunc, nodes.CallFunc)
-            self.assertEquals(callfunc.fromlineno, 4)
-            self.assertEquals(callfunc.tolineno, 5)
-            name = callfunc.node
-            self.assertIsInstance(name, nodes.Name)
-            self.assertEquals(name.fromlineno, 4)
-            self.assertEquals(name.tolineno, 4)
-            strarg = callfunc.args[0]
-            self.assertIsInstance(strarg, nodes.Const)
-            self.assertEquals(strarg.fromlineno, 5) # no way for this one (is 4 actually)
-            self.assertEquals(strarg.tolineno, 5)
-            namearg = callfunc.args[1]
-            self.assertIsInstance(namearg, nodes.Name)
-            self.assertEquals(namearg.fromlineno, 5)
-            self.assertEquals(namearg.tolineno, 5)
-            # on line 10:
-            #    fonction(1,
-            #             2,
-            #             3,
-            #             4)
-            discard = stmts[2]
-            self.assertIsInstance(discard, nodes.Discard)
-            self.assertEquals(discard.fromlineno, 10)
-            self.assertEquals(discard.tolineno, 13)
-            callfunc = discard.expr
-            self.assertIsInstance(callfunc, nodes.CallFunc)
-            self.assertEquals(callfunc.fromlineno, 10)
-            self.assertEquals(callfunc.tolineno, 13)
-            name = callfunc.node
-            self.assertIsInstance(name, nodes.Name)
-            self.assertEquals(name.fromlineno, 10)
-            self.assertEquals(name.tolineno, 10)
-            for i, arg in enumerate(callfunc.args):
-                self.assertIsInstance(arg, nodes.Const)
-                self.assertEquals(arg.fromlineno, 10+i)
-                self.assertEquals(arg.tolineno, 10+i)
-
-        def test_function_lineno(self):
-            stmts = self.astng.getChildNodes()[0].nodes
-            # on line 15:
-            #    def definition(a,
-            #                   b,
-            #                   c):
-            #        return a + b + c
-            function = stmts[3]
-            self.assertIsInstance(function, nodes.Function)
-            self.assertEquals(function.fromlineno, 15)
-            self.assertEquals(function.tolineno, 17)
-            code = function.code
-            self.assertIsInstance(code, nodes.Stmt)
-    ##         self.assertEquals(code.fromlineno, 18)
-    ##         self.assertEquals(code.tolineno, 18)
-            return_ = code.nodes[0]
-            self.assertIsInstance(return_, nodes.Return)
-            self.assertEquals(return_.fromlineno, 18)
-            self.assertEquals(return_.tolineno, 18)
-
-        def test_class_lineno(self):
-            stmts = self.astng.getChildNodes()[0].nodes
-            # on line 20:
-            #    class debile(dict,
-            #                 object):
-            #       pass
-            class_ = stmts[4]
-            self.assertIsInstance(class_, nodes.Class)
-            self.assertEquals(class_.fromlineno, 20)
-            self.assertEquals(class_.tolineno, 21)
-            code = class_.code
-            self.assertIsInstance(code, nodes.Stmt)
-    ##         self.assertEquals(code.fromlineno, 18)
-    ##         self.assertEquals(code.tolineno, 18)
-            pass_ = code.nodes[0]
-            self.assertIsInstance(pass_, nodes.Pass)
-            self.assertEquals(pass_.fromlineno, 22)
-            self.assertEquals(pass_.tolineno, 22)
-
-        def test_if_lineno(self):
-            stmts = self.astng.getChildNodes()[0].nodes
-            # on line 20:
-            #    if aaaa: pass
-            #    else:
-            #        aaaa,bbbb = 1,2
-            #        aaaa,bbbb = bbbb,aaaa
-            if_ = stmts[5]
-            self.assertIsInstance(if_, nodes.If)
-            self.assertEquals(if_.fromlineno, 24)
-            self.assertEquals(if_.tolineno, 24)
-            else_ = if_.else_
-            self.assertIsInstance(else_, nodes.Stmt)
-            self.assertEquals(else_.fromlineno, 25)
-            self.assertEquals(else_.tolineno, 27)
+class FromToLineNoTC(TestCase):
+
+    def setUp(self):
+        self.astng = builder.ASTNGBuilder().file_build('data/format.py')
+
+    def test_callfunc_lineno(self):
+        stmts = self.astng.body
+        # on line 4:
+        #    function('aeozrijz\
+        #    earzer', hop)
+        discard = stmts[0]
+        self.assertIsInstance(discard, nodes.Discard)
+        self.assertEquals(discard.fromlineno, 4)
+        self.assertEquals(discard.tolineno, 5)
+        callfunc = discard.value
+        self.assertIsInstance(callfunc, nodes.CallFunc)
+        self.assertEquals(callfunc.fromlineno, 4)
+        self.assertEquals(callfunc.tolineno, 5)
+        name = callfunc.func
+        self.assertIsInstance(name, nodes.Name)
+        self.assertEquals(name.fromlineno, 4)
+        self.assertEquals(name.tolineno, 4)
+        strarg = callfunc.args[0]
+        self.assertIsInstance(strarg, nodes.Const)
+        self.assertEquals(strarg.fromlineno, 5) # no way for this one (is 4 actually)
+        self.assertEquals(strarg.tolineno, 5)
+        namearg = callfunc.args[1]
+        self.assertIsInstance(namearg, nodes.Name)
+        self.assertEquals(namearg.fromlineno, 5)
+        self.assertEquals(namearg.tolineno, 5)
+        # on line 10:
+        #    fonction(1,
+        #             2,
+        #             3,
+        #             4)
+        discard = stmts[2]
+        self.assertIsInstance(discard, nodes.Discard)
+        self.assertEquals(discard.fromlineno, 10)
+        self.assertEquals(discard.tolineno, 13)
+        callfunc = discard.value
+        self.assertIsInstance(callfunc, nodes.CallFunc)
+        self.assertEquals(callfunc.fromlineno, 10)
+        self.assertEquals(callfunc.tolineno, 13)
+        name = callfunc.func
+        self.assertIsInstance(name, nodes.Name)
+        self.assertEquals(name.fromlineno, 10)
+        self.assertEquals(name.tolineno, 10)
+        for i, arg in enumerate(callfunc.args):
+            self.assertIsInstance(arg, nodes.Const)
+            self.assertEquals(arg.fromlineno, 10+i)
+            self.assertEquals(arg.tolineno, 10+i)
+
+    def test_function_lineno(self):
+        stmts = self.astng.body
+        # on line 15:
+        #    def definition(a,
+        #                   b,
+        #                   c):
+        #        return a + b + c
+        function = stmts[3]
+        self.assertIsInstance(function, nodes.Function)
+        self.assertEquals(function.fromlineno, 15)
+        self.assertEquals(function.tolineno, 18)
+        self.assertEquals(function.blockstart_tolineno, 17)
+        return_ = function.body[0]
+        self.assertIsInstance(return_, nodes.Return)
+        self.assertEquals(return_.fromlineno, 18)
+        self.assertEquals(return_.tolineno, 18)
+
+    def test_class_lineno(self):
+        stmts = self.astng.body
+        # on line 20:
+        #    class debile(dict,
+        #                 object):
+        #       pass
+        class_ = stmts[4]
+        self.assertIsInstance(class_, nodes.Class)
+        self.assertEquals(class_.fromlineno, 20)
+        self.assertEquals(class_.tolineno, 22)
+        self.assertEquals(class_.blockstart_tolineno, 21)
+        pass_ = class_.body[0]
+        self.assertIsInstance(pass_, nodes.Pass)
+        self.assertEquals(pass_.fromlineno, 22)
+        self.assertEquals(pass_.tolineno, 22)
+
+    def test_if_lineno(self):
+        stmts = self.astng.body
+        # on line 20:
+        #    if aaaa: pass
+        #    else:
+        #        aaaa,bbbb = 1,2
+        #        aaaa,bbbb = bbbb,aaaa
+        if_ = stmts[5]
+        self.assertIsInstance(if_, nodes.If)
+        self.assertEquals(if_.fromlineno, 24)
+        self.assertEquals(if_.tolineno, 27)
+        self.assertEquals(if_.blockstart_tolineno, 24)
+        self.assertEquals(if_.orelse[0].fromlineno, 26)
+        self.assertEquals(if_.orelse[1].tolineno, 27)
 
         
 class BuilderTC(TestCase):
@@ -289,8 +278,8 @@ def global_no_effect():
         astng = self.builder.string_build(data, __name__, __file__)
         self.failUnlessEqual(len(astng.getattr('CSTE')), 2)
         self.failUnless(isinstance(astng.getattr('CSTE')[0], nodes.AssName))
-        self.failUnlessEqual(astng.getattr('CSTE')[0].source_line(), 2)
-        self.failUnlessEqual(astng.getattr('CSTE')[1].source_line(), 6)
+        self.failUnlessEqual(astng.getattr('CSTE')[0].fromlineno, 2)
+        self.failUnlessEqual(astng.getattr('CSTE')[1].fromlineno, 6)
         self.assertRaises(nodes.NotFoundError,
                           astng.getattr, 'CSTE2')
         self.assertRaises(InferenceError,
@@ -333,7 +322,7 @@ class FileBuildTC(TestCase):
         module = self.module
         self.assertEquals(module.name, 'data.module')
         self.assertEquals(module.doc, "test module for astng\n")
-        self.assertEquals(module.source_line(), 0)
+        self.assertEquals(module.fromlineno, 0)
         self.assertEquals(module.parent, None)
         self.assertEquals(module.frame(), module)
         self.assertEquals(module.root(), module)
@@ -365,12 +354,12 @@ class FileBuildTC(TestCase):
         function = module['global_access']
         self.assertEquals(function.name, 'global_access')
         self.assertEquals(function.doc, 'function test')
-        self.assertEquals(function.source_line(), 15)
+        self.assertEquals(function.fromlineno, 15)
         self.assert_(function.parent)
         self.assertEquals(function.frame(), function)
         self.assertEquals(function.parent.frame(), module)
         self.assertEquals(function.root(), module)
-        self.assertEquals(function.argnames, ['key', 'val'])
+        self.assertEquals([n.name for n in function.args.args], ['key', 'val'])
         self.assertEquals(function.type, 'function')
 
     def test_function_locals(self):
@@ -387,7 +376,7 @@ class FileBuildTC(TestCase):
         klass = module['YO']
         self.assertEquals(klass.name, 'YO')
         self.assertEquals(klass.doc, 'hehe')
-        self.assertEquals(klass.source_line(), 28)
+        self.assertEquals(klass.fromlineno, 28)
         self.assert_(klass.parent)
         self.assertEquals(klass.frame(), klass)
         self.assertEquals(klass.parent.frame(), module)
@@ -433,17 +422,17 @@ class FileBuildTC(TestCase):
         # "normal" method
         method = klass2['method']
         self.assertEquals(method.name, 'method')
-        self.assertEquals(method.argnames, ['self'])
+        self.assertEquals([n.name for n in method.args.args], ['self'])
         self.assertEquals(method.doc, 'method test')
-        self.assertEquals(method.source_line(), 48)
+        self.assertEquals(method.fromlineno, 48)
         self.assertEquals(method.type, 'method')
         # class method
         method = klass2['class_method']
-        self.assertEquals(method.argnames, ['cls'])
+        self.assertEquals([n.name for n in method.args.args], ['cls'])
         self.assertEquals(method.type, 'classmethod')
         # static method
         method = klass2['static_method']
-        self.assertEquals(method.argnames, [])
+        self.assertEquals(method.args.args, [])
         self.assertEquals(method.type, 'staticmethod')
 
     def test_method_locals(self):
@@ -465,19 +454,32 @@ class ModuleBuildTC(FileBuildTC):
 
 
 class InferedBuildTC(TestCase):
-
-    def test(self):
-        from logilab import astng
-        import compiler
-        sn = astng.MANAGER.astng_from_file(join(astng.__path__[0], 'scoped_nodes.py'))
-        astastng = astng.MANAGER.astng_from_file(join(compiler.__path__[0], 'ast.py'))
-        # check monkey patching of the compiler module has been infered
-        lclass = list(astastng.igetattr('Lambda'))
+    code = '''class A: pass
+A.type = "class"
+
+def A_ass_type(self):
+    print self
+A.ass_type = A_ass_type
+    '''
+    def test_0(self):
+        astng = builder.ASTNGBuilder().string_build(self.code)
+        lclass = list(astng.igetattr('A'))
         self.assertEquals(len(lclass), 1)
         lclass = lclass[0]
-        self.assert_('format_args' in lclass.locals, lclass.locals.keys())
+        self.assert_('ass_type' in lclass.locals, lclass.locals.keys())
         self.assert_('type' in lclass.locals.keys())
         
+#     def test_1(self):
+#         from logilab import astng
+#         import compiler
+#         sn = astng.MANAGER.astng_from_file(join(astng.__path__[0], 'inference.py'))
+#         astastng = astng.MANAGER.astng_from_file(join(compiler.__path__[0], 'ast.py'))
+#         # check monkey patching of the compiler module has been infered
+#         lclass = list(astastng.igetattr('Function'))
+#         self.assertEquals(len(lclass), 1)
+#         lclass = lclass[0]
+#         self.assert_('ass_type' in lclass.locals, lclass.locals.keys())
+        
 #__all__ = ('BuilderModuleBuildTC', 'BuilderFileBuildTC', 'BuilderTC')
 
 if __name__ == '__main__':