Fixed new fortran reader and parser bugs.

4 files changed, 144 insertions, 68 deletions

diff --git a/numpy/f2py/lib/block.py b/numpy/f2py/lib/block.py
index f7cdb7dd3..b9bafdcff 100644
--- a/numpy/f2py/lib/block.py
+++ b/numpy/f2py/lib/block.py
@@ -70,7 +70,8 @@ class Block:
                         'assuming the end of undefined PROGRAM statement',
                         item.span[0],item.span[1])
                     print >> sys.stderr, message
-                    p = Program(self,Program.start_re.match('program UNDEFINED'))
+                    i = Line('program UNDEFINED',(0,0),None,self.reader)
+                    p = Program(self,Program.start_re.match(i.get_line()),i)
                     p.content.extend(self.content)
                     self.content[:] = [p]
                     return None
@@ -81,6 +82,7 @@ class Block:
 
         if m.groupdict().has_key('name'):
             end_name = m.group('name')
+            if end_name: end_name = end_name.strip()
             name = self.get_name()
             if end_name and name != end_name:
                 message = self.reader.format_message(\
@@ -103,12 +105,14 @@ class Block:
         return False
 
     def fill(self):
-        item = self.get_item()
+        end_flag = self.__class__ is Block
+        item = startitem = self.get_item()
         while item is not None:
             if isinstance(item, Line):
                 # handle end of a block
                 flag = self.isenditem(item)
                 if flag: # end of block
+                    end_flag = True
                     break
                 if flag is None: # fixing the end of undefined start
                     item = self.get_item()
@@ -117,73 +121,90 @@ class Block:
                 if self.isblock(item):
                     item = self.get_item()
                     continue                    
-
                 # line contains something else
                 self.content.append(item)
             item = self.get_item()
+        if not end_flag:
+            message = self.reader.format_message(\
+                        'WARNING',
+                        'failed to find the end of block',
+                        self.item.span[0],self.item.span[1])
+            print >> sys.stderr, message
+            sys.stderr.flush()
         return
 
-class Program(Block):
-    classes = []
-    start_re = re.compile(r'\s*program\s*((?P<name>\w+)|)', re.I)
-    end_re = re.compile(r'\s*end(\s*program(\s*(?P<name>\w+)|)|)\s*\Z', re.I)
+class CodeBlock(Block):
     def __init__(self, parent, start_re_match, item):
         Block.__init__(self, parent)
         self.name = start_re_match.group('name')
+        self.item = item
 
-class Module(Block):
+class Program(CodeBlock):
+    classes = []
+    start_re = re.compile(r'\s*program\s*((?P<name>\w+)|)', re.I)
+    end_re = re.compile(r'\s*end(\s*program(\s*(?P<name>\w+)|)|)\s*\Z', re.I)
+
+
+class Module(CodeBlock):
     classes = []
     start_re = re.compile(r'\s*module\s*(?P<name>\w+)\s*\Z', re.I)
     end_re = re.compile(r'\s*end(\s*module(\s*(?P<name>\w+)|)|)\s*\Z', re.I)
     def __init__(self, parent, start_re_match, item):
         Block.__init__(self, parent)
         self.name = start_re_match.group('name')
-
-class Interface(Block):
+        self.item = item
+        
+class Interface(CodeBlock):
     classes = []
     start_re = re.compile(r'\s*interface(\s*(?P<name>\w+)|)', re.I)
     end_re = re.compile(r'\s*end(\s*interface(\s*(?P<name>\w+)|)|)\s*\Z', re.I)
-    def __init__(self, parent, start_re_match, item):
-        Block.__init__(self, parent)
-        self.name = start_re_match.group('name')
 
-
-class PythonModule(Block):
+class PythonModule(CodeBlock):
     classes = []
     start_re = re.compile(r'\s*python\s*module\s*(?P<name>\w+)', re.I)
     end_re = re.compile(r'\s*end(\s*python\s*module(\s*(?P<name>\w+)|)|)\s*\Z', re.I)
-    def __init__(self, parent, start_re_match, item):
-        Block.__init__(self, parent)
-        self.name = start_re_match.group('name')
     
-class Subroutine(Block):
+class Subroutine(CodeBlock):
     classes = []
     start_re = re.compile(r'\s*subroutine\s*(?P<name>\w+)', re.I)
     end_re = re.compile(r'\s*end(\s*subroutine(\s*(?P<name>\w+)|)|)\s*\Z', re.I)
-    def __init__(self, parent, start_re_match, item):
-        Block.__init__(self, parent)
-        self.name = start_re_match.group('name')
 
-class Function(Block):
+class Function(CodeBlock):
     classes = []
-    start_re = re.compile(r'\s*function\s*(?P<name>\w+)', re.I)
+    start_re = re.compile(r'\s*(?P<prefix>[\w,\s=()]*)\s*function\s*(?P<name>\w+)', re.I)
     end_re = re.compile(r'\s*end(\s*function(\s*(?P<name>\w+)|)|)\s*\Z')
-    def __init__(self, parent, start_re_match, item):
-        Block.__init__(self, parent)
-        self.name = start_re_match.group('name')
 
-class Type(Block):
+class Type(CodeBlock):
     classes = []
     start_re = re.compile(r'\s*type(?!\s*\()(.*::|)\s*(?P<name>\w+)\s*\Z', re.I)
     end_re = re.compile(r'\s*end(\s*type(\s*(?P<name>\w+)|)|)\s*\Z', re.I)
-    def __init__(self, parent, start_re_match, item):
-        Block.__init__(self, parent)
-        self.name = start_re_match.group('name')
-
 
 class StatementBlock(Block):
     classes = []
 
+    def __init__(self, parent, start_re_match, item):
+        Block.__init__(self, parent)
+        self.item = item
+
+    def isenditem(self, item):
+        line,sline = split2(item.get_line())
+        if sline: return False # end statement never contains strings 
+        m = self.end_re.match(line)
+        if not m: return False
+        # check if the block start name matches with the block end name
+        if m.groupdict().has_key('name'):
+            end_name = m.group('name')
+            if end_name: end_name = end_name.strip()
+            name = self.get_name()
+            if end_name and name != end_name:
+                message = self.reader.format_message(\
+                        'WARNING',
+                        'expected the end of %r block but got end of %r'\
+                        % (name, end_name),
+                        item.span[0],item.span[1])
+                print >> sys.stderr, message
+        return True
+
     def fill(self):
         item = self.get_item()
         while item is not None:
@@ -200,17 +221,38 @@ class StatementBlock(Block):
                 # line contains something else
                 self.content.append(item)
             item = self.get_item()
+        if item is None:
+            message = self.reader.format_message(\
+                        'WARNING',
+                        'failed to find the end of block',
+                        self.item.span[0],self.item.span[1])
+            print >> sys.stderr, message
+            sys.stderr.flush()
         return
 
 class DoBlock(StatementBlock):
 
-    start_re = re.compile(r'\s*do\b', re.I)
+    start_re = re.compile(r'\s*do\b\s*(?P<label>\d*)', re.I)
     end_re = re.compile(r'\s*end\s*do(\s*(?P<name>.*)|)\s*\Z', re.I)
-    
     def __init__(self, parent, start_re_match, item):
-        Block.__init__(self, parent)
+        StatementBlock.__init__(self, parent, start_re_match, item)
+        label = start_re_match.group('label').strip()
+        if label.endswith(':'): label = label[:-1].strip()
+        self.endlabel = label
         self.name = item.label
 
+    def isenditem(self, item):
+        if self.endlabel:
+            if item.label==self.endlabel:
+                # item may contain computational statemets
+                self.content.append(item)
+                # the same item label may be used for different block ends
+                self.put_item(item)       
+                return True
+        else:
+            return StatementBlock.isenditem(self, item)
+        return False
+
 class IfThenBlock(StatementBlock):
 
     start_re = re.compile(r'\s*if\b.*?\bthen\s*\Z', re.I)
@@ -218,7 +260,7 @@ class IfThenBlock(StatementBlock):
     end_re = re.compile(r'\s*end\s*if(\s*(?P<name>.*)|)\s*\Z', re.I)
     
     def __init__(self, parent, start_re_match, item):
-        Block.__init__(self, parent)
+        StatementBlock.__init__(self, parent, start_re_match, item)
         self.name = item.label
 
 
diff --git a/numpy/f2py/lib/parsefortran.py b/numpy/f2py/lib/parsefortran.py
index 2341a0f22..89d793a52 100644
--- a/numpy/f2py/lib/parsefortran.py
+++ b/numpy/f2py/lib/parsefortran.py
@@ -12,6 +12,8 @@ Created: May 2006
 
 import re
 
+from numpy.distutils.misc_util import yellow_text
+
 from readfortran import FortranFileReader, FortranStringReader
 from block import Block
 
@@ -66,12 +68,12 @@ c      program foo
 def simple_main():
     import sys
     for filename in sys.argv[1:]:
-        print 'Processing',filename
+        print yellow_text('Processing '+filename)
         reader = FortranFileReader(filename)
         parser = FortranParser(reader)
         block = parser.parse()
-        print block
-
+        #print block
+        
 if __name__ == "__main__":
     #test_f77()
     #test_pyf()
diff --git a/numpy/f2py/lib/readfortran.py b/numpy/f2py/lib/readfortran.py
index 4937cb4b3..658d69804 100644
--- a/numpy/f2py/lib/readfortran.py
+++ b/numpy/f2py/lib/readfortran.py
@@ -32,7 +32,7 @@ from splitline import LineSplitter, String, string_replace_map
 
 _spacedigits=' 0123456789'
 _cf2py_re = re.compile(r'(?P<indent>\s*)!f2py(?P<rest>.*)',re.I)
-_is_fix_cont = lambda line: line and len(line)>5 and line[5]!=' ' and line[0]==' '
+_is_fix_cont = lambda line: line and len(line)>5 and line[5]!=' ' and line[:5]==5*' '
 _is_f90_cont = lambda line: line and '&' in line and line.rstrip()[-1]=='&'
 _f90label_re = re.compile(r'\s*(?P<label>(\w+\s*:|\d+))\s*(\b|(?=&)|\Z)',re.I)
 
@@ -40,6 +40,7 @@ class FortranReaderError: # TODO: may be derive it from Exception
     def __init__(self, message):
         self.message = message
         print >> sys.stderr,message
+        sys.stderr.flush()
 
 class Line:
     """ Holds a Fortran source line.
@@ -52,9 +53,9 @@ class Line:
         self.strline = None
     def __repr__(self):
         return self.__class__.__name__+'(%r,%s,%r)' \
-               % (self.get_line(), self.span, self.label)
+               % (self.line, self.span, self.label)
     def isempty(self, ignore_comments=False):
-        return not (self.line.strip() or (self.label and self.label.strip()))
+        return not (self.line.strip() or self.label)
     def get_line(self):
         if self.strline is not None:
             return self.strline
@@ -124,6 +125,12 @@ class FortranReaderBase:
         self.isfree  = isfree
         self.isfix   = not isfree
 
+        if self.isfree90: mode = 'free90'
+        elif self.isfix90: mode = 'fix90'
+        elif self.isfix77: mode = 'fix77'
+        else: mode = 'pyf'
+        self.mode = mode
+
         self.linecount = 0
         self.source = source
         self.isclosed = False
@@ -133,6 +140,7 @@ class FortranReaderBase:
         self.source_lines = []
 
     def close_source(self):
+        # called when self.source.next() raises StopIteration.
         pass
 
     # For handling raw source lines:
@@ -174,6 +182,19 @@ class FortranReaderBase:
         return self
 
     def next(self, ignore_comments = False):
+        try:
+            return self._next(ignore_comments)
+        except StopIteration:
+            raise
+        except Exception, msg:
+            message = self.format_message('FATAL ERROR',
+                                          'while processing got exception: %s'\
+                                          '\nSTOP READING' % msg,
+                                          self.linecount, self.linecount)
+            self.show_message(message, sys.stdout)
+            raise StopIteration
+
+    def _next(self, ignore_comments = False):
         fifo_item_pop = self.fifo_item.pop
         while 1:
             try:
@@ -226,9 +247,13 @@ class FortranReaderBase:
 
     # For handling messages:
 
+    def show_message(self, message, stream = sys.stdout):
+        stream.write(message+'\n')
+        stream.flush()
+
     def format_message(self, kind, message, startlineno, endlineno,
                        startcolno=0, endcolno=-1):
-        r = ['%s while processing %s..' % (kind, self.source)]
+        r = ['%s while processing %s (mode=%r)..' % (kind, self.source, self.mode)]
         for i in range(max(1,startlineno-3),startlineno):
             r.append('%5d:%s' % (i,self.source_lines[i-1]))
         for i in range(startlineno,min(endlineno+3,len(self.source_lines))+1):
@@ -305,7 +330,6 @@ class FortranReaderBase:
                 noncomment_items_append(item)
                 continue
             j = item.find('!')
-            # TODO: handle f2py directives in inline comments.
             noncomment_items_append(item[:j])
             items[k] = item[j:]
             commentline = ''.join(items[k:])
@@ -336,7 +360,7 @@ class FortranReaderBase:
                             'multiline prefix contains odd number of %r characters' \
                             % (quote), startlineno, startlineno,
                             0, len(prefix))
-                    print >> sys.stderr, message
+                    self.show_message(message, sys.stderr)
 
             suffix = None
             multilines = []
@@ -363,7 +387,7 @@ class FortranReaderBase:
                             'ASSERTION FAILURE(pyf)',
                         'following character continuation: %r, expected None.' % (qc),
                             startlineno, self.linecount)
-                print >> sys.stderr, message
+                self.show_message(message, sys.stderr)
             # XXX: should we do line.replace('\\'+mlstr[0],mlstr[0])
             #      for line in multilines?
             return self.multiline_item(prefix,multilines,suffix,
@@ -390,9 +414,9 @@ class FortranReaderBase:
             for mlstr in ['"""',"'''"]:
                 r = self.handle_multilines(line, startlineno, mlstr)
                 if r: return r
-
         if self.isfix:
-            label = line[:5]
+            label = line[:5].strip().lower()
+            if label.endswith(':'): label = label[:-1].strip()
             if not line.strip():
                 # empty line
                 return self.line_item(line[6:],startlineno,self.linecount,label)
@@ -406,7 +430,7 @@ class FortranReaderBase:
                         message = message + ', switching to free format mode'
                         message = self.format_warning_message(\
                             message,startlineno, self.linecount)
-                        print >> sys.stderr, message
+                        self.show_message(message, sys.stderr)
                         self.isfree = True
                         self.isfix90 = False
                         self.isfree90 = True
@@ -439,9 +463,9 @@ class FortranReaderBase:
             if qc is not None:
                 message = self.format_message(\
                             'ASSERTION FAILURE(fix90)',
-                            'following character continuation: %r, expected None.' % (qc),
-                            startlineno, self.linecount)
-                print >> sys.stderr, message
+                            'following character continuation: %r, expected None.'\
+                            % (qc), startlineno, self.linecount)
+                self.show_message(message, sys.stderr)
             for i in range(len(lines)):
                 l = lines[i]
                 if l.rstrip().endswith('&'):
@@ -449,7 +473,7 @@ class FortranReaderBase:
                         'f90 line continuation character `&\' detected'\
                         ' in fix format code',
                         startlineno + i, startlineno + i, l.rfind('&')+5)
-                    print >> sys.stderr, message
+                    self.show_message(message, sys.stderr)
             return self.line_item(''.join(lines),startlineno,self.linecount,label)
 
         start_index = 0
@@ -466,18 +490,21 @@ class FortranReaderBase:
                                                 self.linecount,qc)
             else:
                 line_lstrip = line.lstrip()
-                if lines and line_lstrip.startswith('!'):
-                    # check for comment line within line continuation
-                    put_item(self.comment_item(line_lstrip,
-                                               self.linecount, self.linecount))
-                    line = get_single_line()
-                    continue
+                if lines:
+                    if line_lstrip.startswith('!'):
+                        # check for comment line within line continuation
+                        put_item(self.comment_item(line_lstrip,
+                                                   self.linecount, self.linecount))
+                        line = get_single_line()
+                        continue
                 else:
                     # first line, check for a f90 label
                     m = _f90label_re.match(line)
                     if m:
-                        assert label is None,`label`
-                        label = m.group('label')
+                        assert not label,`label,m.group('label')`
+                        label = m.group('label').strip()
+                        if label.endswith(':'): label = label[:-1].strip()
+                        if not self.ispyf: label = label.lower()
                         line = line[m.end():]
                 line,qc = handle_inline_comment(line, self.linecount, qc)
 
@@ -510,7 +537,7 @@ class FortranReaderBase:
             message = self.format_message('ASSERTION FAILURE(free)',
                 'following character continuation: %r, expected None.' % (qc),
                 startlineno, self.linecount)
-            print >> sys.stderr, message
+            self.show_message(message, sys.stderr)
         return self.line_item(''.join(lines),startlineno,self.linecount,label)
 
     ##  FortranReaderBase
@@ -596,6 +623,10 @@ cComment
      !4!line cont. with comment symbol
      &5
       a = 3!f2py.14 ! pi!
+!   KDMO
+     write (obj%print_lun, *) ' KDMO : '
+     write (obj%print_lun, *) '  COORD = ',coord, '  BIN_WID = ',             &
+       obj%bin_wid,'  VEL_DMO = ', obj%vel_dmo
       end subroutine foo
 """
     reader = FortranStringReader(string_fix90,False, False)
@@ -607,7 +638,8 @@ def simple_main():
         print 'Processing',filename
         reader = FortranFileReader(filename)
         for item in reader:
-            #print item
+            #print >> sys.stdout, item
+            #sys.stdout.flush()
             pass
 
 def profile_main():
diff --git a/numpy/f2py/lib/sourceinfo.py b/numpy/f2py/lib/sourceinfo.py
index 457c3d8b3..a31c99d73 100644
--- a/numpy/f2py/lib/sourceinfo.py
+++ b/numpy/f2py/lib/sourceinfo.py
@@ -54,13 +54,13 @@ def is_free_format(file):
     while n>0 and line:
         if line[0]!='!' and line.strip():
             n -= 1
-            if not contline and (line[0]!='\t' and _free_f90_start(line[:5])):
+            if line[0]!='\t' and _free_f90_start(line[:5]) or line[-2:-1]=='&':
                 isfree = True
                 break
-            elif line[-2:-1]=='&':
-                contline = True
-            else:
-                contline = False
+            #elif line[-2:-1]=='&':
+            #    contline = True
+            #else:
+            #    contline = False
         line = f.readline()
     f.close()
     return isfree