merge with bitbucket.org/andyli/pygments-main (Haxe3 lexer), pull request #174

5 files changed, 935 insertions, 178 deletions

diff --git a/AUTHORS b/AUTHORS
index 83db4cec..f88dfddb 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -30,7 +30,7 @@ Other contributors, listed alphabetically, are:
 * Christopher Creutzig -- MuPAD lexer
 * Pete Curry -- bugfixes
 * Bryan Davis -- EBNF lexer
-* Owen Durni -- haXe lexer
+* Owen Durni -- Haxe lexer
 * Nick Efford -- Python 3 lexer
 * Sven Efftinge -- Xtend lexer
 * Artem Egorkine -- terminal256 formatter
diff --git a/CHANGES b/CHANGES
index 781f8857..3cad33ea 100644
--- a/CHANGES
+++ b/CHANGES
@@ -45,6 +45,8 @@ Version 1.7
 
 - Koka lexer: update to newest language spec (PR#201).
 
+- Haxe lexer: rewrite and support for Haxe 3 (PR#174).
+
 
 Version 1.6
 -----------
@@ -299,7 +301,7 @@ Version 1.3
   * Ada
   * Coldfusion
   * Modula-2
-  * haXe
+  * Haxe
   * R console
   * Objective-J
   * Haml and Sass
@@ -358,7 +360,7 @@ Version 1.2
   * CMake
   * Ooc
   * Coldfusion
-  * haXe
+  * Haxe
   * R console
 
 - Added options for rendering LaTeX in source code comments in the
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 3864b0c1..59e1f999 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -126,7 +126,7 @@ LEXERS = {
     'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy',), ('text/x-groovy',)),
     'HamlLexer': ('pygments.lexers.web', 'Haml', ('haml', 'HAML'), ('*.haml',), ('text/x-haml',)),
     'HaskellLexer': ('pygments.lexers.functional', 'Haskell', ('haskell', 'hs'), ('*.hs',), ('text/x-haskell',)),
-    'HaxeLexer': ('pygments.lexers.web', 'haXe', ('hx', 'haXe', 'haxe'), ('*.hx',), ('text/haxe',)),
+    'HaxeLexer': ('pygments.lexers.web', 'Haxe', ('hx', 'Haxe', 'haxe', 'haXe', 'hxsl'), ('*.hx', '*.hxsl'), ('text/haxe', 'text/x-haxe', 'text/x-hx')),
     'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja', 'htmldjango'), (), ('text/html+django', 'text/html+jinja')),
     'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ('text/html+genshi',)),
     'HtmlLexer': ('pygments.lexers.web', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')),
diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py
index 10fb7672..7d2c812a 100644
--- a/pygments/lexers/web.py
+++ b/pygments/lexers/web.py
@@ -1125,227 +1125,840 @@ class MxmlLexer(RegexLexer):
         }
 
 
-class HaxeLexer(RegexLexer):
+class HaxeLexer(ExtendedRegexLexer):
     """
-    For haXe source code (http://haxe.org/).
+    For Haxe source code (http://haxe.org/).
 
     *New in Pygments 1.3.*
     """
 
-    name = 'haXe'
-    aliases = ['hx', 'haXe', 'haxe']
-    filenames = ['*.hx']
-    mimetypes = ['text/haxe']
+    name = 'Haxe'
+    aliases = ['hx', 'Haxe', 'haxe', 'haXe', 'hxsl']
+    filenames = ['*.hx', '*.hxsl']
+    mimetypes = ['text/haxe', 'text/x-haxe', 'text/x-hx']
 
-    ident = r'(?:[a-zA-Z_][a-zA-Z0-9_]*)'
-    typeid = r'(?:(?:[a-z0-9_\.])*[A-Z_][A-Za-z0-9_]*)'
-    key_prop = r'(?:default|null|never)'
-    key_decl_mod = r'(?:public|private|override|static|inline|extern|dynamic)'
+    # keywords extracted from lexer.mll in the haxe compiler source
+    keyword = (r'(?:function|class|static|var|if|else|while|do|for|'
+               r'break|return|continue|extends|implements|import|'
+               r'switch|case|default|public|private|try|untyped|'
+               r'catch|new|this|throw|extern|enum|in|interface|'
+               r'cast|override|dynamic|typedef|package|'
+               r'inline|using|null|true|false|abstract)\b')
+
+    # idtype in lexer.mll
+    typeid = r'_*[A-Z][_a-zA-Z0-9]*'
+
+    # combined ident and dollar and idtype
+    ident = r'(?:_*[a-z][_a-zA-Z0-9]*|_+[0-9][_a-zA-Z0-9]*|' + typeid + \
+        '|_+|\$[_a-zA-Z0-9]+)'
+
+    binop = (r'(?:%=|&=|\|=|\^=|\+=|\-=|\*=|/=|<<=|>\s*>\s*=|>\s*>\s*>\s*=|==|'
+             r'!=|<=|>\s*=|&&|\|\||<<|>>>|>\s*>|\.\.\.|<|>|%|&|\||\^|\+|\*|'
+             r'/|\-|=>|=)')
+
+    # ident except keywords
+    ident_no_keyword = r'(?!' + keyword + ')' + ident
 
     flags = re.DOTALL | re.MULTILINE
 
+    preproc_stack = []
+
+    def preproc_callback(self, match, ctx):
+        proc = match.group(2)
+
+        if proc == 'if':
+            # store the current stack
+            self.preproc_stack.append(ctx.stack[:])
+        elif proc in ['else', 'elseif']:
+            # restore the stack back to right before #if
+            if self.preproc_stack: ctx.stack = self.preproc_stack[-1][:]
+        elif proc == 'end':
+            # remove the saved stack of previous #if
+            if self.preproc_stack: self.preproc_stack.pop()
+
+        # #if and #elseif should follow by an expr
+        if proc in ['if', 'elseif']:
+            ctx.stack.append('preproc-expr')
+
+        # #error can be optionally follow by the error msg
+        if proc in ['error']:
+            ctx.stack.append('preproc-error')
+
+        yield match.start(), Comment.Preproc, '#' + proc
+        ctx.pos = match.end()
+
+
     tokens = {
         'root': [
-            include('whitespace'),
-            include('comments'),
-            (key_decl_mod, Keyword.Declaration),
-            include('enumdef'),
-            include('typedef'),
-            include('classdef'),
-            include('imports'),
+            include('spaces'),
+            include('meta'),
+            (r'(?:package)\b', Keyword.Namespace, ('semicolon', 'package')),
+            (r'(?:import)\b', Keyword.Namespace, ('semicolon', 'import')),
+            (r'(?:using)\b', Keyword.Namespace, ('semicolon', 'using')),
+            (r'(?:extern|private)\b', Keyword.Declaration),
+            (r'(?:abstract)\b', Keyword.Declaration, 'abstract'),
+            (r'(?:class|interface)\b', Keyword.Declaration, 'class'),
+            (r'(?:enum)\b', Keyword.Declaration, 'enum'),
+            (r'(?:typedef)\b', Keyword.Declaration, 'typedef'),
+
+            # top-level expression
+            # although it is not supported in haxe, but it is common to write
+            # expression in web pages the positive lookahead here is to prevent
+            # an infinite loop at the EOF
+            (r'(?=.)', Text, 'expr-statement'),
+        ],
+
+        # space/tab/comment/preproc
+        'spaces': [
+            (r'\s+', Text),
+            (r'//[^\n\r]*', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'(#)(if|elseif|else|end|error)\b', preproc_callback),
         ],
 
-        # General constructs
-        'comments': [
-            (r'//.*?\n', Comment.Single),
-            (r'/\*.*?\*/', Comment.Multiline),
-            (r'#[^\n]*', Comment.Preproc),
+        'string-single-interpol': [
+            (r'\$\{', String.Interpol, ('string-interpol-close', 'expr')),
+            (r'\$\$', String.Escape),
+            (r'\$(?=' + ident + ')', String.Interpol, 'ident'),
+            include('string-single'),
         ],
-        'whitespace': [
-            include('comments'),
-            (r'\s+', Text),
+
+        'string-single': [
+            (r"'", String.Single, '#pop'),
+            (r'\\.', String.Escape),
+            (r'.', String.Single),
         ],
-        'codekeywords': [
-            (r'\b(if|else|while|do|for|in|break|continue|'
-             r'return|switch|case|try|catch|throw|null|trace|'
-             r'new|this|super|untyped|cast|callback|here)\b',
-             Keyword.Reserved),
+
+        'string-double': [
+            (r'"', String.Double, '#pop'),
+            (r'\\.', String.Escape),
+            (r'.', String.Double),
         ],
-        'literals': [
-            (r'0[xX][0-9a-fA-F]+', Number.Hex),
-            (r'[0-9]+', Number.Integer),
-            (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
-            (r"'(\\\\|\\'|[^'])*'", String.Single),
-            (r'"(\\\\|\\"|[^"])*"', String.Double),
-            (r'~/([^\n])*?/[gisx]*', String.Regex),
-            (r'\b(true|false|null)\b', Keyword.Constant),
-        ],
-        'codeblock': [
-          include('whitespace'),
-          include('new'),
-          include('case'),
-          include('anonfundef'),
-          include('literals'),
-          include('vardef'),
-          include('codekeywords'),
-          (r'[();,\[\]]', Punctuation),
-          (r'(?:=|\+=|-=|\*=|/=|%=|&=|\|=|\^=|<<=|>>=|>>>=|\|\||&&|'
-           r'\.\.\.|==|!=|>|<|>=|<=|\||&|\^|<<|>>>|>>|\+|\-|\*|/|%|'
-           r'!|\+\+|\-\-|~|\.|\?|\:)',
-           Operator),
-          (ident, Name),
-
-          (r'}', Punctuation,'#pop'),
-          (r'{', Punctuation,'#push'),
-        ],
-
-        # Instance/Block level constructs
-        'propertydef': [
-            (r'(\()(' + key_prop + ')(,)(' + key_prop + ')(\))',
-             bygroups(Punctuation, Keyword.Reserved, Punctuation,
-                      Keyword.Reserved, Punctuation)),
+
+        'string-interpol-close': [
+            (r'\$'+ident, String.Interpol),
+            (r'\}', String.Interpol, '#pop'),
         ],
-        'new': [
-            (r'\bnew\b', Keyword, 'typedecl'),
+
+        'package': [
+            include('spaces'),
+            (ident, Name.Namespace),
+            (r'\.', Punctuation, 'import-ident'),
+            (r'', Text, '#pop'),
         ],
-        'case': [
-            (r'\b(case)(\s+)(' + ident + ')(\s*)(\()',
-             bygroups(Keyword.Reserved, Text, Name, Text, Punctuation),
-             'funargdecl'),
+
+        'import': [
+            include('spaces'),
+            (ident, Name.Namespace),
+            (r'\*', Keyword), # wildcard import
+            (r'\.', Punctuation, 'import-ident'),
+            (r'in', Keyword.Namespace, 'ident'),
+            (r'', Text, '#pop'),
         ],
-        'vardef': [
-            (r'\b(var)(\s+)(' + ident + ')',
-             bygroups(Keyword.Declaration, Text, Name.Variable), 'vardecl'),
+
+        'import-ident': [
+            include('spaces'),
+            (r'\*', Keyword, '#pop'), # wildcard import
+            (ident, Name.Namespace, '#pop'),
         ],
-        'vardecl': [
-            include('whitespace'),
-            include('typelabel'),
-            (r'=', Operator,'#pop'),
-            (r';', Punctuation,'#pop'),
+
+        'using': [
+            include('spaces'),
+            (ident, Name.Namespace),
+            (r'\.', Punctuation, 'import-ident'),
+            (r'', Text, '#pop'),
         ],
-        'instancevardef': [
-            (key_decl_mod,Keyword.Declaration),
-            (r'\b(var)(\s+)(' + ident + ')',
-             bygroups(Keyword.Declaration, Text, Name.Variable.Instance),
-             'instancevardecl'),
+
+        'preproc-error': [
+            (r'\s+', Comment.Preproc),
+            (r"'", String.Single, ('#pop', 'string-single')),
+            (r'"', String.Double, ('#pop', 'string-double')),
+            (r'', Text, '#pop'),
         ],
-        'instancevardecl': [
-            include('vardecl'),
-            include('propertydef'),
+
+        'preproc-expr': [
+            (r'\s+', Comment.Preproc),
+            (r'\!', Comment.Preproc),
+            (r'\(', Comment.Preproc, ('#pop', 'preproc-parenthesis')),
+
+            (ident, Comment.Preproc, '#pop'),
+            (r"'", String.Single, ('#pop', 'string-single')),
+            (r'"', String.Double, ('#pop', 'string-double')),
         ],
 
-        'anonfundef': [
-            (r'\bfunction\b', Keyword.Declaration, 'fundecl'),
+        'preproc-parenthesis': [
+            (r'\s+', Comment.Preproc),
+            (r'\)', Comment.Preproc, '#pop'),
+            ('', Text, 'preproc-expr-in-parenthesis'),
         ],
-        'instancefundef': [
-            (key_decl_mod, Keyword.Declaration),
-            (r'\b(function)(\s+)(' + ident + ')',
-             bygroups(Keyword.Declaration, Text, Name.Function), 'fundecl'),
+
+        'preproc-expr-chain': [
+            (r'\s+', Comment.Preproc),
+            (binop, Comment.Preproc, ('#pop', 'preproc-expr-in-parenthesis')),
+            (r'', Text, '#pop'),
         ],
-        'fundecl': [
-            include('whitespace'),
-            include('typelabel'),
-            include('generictypedecl'),
-            (r'\(',Punctuation,'funargdecl'),
-            (r'(?=[a-zA-Z0-9_])',Text,'#pop'),
-            (r'{',Punctuation,('#pop','codeblock')),
-            (r';',Punctuation,'#pop'),
-        ],
-        'funargdecl': [
-            include('whitespace'),
-            (ident, Name.Variable),
-            include('typelabel'),
-            include('literals'),
-            (r'=', Operator),
+
+        # same as 'preproc-expr' but able to chain 'preproc-expr-chain'
+        'preproc-expr-in-parenthesis': [
+            (r'\s+', Comment.Preproc),
+            (r'\!', Comment.Preproc),
+            (r'\(', Comment.Preproc,
+             ('#pop', 'preproc-expr-chain', 'preproc-parenthesis')),
+
+            (ident, Comment.Preproc, ('#pop', 'preproc-expr-chain')),
+            (r"'", String.Single,
+             ('#pop', 'preproc-expr-chain', 'string-single')),
+            (r'"', String.Double,
+             ('#pop', 'preproc-expr-chain', 'string-double')),
+        ],
+
+        'abstract' : [
+            include('spaces'),
+            (r'', Text, ('#pop', 'abstract-body', 'abstract-relation',
+                    'abstract-opaque', 'type-param-constraint', 'type-name')),
+        ],
+
+        'abstract-body' : [
+            include('spaces'),
+            (r'\{', Punctuation, ('#pop', 'class-body')),
+        ],
+
+        'abstract-opaque' : [
+            include('spaces'),
+            (r'\(', Punctuation, ('#pop', 'parenthesis-close', 'type')),
+            (r'', Text, '#pop'),
+        ],
+
+        'abstract-relation': [
+            include('spaces'),
+            (r'(?:to|from)', Keyword.Declaration, 'type'),
             (r',', Punctuation),
+            (r'', Text, '#pop'),
+        ],
+
+        'meta': [
+            include('spaces'),
+            (r'@', Name.Decorator, ('meta-body', 'meta-ident', 'meta-colon')),
+        ],
+
+        # optional colon
+        'meta-colon': [
+            include('spaces'),
+            (r':', Name.Decorator, '#pop'),
+            (r'', Text, '#pop'),
+        ],
+
+        # same as 'ident' but set token as Name.Decorator instead of Name
+        'meta-ident': [
+            include('spaces'),
+            (ident, Name.Decorator, '#pop'),
+        ],
+
+        'meta-body': [
+            include('spaces'),
+            (r'\(', Name.Decorator, ('#pop', 'meta-call')),
+            (r'', Text, '#pop'),
+        ],
+
+        'meta-call': [
+            include('spaces'),
+            (r'\)', Name.Decorator, '#pop'),
+            (r'', Text, ('#pop', 'meta-call-sep', 'expr')),
+        ],
+
+        'meta-call-sep': [
+            include('spaces'),
+            (r'\)', Name.Decorator, '#pop'),
+            (r',', Punctuation, ('#pop', 'meta-call')),
+        ],
+
+        'typedef': [
+            include('spaces'),
+            (r'', Text, ('#pop', 'typedef-body', 'type-param-constraint',
+                         'type-name')),
+        ],
+
+        'typedef-body': [
+            include('spaces'),
+            (r'=', Operator, ('#pop', 'optional-semicolon', 'type')),
+        ],
+
+        'enum': [
+            include('spaces'),
+            (r'', Text, ('#pop', 'enum-body', 'bracket-open',
+                         'type-param-constraint', 'type-name')),
+        ],
+
+        'enum-body': [
+            include('spaces'),
+            include('meta'),
+            (r'\}', Punctuation, '#pop'),
+            (ident_no_keyword, Name, ('enum-member', 'type-param-constraint')),
+        ],
+
+        'enum-member': [
+            include('spaces'),
+            (r'\(', Punctuation,
+             ('#pop', 'semicolon', 'flag', 'function-param')),
+            (r'', Punctuation, ('#pop', 'semicolon', 'flag')),
+        ],
+
+        'class': [
+            include('spaces'),
+            (r'', Text, ('#pop', 'class-body', 'bracket-open', 'extends',
+                         'type-param-constraint', 'type-name')),
+        ],
+
+        'extends': [
+            include('spaces'),
+            (r'(?:extends|implements)\b', Keyword.Declaration, 'type'),
+            (r',', Punctuation), # the comma is made optional here, since haxe2
+                                 # requires the comma but haxe3 does not allow it
+            (r'', Text, '#pop'),
+        ],
+
+        'bracket-open': [
+            include('spaces'),
+            (r'\{', Punctuation, '#pop'),
+        ],
+
+        'bracket-close': [
+            include('spaces'),
+            (r'\}', Punctuation, '#pop'),
+        ],
+
+        'class-body': [
+            include('spaces'),
+            include('meta'),
+            (r'\}', Punctuation, '#pop'),
+            (r'(?:static|public|private|override|dynamic|inline|macro)\b',
+             Keyword.Declaration),
+            (r'', Text, 'class-member'),
+        ],
+
+        'class-member': [
+            include('spaces'),
+            (r'(var)\b', Keyword.Declaration,
+             ('#pop', 'optional-semicolon', 'prop')),
+            (r'(function)\b', Keyword.Declaration,
+             ('#pop', 'optional-semicolon', 'class-method')),
+        ],
+
+        # local function, anonymous or not
+        'function-local': [
+            include('spaces'),
+            (r'(' + ident_no_keyword + ')?', Name.Function,
+             ('#pop', 'expr', 'flag', 'function-param',
+              'parenthesis-open', 'type-param-constraint')),
+        ],
+
+        'optional-expr': [
+            include('spaces'),
+            include('expr'),
+            (r'', Text, '#pop'),
+        ],
+
+        'class-method': [
+            include('spaces'),
+            (ident, Name.Function, ('#pop', 'optional-expr', 'flag',
+                                    'function-param', 'parenthesis-open',
+                                    'type-param-constraint')),
+        ],
+
+        # function arguments
+        'function-param': [
+            include('spaces'),
+            (r'\)', Punctuation, '#pop'),
             (r'\?', Punctuation),
+            (ident_no_keyword, Name,
+             ('#pop', 'function-param-sep', 'assign', 'flag')),
+        ],
+
+        'function-param-sep': [
+            include('spaces'),
             (r'\)', Punctuation, '#pop'),
+            (r',', Punctuation, ('#pop', 'function-param')),
         ],
 
-        'typelabel': [
-            (r':', Punctuation, 'type'),
+        # class property
+        # eg. var prop(default, null):String;
+        'prop': [
+            include('spaces'),
+            (ident_no_keyword, Name, ('#pop', 'assign', 'flag', 'prop-get-set')),
         ],
-        'typedecl': [
-            include('whitespace'),
-            (typeid, Name.Class),
-            (r'<', Punctuation, 'generictypedecl'),
-            (r'(?=[{}()=,a-z])', Text,'#pop'),
+
+        'prop-get-set': [
+            include('spaces'),
+            (r'\(', Punctuation, ('#pop', 'parenthesis-close',
+                                  'prop-get-set-opt', 'comma', 'prop-get-set-opt')),
+            (r'', Text, '#pop'),
         ],
+
+        'prop-get-set-opt': [
+            include('spaces'),
+            (r'(?:default|null|never|dynamic|get|set)\b', Keyword, '#pop'),
+            (ident_no_keyword, Text, '#pop'), #custom getter/setter
+        ],
+
+        'expr-statement': [
+            include('spaces'),
+            # makes semicolon optional here, just to avoid checking the last
+            # one is bracket or not.
+            (r'', Text, ('#pop', 'optional-semicolon', 'expr')),
+        ],
+
+        'expr': [
+            include('spaces'),
+            (r'@', Name.Decorator, ('#pop', 'optional-expr', 'meta-body',
+                                    'meta-ident', 'meta-colon')),
+            (r'(?:\+\+|\-\-|~(?!/)|!|\-)', Operator),
+            (r'\(', Punctuation, ('#pop', 'expr-chain', 'parenthesis')),
+            (r'(?:inline)\b', Keyword.Declaration),
+            (r'(?:function)\b', Keyword.Declaration, ('#pop', 'expr-chain',
+                                                      'function-local')),
+            (r'\{', Punctuation, ('#pop', 'expr-chain', 'bracket')),
+            (r'(?:true|false|null)\b', Keyword.Constant, ('#pop', 'expr-chain')),
+            (r'(?:this)\b', Keyword, ('#pop', 'expr-chain')),
+            (r'(?:cast)\b', Keyword, ('#pop', 'expr-chain', 'cast')),
+            (r'(?:try)\b', Keyword, ('#pop', 'catch', 'expr')),
+            (r'(?:var)\b', Keyword.Declaration, ('#pop', 'var')),
+            (r'(?:new)\b', Keyword, ('#pop', 'expr-chain', 'new')),
+            (r'(?:switch)\b', Keyword, ('#pop', 'switch')),
+            (r'(?:if)\b', Keyword, ('#pop', 'if')),
+            (r'(?:do)\b', Keyword, ('#pop', 'do')),
+            (r'(?:while)\b', Keyword, ('#pop', 'while')),
+            (r'(?:for)\b', Keyword, ('#pop', 'for')),
+            (r'(?:untyped|throw)\b', Keyword),
+            (r'(?:return)\b', Keyword, ('#pop', 'optional-expr')),
+            (r'(?:macro)\b', Keyword, ('#pop', 'macro')),
+            (r'(?:continue|break)\b', Keyword, '#pop'),
+            (r'(?:\$\s*[a-z]\b|\$(?!'+ident+'))', Name, ('#pop', 'dollar')),
+            (ident_no_keyword, Name, ('#pop', 'expr-chain')),
+
+            # Float
+            (r'\.[0-9]+', Number.Float, ('#pop', 'expr-chain')),
+            (r'[0-9]+[eE][\+\-]?[0-9]+', Number.Float, ('#pop', 'expr-chain')),
+            (r'[0-9]+\.[0-9]*[eE][\+\-]?[0-9]+', Number.Float, ('#pop', 'expr-chain')),
+            (r'[0-9]+\.[0-9]+', Number.Float, ('#pop', 'expr-chain')),
+            (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, ('#pop', 'expr-chain')),
+
+            # Int
+            (r'0x[0-9a-fA-F]+', Number.Hex, ('#pop', 'expr-chain')),
+            (r'[0-9]+', Number.Integer, ('#pop', 'expr-chain')),
+
+            # String
+            (r"'", String.Single, ('#pop', 'expr-chain', 'string-single-interpol')),
+            (r'"', String.Double, ('#pop', 'expr-chain', 'string-double')),
+
+            # EReg
+            (r'~/(\\\\|\\/|[^/\n])*/[gimsu]*', String.Regex, ('#pop', 'expr-chain')),
+
+            # Array
+            (r'\[', Punctuation, ('#pop', 'expr-chain', 'array-decl')),
+        ],
+
+        'expr-chain': [
+            include('spaces'),
+            (r'(?:\+\+|\-\-)', Operator),
+            (binop, Operator, ('#pop', 'expr')),
+            (r'(?:in)\b', Keyword, ('#pop', 'expr')),
+            (r'\?', Operator, ('#pop', 'expr', 'ternary', 'expr')),
+            (r'(\.)(' + ident_no_keyword + ')', bygroups(Punctuation, Name)),
+            (r'\[', Punctuation, 'array-access'),
+            (r'\(', Punctuation, 'call'),
+            (r'', Text, '#pop'),
+        ],
+
+        # macro reification
+        'macro': [
+            include('spaces'),
+            (r':', Punctuation, ('#pop', 'type')),
+            (r'', Text, ('#pop', 'expr')),
+        ],
+
+        # cast can be written as "cast expr" or "cast(expr, type)"
+        'cast': [
+            include('spaces'),
+            (r'\(', Punctuation, ('#pop', 'parenthesis-close',
+                                  'cast-type', 'expr')),
+            (r'', Text, ('#pop', 'expr')),
+        ],
+
+        # optionally give a type as the 2nd argument of cast()
+        'cast-type': [
+            include('spaces'),
+            (r',', Punctuation, ('#pop', 'type')),
+            (r'', Text, '#pop'),
+        ],
+
+        'catch': [
+            include('spaces'),
+            (r'(?:catch)\b', Keyword, ('expr', 'function-param',
+                                       'parenthesis-open')),
+            (r'', Text, '#pop'),
+        ],
+
+        # do-while loop
+        'do': [
+            include('spaces'),
+            (r'', Punctuation, ('#pop', 'do-while', 'expr')),
+        ],
+
+        # the while after do
+        'do-while': [
+            include('spaces'),
+            (r'(?:while)\b', Keyword, ('#pop', 'parenthesis',
+                                       'parenthesis-open')),
+        ],
+
+        'while': [
+            include('spaces'),
+            (r'\(', Punctuation, ('#pop', 'expr', 'parenthesis')),
+        ],
+
+        'for': [
+            include('spaces'),
+            (r'\(', Punctuation, ('#pop', 'expr', 'parenthesis')),
+        ],
+
+        'if': [
+            include('spaces'),
+            (r'\(', Punctuation, ('#pop', 'else', 'optional-semicolon', 'expr',
+                                  'parenthesis')),
+        ],
+
+        'else': [
+            include('spaces'),
+            (r'(?:else)\b', Keyword, ('#pop', 'expr')),
+            (r'', Text, '#pop'),
+        ],
+
+        'switch': [
+            include('spaces'),
+            (r'', Text, ('#pop', 'switch-body', 'bracket-open', 'expr')),
+        ],
+
+        'switch-body': [
+            include('spaces'),
+            (r'(?:case|default)\b', Keyword, ('case-block', 'case')),
+            (r'\}', Punctuation, '#pop'),
+        ],
+
+        'case': [
+            include('spaces'),
+            (r':', Punctuation, '#pop'),
+            (r'', Text, ('#pop', 'case-sep', 'case-guard', 'expr')),
+        ],
+
+        'case-sep': [
+            include('spaces'),
+            (r':', Punctuation, '#pop'),
+            (r',', Punctuation, ('#pop', 'case')),
+        ],
+
+        'case-guard': [
+            include('spaces'),
+            (r'(?:if)\b', Keyword, ('#pop', 'parenthesis', 'parenthesis-open')),
+            (r'', Text, '#pop'),
+        ],
+
+        # optional multiple expr under a case
+        'case-block': [
+            include('spaces'),
+            (r'(?!(?:case|default)\b|\})', Keyword, 'expr-statement'),
+            (r'', Text, '#pop'),
+        ],
+
+        'new': [
+            include('spaces'),
+            (r'', Text, ('#pop', 'call', 'parenthesis-open', 'type')),
+        ],
+
+        'array-decl': [
+            include('spaces'),
+            (r'\]', Punctuation, '#pop'),
+            (r'', Text, ('#pop', 'array-decl-sep', 'expr')),
+        ],
+
+        'array-decl-sep': [
+            include('spaces'),
+            (r'\]', Punctuation, '#pop'),
+            (r',', Punctuation, ('#pop', 'array-decl')),
+        ],
+
+        'array-access': [
+            include('spaces'),
+            (r'', Text, ('#pop', 'array-access-close', 'expr')),
+        ],
+
+        'array-access-close': [
+            include('spaces'),
+            (r'\]', Punctuation, '#pop'),
+        ],
+
+        'comma': [
+            include('spaces'),
+            (r',', Punctuation, '#pop'),
+        ],
+
+        'colon': [
+            include('spaces'),
+            (r':', Punctuation, '#pop'),
+        ],
+
+        'semicolon': [
+            include('spaces'),
+            (r';', Punctuation, '#pop'),
+        ],
+
+        'optional-semicolon': [
+            include('spaces'),
+            (r';', Punctuation, '#pop'),
+            (r'', Text, '#pop'),
+        ],
+
+        # identity that CAN be a Haxe keyword
+        'ident': [
+            include('spaces'),
+            (ident, Name, '#pop'),
+        ],
+
+        'dollar': [
+            include('spaces'),
+            (r'\{', Keyword, ('#pop', 'bracket-close', 'expr')),
+            (r'', Text, ('#pop', 'expr-chain')),
+        ],
+
+        'type-name': [
+            include('spaces'),
+            (typeid, Name, '#pop'),
+        ],
+
+        'type-full-name': [
+            include('spaces'),
+            (r'\.', Punctuation, 'ident'),
+            (r'', Text, '#pop'),
+        ],
+
         'type': [
-            include('whitespace'),
-            (typeid, Name.Class),
-            (r'<', Punctuation, 'generictypedecl'),
-            (r'->', Keyword.Type),
-            (r'(?=[{}(),;=])', Text, '#pop'),
+            include('spaces'),
+            (r'\?', Punctuation),
+            (ident, Name, ('#pop', 'type-check', 'type-full-name')),
+            (r'\{', Punctuation, ('#pop', 'type-check', 'type-struct')),
+            (r'\(', Punctuation, ('#pop', 'type-check', 'type-parenthesis')),
         ],
-        'generictypedecl': [
-            include('whitespace'),
-            (typeid, Name.Class),
-            (r'<', Punctuation, '#push'),
+
+        'type-parenthesis': [
+            include('spaces'),
+            (r'', Text, ('#pop', 'parenthesis-close', 'type')),
+        ],
+
+        'type-check': [
+            include('spaces'),
+            (r'->', Punctuation, ('#pop', 'type')),
+            (r'<(?!=)', Punctuation, 'type-param'),
+            (r'', Text, '#pop'),
+        ],
+
+        'type-struct': [
+            include('spaces'),
+            (r'\}', Punctuation, '#pop'),
+            (r'\?', Punctuation),
+            (r'>', Punctuation, ('comma', 'type')),
+            (ident_no_keyword, Name, ('#pop', 'type-struct-sep', 'type', 'colon')),
+            include('class-body'),
+        ],
+
+        'type-struct-sep': [
+            include('spaces'),
+            (r'\}', Punctuation, '#pop'),
+            (r',', Punctuation, ('#pop', 'type-struct')),
+        ],
+
+        # type-param can be a normal type or a constant literal...
+        'type-param-type': [
+            # Float
+            (r'\.[0-9]+', Number.Float, '#pop'),
+            (r'[0-9]+[eE][\+\-]?[0-9]+', Number.Float, '#pop'),
+            (r'[0-9]+\.[0-9]*[eE][\+\-]?[0-9]+', Number.Float, '#pop'),
+            (r'[0-9]+\.[0-9]+', Number.Float, '#pop'),
+            (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, '#pop'),
+
+            # Int
+            (r'0x[0-9a-fA-F]+', Number.Hex, '#pop'),
+            (r'[0-9]+', Number.Integer, '#pop'),
+
+            # String
+            (r"'", String.Single, ('#pop', 'string-single')),
+            (r'"', String.Double, ('#pop', 'string-double')),
+
+            # EReg
+            (r'~/(\\\\|\\/|[^/\n])*/[gim]*', String.Regex, '#pop'),
+
+            # Array
+            (r'\[', Operator, ('#pop', 'array-decl')),
+
+            include('type'),
+        ],
+
+        # type-param part of a type
+        # ie. the <A,B> path in Map<A,B>
+        'type-param': [
+            include('spaces'),
+            (r'', Text, ('#pop', 'type-param-sep', 'type-param-type')),
+        ],
+
+        'type-param-sep': [
+            include('spaces'),
             (r'>', Punctuation, '#pop'),
-            (r',', Punctuation),
+            (r',', Punctuation, ('#pop', 'type-param')),
         ],
 
-        # Top level constructs
-        'imports': [
-            (r'(package|import|using)(\s+)([^;]+)(;)',
-             bygroups(Keyword.Namespace, Text, Name.Namespace,Punctuation)),
+        # optional type-param that may include constraint
+        # ie. <T:Constraint, T2:(ConstraintA,ConstraintB)>
+        'type-param-constraint': [
+            include('spaces'),
+            (r'<(?!=)', Punctuation, ('#pop', 'type-param-constraint-sep',
+                                      'type-param-constraint-flag', 'type-name')),
+            (r'', Text, '#pop'),
         ],
-        'typedef': [
-            (r'typedef', Keyword.Declaration, ('typedefprebody', 'typedecl')),
+
+        'type-param-constraint-sep': [
+            include('spaces'),
+            (r'>', Punctuation, '#pop'),
+            (r',', Punctuation, ('#pop', 'type-param-constraint-sep',
+                                 'type-param-constraint-flag', 'type-name')),
         ],
-        'typedefprebody': [
-            include('whitespace'),
-            (r'(=)(\s*)({)', bygroups(Punctuation, Text, Punctuation),
-             ('#pop', 'typedefbody')),
+
+        # the optional constraint inside type-param
+        'type-param-constraint-flag': [
+            include('spaces'),
+            (r':', Punctuation, ('#pop', 'type-param-constraint-flag-type')),
+            (r'', Text, '#pop'),
         ],
-        'enumdef': [
-            (r'enum', Keyword.Declaration, ('enumdefprebody', 'typedecl')),
+
+        'type-param-constraint-flag-type': [
+            include('spaces'),
+            (r'\(', Punctuation, ('#pop', 'type-param-constraint-flag-type-sep',
+                                  'type')),
+            (r'', Text, ('#pop', 'type')),
         ],
-        'enumdefprebody': [
-            include('whitespace'),
-            (r'{', Punctuation, ('#pop','enumdefbody')),
+
+        'type-param-constraint-flag-type-sep': [
+            include('spaces'),
+            (r'\)', Punctuation, '#pop'),
+            (r',', Punctuation, 'type'),
         ],
-        'classdef': [
-            (r'class', Keyword.Declaration, ('classdefprebody', 'typedecl')),
+
+        # a parenthesis expr that contain exactly one expr
+        'parenthesis': [
+            include('spaces'),
+            (r'', Text, ('#pop', 'parenthesis-close', 'expr')),
         ],
-        'classdefprebody': [
-            include('whitespace'),
-            (r'(extends|implements)', Keyword.Declaration,'typedecl'),
-            (r'{', Punctuation, ('#pop', 'classdefbody')),
+
+        'parenthesis-open': [
+            include('spaces'),
+            (r'\(', Punctuation, '#pop'),
         ],
-        'interfacedef': [
-            (r'interface', Keyword.Declaration,
-             ('interfacedefprebody', 'typedecl')),
+
+        'parenthesis-close': [
+            include('spaces'),
+            (r'\)', Punctuation, '#pop'),
         ],
-        'interfacedefprebody': [
-            include('whitespace'),
-            (r'(extends)', Keyword.Declaration, 'typedecl'),
-            (r'{', Punctuation, ('#pop', 'classdefbody')),
-        ],
-
-        'typedefbody': [
-          include('whitespace'),
-          include('instancevardef'),
-          include('instancefundef'),
-          (r'>', Punctuation, 'typedecl'),
-          (r',', Punctuation),
-          (r'}', Punctuation, '#pop'),
-        ],
-        'enumdefbody': [
-          include('whitespace'),
-          (ident, Name.Variable.Instance),
-          (r'\(', Punctuation, 'funargdecl'),
-          (r';', Punctuation),
-          (r'}', Punctuation, '#pop'),
-        ],
-        'classdefbody': [
-          include('whitespace'),
-          include('instancevardef'),
-          include('instancefundef'),
-          (r'}', Punctuation, '#pop'),
-          include('codeblock'),
+
+        'var': [
+            include('spaces'),
+            (ident_no_keyword, Text, ('#pop', 'var-sep', 'assign', 'flag')),
         ],
+
+        # optional more var decl.
+        'var-sep': [
+            include('spaces'),
+            (r',', Punctuation, ('#pop', 'var')),
+            (r'', Text, '#pop'),
+        ],
+
+        # optional assignment
+        'assign': [
+            include('spaces'),
+            (r'=', Operator, ('#pop', 'expr')),
+            (r'', Text, '#pop'),
+        ],
+
+        # optional type flag
+        'flag': [
+            include('spaces'),
+            (r':', Punctuation, ('#pop', 'type')),
+            (r'', Text, '#pop'),
+        ],
+
+        # colon as part of a ternary operator (?:)
+        'ternary': [
+            include('spaces'),
+            (r':', Operator, '#pop'),
+        ],
+
+        # function call
+        'call': [
+            include('spaces'),
+            (r'\)', Punctuation, '#pop'),
+            (r'', Text, ('#pop', 'call-sep', 'expr')),
+        ],
+
+        # after a call param
+        'call-sep': [
+            include('spaces'),
+            (r'\)', Punctuation, '#pop'),
+            (r',', Punctuation, ('#pop', 'call')),
+        ],
+
+        # bracket can be block or object
+        'bracket': [
+            include('spaces'),
+            (r'(?!(?:\$\s*[a-z]\b|\$(?!'+ident+')))' + ident_no_keyword, Name,
+             ('#pop', 'bracket-check')),
+            (r"'", String.Single, ('#pop', 'bracket-check', 'string-single')),
+            (r'"', String.Double, ('#pop', 'bracket-check', 'string-double')),
+            (r'', Text, ('#pop', 'block')),
+        ],
+
+        'bracket-check': [
+            include('spaces'),
+            (r':', Punctuation, ('#pop', 'object-sep', 'expr')), #is object
+            (r'', Text, ('#pop', 'block', 'optional-semicolon', 'expr-chain')), #is block
+        ],
+
+        # code block
+        'block': [
+            include('spaces'),
+            (r'\}', Punctuation, '#pop'),
+            (r'', Text, 'expr-statement'),
+        ],
+
+        # object in key-value pairs
+        'object': [
+            include('spaces'),
+            (r'\}', Punctuation, '#pop'),
+            (r'', Text, ('#pop', 'object-sep', 'expr', 'colon', 'ident-or-string'))
+        ],
+
+        # a key of an object
+        'ident-or-string': [
+            include('spaces'),
+            (ident_no_keyword, Name, '#pop'),
+            (r"'", String.Single, ('#pop', 'string-single')),
+            (r'"', String.Double, ('#pop', 'string-double')),
+        ],
+
+        # after a key-value pair in object
+        'object-sep': [
+            include('spaces'),
+            (r'\}', Punctuation, '#pop'),
+            (r',', Punctuation, ('#pop', 'object')),
+        ],
+
+
+
     }
 
     def analyse_text(text):
diff --git a/tests/examplefiles/example.hx b/tests/examplefiles/example.hx
new file mode 100644
index 00000000..fd93bb49
--- /dev/null
+++ b/tests/examplefiles/example.hx
@@ -0,0 +1,142 @@
+/**
+ * This is not really a valid Haxe file, but just an demo...
+ */
+
+package;
+package net.onthewings;
+
+import net.onthewings.Test;
+import net.onthewings.*;
+
+using Lambda;
+using net.onthewings.Test;
+
+#if flash8
+// Haxe code specific for flash player 8
+#elseif flash
+// Haxe code specific for flash platform (any version)
+#elseif js
+// Haxe code specific for javascript plaform
+#elseif neko
+// Haxe code specific for neko plaform
+#else 
+// do something else
+    #error  // will display an error "Not implemented on this platform"
+    #error "Custom error message" // will display an error "Custom error message"
+#end
+
+0; // Int
+-134; // Int
+0xFF00; // Int
+
+123.0; // Float
+.14179; // Float
+13e50; // Float
+-1e-99; // Float
+
+"hello"; // String
+"hello \"world\" !"; // String
+'hello "world" !'; // String
+
+true; // Bool
+false; // Bool
+
+null; // Unknown<0>
+
+~/[a-z]+/i; // EReg : regular expression
+
+var point = { "x" : 1, "y" : -5 };
+
+{
+    var x;
+    var y = 3;
+    var z : String;
+    var w : String = "";
+    var a, b : Bool, c : Int = 0;
+}
+
+//haxe3 pattern matching
+switch(e.expr) {
+	case EConst(CString(s)) if (StringTools.startsWith(s, "foo")):
+		"1";
+	case EConst(CString(s)) if (StringTools.startsWith(s, "bar")):
+		"2";
+	case EConst(CInt(i)) if (switch(Std.parseInt(i) * 2) { case 4: true; case _: false; }):
+		"3";
+	case EConst(_):
+		"4";
+	case _:
+		"5";
+}
+
+switch [true, 1, "foo"] {
+	case [true, 1, "foo"]: "0";
+	case [true, 1, _]: "1";
+	case _: "_";
+}
+
+
+class Test <T:Void->Void> {
+	private function new():Void {
+		inline function innerFun(a:Int, b:Int):Int {
+			return readOnlyField = a + b;
+		}
+		
+		_innerFun(1, 2.3);
+	}
+	
+	static public var instance(get,null):Test;
+	static function get_instance():Test {
+		return instance != null ? instance : instance = new Test();
+	}
+}
+
+@:native("Test") private class Test2 {}
+
+extern class Ext {}
+
+@:macro class M {
+	@:macro static function test(e:Array<Expr>):ExprOf<String> {
+		return macro "ok";
+	}
+}
+
+enum Color {
+    Red;
+    Green;
+    Blue;
+    Grey( v : Int );
+    Rgb( r : Int, g : Int, b : Int );
+    Alpha( a : Int, col : Color );
+}
+
+class Colors {
+    static function toInt( c : Color ) : Int {
+        return switch( c ) {
+            case Red: 0xFF0000;
+            case Green: 0x00FF00;
+            case Blue: 0x0000FF;
+            case Grey(v): (v << 16) | (v << 8) | v;
+            case Rgb(r,g,b): (r << 16) | (g << 8) | b;
+            case Alpha(a,c): (a << 24) | (toInt(c) & 0xFFFFFF);
+        }
+    }
+}
+
+class EvtQueue<T : (Event, EventDispatcher)> {
+    var evt : T;
+}
+
+typedef DS = Dynamic<String>;
+typedef Pt = {
+	var x:Float;
+	var y:Float;
+	@:optional var z:Float; /* optional z */
+	function add(pt:Pt):Void;
+}
+typedef Pt2 = {
+	x:Float,
+	y:Float,
+	?z:Float, //optional z
+	add : Point -> Void,
+}
\ No newline at end of file