Merged in camilstaps/pygments-main (pull request #503)

Adds a Clean (http://clean.cs.ru.nl/Clean) Lexer and example file

4 files changed, 370 insertions, 0 deletions

diff --git a/AUTHORS b/AUTHORS
index 9f45d8ea..c5ed18b7 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -174,6 +174,7 @@ Other contributors, listed alphabetically, are:
 * Alexander Smishlajev -- Visual FoxPro lexer
 * Steve Spigarelli -- XQuery lexer
 * Jerome St-Louis -- eC lexer
+* Camil Staps -- Clean lexer
 * James Strachan -- Kotlin lexer
 * Tom Stuart -- Treetop lexer
 * Colin Sullivan -- SuperCollider lexer
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index a414a565..c5728cf2 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -79,6 +79,7 @@ LEXERS = {
     'CheetahXmlLexer': ('pygments.lexers.templates', 'XML+Cheetah', ('xml+cheetah', 'xml+spitfire'), (), ('application/xml+cheetah', 'application/xml+spitfire')),
     'CirruLexer': ('pygments.lexers.webmisc', 'Cirru', ('cirru',), ('*.cirru',), ('text/x-cirru',)),
     'ClayLexer': ('pygments.lexers.c_like', 'Clay', ('clay',), ('*.clay',), ('text/x-clay',)),
+    'CleanLexer': ('pygments.lexers.clean', 'CleanLexer', ('Clean', 'clean'), ('*.icl', '*.dcl'), ()),
     'ClojureLexer': ('pygments.lexers.jvm', 'Clojure', ('clojure', 'clj'), ('*.clj',), ('text/x-clojure', 'application/x-clojure')),
     'ClojureScriptLexer': ('pygments.lexers.jvm', 'ClojureScript', ('clojurescript', 'cljs'), ('*.cljs',), ('text/x-clojurescript', 'application/x-clojurescript')),
     'CobolFreeformatLexer': ('pygments.lexers.business', 'COBOLFree', ('cobolfree',), ('*.cbl', '*.CBL'), ()),
diff --git a/pygments/lexers/clean.py b/pygments/lexers/clean.py
new file mode 100644
index 00000000..b33bf884
--- /dev/null
+++ b/pygments/lexers/clean.py
@@ -0,0 +1,276 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.clean
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Clean language.
+
+    :copyright: Copyright 2016 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import ExtendedRegexLexer, LexerContext,\
+        bygroups, words, include, default
+from pygments.token import Comment, Keyword, Literal, Name, Number, Operator,\
+        Punctuation, String, Text, Whitespace
+
+__all__ = ['CleanLexer']
+
+class CleanLexer(ExtendedRegexLexer):
+    """
+    Lexer for the general purpose, state-of-the-art, pure and lazy functional
+    programming language Clean (http://clean.cs.ru.nl/Clean).
+
+    .. versionadded: 2.1
+    """
+    name = 'CleanLexer'
+    aliases = ['Clean', 'clean']
+    filenames = ['*.icl', '*.dcl']
+
+    def __init__(self, *args, **kwargs):
+        super(CleanLexer, self).__init__(*args, **kwargs)
+
+    def get_tokens_unprocessed(self, text=None, context=None):
+        ctx = LexerContext(text, 0)
+        ctx.indent = 0
+        return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context=ctx)
+
+    def check_class_not_import(lexer, match, ctx):
+        if match.group(0) == 'import':
+            yield match.start(), Keyword.Namespace, match.group(0)
+            ctx.stack = ctx.stack[:-1] + ['fromimportfunc']
+        else:
+            yield match.start(), Name.Class, match.group(0)
+        ctx.pos = match.end()
+
+    def check_instance_class(lexer, match, ctx):
+        if match.group(0) == 'instance' or match.group(0) == 'class':
+            yield match.start(), Keyword, match.group(0)
+        else:
+            yield match.start(), Name.Function, match.group(0)
+            ctx.stack = ctx.stack + ['fromimportfunctype']
+        ctx.pos = match.end()
+
+    def store_indent(lexer, match, ctx):
+        # Tabs are four spaces: 
+        # https://svn.cs.ru.nl/repos/clean-platform/trunk/doc/STANDARDS.txt
+        ctx.indent = len(match.group(0).replace('\t','    '))
+        ctx.pos = match.end()
+        yield match.start(), Text, match.group(0)
+
+    def check_indent1(lexer, match, ctx):
+        indent = len(match.group(0)) - 1
+        if indent > ctx.indent:
+            yield match.start(), Whitespace, match.group(0)
+            ctx.pos = match.start() + indent + 1
+        else:
+            ctx.indent = 0
+            ctx.pos = match.start()
+            ctx.stack = ctx.stack[:-1]
+            yield match.start(), Whitespace, match.group(0)[1:]
+
+    def check_indent2(lexer, match, ctx):
+        indent = len(match.group(0)) - 1
+        if indent > ctx.indent:
+            yield match.start(), Whitespace, match.group(0)
+            ctx.pos = match.start() + indent + 1
+        else:
+            ctx.indent = 0
+            ctx.pos = match.start()
+            ctx.stack = ctx.stack[:-2]
+            yield match.start(), Whitespace, match.group(0)[1:]
+            if match.group(0) == '\n\n':
+                ctx.pos = ctx.pos + 1
+
+    def check_indent3(lexer, match, ctx):
+        indent = len(match.group(0)) - 1
+        if indent > ctx.indent:
+            yield match.start(), Whitespace, match.group(0)
+            ctx.pos = match.start() + indent + 1
+        else:
+            ctx.indent = 0
+            ctx.pos = match.start()
+            ctx.stack = ctx.stack[:-3]
+            yield match.start(), Whitespace, match.group(0)[1:]
+            if match.group(0) == '\n\n':
+                ctx.pos = ctx.pos + 1
+
+    def skip(lexer, match, ctx):
+        ctx.stack = ctx.stack[:-1]
+        ctx.pos = match.end()
+        yield match.start(), Comment, match.group(0)
+
+    tokens = {
+        'common': [
+            (r';', Punctuation, '#pop'),
+            (r'//', Comment, 'singlecomment')
+        ],
+        'root': [
+            # Comments
+            (r'//.*\n', Comment.Single),
+            (r'(?s)/\*\*.*?\*/', Comment.Special),
+            (r'(?s)/\*.*?\*/', Comment.Multi),
+
+            # Modules, imports, etc.
+            (r'\b((?:implementation|definition|system)\s+)?(module)(\s+)([\w`]+)', 
+                bygroups(Keyword.Namespace, Keyword.Namespace, Text, Name.Class)),
+            (r'(?<=\n)import(?=\s)', Keyword.Namespace, 'import'),
+            (r'(?<=\n)from(?=\s)', Keyword.Namespace, 'fromimport'),
+
+            # Keywords
+            # We cannot use (?s)^|(?<=\s) as prefix, so need to repeat this
+            (words(('class','instance','where','with','let','let!','with','in',
+                'case','of','infix','infixr','infixl','generic','derive',
+                'otherwise', 'code', 'inline'), 
+                prefix=r'(?<=\s)', suffix=r'(?=\s)'), Keyword),
+            (words(('class','instance','where','with','let','let!','with','in',
+                'case','of','infix','infixr','infixl','generic','derive',
+                'otherwise', 'code', 'inline'), 
+                prefix=r'^', suffix=r'(?=\s)'), Keyword),
+
+            # Function definitions
+            (r'(?=\{\|)', Whitespace, 'genericfunction'),
+            (r'(?<=\n)(\s*)([\w`\$\(\)=\-<>~*\^\|\+&%]+)(\s+[\w])*(\s*)(::)',
+                bygroups(store_indent, Name.Function, Keyword.Type, Whitespace, Punctuation),
+                'functiondefargs'),
+
+            # Type definitions
+            (r'(?<=\n)([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'),
+            (r'^([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'),
+
+            # Literals
+            (r'\'\\?.(?<!\\)\'', String.Char),
+            (r'\'\\\d+\'', String.Char),
+            (r'\'\\\\\'', String.Char), # (special case for '\\')
+            (r'[\+\-~]?\s\d+\.\d+(E[+-~]?\d+)?\b', Number.Float),
+            (r'[\+\-~]?\s0[0-7]\b', Number.Oct),
+            (r'[\+\-~]?\s0x[0-9a-fA-F]\b', Number.Hex),
+            (r'[\+\-~]?\s\d+\b', Number.Integer),
+            (r'"', String.Double, 'doubleqstring'),
+            (words(('True', 'False'), prefix=r'(?<=\s)', suffix=r'(?=\s)'), Literal),
+
+            # Everything else is some name
+            (r'([\w`\$%]+\.?)*[\w`\$%]+', Name),
+
+            # Punctuation
+            (r'[{}()\[\],:;\.#]', Punctuation),
+            (r'[\+\-=!<>\|&~*\^/]', Operator),
+            (r'\\\\', Operator),
+
+            # Lambda expressions
+            (r'\\.*?(->|\.|=)', Name.Function),
+
+            # Whitespace
+            (r'\s', Whitespace),
+
+            include('common')
+        ],
+        'fromimport': [
+            include('common'),
+            (r'([\w`]+)', check_class_not_import),
+            (r'\n', Whitespace, '#pop'),
+            (r'\s', Whitespace)
+        ],
+        'fromimportfunc': [
+            include('common'),
+            (r'([\w`\$\(\)=\-<>~*\^\|\+&%]+)', check_instance_class),
+            (r',', Punctuation),
+            (r'\n', Whitespace, '#pop'),
+            (r'\s', Whitespace)
+        ],
+        'fromimportfunctype': [
+            include('common'),
+            (r'[{(\[]', Punctuation, 'combtype'),
+            (r',', Punctuation, '#pop'),
+            (r'[:;\.#]', Punctuation),
+            (r'\n', Whitespace, '#pop:2'),
+            (r'[^\S\n]+', Whitespace),
+            (r'\S+', Keyword.Type)
+        ],
+        'combtype': [
+            include('common'),
+            (r'[})\]]', Punctuation, '#pop'),
+            (r'[{(\[]', Punctuation, '#pop'),
+            (r'[,:;\.#]', Punctuation),
+            (r'\s+', Whitespace),
+            (r'\S+', Keyword.Type)
+        ],
+        'import': [
+            include('common'),
+            (words(('from', 'import', 'as', 'qualified'), 
+                prefix='(?<=\s)', suffix='(?=\s)'), Keyword.Namespace),
+            (r'[\w`]+', Name.Class),
+            (r'\n', Whitespace, '#pop'),
+            (r',', Punctuation),
+            (r'[^\S\n]+', Whitespace)
+        ],
+        'singlecomment': [
+            (r'(.)(?=\n)', skip),
+            (r'.+(?!\n)', Comment)
+        ],
+        'doubleqstring': [
+            (r'[^\\"]+', String.Double),
+            (r'"', String.Double, '#pop'),
+            (r'\\.', String.Double)
+        ],
+        'typedef': [
+            include('common'),
+            (r'[\w`]+', Keyword.Type),
+            (r'[:=\|\(\),\[\]\{\}\!\*]', Punctuation),
+            (r'->', Punctuation),
+            (r'\n(?=[^\s\|])', Whitespace, '#pop'),
+            (r'\s', Whitespace),
+            (r'.', Keyword.Type)
+        ],
+        'genericfunction': [
+            include('common'),
+            (r'\{\|', Punctuation),
+            (r'\|\}', Punctuation, '#pop'),
+            (r',', Punctuation),
+            (r'->', Punctuation),
+            (r'(\s+of\s+)(\{)', bygroups(Keyword, Punctuation), 'genericftypes'),
+            (r'\s', Whitespace),
+            (r'[\w`]+', Keyword.Type),
+            (r'[\*\(\)]', Punctuation)
+        ],
+        'genericftypes': [
+            include('common'),
+            (r'[\w`]+', Keyword.Type),
+            (r',', Punctuation),
+            (r'\s', Whitespace),
+            (r'\}', Punctuation, '#pop')
+        ],
+        'functiondefargs': [
+            include('common'),
+            (r'\n(\s*)', check_indent1),
+            (r'[!{}()\[\],:;\.#]', Punctuation),
+            (r'->', Punctuation, 'functiondefres'),
+            (r'^(?=\S)', Whitespace, '#pop'),
+            (r'\S', Keyword.Type),
+            (r'\s', Whitespace)
+        ],
+        'functiondefres': [
+            include('common'),
+            (r'\n(\s*)', check_indent2),
+            (r'^(?=\S)', Whitespace, '#pop:2'),
+            (r'[!{}()\[\],:;\.#]', Punctuation),
+            (r'\|', Punctuation, 'functiondefclasses'),
+            (r'\S', Keyword.Type),
+            (r'\s', Whitespace)
+        ],
+        'functiondefclasses': [
+            include('common'),
+            (r'\n(\s*)', check_indent3),
+            (r'^(?=\S)', Whitespace, '#pop:3'),
+            (r'[,&]', Punctuation),
+            (r'[\w`\$\(\)=\-<>~*\^\|\+&%]', Name.Function, 'functionname'),
+            (r'\s', Whitespace)
+        ],
+        'functionname': [
+            include('common'),
+            (r'[\w`\$\(\)=\-<>~*\^\|\+&%]+', Name.Function),
+            (r'(?=\{\|)', Punctuation, 'genericfunction'),
+            default('#pop')
+        ]
+    }
+
diff --git a/tests/examplefiles/StdGeneric.icl b/tests/examplefiles/StdGeneric.icl
new file mode 100644
index 00000000..2e6c3931
--- /dev/null
+++ b/tests/examplefiles/StdGeneric.icl
@@ -0,0 +1,92 @@
+implementation module StdGeneric
+
+import StdInt, StdMisc, StdClass, StdFunc
+
+generic bimap a b :: Bimap .a .b
+
+bimapId :: Bimap .a .a
+bimapId = { map_to = id, map_from = id }
+
+bimap{|c|} = { map_to = id, map_from = id }
+
+bimap{|PAIR|} bx by = { map_to= map_to, map_from=map_from }
+where
+	map_to (PAIR x y) 	= PAIR (bx.map_to x) (by.map_to y)
+	map_from (PAIR x y) 	= PAIR (bx.map_from x) (by.map_from y)
+bimap{|EITHER|} bl br = { map_to= map_to, map_from=map_from }
+where	
+	map_to (LEFT x) 	= LEFT (bl.map_to x)
+	map_to (RIGHT x)	= RIGHT (br.map_to x)
+	map_from (LEFT x) 	= LEFT (bl.map_from x)
+	map_from (RIGHT x) 	= RIGHT (br.map_from x)
+
+bimap{|(->)|} barg bres = { map_to = map_to, map_from = map_from }
+where
+	map_to f = comp3 bres.map_to f barg.map_from
+	map_from f = comp3 bres.map_from f barg.map_to
+
+bimap{|CONS|} barg = { map_to= map_to, map_from=map_from }
+where
+	map_to   (CONS x) = CONS (barg.map_to x)
+	map_from (CONS x) = CONS (barg.map_from x)
+
+bimap{|FIELD|} barg = { map_to= map_to, map_from=map_from }
+where
+	map_to   (FIELD x) = FIELD (barg.map_to x)
+	map_from (FIELD x) = FIELD (barg.map_from x)
+
+bimap{|OBJECT|} barg = { map_to= map_to, map_from=map_from }
+where
+	map_to   (OBJECT x) = OBJECT (barg.map_to x)
+	map_from (OBJECT x) = OBJECT (barg.map_from x)
+
+bimap{|Bimap|} x y = {map_to = map_to, map_from = map_from}
+where
+	map_to 	{map_to, map_from} = 
+		{ map_to 	= comp3 y.map_to map_to x.map_from
+		, map_from 	= comp3 x.map_to map_from y.map_from
+		}
+	map_from {map_to, map_from} = 
+		{ map_to 	= comp3 y.map_from map_to x.map_to
+		, map_from 	= comp3 x.map_from map_from y.map_to
+		}
+
+comp3 :: !(.a -> .b) u:(.c -> .a) !(.d -> .c) -> u:(.d -> .b)
+comp3 f g h
+	| is_id f
+		| is_id h
+			= cast g
+			= cast (\x -> g (h x))
+		| is_id h
+			= cast (\x -> f (g x))
+			= \x -> f (g (h x))
+where
+	is_id :: !.(.a -> .b) -> Bool
+	is_id f = code inline
+	{
+		eq_desc e_StdFunc_did 0 0
+		pop_a 1
+	}
+	
+	cast :: !u:a -> u:b
+	cast f = code inline
+	{
+		pop_a 0
+	}
+
+getConsPath :: !GenericConsDescriptor -> [ConsPos]
+getConsPath {gcd_index, gcd_type_def={gtd_num_conses}}
+	= doit gcd_index gtd_num_conses
+where
+	doit i n
+		| n == 0 	
+			= abort "getConsPath: zero conses\n"
+		| i >= n	
+			= abort "getConsPath: cons index >= number of conses"
+		| n == 1
+			= []
+		| i < (n/2)
+			= [ ConsLeft : doit i (n/2) ]
+		| otherwise
+			= [ ConsRight : doit (i - (n/2)) (n - (n/2)) ]
+			  	 							 	
\ No newline at end of file