Add func lexer (#2232)

5 files changed, 634 insertions, 0 deletions

diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 3c5f043a..89a286f9 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -167,6 +167,7 @@ LEXERS = {
     'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran', 'f90'), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)),
     'FoxProLexer': ('pygments.lexers.foxpro', 'FoxPro', ('foxpro', 'vfp', 'clipper', 'xbase'), ('*.PRG', '*.prg'), ()),
     'FreeFemLexer': ('pygments.lexers.freefem', 'Freefem', ('freefem',), ('*.edp',), ('text/x-freefem',)),
+    'FuncLexer': ('pygments.lexers.func', 'FunC', ('func', 'fc'), ('*.fc', '*.func'), ()),
     'FutharkLexer': ('pygments.lexers.futhark', 'Futhark', ('futhark',), ('*.fut',), ('text/x-futhark',)),
     'GAPConsoleLexer': ('pygments.lexers.algebra', 'GAP session', ('gap-console', 'gap-repl'), ('*.tst',), ()),
     'GAPLexer': ('pygments.lexers.algebra', 'GAP', ('gap',), ('*.g', '*.gd', '*.gi', '*.gap'), ()),
diff --git a/pygments/lexers/func.py b/pygments/lexers/func.py
new file mode 100644
index 00000000..6018f85b
--- /dev/null
+++ b/pygments/lexers/func.py
@@ -0,0 +1,108 @@
+"""
+    pygments.lexers.func
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for FunC.
+
+    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Whitespace, Punctuation
+
+__all__ = ['FuncLexer']
+
+
+class FuncLexer(RegexLexer):
+    """
+    For FunC source code.
+    """
+
+    name = 'FunC'
+    aliases = ['func', 'fc']
+    filenames = ['*.fc', '*.func']
+
+    # 1. Does not start from "
+    # 2. Can start from ` and end with `, containing any character
+    # 3. Starts with underscore or { or } and have more than 1 character after it
+    # 4. Starts with letter, contains letters, numbers and underscores
+    identifier = r'(?!")(`([^`]+)`|((?=_)_|(?=\{)\{|(?=\})\}|(?![_`{}]))([^;,\[\]\(\)\s~.]+))'
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'\s+', Whitespace),
+
+            include('keywords'),
+            include('strings'),
+            include('directives'),
+            include('numeric'),
+            include('comments'),
+            include('storage'),
+            include('functions'),
+            include('variables'),
+
+            (r'[.;(),\[\]~{}]', Punctuation)
+        ],
+        'keywords': [
+            (words((
+                '<=>', '>=', '<=', '!=', '==', '^>>', '~>>',
+                '>>', '<<', '/%', '^%', '~%', '^/', '~/', '+=',
+                '-=', '*=', '/=', '~/=', '^/=', '%=', '^%=', '<<=',
+                '>>=', '~>>=', '^>>=', '&=', '|=', '^=', '^', '=',
+                '~', '/', '%', '-', '*', '+','>',
+                '<', '&', '|', ':', '?'), prefix=r'(?<=\s)', suffix=r'(?=\s)'),
+             Operator),
+            (words((
+                'if', 'ifnot',
+                'else', 'elseif', 'elseifnot',
+                'while', 'do', 'until', 'repeat',
+                'return', 'impure', 'method_id',
+                'forall', 'asm', 'inline', 'inline_ref'), prefix=r'\b', suffix=r'\b'),
+             Keyword),
+            (words(('true', 'false'), prefix=r'\b', suffix=r'\b'), Keyword.Constant),
+        ],
+        'directives': [
+            (r'#include|#pragma', Keyword, 'directive'),
+        ],
+        'directive': [
+            include('strings'),
+            (r'\s+', Whitespace),
+            (r'version|not-version', Keyword),
+            (r'(>=|<=|=|>|<|\^)?([0-9]+)(.[0-9]+)?(.[0-9]+)?', Number), # version
+            (r';', Text, '#pop')
+        ],
+        'strings': [
+            (r'\"([^\n\"]+)\"[Hhcusa]?', String),
+        ],
+        'numeric': [
+            (r'\b(-?(?!_)([\d_]+|0x[\d_a-fA-F]+)|0b[1_0]+)(?<!_)(?=[\s\)\],;])', Number)
+        ],
+        'comments': [
+            (r';;([^\n]*)', Comment.Singleline),
+            (r'\{-', Comment.Multiline, 'comment'),
+        ],
+        'comment': [
+            (r'[^-}{]+', Comment.Multiline),
+            (r'\{-', Comment.Multiline, '#push'),
+            (r'-\}', Comment.Multiline, '#pop'),
+            (r'[-}{]', Comment.Multiline),
+        ],
+        'storage': [
+            (words((
+                'var', 'int', 'slice', 'tuple',
+                'cell', 'builder', 'cont', '_'),
+                prefix=r'\b', suffix=r'(?=[\s\(\),\[\]])'),
+             Keyword.Type),
+            (words(('global', 'const'), prefix=r'\b', suffix=r'\b'), Keyword.Constant),
+        ],
+        'variables': [
+            (identifier, Name.Variable),
+        ],
+        'functions': [
+            # identifier followed by (
+            (identifier + r'(?=[\(])', Name.Function),
+        ]
+    }
diff --git a/tests/examplefiles/func/test.fc b/tests/examplefiles/func/test.fc
new file mode 100644
index 00000000..b196a6b4
--- /dev/null
+++ b/tests/examplefiles/func/test.fc
@@ -0,0 +1,52 @@
+#include "../";
+#pragma version >=1.0.0;
+
+global int k_const;
+const int k = 1;
+
+() recv_internal(int my_balance, int msg_value, cell in_msg_full, slice in_msg_body) impure {
+    slice cs = in_msg_full.begin_parse();
+    int flags = cs~load_uint(0x4_1_0);
+
+    if ((flags & 1) == true) { ;; ignore all bounced messages
+        return ();
+    }
+
+    slice sender_address = cs~load_msg_addr();
+
+    {-
+    {-
+    test - is test
+    -}
+    -}
+
+    ;; Send message
+    var message = begin_cell()
+        .store_uint(0x18, 6)
+        .store_slice(sender_address)
+        .store_coins(0)
+        .store_uint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1)
+        .store_slice("Hello, world!"s)
+        .end_cell();
+
+    send_raw_message(message, 64);
+
+    ;; Update counter
+    var cs = get_data().begin_parse();
+    var counter = data~load_uint(32);
+
+    store_data(
+        begin_cell()
+            .store_uint(counter + 1, 32)
+        .end_cell()
+    );
+}
+
+() recv_external(slice in_msg) impure {
+    throw(0xffff);
+}
+
+int counter() method_id {
+    var data = get_data().begin_parse();
+    return data~load_uint(32);
+}
\ No newline at end of file
diff --git a/tests/examplefiles/func/test.fc.output b/tests/examplefiles/func/test.fc.output
new file mode 100644
index 00000000..39358e23
--- /dev/null
+++ b/tests/examplefiles/func/test.fc.output
@@ -0,0 +1,429 @@
+'#include'    Keyword
+' '           Text.Whitespace
+'"../"'       Literal.String
+';'           Text
+'\n'          Text.Whitespace
+
+'#pragma'     Keyword
+' '           Text.Whitespace
+'version'     Keyword
+' '           Text.Whitespace
+'>=1.0.0'     Literal.Number
+';'           Text
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'global'      Keyword.Constant
+' '           Text.Whitespace
+'int'         Keyword.Type
+' '           Text.Whitespace
+'k_const'     Name.Variable
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'const'       Keyword.Constant
+' '           Text.Whitespace
+'int'         Keyword.Type
+' '           Text.Whitespace
+'k'           Name.Variable
+' '           Text.Whitespace
+'='           Operator
+' '           Text.Whitespace
+'1'           Literal.Number
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'('           Punctuation
+')'           Punctuation
+' '           Text.Whitespace
+'recv_internal' Name.Function
+'('           Punctuation
+'int'         Keyword.Type
+' '           Text.Whitespace
+'my_balance'  Name.Variable
+','           Punctuation
+' '           Text.Whitespace
+'int'         Keyword.Type
+' '           Text.Whitespace
+'msg_value'   Name.Variable
+','           Punctuation
+' '           Text.Whitespace
+'cell'        Keyword.Type
+' '           Text.Whitespace
+'in_msg_full' Name.Variable
+','           Punctuation
+' '           Text.Whitespace
+'slice'       Keyword.Type
+' '           Text.Whitespace
+'in_msg_body' Name.Variable
+')'           Punctuation
+' '           Text.Whitespace
+'impure'      Keyword
+' '           Text.Whitespace
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'slice'       Keyword.Type
+' '           Text.Whitespace
+'cs'          Name.Variable
+' '           Text.Whitespace
+'='           Operator
+' '           Text.Whitespace
+'in_msg_full' Name.Variable
+'.'           Punctuation
+'begin_parse' Name.Function
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'int'         Keyword.Type
+' '           Text.Whitespace
+'flags'       Name.Variable
+' '           Text.Whitespace
+'='           Operator
+' '           Text.Whitespace
+'cs'          Name.Variable
+'~'           Punctuation
+'load_uint'   Name.Function
+'('           Punctuation
+'0x4_1_0'     Literal.Number
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'if'          Keyword
+' '           Text.Whitespace
+'('           Punctuation
+'('           Punctuation
+'flags'       Name.Variable
+' '           Text.Whitespace
+'&'           Operator
+' '           Text.Whitespace
+'1'           Literal.Number
+')'           Punctuation
+' '           Text.Whitespace
+'=='          Operator
+' '           Text.Whitespace
+'true'        Keyword.Constant
+')'           Punctuation
+' '           Text.Whitespace
+'{'           Punctuation
+' '           Text.Whitespace
+';; ignore all bounced messages' Comment.Singleline
+'\n'          Text.Whitespace
+
+'        '    Text.Whitespace
+'return'      Keyword
+' '           Text.Whitespace
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'slice'       Keyword.Type
+' '           Text.Whitespace
+'sender_address' Name.Variable
+' '           Text.Whitespace
+'='           Operator
+' '           Text.Whitespace
+'cs'          Name.Variable
+'~'           Punctuation
+'load_msg_addr' Name.Function
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'{-'          Comment.Multiline
+'\n    '      Comment.Multiline
+'{-'          Comment.Multiline
+'\n    test ' Comment.Multiline
+'-'           Comment.Multiline
+' is test\n    ' Comment.Multiline
+'-}'          Comment.Multiline
+'\n    '      Comment.Multiline
+'-}'          Comment.Multiline
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+';; Send message' Comment.Singleline
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'var'         Keyword.Type
+' '           Text.Whitespace
+'message'     Name.Variable
+' '           Text.Whitespace
+'='           Operator
+' '           Text.Whitespace
+'begin_cell'  Name.Function
+'('           Punctuation
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'        '    Text.Whitespace
+'.'           Punctuation
+'store_uint'  Name.Function
+'('           Punctuation
+'0x18'        Literal.Number
+','           Punctuation
+' '           Text.Whitespace
+'6'           Literal.Number
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'        '    Text.Whitespace
+'.'           Punctuation
+'store_slice' Name.Function
+'('           Punctuation
+'sender_address' Name.Variable
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'        '    Text.Whitespace
+'.'           Punctuation
+'store_coins' Name.Function
+'('           Punctuation
+'0'           Literal.Number
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'        '    Text.Whitespace
+'.'           Punctuation
+'store_uint'  Name.Function
+'('           Punctuation
+'0'           Literal.Number
+','           Punctuation
+' '           Text.Whitespace
+'1'           Literal.Number
+' '           Text.Whitespace
+'+'           Operator
+' '           Text.Whitespace
+'4'           Literal.Number
+' '           Text.Whitespace
+'+'           Operator
+' '           Text.Whitespace
+'4'           Literal.Number
+' '           Text.Whitespace
+'+'           Operator
+' '           Text.Whitespace
+'64'          Literal.Number
+' '           Text.Whitespace
+'+'           Operator
+' '           Text.Whitespace
+'32'          Literal.Number
+' '           Text.Whitespace
+'+'           Operator
+' '           Text.Whitespace
+'1'           Literal.Number
+' '           Text.Whitespace
+'+'           Operator
+' '           Text.Whitespace
+'1'           Literal.Number
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'        '    Text.Whitespace
+'.'           Punctuation
+'store_slice' Name.Function
+'('           Punctuation
+'"Hello, world!"s' Literal.String
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'        '    Text.Whitespace
+'.'           Punctuation
+'end_cell'    Name.Function
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'send_raw_message' Name.Function
+'('           Punctuation
+'message'     Name.Variable
+','           Punctuation
+' '           Text.Whitespace
+'64'          Literal.Number
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+';; Update counter' Comment.Singleline
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'var'         Keyword.Type
+' '           Text.Whitespace
+'cs'          Name.Variable
+' '           Text.Whitespace
+'='           Operator
+' '           Text.Whitespace
+'get_data'    Name.Function
+'('           Punctuation
+')'           Punctuation
+'.'           Punctuation
+'begin_parse' Name.Function
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'var'         Keyword.Type
+' '           Text.Whitespace
+'counter'     Name.Variable
+' '           Text.Whitespace
+'='           Operator
+' '           Text.Whitespace
+'data'        Name.Variable
+'~'           Punctuation
+'load_uint'   Name.Function
+'('           Punctuation
+'32'          Literal.Number
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'store_data'  Name.Function
+'('           Punctuation
+'\n'          Text.Whitespace
+
+'        '    Text.Whitespace
+'begin_cell'  Name.Function
+'('           Punctuation
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'            ' Text.Whitespace
+'.'           Punctuation
+'store_uint'  Name.Function
+'('           Punctuation
+'counter'     Name.Variable
+' '           Text.Whitespace
+'+'           Operator
+' '           Text.Whitespace
+'1'           Literal.Number
+','           Punctuation
+' '           Text.Whitespace
+'32'          Literal.Number
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'        '    Text.Whitespace
+'.'           Punctuation
+'end_cell'    Name.Function
+'('           Punctuation
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'('           Punctuation
+')'           Punctuation
+' '           Text.Whitespace
+'recv_external' Name.Function
+'('           Punctuation
+'slice'       Keyword.Type
+' '           Text.Whitespace
+'in_msg'      Name.Variable
+')'           Punctuation
+' '           Text.Whitespace
+'impure'      Keyword
+' '           Text.Whitespace
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'throw'       Name.Function
+'('           Punctuation
+'0xffff'      Literal.Number
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'\n'          Text.Whitespace
+
+'int'         Keyword.Type
+' '           Text.Whitespace
+'counter'     Name.Function
+'('           Punctuation
+')'           Punctuation
+' '           Text.Whitespace
+'method_id'   Keyword
+' '           Text.Whitespace
+'{'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'var'         Keyword.Type
+' '           Text.Whitespace
+'data'        Name.Variable
+' '           Text.Whitespace
+'='           Operator
+' '           Text.Whitespace
+'get_data'    Name.Function
+'('           Punctuation
+')'           Punctuation
+'.'           Punctuation
+'begin_parse' Name.Function
+'('           Punctuation
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'    '        Text.Whitespace
+'return'      Keyword
+' '           Text.Whitespace
+'data'        Name.Variable
+'~'           Punctuation
+'load_uint'   Name.Function
+'('           Punctuation
+'32'          Literal.Number
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
diff --git a/tests/test_func.py b/tests/test_func.py
new file mode 100644
index 00000000..c494b9cc
--- /dev/null
+++ b/tests/test_func.py
@@ -0,0 +1,44 @@
+import pytest
+from pygments.lexers.func import FuncLexer
+from pygments.token import Token, Name
+
+@pytest.fixture(scope='module')
+def lexer_func():
+    yield FuncLexer()
+
+
+@pytest.mark.parametrize('text', (
+    'take(first)Entry', '"not_a_string', 'msg.sender', 'send_message,then_terminate', '_'))
+def test_func_not_identifier(lexer_func, text):
+    """Test text that should **not** be tokenized as identifier."""
+    assert list(lexer_func.get_tokens(text))[0] != (Name.Variable, text)
+
+
+@pytest.mark.parametrize('text', (
+    '`test identifier`', 'simple_identifier', 'query\'\'', 
+    '_internal_value', 'get_pubkeys&signatures',
+    'dict::udict_set_builder', '2+2=2*2', '-alsovalidname', '{hehehe}'))
+def test_func_identifier(lexer_func, text):
+    """Test text that should be tokenized as identifier."""
+    assert list(lexer_func.get_tokens(text))[0] == (Name.Variable, text)
+
+
+@pytest.mark.parametrize('text', (
+'`test identifier`(', 'simple_identifier(', 'query\'\'(', 
+'_internal_value(', 'get_pubkeys&signatures(',
+'dict::udict_set_builder(', '2+2=2*2(', '-alsovalidname(', '{hehehe}('))
+def test_func_function(lexer_func, text):
+    """Test text that should be tokenized as identifier."""
+    assert list(lexer_func.get_tokens(text))[0] == (Name.Function, text[:-1])
+
+
+@pytest.mark.parametrize('text', ('0x0f', '0x1_2', '123', '0b10', '0xffff_fff', '1'))
+def test_func_number(lexer_func, text):
+    """Test text that should be tokenized as number."""
+    assert list(lexer_func.get_tokens(text))[0] == (Token.Literal.Number, text)
+
+
+@pytest.mark.parametrize('text', ('0x0f_m', '0X1_2', '12d3', '0b1_0f', '0bff_fff', '0b'))
+def test_func_not_number(lexer_func, text):
+    """Test text that should *not* be tokenized as number."""
+    assert list(lexer_func.get_tokens(text))[0] != (Token.Literal.Number, text)