diff options
10 files changed, 280 insertions, 158 deletions
diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index 557318c8..f6e12031 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -1036,74 +1036,137 @@ class KotlinLexer(RegexLexer): kt_space_name = ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' + '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf', - 'Mn', 'Mc', 'Zs') + ',-]*') + 'Mn', 'Mc', 'Zs') + + r'\'~!%^&*()+=|\[\]:;,.<>/\?-]*') kt_id = '(' + kt_name + '|`' + kt_space_name + '`)' + modifiers = (r'actual|abstract|annotation|companion|const|crossinline|' + r'data|enum|expect|external|final|infix|inline|inner|' + r'internal|lateinit|noinline|open|operator|override|private|' + r'protected|public|sealed|suspend|tailrec') + tokens = { 'root': [ - (r'^\s*\[.*?\]', Name.Attribute), + # Whitespaces (r'[^\S\n]+', Text), (r'\s+', Text), (r'\\\n', Text), # line continuation + (r'\n', Text), + # Comments (r'//.*?\n', Comment.Single), (r'^#!/.+?\n', Comment.Single), # shebang for kotlin scripts (r'/[*].*?[*]/', Comment.Multiline), - (r'""".*?"""', String), - (r'\n', Text), + # Keywords + (r'as\?', Keyword), + (r'(as|break|by|catch|constructor|continue|do|dynamic|else|finally|' + r'get|for|if|init|[!]*in|[!]*is|out|reified|return|set|super|this|' + r'throw|try|typealias|typeof|vararg|when|where|while)\b', Keyword), + (r'it\b', Name.Builtin), + # Built-in types + (words(('Boolean?', 'Byte?', 'Char?', 'Double?', 'Float?', + 'Int?', 'Long?', 'Short?', 'String?', 'Any?', 'Unit?')), Keyword.Type), + (words(('Boolean', 'Byte', 'Char', 'Double', 'Float', + 'Int', 'Long', 'Short', 'String', 'Any', 'Unit'), suffix=r'\b'), Keyword.Type), + # Constants + (r'(true|false|null)\b', Keyword.Constant), + # Imports + (r'(package|import)(\s+)(\S+)', bygroups(Keyword, Text, Name.Namespace)), + # Dot access + (r'(\?\.)((?:[^\W\d]|\$)[\w$]*)', bygroups(Operator, Name.Attribute)), + (r'(\.)((?:[^\W\d]|\$)[\w$]*)', bygroups(Punctuation, Name.Attribute)), + # Annotations + (r'@[^\W\d][\w.]*', Name.Decorator), + # Labels + (r'[^\W\d][\w.]+@', Name.Decorator), + # Object expression + (r'(object)(\s+)(:)(\s+)', bygroups(Keyword, Text, Punctuation, Text), 'class'), + # Types + (r'((?:(?:' + modifiers + r'|fun)\s+)*)(class|interface|object)(\s+)', + bygroups(using(this, state='modifiers'), Keyword.Declaration, Text), 'class'), + # Variables + (r'(var|val)(\s+)(\()', bygroups(Keyword.Declaration, Text, Punctuation), + 'destructuring_assignment'), + (r'((?:(?:' + modifiers + r')\s+)*)(var|val)(\s+)', + bygroups(using(this, state='modifiers'), Keyword.Declaration, Text), 'variable'), + # Functions + (r'((?:(?:' + modifiers + r')\s+)*)(fun)(\s+)', + bygroups(using(this, state='modifiers'), Keyword.Declaration, Text), 'function'), + # Operators (r'::|!!|\?[:.]', Operator), - (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation), - (r'[{}]', Punctuation), - (r'@"(""|[^"])*"', String), - (r'"(\\\\|\\[^\\]|[^"\\\n])*["\n]', String), + (r'[~^*!%&\[\]<>|+=/?-]', Operator), + # Punctuation + (r'[{}();:.,]', Punctuation), + # Strings + (r'"""', String, 'multiline_string'), + (r'"', String, 'string'), (r"'\\.'|'[^\\]'", String.Char), + # Numbers (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?[flFL]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), - (r'(object)(\s+)(:)(\s+)', bygroups(Keyword, Text, Punctuation, Text), 'class'), - (r'(companion)(\s+)(object)', bygroups(Keyword, Text, Keyword)), - (r'(class|interface|object)(\s+)', bygroups(Keyword, Text), 'class'), - (r'(package|import)(\s+)', bygroups(Keyword, Text), 'package'), - (r'(val|var)(\s+)([(])', bygroups(Keyword, Text, Punctuation), 'property_dec'), - (r'(val|var)(\s+)', bygroups(Keyword, Text), 'property'), - (r'(fun)(\s+)', bygroups(Keyword, Text), 'function'), - (r'(inline fun)(\s+)', bygroups(Keyword, Text), 'function'), - (r'(abstract|annotation|as|break|by|catch|class|companion|const|' - r'constructor|continue|crossinline|data|do|dynamic|else|enum|' - r'external|false|final|finally|for|fun|get|if|import|in|infix|' - r'inline|inner|interface|internal|is|lateinit|noinline|null|' - r'object|open|operator|out|override|package|private|protected|' - r'public|reified|return|sealed|set|super|tailrec|this|throw|' - r'true|try|val|var|vararg|when|where|while)\b', Keyword), - (kt_id, Name), - ], - 'package': [ - (r'\S+', Name.Namespace, '#pop') + # Identifiers + (r'' + kt_id + r'((\?[^.])?)', Name) # additionally handle nullable types ], 'class': [ (kt_id, Name.Class, '#pop') ], - 'property': [ - (kt_id, Name.Property, '#pop') + 'variable': [ + (kt_id, Name.Variable, '#pop') ], - 'property_dec': [ - (r'(,)(\s*)', bygroups(Punctuation, Text)), - (r'(:)(\s*)', bygroups(Punctuation, Text)), - (r'<', Punctuation, 'generic'), - (r'([)])', Punctuation, '#pop'), - (kt_id, Name.Property) + 'destructuring_assignment': [ + (r',', Punctuation), + (r'\s+', Text), + (kt_id, Name.Variable), + (r'(:)(\s+)(' + kt_id + ')', bygroups(Punctuation, Text, Name)), + (r'<', Operator, 'generic'), + (r'\)', Punctuation, '#pop') ], 'function': [ - (r'<', Punctuation, 'generic'), - (r''+kt_id+'([.])'+kt_id, bygroups(Name.Class, Punctuation, Name.Function), '#pop'), + (r'<', Operator, 'generic'), + (r'' + kt_id + r'(\.)' + kt_id, bygroups(Name, Punctuation, Name.Function), '#pop'), (kt_id, Name.Function, '#pop') ], 'generic': [ - (r'(>)(\s*)', bygroups(Punctuation, Text), '#pop'), - (r':',Punctuation), + (r'(>)(\s*)', bygroups(Operator, Text), '#pop'), + (r':', Punctuation), (r'(reified|out|in)\b', Keyword), - (r',',Text), - (r'\s+',Text), - (kt_id,Name) + (r',', Punctuation), + (r'\s+', Text), + (kt_id, Name) + ], + 'modifiers': [ + (r'\w+', Keyword.Declaration), + (r'\s+', Text), + default('#pop') + ], + 'string': [ + (r'"', String, '#pop'), + include('string_common') + ], + 'multiline_string': [ + (r'"""', String, '#pop'), + (r'"', String), + include('string_common') + ], + 'string_common': [ + (r'\\\\', String), # escaped backslash + (r'\\"', String), # escaped quote + (r'\\', String), # bare backslash + (r'\$\{', String.Interpol, 'interpolation'), + (r'(\$)(\w+)', bygroups(String.Interpol, Name)), + (r'[^\\"$]+', String) + ], + 'interpolation': [ + (r'"', String), + (r'\$\{', String.Interpol, 'interpolation'), + (r'\{', Punctuation, 'scope'), + (r'\}', String.Interpol, '#pop'), + include('root') + ], + 'scope': [ + (r'\{', Punctuation, 'scope'), + (r'\}', Punctuation, '#pop'), + include('root') ] } diff --git a/tests/examplefiles/kotlin/example.kt.output b/tests/examplefiles/kotlin/example.kt.output index e42768fa..0e911d9d 100644 --- a/tests/examplefiles/kotlin/example.kt.output +++ b/tests/examplefiles/kotlin/example.kt.output @@ -3,130 +3,128 @@ 'addressbook' Name.Namespace '\n\n' Text -'class' Keyword +'class' Keyword.Declaration ' ' Text 'Contact' Name.Class '(' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'name' Name.Property +'name' Name.Variable ' ' Text ':' Punctuation ' ' Text -'String' Name +'String' Keyword.Type ',' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'emails' Name.Property +'emails' Name.Variable ' ' Text ':' Punctuation ' ' Text 'List' Name -'<' Punctuation +'<' Operator 'EmailAddress' Name -'>' Punctuation +'>' Operator ',' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'addresses' Name.Property +'addresses' Name.Variable ' ' Text ':' Punctuation ' ' Text 'List' Name -'<' Punctuation +'<' Operator 'PostalAddress' Name -'>' Punctuation +'>' Operator ',' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'phonenums' Name.Property +'phonenums' Name.Variable ' ' Text ':' Punctuation ' ' Text 'List' Name -'<' Punctuation +'<' Operator 'PhoneNumber' Name -'>' Punctuation +'>' Operator '\n' Text ')' Punctuation '\n\n' Text -'class' Keyword +'class' Keyword.Declaration ' ' Text 'EmailAddress' Name.Class '(' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'user' Name.Property +'user' Name.Variable ' ' Text ':' Punctuation ' ' Text -'String' Name +'String' Keyword.Type ',' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'host' Name.Property +'host' Name.Variable ' ' Text ':' Punctuation ' ' Text -'String' Name +'String' Keyword.Type '\n' Text ')' Punctuation '\n\n' Text -'class' Keyword +'class' Keyword.Declaration ' ' Text 'PostalAddress' Name.Class '(' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'streetAddress' Name.Property +'streetAddress' Name.Variable ' ' Text ':' Punctuation ' ' Text -'String' Name +'String' Keyword.Type ',' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'city' Name.Property +'city' Name.Variable ' ' Text ':' Punctuation ' ' Text -'String' Name +'String' Keyword.Type ',' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'zip' Name.Property +'zip' Name.Variable ' ' Text ':' Punctuation ' ' Text -'String' Name +'String' Keyword.Type ',' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'state' Name.Property +'state' Name.Variable ' ' Text ':' Punctuation ' ' Text -'USState' Name -'?' Punctuation -',' Punctuation +'USState?,' Name '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'country' Name.Property +'country' Name.Variable ' ' Text ':' Punctuation ' ' Text @@ -143,10 +141,10 @@ '(' Punctuation 'state' Name ' ' Text -'=' Punctuation -'=' Punctuation +'=' Operator +'=' Operator ' ' Text -'null' Keyword +'null' Keyword.Constant ')' Punctuation ' ' Text 'xor' Name @@ -154,13 +152,15 @@ '(' Punctuation 'country' Name ' ' Text -'=' Punctuation -'=' Punctuation +'=' Operator +'=' Operator ' ' Text 'Countries' Name -'[' Punctuation -'"US"' Literal.String -']' Punctuation +'[' Operator +'"' Literal.String +'US' Literal.String +'"' Literal.String +']' Operator ')' Punctuation ' ' Text '}' Punctuation @@ -169,48 +169,48 @@ '}' Punctuation '\n\n' Text -'class' Keyword +'class' Keyword.Declaration ' ' Text 'PhoneNumber' Name.Class '(' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'country' Name.Property +'country' Name.Variable ' ' Text ':' Punctuation ' ' Text 'Country' Name ',' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'areaCode' Name.Property +'areaCode' Name.Variable ' ' Text ':' Punctuation ' ' Text -'Int' Name +'Int' Keyword.Type ',' Punctuation '\n ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'number' Name.Property +'number' Name.Variable ' ' Text ':' Punctuation ' ' Text -'Long' Name +'Long' Keyword.Type '\n' Text ')' Punctuation '\n\n' Text -'object' Keyword +'object' Keyword.Declaration ' ' Text 'Countries' Name.Class ' ' Text '{' Punctuation '\n ' Text -'fun' Keyword +'fun' Keyword.Declaration ' ' Text 'get' Name.Function '(' Punctuation @@ -225,49 +225,49 @@ ' ' Text 'Country' Name ' ' Text -'=' Punctuation +'=' Operator ' ' Text 'countryTable' Name -'[' Punctuation +'[' Operator 'id' Name -']' Punctuation +']' Operator '\n \n ' Text -'private' Keyword +'private' Keyword.Declaration ' ' Text -'var' Keyword +'var' Keyword.Declaration ' ' Text -'table' Name.Property +'table' Name.Variable ' ' Text ':' Punctuation ' ' Text 'Map' Name -'<' Punctuation -'String' Name +'<' Operator +'String' Keyword.Type ',' Punctuation ' ' Text 'Country' Name -'>' Punctuation -'?' Punctuation +'>' Operator +'?' Operator ' ' Text -'=' Punctuation +'=' Operator ' ' Text -'null' Keyword +'null' Keyword.Constant '\n ' Text -'private' Keyword +'private' Keyword.Declaration ' ' Text -'val' Keyword +'val' Keyword.Declaration ' ' Text -'countryTable' Name.Property +'countryTable' Name.Variable ' ' Text ':' Punctuation ' ' Text 'Map' Name -'<' Punctuation -'String' Name +'<' Operator +'String' Keyword.Type ',' Punctuation ' ' Text 'Country' Name -'>' Punctuation +'>' Operator '\n ' Text 'get' Keyword '(' Punctuation @@ -280,17 +280,17 @@ '(' Punctuation 'table' Name ' ' Text -'=' Punctuation -'=' Punctuation +'=' Operator +'=' Operator ' ' Text -'null' Keyword +'null' Keyword.Constant ')' Punctuation ' ' Text '{' Punctuation '\n ' Text 'table' Name ' ' Text -'=' Punctuation +'=' Operator ' ' Text 'HashMap' Name '(' Punctuation @@ -305,27 +305,29 @@ ' ' Text 'TextFile' Name '(' Punctuation -'"countries.txt"' Literal.String +'"' Literal.String +'countries.txt' Literal.String +'"' Literal.String ')' Punctuation '.' Punctuation -'lines' Name +'lines' Name.Attribute '(' Punctuation 'stripWhiteSpace' Name ' ' Text -'=' Punctuation +'=' Operator ' ' Text -'true' Keyword +'true' Keyword.Constant ')' Punctuation ')' Punctuation ' ' Text '{' Punctuation '\n ' Text 'table' Name -'[' Punctuation +'[' Operator 'line' Name -']' Punctuation +']' Operator ' ' Text -'=' Punctuation +'=' Operator ' ' Text 'Country' Name '(' Punctuation @@ -346,16 +348,16 @@ '}' Punctuation '\n\n' Text -'class' Keyword +'class' Keyword.Declaration ' ' Text 'Country' Name.Class '(' Punctuation -'val' Keyword +'val' Keyword.Declaration ' ' Text -'name' Name.Property +'name' Name.Variable ' ' Text ':' Punctuation ' ' Text -'String' Name +'String' Keyword.Type ')' Punctuation '\n' Text diff --git a/tests/snippets/kotlin/test_can_cope_generics_in_destructuring.txt b/tests/snippets/kotlin/test_can_cope_generics_in_destructuring.txt index 7057ea62..8a880a55 100644 --- a/tests/snippets/kotlin/test_can_cope_generics_in_destructuring.txt +++ b/tests/snippets/kotlin/test_can_cope_generics_in_destructuring.txt @@ -2,26 +2,26 @@ val (a: List<Something>, b: Set<Wobble>) = ---tokens--- -'val' Keyword +'val' Keyword.Declaration ' ' Text '(' Punctuation -'a' Name.Property +'a' Name.Variable ':' Punctuation ' ' Text -'List' Name.Property -'<' Punctuation +'List' Name +'<' Operator 'Something' Name -'>' Punctuation +'>' Operator ',' Punctuation ' ' Text -'b' Name.Property +'b' Name.Variable ':' Punctuation ' ' Text -'Set' Name.Property -'<' Punctuation +'Set' Name +'<' Operator 'Wobble' Name -'>' Punctuation +'>' Operator ')' Punctuation ' ' Text -'=' Punctuation +'=' Operator '\n' Text diff --git a/tests/snippets/kotlin/test_can_cope_with_backtick_names_in_functions.txt b/tests/snippets/kotlin/test_can_cope_with_backtick_names_in_functions.txt index 48447109..ab2ef6d8 100644 --- a/tests/snippets/kotlin/test_can_cope_with_backtick_names_in_functions.txt +++ b/tests/snippets/kotlin/test_can_cope_with_backtick_names_in_functions.txt @@ -2,7 +2,7 @@ fun `wo bble` ---tokens--- -'fun' Keyword +'fun' Keyword.Declaration ' ' Text '`wo bble`' Name.Function '\n' Text diff --git a/tests/snippets/kotlin/test_can_cope_with_commas_and_dashes_in_backtick_Names.txt b/tests/snippets/kotlin/test_can_cope_with_commas_and_dashes_in_backtick_Names.txt index 4df4e397..6d3c4ec3 100644 --- a/tests/snippets/kotlin/test_can_cope_with_commas_and_dashes_in_backtick_Names.txt +++ b/tests/snippets/kotlin/test_can_cope_with_commas_and_dashes_in_backtick_Names.txt @@ -2,7 +2,7 @@ fun `wo,-bble` ---tokens--- -'fun' Keyword +'fun' Keyword.Declaration ' ' Text '`wo,-bble`' Name.Function '\n' Text diff --git a/tests/snippets/kotlin/test_can_cope_with_destructuring.txt b/tests/snippets/kotlin/test_can_cope_with_destructuring.txt index e9fab5e9..b51c1e04 100644 --- a/tests/snippets/kotlin/test_can_cope_with_destructuring.txt +++ b/tests/snippets/kotlin/test_can_cope_with_destructuring.txt @@ -2,15 +2,15 @@ val (a, b) = ---tokens--- -'val' Keyword +'val' Keyword.Declaration ' ' Text '(' Punctuation -'a' Name.Property +'a' Name.Variable ',' Punctuation ' ' Text -'b' Name.Property +'b' Name.Variable ')' Punctuation ' ' Text -'=' Punctuation +'=' Operator ' ' Text '\n' Text diff --git a/tests/snippets/kotlin/test_can_cope_with_generics.txt b/tests/snippets/kotlin/test_can_cope_with_generics.txt index 554a820a..ed616ec3 100644 --- a/tests/snippets/kotlin/test_can_cope_with_generics.txt +++ b/tests/snippets/kotlin/test_can_cope_with_generics.txt @@ -2,9 +2,11 @@ inline fun <reified T : ContractState> VaultService.queryBy(): Vault.Page<T> { ---tokens--- -'inline fun' Keyword +'inline' Keyword.Declaration ' ' Text -'<' Punctuation +'fun' Keyword.Declaration +' ' Text +'<' Operator 'reified' Keyword ' ' Text 'T' Name @@ -12,9 +14,9 @@ inline fun <reified T : ContractState> VaultService.queryBy(): Vault.Page<T> { ':' Punctuation ' ' Text 'ContractState' Name -'>' Punctuation +'>' Operator ' ' Text -'VaultService' Name.Class +'VaultService' Name '.' Punctuation 'queryBy' Name.Function '(' Punctuation @@ -23,10 +25,10 @@ inline fun <reified T : ContractState> VaultService.queryBy(): Vault.Page<T> { ' ' Text 'Vault' Name '.' Punctuation -'Page' Name -'<' Punctuation +'Page' Name.Attribute +'<' Operator 'T' Name -'>' Punctuation +'>' Operator ' ' Text '{' Punctuation '\n' Text diff --git a/tests/snippets/kotlin/test_modifier_keyword.txt b/tests/snippets/kotlin/test_modifier_keyword.txt new file mode 100644 index 00000000..b30f65b9 --- /dev/null +++ b/tests/snippets/kotlin/test_modifier_keyword.txt @@ -0,0 +1,18 @@ +---input--- +data class A(val data: String) + +---tokens--- +'data' Keyword.Declaration +' ' Text +'class' Keyword.Declaration +' ' Text +'A' Name.Class +'(' Punctuation +'val' Keyword.Declaration +' ' Text +'data' Name.Variable +':' Punctuation +' ' Text +'String' Keyword.Type +')' Punctuation +'\n' Text diff --git a/tests/snippets/kotlin/test_should_cope_with_multiline_comments.txt b/tests/snippets/kotlin/test_should_cope_with_multiline_comments.txt index 3de13278..ee7c3275 100644 --- a/tests/snippets/kotlin/test_should_cope_with_multiline_comments.txt +++ b/tests/snippets/kotlin/test_should_cope_with_multiline_comments.txt @@ -6,5 +6,7 @@ a comment""" ---tokens--- -'"""\nthis\nis\na\ncomment"""' Literal.String +'"""' Literal.String +'\nthis\nis\na\ncomment' Literal.String +'"""' Literal.String '\n' Text diff --git a/tests/snippets/kotlin/test_string_interpolation.txt b/tests/snippets/kotlin/test_string_interpolation.txt new file mode 100644 index 00000000..c254f112 --- /dev/null +++ b/tests/snippets/kotlin/test_string_interpolation.txt @@ -0,0 +1,35 @@ +---input--- +val something = "something" +"Here is $something" +"Here is ${something.toUpperList()}" + +---tokens--- +'val' Keyword.Declaration +' ' Text +'something' Name.Variable +' ' Text +'=' Operator +' ' Text +'"' Literal.String +'something' Literal.String +'"' Literal.String +'\n' Text + +'"' Literal.String +'Here is ' Literal.String +'$' Literal.String.Interpol +'something' Name +'"' Literal.String +'\n' Text + +'"' Literal.String +'Here is ' Literal.String +'${' Literal.String.Interpol +'something' Name +'.' Punctuation +'toUpperList' Name.Attribute +'(' Punctuation +')' Punctuation +'}' Literal.String.Interpol +'"' Literal.String +'\n' Text |