diff options
author | jmzambon <jeanmarczambon@gmail.com> | 2022-09-25 22:56:48 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-09-25 22:56:48 +0200 |
commit | a38cb38e93c9635240b3ae89d78d38cf182745da (patch) | |
tree | 3309e48108388a4d349d622d11c5a0239d0bfa83 | |
parent | fafb6d7211722068f5fe42db56437e659142d269 (diff) | |
download | pygments-git-a38cb38e93c9635240b3ae89d78d38cf182745da.tar.gz |
Fix: Issues with .properties format using whitespace delimited key (#2241)
Added:
- support for space delimitor in every case, included multiline value
- check for odd number of backslash escapes
- "!" as comment start
- support for escape of spaces and separators
Dropped:
- undocumented ";" and "//" comment start
-rw-r--r-- | pygments/lexers/configs.py | 28 | ||||
-rw-r--r-- | tests/examplefiles/properties/java.properties | 38 | ||||
-rw-r--r-- | tests/examplefiles/properties/java.properties.output | 88 | ||||
-rw-r--r-- | tests/snippets/properties/test_comments.txt | 4 | ||||
-rw-r--r-- | tests/snippets/properties/test_leading_whitespace_comments.txt | 2 | ||||
-rw-r--r-- | tests/snippets/properties/test_space_delimited_kv_pair.txt | 3 |
6 files changed, 114 insertions, 49 deletions
diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py index 4e0e7f12..6f61d98e 100644 --- a/pygments/lexers/configs.py +++ b/pygments/lexers/configs.py @@ -128,14 +128,26 @@ class PropertiesLexer(RegexLexer): tokens = { 'root': [ - (r'^(\w+)([ \t])(\w+\s*)$', bygroups(Name.Attribute, Whitespace, String)), - (r'^\w+(\\[ \t]\w*)*$', Name.Attribute), - (r'(^ *)([#!].*)', bygroups(Whitespace, Comment)), - # More controversial comments - (r'(^ *)((?:;|//).*)', bygroups(Whitespace, Comment)), - (r'(.*?)([ \t]*)([=:])([ \t]*)(.*(?:(?<=\\)\n.*)*)', - bygroups(Name.Attribute, Whitespace, Operator, Whitespace, String)), - (r'\s', Whitespace), + (r'\s+', Whitespace), + (r'[!#].*|/{2}.*', Comment.Single), + # search for first separator + (r'([^\\\n]|\\.)*?(?=[ \f\t=:])', Name.Attribute, "separator"), + # empty key + (r'.+?$', Name.Attribute), + ], + 'separator': [ + # search for line continuation escape + (r'([ \f\t]*)([=:]*)([ \f\t]*)(.*(?<!\\)(?:\\{2})*)(\\)(?!\\)$', + bygroups(Whitespace, Operator, Whitespace, String, Text), "value", "#pop"), + (r'([ \f\t]*)([=:]*)([ \f\t]*)(.*)', + bygroups(Whitespace, Operator, Whitespace, String), "#pop"), + ], + 'value': [ # line continuation + (r'\s+', Whitespace), + # search for line continuation escape + (r'(\s*)(.*(?<!\\)(?:\\{2})*)(\\)(?!\\)([ \t]*)', + bygroups(Whitespace, String, Text, Whitespace)), + (r'.*$', String, "#pop"), ], } diff --git a/tests/examplefiles/properties/java.properties b/tests/examplefiles/properties/java.properties index 72ad0f96..d5b594e3 100644 --- a/tests/examplefiles/properties/java.properties +++ b/tests/examplefiles/properties/java.properties @@ -1,16 +1,24 @@ -foo = bar -foo: bar -foo.oof: \ - bar=baz; \ - asdf +#https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/Properties.htm +# mixing spaces + Truth = Beauty + Truth:Beauty +Truth Beauty +Truth :Beauty + +! line continuations and escapes + fruits apple, banana, pear, \ + cantaloupe, watermelon, \ + kiwi, mango +key = \ + value1 \\\ + and value2\\ +key\ 2 = value +key\\ 3 = value3 -// comment -# comment -; comment - -x:a\ -b -x: a \ - b - -x = \ +! empty keys and edge cases +key1 = +key2 +key3 the value3 +key4 the:value4 +key5 the=value5 +key6=the value6 diff --git a/tests/examplefiles/properties/java.properties.output b/tests/examplefiles/properties/java.properties.output index 3efd7e2a..0c1fdeeb 100644 --- a/tests/examplefiles/properties/java.properties.output +++ b/tests/examplefiles/properties/java.properties.output @@ -1,50 +1,94 @@ -'foo' Name.Attribute +'#https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/Properties.htm' Comment.Single +'\n' Text.Whitespace + +'# mixing spaces' Comment.Single +'\n\t' Text.Whitespace +'Truth' Name.Attribute ' ' Text.Whitespace '=' Operator ' ' Text.Whitespace -'bar' Literal.String +'Beauty' Literal.String +'\n ' Text.Whitespace +'Truth' Name.Attribute +':' Operator +'Beauty' Literal.String '\n' Text.Whitespace -'foo' Name.Attribute -':' Operator -' ' Text.Whitespace -'bar' Literal.String +'Truth' Name.Attribute +'\t' Text.Whitespace +'Beauty' Literal.String '\n' Text.Whitespace -'foo.oof' Name.Attribute +'Truth' Name.Attribute +' ' Text.Whitespace ':' Operator -' ' Text.Whitespace -'\\\n bar=baz; \\\n asdf' Literal.String -'\n' Text.Whitespace +'Beauty' Literal.String +'\n \n' Text.Whitespace +'! line continuations and escapes' Comment.Single +'\n ' Text.Whitespace +'fruits' Name.Attribute +' ' Text.Whitespace +'apple, banana, pear, ' Literal.String +'\\' Text +'\n ' Text.Whitespace +'cantaloupe, watermelon, ' Literal.String +'\\' Text +'\n ' Text.Whitespace +'kiwi, mango' Literal.String '\n' Text.Whitespace -'// comment' Comment +'key' Name.Attribute +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'\\' Text +'\n ' Text.Whitespace +'value1 \\\\' Literal.String +'\\' Text +'\n ' Text.Whitespace +'and value2\\\\' Literal.String '\n' Text.Whitespace -'# comment' Comment +'key\\ 2' Name.Attribute +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'value' Literal.String '\n' Text.Whitespace -'; comment' Comment +'key\\\\' Name.Attribute +' ' Text.Whitespace +'3 = value3' Literal.String +'\n\n' Text.Whitespace + +'! empty keys and edge cases' Comment.Single '\n' Text.Whitespace +'key1' Name.Attribute +' ' Text.Whitespace +'=' Operator '\n' Text.Whitespace -'x' Name.Attribute -':' Operator -'a\\\nb' Literal.String +'key2' Name.Attribute '\n' Text.Whitespace -'x' Name.Attribute -':' Operator +'key3' Name.Attribute ' ' Text.Whitespace -'a \\\n b' Literal.String +'the value3' Literal.String '\n' Text.Whitespace +'key4' Name.Attribute +' ' Text.Whitespace +'the:value4' Literal.String '\n' Text.Whitespace -'x' Name.Attribute +'key5' Name.Attribute ' ' Text.Whitespace +'the=value5' Literal.String +'\n' Text.Whitespace + +'key6' Name.Attribute '=' Operator -' ' Text.Whitespace -'\\\n' Literal.String +'the value6' Literal.String +'\n' Text.Whitespace diff --git a/tests/snippets/properties/test_comments.txt b/tests/snippets/properties/test_comments.txt index b6bc8fb9..9bc65860 100644 --- a/tests/snippets/properties/test_comments.txt +++ b/tests/snippets/properties/test_comments.txt @@ -5,8 +5,8 @@ # also a comment ---tokens--- -'! a comment' Comment +'! a comment' Comment.Single '\n' Text.Whitespace -'# also a comment' Comment +'# also a comment' Comment.Single '\n' Text.Whitespace diff --git a/tests/snippets/properties/test_leading_whitespace_comments.txt b/tests/snippets/properties/test_leading_whitespace_comments.txt index be77792b..3a36afc9 100644 --- a/tests/snippets/properties/test_leading_whitespace_comments.txt +++ b/tests/snippets/properties/test_leading_whitespace_comments.txt @@ -2,5 +2,5 @@ # comment ---tokens--- -'# comment' Comment +'# comment' Comment.Single '\n' Text.Whitespace diff --git a/tests/snippets/properties/test_space_delimited_kv_pair.txt b/tests/snippets/properties/test_space_delimited_kv_pair.txt index fe12d48e..98961e42 100644 --- a/tests/snippets/properties/test_space_delimited_kv_pair.txt +++ b/tests/snippets/properties/test_space_delimited_kv_pair.txt @@ -4,4 +4,5 @@ key value ---tokens--- 'key' Name.Attribute ' ' Text.Whitespace -'value\n' Literal.String +'value' Literal.String +'\n' Text.Whitespace |