diff options
author | Jean Abou-Samra <jean@abou-samra.fr> | 2023-02-23 16:41:01 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-02-23 16:41:01 +0100 |
commit | b1f8dddc0b72923f05ec07dc9856f7fc143f6b97 (patch) | |
tree | 275be16203ffafcc9775c1409646f9fb0ac12d53 /tests | |
parent | e589fee754a76e9600dcb42baaeb3372d9e163d7 (diff) | |
download | pygments-git-b1f8dddc0b72923f05ec07dc9856f7fc143f6b97.tar.gz |
More tests moved to snippets/ (#2351)
Diffstat (limited to 'tests')
31 files changed, 1429 insertions, 635 deletions
diff --git a/tests/snippets/crystal/test_empty_percent_strings.txt b/tests/snippets/crystal/test_empty_percent_strings.txt new file mode 100644 index 00000000..3cc0844e --- /dev/null +++ b/tests/snippets/crystal/test_empty_percent_strings.txt @@ -0,0 +1,27 @@ +---input--- +%() +%[] +%{} +%<> +%|| + +---tokens--- +'%(' Literal.String.Other +')' Literal.String.Other +'\n' Text.Whitespace + +'%[' Literal.String.Other +']' Literal.String.Other +'\n' Text.Whitespace + +'%{' Literal.String.Other +'}' Literal.String.Other +'\n' Text.Whitespace + +'%<' Literal.String.Other +'>' Literal.String.Other +'\n' Text.Whitespace + +'%|' Literal.String.Other +'|' Literal.String.Other +'\n' Text.Whitespace diff --git a/tests/snippets/crystal/test_numbers.txt b/tests/snippets/crystal/test_numbers.txt new file mode 100644 index 00000000..0ddfec2a --- /dev/null +++ b/tests/snippets/crystal/test_numbers.txt @@ -0,0 +1,84 @@ +---input--- +# Integers +0 +1 +1_000_000 +1u8 +11231231231121312i64 + +# Floats +0.0 +1.0_f32 +1_f32 +0f64 +1e+4 +1e111 +1_234.567_890 + +# Error +01 +0b2 +0x129g2 +0o12358 + +---tokens--- +'# Integers' Comment.Single +'\n' Text.Whitespace + +'0' Literal.Number.Integer +'\n' Text.Whitespace + +'1' Literal.Number.Integer +'\n' Text.Whitespace + +'1_000_000' Literal.Number.Integer +'\n' Text.Whitespace + +'1u8' Literal.Number.Integer +'\n' Text.Whitespace + +'11231231231121312i64' Literal.Number.Integer +'\n\n' Text.Whitespace + +'# Floats' Comment.Single +'\n' Text.Whitespace + +'0.0' Literal.Number.Float +'\n' Text.Whitespace + +'1.0_f32' Literal.Number.Float +'\n' Text.Whitespace + +'1_f32' Literal.Number.Float +'\n' Text.Whitespace + +'0f64' Literal.Number.Float +'\n' Text.Whitespace + +'1e+4' Literal.Number.Float +'\n' Text.Whitespace + +'1e111' Literal.Number.Float +'\n' Text.Whitespace + +'1_234.567_890' Literal.Number.Float +'\n\n' Text.Whitespace + +'# Error' Comment.Single +'\n' Text.Whitespace + +'0' Error +'1' Literal.Number.Integer +'\n' Text.Whitespace + +'0' Error +'b2' Name +'\n' Text.Whitespace + +'0' Error +'x129g2' Name +'\n' Text.Whitespace + +'0' Error +'o12358' Name +'\n' Text.Whitespace diff --git a/tests/snippets/crystal/test_string_escapes.txt b/tests/snippets/crystal/test_string_escapes.txt new file mode 100644 index 00000000..3ea12f4d --- /dev/null +++ b/tests/snippets/crystal/test_string_escapes.txt @@ -0,0 +1,58 @@ +---input--- +"a\nz" +"a\az" +"a\xffz" +"a\u1234z" +"a\000z" +"a\u{0}z" +"a\u{10AfF9}z" + +---tokens--- +'"' Literal.String.Double +'a' Literal.String.Double +'\\n' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\a' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\xff' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\u1234' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\000' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\u{0}' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'"' Literal.String.Double +'a' Literal.String.Double +'\\u{10AfF9}' Literal.String.Escape +'z' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace diff --git a/tests/snippets/crystal/test_symbols.txt b/tests/snippets/crystal/test_symbols.txt new file mode 100644 index 00000000..3d2cb0aa --- /dev/null +++ b/tests/snippets/crystal/test_symbols.txt @@ -0,0 +1,20 @@ +---input--- +:sym_bol +:あ +:question? +:"symbol" + +---tokens--- +':sym_bol' Literal.String.Symbol +'\n' Text.Whitespace + +':あ' Literal.String.Symbol +'\n' Text.Whitespace + +':question?' Literal.String.Symbol +'\n' Text.Whitespace + +':"' Literal.String.Symbol +'symbol' Literal.String.Symbol +'"' Literal.String.Symbol +'\n' Text.Whitespace diff --git a/tests/snippets/html/css.txt b/tests/snippets/html/css.txt new file mode 100644 index 00000000..08b7e726 --- /dev/null +++ b/tests/snippets/html/css.txt @@ -0,0 +1,29 @@ +---input--- +<style> +.ui-helper-hidden { display: none } +</style> + +---tokens--- +'<' Punctuation +'style' Name.Tag +'>' Punctuation +'\n' Text.Whitespace + +'.' Punctuation +'ui-helper-hidden' Name.Class +' ' Text.Whitespace +'{' Punctuation +' ' Text.Whitespace +'display' Keyword +':' Punctuation +' ' Text.Whitespace +'none' Keyword.Constant +' ' Text.Whitespace +'}' Punctuation +'\n' Text.Whitespace + +'<' Punctuation +'/' Punctuation +'style' Name.Tag +'>' Punctuation +'\n' Text diff --git a/tests/snippets/html/css_backtracking.txt b/tests/snippets/html/css_backtracking.txt new file mode 100644 index 00000000..7efb4c17 --- /dev/null +++ b/tests/snippets/html/css_backtracking.txt @@ -0,0 +1,93 @@ +---input--- +<style> +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } +.ui-helper-hidden { display: none } + +---tokens--- +'<' Punctuation +'style' Name.Tag +'>' Punctuation +'\n' Text.Whitespace + +'.' Punctuation +'ui-helper-hidden' Name.Class +' ' Text.Whitespace +'{' Punctuation +' ' Text.Whitespace +'display' Keyword +':' Punctuation +' ' Text.Whitespace +'none' Keyword.Constant +' ' Text.Whitespace +'}' Punctuation +'\n' Text.Whitespace + +'.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n' Text diff --git a/tests/snippets/html/javascript.txt b/tests/snippets/html/javascript.txt new file mode 100644 index 00000000..5e12df2e --- /dev/null +++ b/tests/snippets/html/javascript.txt @@ -0,0 +1,28 @@ +---input--- +<script type="text/javascript"> +alert("hi"); +</script> + +---tokens--- +'<' Punctuation +'script' Name.Tag +' ' Text +'type' Name.Attribute +'=' Operator +'"text/javascript"' Literal.String +'>' Punctuation +'' Text +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'<' Punctuation +'/' Punctuation +'script' Name.Tag +'>' Punctuation +'\n' Text diff --git a/tests/snippets/html/javascript_backtracking.txt b/tests/snippets/html/javascript_backtracking.txt new file mode 100644 index 00000000..0bb8fa6f --- /dev/null +++ b/tests/snippets/html/javascript_backtracking.txt @@ -0,0 +1,285 @@ +---input--- +<script type="text/javascript"> +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +// Note the missing </script> tag; this should not cause catastrophic backtracking + +---tokens--- +'<' Punctuation +'script' Name.Tag +' ' Text +'type' Name.Attribute +'=' Operator +'"text/javascript"' Literal.String +'>' Punctuation +'' Text +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'// Note the missing ' Comment.Single +'<' Punctuation +'/' Punctuation +'script' Name.Tag +'>' Punctuation +' tag; this should not cause catastrophic backtracking\n' Text diff --git a/tests/snippets/html/javascript_unclosed.txt b/tests/snippets/html/javascript_unclosed.txt new file mode 100644 index 00000000..5754a4f1 --- /dev/null +++ b/tests/snippets/html/javascript_unclosed.txt @@ -0,0 +1,40 @@ +---input--- +<script type="text/javascript"> +alert("hi"); +alert("hi"); +alert("hi"); +alert("hi"); +<div> +hi +</div> + +---tokens--- +'<' Punctuation +'script' Name.Tag +' ' Text +'type' Name.Attribute +'=' Operator +'"text/javascript"' Literal.String +'>' Punctuation +'' Text +'\n' Text.Whitespace + +'alert' Name.Other +'(' Punctuation +'"hi"' Literal.String.Double +')' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'alert("hi");\nalert("hi");\nalert("hi");\n' Text + +'<' Punctuation +'div' Name.Tag +'>' Punctuation +'\nhi\n' Text + +'<' Punctuation +'/' Punctuation +'div' Name.Tag +'>' Punctuation +'\n' Text diff --git a/tests/snippets/java/test_backtracking.txt b/tests/snippets/java/test_backtracking.txt new file mode 100644 index 00000000..63532b81 --- /dev/null +++ b/tests/snippets/java/test_backtracking.txt @@ -0,0 +1,66 @@ +---input--- +"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ + +---tokens--- +'"' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\\\\' Literal.String +'\n' Literal.String diff --git a/tests/snippets/java/test_string_literals.txt b/tests/snippets/java/test_string_literals.txt new file mode 100644 index 00000000..edb2d111 --- /dev/null +++ b/tests/snippets/java/test_string_literals.txt @@ -0,0 +1,44 @@ +---input--- +"" +"abc" +"ひらがな" +"123" +"\\" +"\t" +"\"" + +---tokens--- +'"' Literal.String +'"' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'abc' Literal.String +'"' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'ひらがな' Literal.String +'"' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'123' Literal.String +'"' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\\\' Literal.String +'"' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\' Literal.String +'t' Literal.String +'"' Literal.String +'\n' Text.Whitespace + +'"' Literal.String +'\\"' Literal.String +'"' Literal.String +'\n' Text.Whitespace diff --git a/tests/snippets/json/test_arrays.txt b/tests/snippets/json/test_arrays.txt new file mode 100644 index 00000000..b4e7724f --- /dev/null +++ b/tests/snippets/json/test_arrays.txt @@ -0,0 +1,15 @@ +---input--- +[] +["a", "b"] + +---tokens--- +'[]' Punctuation +'\n' Text.Whitespace + +'[' Punctuation +'"a"' Literal.String.Double +',' Punctuation +' ' Text.Whitespace +'"b"' Literal.String.Double +']' Punctuation +'\n' Text.Whitespace diff --git a/tests/snippets/json/test_backtracking.txt b/tests/snippets/json/test_backtracking.txt new file mode 100644 index 00000000..5c462f52 --- /dev/null +++ b/tests/snippets/json/test_backtracking.txt @@ -0,0 +1,6 @@ +---input--- +{"\u00D0000\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\63CD + +---tokens--- +'{' Punctuation +'"\\u00D0000\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\63CD\n' Error diff --git a/tests/snippets/json/test_comments.txt b/tests/snippets/json/test_comments.txt new file mode 100644 index 00000000..ba0ed53f --- /dev/null +++ b/tests/snippets/json/test_comments.txt @@ -0,0 +1,65 @@ +---input--- +{"a//b"//C1 +:123/////C2 +} + +/** / **/{"a /**/ b"/* +*/: 123} + +// Invalid comments +/ +1/ +/1 +""/ + +// Unclosed + +""/** + +---tokens--- +'{' Punctuation +'"a//b"' Name.Tag +'//C1' Comment.Single +'\n' Text.Whitespace + +':' Punctuation +'123' Literal.Number.Integer +'/////C2' Comment.Single +'\n' Text.Whitespace + +'}' Punctuation +'\n\n' Text.Whitespace + +'/** / **/' Comment.Multiline +'{' Punctuation +'"a /**/ b"' Name.Tag +'/*\n*/' Comment.Multiline +':' Punctuation +' ' Text.Whitespace +'123' Literal.Number.Integer +'}' Punctuation +'\n\n' Text.Whitespace + +'// Invalid comments' Comment.Single +'\n' Text.Whitespace + +'/' Error +'\n' Text.Whitespace + +'1' Literal.Number.Integer +'/' Error +'\n' Text.Whitespace + +'/' Error +'1' Literal.Number.Integer +'\n' Text.Whitespace + +'""' Literal.String.Double +'/' Error +'\n\n' Text.Whitespace + +'// Unclosed' Comment.Single +'\n\n' Text.Whitespace + +'""' Literal.String.Double +'/**\n' Error diff --git a/tests/snippets/json/test_constants.txt b/tests/snippets/json/test_constants.txt new file mode 100644 index 00000000..3d7e4d50 --- /dev/null +++ b/tests/snippets/json/test_constants.txt @@ -0,0 +1,14 @@ +---input--- +true +false +null + +---tokens--- +'true' Keyword.Constant +'\n' Text.Whitespace + +'false' Keyword.Constant +'\n' Text.Whitespace + +'null' Keyword.Constant +'\n' Text.Whitespace diff --git a/tests/snippets/json/test_escape_sequences.txt b/tests/snippets/json/test_escape_sequences.txt new file mode 100644 index 00000000..becc0236 --- /dev/null +++ b/tests/snippets/json/test_escape_sequences.txt @@ -0,0 +1,79 @@ +---input--- +"\"" +"\\" +"\/" +"\b" +"\f" +"\n" +"\r" +"\t" +"\u0123" +"\u4567" +"\u89ab" +"\ucdef" +"\uABCD" +"\uEF01" + +// Incomplete sequences +"\uz" +"\u1z" +"\u12z" +"\u123z" + +---tokens--- +'"\\""' Literal.String.Double +'\n' Text.Whitespace + +'"\\\\"' Literal.String.Double +'\n' Text.Whitespace + +'"\\/"' Literal.String.Double +'\n' Text.Whitespace + +'"\\b"' Literal.String.Double +'\n' Text.Whitespace + +'"\\f"' Literal.String.Double +'\n' Text.Whitespace + +'"\\n"' Literal.String.Double +'\n' Text.Whitespace + +'"\\r"' Literal.String.Double +'\n' Text.Whitespace + +'"\\t"' Literal.String.Double +'\n' Text.Whitespace + +'"\\u0123"' Literal.String.Double +'\n' Text.Whitespace + +'"\\u4567"' Literal.String.Double +'\n' Text.Whitespace + +'"\\u89ab"' Literal.String.Double +'\n' Text.Whitespace + +'"\\ucdef"' Literal.String.Double +'\n' Text.Whitespace + +'"\\uABCD"' Literal.String.Double +'\n' Text.Whitespace + +'"\\uEF01"' Literal.String.Double +'\n\n' Text.Whitespace + +'// Incomplete sequences' Comment.Single +'\n' Text.Whitespace + +'"\\uz"' Literal.String.Double +'\n' Text.Whitespace + +'"\\u1z"' Literal.String.Double +'\n' Text.Whitespace + +'"\\u12z"' Literal.String.Double +'\n' Text.Whitespace + +'"\\u123z"' Literal.String.Double +'\n' Text.Whitespace diff --git a/tests/snippets/json/test_floats.txt b/tests/snippets/json/test_floats.txt new file mode 100644 index 00000000..1380578f --- /dev/null +++ b/tests/snippets/json/test_floats.txt @@ -0,0 +1,42 @@ +---input--- +123456789.0123456789 +-123456789.0123456789 +1e10 +-1E10 +1e-10 +-1E+10 +1.0e10 +-1.0E10 +1.0e-10 +-1.0E+10 + +---tokens--- +'123456789.0123456789' Literal.Number.Float +'\n' Text.Whitespace + +'-123456789.0123456789' Literal.Number.Float +'\n' Text.Whitespace + +'1e10' Literal.Number.Float +'\n' Text.Whitespace + +'-1E10' Literal.Number.Float +'\n' Text.Whitespace + +'1e-10' Literal.Number.Float +'\n' Text.Whitespace + +'-1E+10' Literal.Number.Float +'\n' Text.Whitespace + +'1.0e10' Literal.Number.Float +'\n' Text.Whitespace + +'-1.0E10' Literal.Number.Float +'\n' Text.Whitespace + +'1.0e-10' Literal.Number.Float +'\n' Text.Whitespace + +'-1.0E+10' Literal.Number.Float +'\n' Text.Whitespace diff --git a/tests/snippets/json/test_integers.txt b/tests/snippets/json/test_integers.txt new file mode 100644 index 00000000..d1125c31 --- /dev/null +++ b/tests/snippets/json/test_integers.txt @@ -0,0 +1,18 @@ +---input--- +0 +-1 +1234567890 +-1234567890 + +---tokens--- +'0' Literal.Number.Integer +'\n' Text.Whitespace + +'-1' Literal.Number.Integer +'\n' Text.Whitespace + +'1234567890' Literal.Number.Integer +'\n' Text.Whitespace + +'-1234567890' Literal.Number.Integer +'\n' Text.Whitespace diff --git a/tests/snippets/json/test_objects.txt b/tests/snippets/json/test_objects.txt new file mode 100644 index 00000000..bb6f623c --- /dev/null +++ b/tests/snippets/json/test_objects.txt @@ -0,0 +1,15 @@ +---input--- +{} +{"a": "b"} + +---tokens--- +'{}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"a"' Name.Tag +':' Punctuation +' ' Text.Whitespace +'"b"' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/snippets/json/test_round_trip.txt b/tests/snippets/json/test_round_trip.txt new file mode 100644 index 00000000..510a6f9d --- /dev/null +++ b/tests/snippets/json/test_round_trip.txt @@ -0,0 +1,29 @@ +---input--- ++ +0 +""0 +a +b +""/- + +---tokens--- +'+' Error +'\n' Text.Whitespace + +'0' Literal.Number.Integer +'\n' Text.Whitespace + +'""' Literal.String.Double +'0' Literal.Number.Integer +'\n' Text.Whitespace + +'a' Error +'\n' Text.Whitespace + +'b' Error +'\n' Text.Whitespace + +'""' Literal.String.Double +'/' Error +'-' Literal.Number.Integer +'\n' Text.Whitespace diff --git a/tests/snippets/json/test_strings.txt b/tests/snippets/json/test_strings.txt new file mode 100644 index 00000000..07468e42 --- /dev/null +++ b/tests/snippets/json/test_strings.txt @@ -0,0 +1,42 @@ +---input--- +"" +"abc" +"ひらがな" +"123" +"[]" +"{}" +"true" +"false" +"nul" +":," + +---tokens--- +'""' Literal.String.Double +'\n' Text.Whitespace + +'"abc"' Literal.String.Double +'\n' Text.Whitespace + +'"ひらがな"' Literal.String.Double +'\n' Text.Whitespace + +'"123"' Literal.String.Double +'\n' Text.Whitespace + +'"[]"' Literal.String.Double +'\n' Text.Whitespace + +'"{}"' Literal.String.Double +'\n' Text.Whitespace + +'"true"' Literal.String.Double +'\n' Text.Whitespace + +'"false"' Literal.String.Double +'\n' Text.Whitespace + +'"nul"' Literal.String.Double +'\n' Text.Whitespace + +'":,"' Literal.String.Double +'\n' Text.Whitespace diff --git a/tests/snippets/json/test_whitespace.txt b/tests/snippets/json/test_whitespace.txt new file mode 100644 index 00000000..c4c86588 --- /dev/null +++ b/tests/snippets/json/test_whitespace.txt @@ -0,0 +1,17 @@ +---input--- +{ + "tabs": "here", +} + +---tokens--- +'{' Punctuation +'\n\t' Text.Whitespace +'"tabs"' Name.Tag +':' Punctuation +' ' Text.Whitespace +'"here"' Literal.String.Double +',' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace diff --git a/tests/snippets/jsonld/test_json_ld.txt b/tests/snippets/jsonld/test_json_ld.txt new file mode 100644 index 00000000..dbd29699 --- /dev/null +++ b/tests/snippets/jsonld/test_json_ld.txt @@ -0,0 +1,251 @@ +---input--- +// JSON-LD keywords +{"@base": ""} +{"@container": ""} +{"@context": ""} +{"@direction": ""} +{"@graph": ""} +{"@id": ""} +{"@import": ""} +{"@included": ""} +{"@index": ""} +{"@json": ""} +{"@language": ""} +{"@list": ""} +{"@nest": ""} +{"@none": ""} +{"@prefix": ""} +{"@propagate": ""} +{"@protected": ""} +{"@reverse": ""} +{"@set": ""} +{"@type": ""} +{"@value": ""} +{"@version": ""} +{"@vocab": ""} + +// Not keywords +{"@bogus": ""} // "@" does not guarantee a keyword match +{"@bases": ""} // keyword is "base" +{"container": ""} // no leading "@" + +---tokens--- +'// JSON-LD keywords' Comment.Single +'\n' Text.Whitespace + +'{' Punctuation +'"@base"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@container"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@context"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@direction"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@graph"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@id"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@import"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@included"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@index"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@json"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@language"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@list"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@nest"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@none"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@prefix"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@propagate"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@protected"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@reverse"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@set"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@type"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@value"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@version"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n' Text.Whitespace + +'{' Punctuation +'"@vocab"' Name.Decorator +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +'\n\n' Text.Whitespace + +'// Not keywords' Comment.Single +'\n' Text.Whitespace + +'{' Punctuation +'"@bogus"' Name.Tag +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +' ' Text.Whitespace +'// "@" does not guarantee a keyword match' Comment.Single +'\n' Text.Whitespace + +'{' Punctuation +'"@bases"' Name.Tag +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +' ' Text.Whitespace +'// keyword is "base"' Comment.Single +'\n' Text.Whitespace + +'{' Punctuation +'"container"' Name.Tag +':' Punctuation +' ' Text.Whitespace +'""' Literal.String.Double +'}' Punctuation +' ' Text.Whitespace +'// no leading "@"' Comment.Single +'\n' Text.Whitespace diff --git a/tests/snippets/thingsdb/basic.txt b/tests/snippets/thingsdb/basic.txt new file mode 100644 index 00000000..f962cf24 --- /dev/null +++ b/tests/snippets/thingsdb/basic.txt @@ -0,0 +1,10 @@ +---input--- +42 +#42 + +---tokens--- +'42' Literal.Number.Integer +'\n' Text.Whitespace + +'#42' Comment.Preproc +'\n' Text.Whitespace diff --git a/tests/snippets/usd/basic.txt b/tests/snippets/usd/basic.txt new file mode 100644 index 00000000..7dd68d98 --- /dev/null +++ b/tests/snippets/usd/basic.txt @@ -0,0 +1,52 @@ +---input--- +# Regular file paths + +@./some/path/to/a/file/foo.usda@ +@/some/path/to/a/file/foo.usda@ +@some/path/to/a/file/foo.usda@ +@file://SPECI__Z-_ALIZED(syntax_here)?with_arbitrary#)(%*&)\characters.tar.gz@ + +# SdfPath syntax +</some/another_one/here> +</some/path/here.property_name> +</some/path/here> +<../some/another_one/here> +<../some/path/here.property_name> +<../some/path/here> + +---tokens--- +'# Regular file paths' Comment.Single +'\n\n' Text.Whitespace + +'@./some/path/to/a/file/foo.usda@' Literal.String.Interpol +'\n' Text.Whitespace + +'@/some/path/to/a/file/foo.usda@' Literal.String.Interpol +'\n' Text.Whitespace + +'@some/path/to/a/file/foo.usda@' Literal.String.Interpol +'\n' Text.Whitespace + +'@file://SPECI__Z-_ALIZED(syntax_here)?with_arbitrary#)(%*&)\\characters.tar.gz@' Literal.String.Interpol +'\n\n' Text.Whitespace + +'# SdfPath syntax' Comment.Single +'\n' Text.Whitespace + +'</some/another_one/here>' Name.Namespace +'\n' Text.Whitespace + +'</some/path/here.property_name>' Name.Namespace +'\n' Text.Whitespace + +'</some/path/here>' Name.Namespace +'\n' Text.Whitespace + +'<../some/another_one/here>' Name.Namespace +'\n' Text.Whitespace + +'<../some/path/here.property_name>' Name.Namespace +'\n' Text.Whitespace + +'<../some/path/here>' Name.Namespace +'\n' Text.Whitespace diff --git a/tests/test_crystal.py b/tests/test_crystal.py deleted file mode 100644 index 9bc2f11a..00000000 --- a/tests/test_crystal.py +++ /dev/null @@ -1,80 +0,0 @@ -""" - Basic CrystalLexer Test - ~~~~~~~~~~~~~~~~~~~~~~~ - - :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import pytest - -from pygments.token import String, Number, Punctuation, Error, Whitespace -from pygments.lexers import CrystalLexer - - -@pytest.fixture(scope='module') -def lexer(): - yield CrystalLexer() - - -def test_numbers(lexer): - for kind, testset in [ - (Number.Integer, '0 1 1_000_000 1u8 11231231231121312i64'), - (Number.Float, '0.0 1.0_f32 1_f32 0f64 1e+4 1e111 1_234.567_890'), - (Number.Bin, '0b1001_0110 0b0u8'), - (Number.Oct, '0o17 0o7_i32'), - (Number.Hex, '0xdeadBEEF'), - ]: - for fragment in testset.split(): - assert list(lexer.get_tokens(fragment + '\n')) == \ - [(kind, fragment), (Whitespace, '\n')] - - for fragment in '01 0b2 0x129g2 0o12358'.split(): - assert next(lexer.get_tokens(fragment + '\n'))[0] == Error - - -def test_symbols(lexer): - for fragment in [':sym_bol', ':\u3042', ':question?']: - assert list(lexer.get_tokens(fragment + '\n')) == \ - [(String.Symbol, fragment), (Whitespace, '\n')] - - fragment = ':"sym bol"\n' - tokens = [ - (String.Symbol, ':"'), - (String.Symbol, 'sym bol'), - (String.Symbol, '"'), - (Whitespace, '\n'), - ] - assert list(lexer.get_tokens(fragment)) == tokens - - -def test_chars(lexer): - for fragment in ["'a'", "'я'", "'\\u{1234}'", "'\n'"]: - assert list(lexer.get_tokens(fragment + '\n')) == \ - [(String.Char, fragment), (Whitespace, '\n')] - assert next(lexer.get_tokens("'abc'"))[0] == Error - - -def test_string_escapes(lexer): - for body in ['\\n', '\\a', '\\xff', '\\u1234', '\\000', '\\u{0}', '\\u{10AfF9}']: - fragment = '"a' + body + 'z"\n' - assert list(lexer.get_tokens(fragment)) == [ - (String.Double, '"'), - (String.Double, 'a'), - (String.Escape, body), - (String.Double, 'z'), - (String.Double, '"'), - (Whitespace, '\n'), - ] - - -def test_empty_percent_strings(lexer): - for body in ['%()', '%[]', '%{}', '%<>', '%||']: - fragment = '(' + body + ')\n' - assert list(lexer.get_tokens(fragment)) == [ - (Punctuation, '('), - (String.Other, body[:-1]), - (String.Other, body[-1]), - (Punctuation, ')'), - (Whitespace, '\n'), - ] diff --git a/tests/test_data.py b/tests/test_data.py deleted file mode 100644 index 719f8bfb..00000000 --- a/tests/test_data.py +++ /dev/null @@ -1,285 +0,0 @@ -""" - Data Tests - ~~~~~~~~~~ - - :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import time - -import pytest - -from pygments.lexers.data import JsonLexer, JsonBareObjectLexer, JsonLdLexer -from pygments.token import Comment, Error, Token, Punctuation, Number, String, \ - Keyword, Name, Whitespace - - -@pytest.fixture(scope='module') -def lexer_json(): - yield JsonLexer() - - -@pytest.fixture(scope='module') -def lexer_bare(): - yield JsonBareObjectLexer() - - -@pytest.fixture(scope='module') -def lexer_json_ld(): - yield JsonLdLexer() - - -@pytest.mark.parametrize( - 'text, expected_token_types', - ( - # Integers - ('0', (Number.Integer,)), - ('-1', (Number.Integer,)), - ('1234567890', (Number.Integer,)), - ('-1234567890', (Number.Integer,)), - - # Floats, including scientific notation - ('123456789.0123456789', (Number.Float,)), - ('-123456789.0123456789', (Number.Float,)), - ('1e10', (Number.Float,)), - ('-1E10', (Number.Float,)), - ('1e-10', (Number.Float,)), - ('-1E+10', (Number.Float,)), - ('1.0e10', (Number.Float,)), - ('-1.0E10', (Number.Float,)), - ('1.0e-10', (Number.Float,)), - ('-1.0E+10', (Number.Float,)), - - # Strings (escapes are tested elsewhere) - ('""', (String.Double,)), - ('"abc"', (String.Double,)), - ('"ひらがな"', (String.Double,)), - ('"123"', (String.Double,)), - ('"[]"', (String.Double,)), - ('"{}"', (String.Double,)), - ('"true"', (String.Double,)), - ('"false"', (String.Double,)), - ('"null"', (String.Double,)), - ('":,"', (String.Double,)), - - # Constants - ('true', (Keyword.Constant, )), - ('false', (Keyword.Constant, )), - ('null', (Keyword.Constant, )), - - # Arrays - ('[]', (Punctuation,)), - ('["a", "b"]', (Punctuation, String.Double, Punctuation, - Whitespace, String.Double, Punctuation)), - - # Objects - ('{}', (Punctuation,)), - ('{"a": "b"}', (Punctuation, Name.Tag, Punctuation, - Whitespace, String.Double, Punctuation)), - ) -) -@pytest.mark.parametrize('end', ('', '\n')) -def test_json_literals_positive_match(lexer_json, text, expected_token_types, end): - """Validate that syntactically-correct JSON literals are parsed correctly.""" - - tokens = list(lexer_json.get_tokens_unprocessed(text + end)) - assert len(tokens) == len(expected_token_types) + bool(end) - assert all(token[1] is expected_token - for token, expected_token in zip(tokens, expected_token_types)) - assert ''.join(token[2] for token in tokens) == text + end - - -@pytest.mark.parametrize( - 'text, expected', - ( - ('\u0020', Whitespace), # space - ('\u000a', Whitespace), # newline - ('\u000d', Whitespace), # carriage return - ('\u0009', Whitespace), # tab - ) -) -def test_json_whitespace_positive_match(lexer_json, text, expected): - """Validate that whitespace is parsed correctly.""" - - tokens = list(lexer_json.get_tokens_unprocessed(text)) - assert tokens == [(0, expected, text)] - - # Expand the whitespace and verify parsing again. - tokens = list(lexer_json.get_tokens_unprocessed(text * 2 + ' ')) - assert tokens == [(0, expected, text * 2 + ' ')] - - -@pytest.mark.parametrize( - 'text', - ( - '"', '\\', '/', 'b', 'f', 'n', 'r', 't', - 'u0123', 'u4567', 'u89ab', 'ucdef', 'uABCD', 'uEF01', - ) -) -def test_json_object_key_escapes_positive_match(lexer_json, text): - """Validate that escape sequences in JSON object keys are parsed correctly.""" - - tokens = list(lexer_json.get_tokens_unprocessed('{"\\%s": 1}' % text)) - assert len(tokens) == 6 - assert tokens[1][1] is Name.Tag - assert tokens[1][2] == '"\\%s"' % text - - -@pytest.mark.parametrize('text', ('uz', 'u1z', 'u12z', 'u123z')) -def test_json_string_unicode_escapes_negative_match(lexer_json, text): - """Validate that if unicode escape sequences end abruptly there's no problem.""" - - tokens = list(lexer_json.get_tokens_unprocessed('"\\%s"' % text)) - assert len(tokens) == 1 - assert tokens[0] == (0, String.Double, '"\\%s"' % text) - - -@pytest.mark.parametrize( - 'text', - ( - '"', '\\', '/', 'b', 'f', 'n', 'r', 't', - 'u0123', 'u4567', 'u89ab', 'ucdef', 'uABCD', 'uEF01', - ) -) -def test_json_string_escapes_positive_match(lexer_json, text): - """Validate that escape sequences in JSON string values are parsed correctly.""" - - text = '"\\%s"' % text - tokens = list(lexer_json.get_tokens_unprocessed(text)) - assert len(tokens) == 1 - assert tokens[0][1] is String.Double - assert tokens[0][2] == text - - -@pytest.mark.parametrize('text', ('+\n', '0\n', '""0\n', 'a\nb\n', '""/-')) -def test_json_round_trip_errors(lexer_json, text): - """Validate that past round-trip errors never crop up again.""" - - tokens = list(lexer_json.get_tokens_unprocessed(text)) - assert ''.join(t[2] for t in tokens) == text - - -def test_json_comments_single_line_positive_matches(lexer_json): - """Verify that single-line comments are tokenized correctly.""" - - text = '{"a//b"//C1\n:123/////C2\n}\n// // C3' - tokens = list(lexer_json.get_tokens_unprocessed(text)) - assert tokens[2] == (7, Comment.Single, "//C1") - assert tokens[6] == (16, Comment.Single, "/////C2") - assert tokens[10] == (26, Comment.Single, "// // C3") - - comment_count = sum(1 for token in tokens if token[1] == Comment or token[1].parent == Comment) - assert comment_count == 3 - - parsed_text = ''.join(token[2] for token in tokens) - assert parsed_text == text, 'Input and output texts must match!' - - -def test_json_comments_multiline_positive_matches(lexer_json): - """Verify that multiline comments are tokenized correctly.""" - - text = '/** / **/{"a /**/ b"/* \n */:123}' - tokens = list(lexer_json.get_tokens_unprocessed(text)) - assert tokens[0] == (0, Comment.Multiline, "/** / **/") - assert tokens[3] == (20, Comment.Multiline, "/* \n */") - - comment_count = sum(1 for token in tokens if token[1] == Comment or token[1].parent == Comment) - assert comment_count == 2 - - parsed_text = ''.join(token[2] for token in tokens) - assert parsed_text == text, 'Input and output texts must match!' - - -@pytest.mark.parametrize( - "text, expected", - ( - # Unfinished comment openers - ('/', (0, Error, '/')), - ('1/', (1, Error, '/')), - ('/1', (0, Error, '/')), - ('""/', (2, Error, '/')), - # Unclosed multiline comments - ('/*', (0, Error, '/*')), - ('/**', (0, Error, '/**')), - ('/*/', (0, Error, '/*/')), - ('1/*', (1, Error, '/*')), - ('""/*', (2, Error, '/*')), - ('""/**', (2, Error, '/**')), - ) -) -def test_json_comments_negative_matches(lexer_json, text, expected): - """Verify that the unfinished or unclosed comments are parsed as errors.""" - - tokens = list(lexer_json.get_tokens_unprocessed(text)) - assert expected in tokens - - parsed_text = ''.join(token[2] for token in tokens) - assert parsed_text == text, 'Input and output texts must match!' - - -def test_json_escape_backtracking(lexer_json): - """Confirm that there is no catastrophic backtracking in the lexer. - - This no longer applies because the JSON lexer doesn't use regular expressions, - but the test is included to ensure no loss of functionality now or in the future. - """ - - fragment = r'{"\u00D0000\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\63CD' - start_time = time.time() - list(lexer_json.get_tokens(fragment)) - assert time.time() - start_time < 1, 'The JSON lexer may have catastrophic backtracking' - - -@pytest.mark.parametrize( - 'keyword', - ( - 'base', - 'container', - 'context', - 'direction', - 'graph', - 'id', - 'import', - 'included', - 'index', - 'json', - 'language', - 'list', - 'nest', - 'none', - 'prefix', - 'propagate', - 'protected', - 'reverse', - 'set', - 'type', - 'value', - 'version', - 'vocab', - ) -) -def test_json_ld_keywords_positive_match(lexer_json_ld, keyword): - """Validate that JSON-LD keywords are parsed correctly.""" - - tokens = list(lexer_json_ld.get_tokens_unprocessed('{"@%s": ""}' % keyword)) - assert len(tokens) == 6 - assert tokens[1][1] is Token.Name.Decorator - assert tokens[1][2] == '"@%s"' % keyword - - -@pytest.mark.parametrize( - 'keyword', - ( - '@bogus', # "@" does not guarantee a keyword match - '@bases', # Begins with the keyword "@base" - 'container', # Matches "container" but has no leading "@" - ) -) -def test_json_ld_keywords_negative_match(lexer_json_ld, keyword): - """Validate that JSON-LD non-keywords are parsed correctly.""" - - tokens = list(lexer_json_ld.get_tokens_unprocessed('{"%s": ""}' % keyword)) - assert len(tokens) == 6 - assert tokens[1][1] is Token.Name.Tag - assert tokens[1][2] == '"%s"' % keyword diff --git a/tests/test_html_lexer.py b/tests/test_html_lexer.py deleted file mode 100644 index fe991498..00000000 --- a/tests/test_html_lexer.py +++ /dev/null @@ -1,131 +0,0 @@ -""" - HTML Lexer Tests - ~~~~~~~~~~~~~~~~ - - :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import time - -import pytest - -from pygments.lexers.html import HtmlLexer -from pygments.token import Token - -MAX_HL_TIME = 10 - - -@pytest.fixture(scope='module') -def lexer_html(): - yield HtmlLexer() - - -def test_happy_javascript_fragment(lexer_html): - """valid, even long Javascript fragments should still get parsed ok""" - - fragment = "<script type=\"text/javascript\">"+"alert(\"hi\");"*2000+"</script>" - start_time = time.time() - tokens = list(lexer_html.get_tokens(fragment)) - assert all(x[1] != Token.Error for x in tokens) - assert time.time() - start_time < MAX_HL_TIME, \ - 'The HTML lexer might have an expensive happy-path script case' - - -def test_happy_css_fragment(lexer_html): - """valid, even long CSS fragments should still get parsed ok""" - - fragment = "<style>"+".ui-helper-hidden{display:none}"*2000+"</style>" - start_time = time.time() - tokens = list(lexer_html.get_tokens(fragment)) - assert all(x[1] != Token.Error for x in tokens) - assert time.time() - start_time < MAX_HL_TIME, \ - 'The HTML lexer might have an expensive happy-path style case' - - -def test_long_unclosed_javascript_fragment(lexer_html): - """unclosed, long Javascript fragments should parse quickly""" - - reps = 2000 - fragment = "<script type=\"text/javascript\">"+"alert(\"hi\");"*reps - start_time = time.time() - tokens = list(lexer_html.get_tokens(fragment)) - assert time.time() - start_time < MAX_HL_TIME, \ - 'The HTML lexer might have an expensive error script case' - tokens_intro = [ - (Token.Punctuation, '<'), - (Token.Name.Tag, 'script'), - (Token.Text, ' '), - (Token.Name.Attribute, 'type'), - (Token.Operator, '='), - (Token.Literal.String, '"text/javascript"'), - (Token.Punctuation, '>'), - ] - tokens_body = [ - (Token.Name.Other, 'alert'), - (Token.Punctuation, '('), - (Token.Literal.String.Double, '"hi"'), - (Token.Punctuation, ')'), - (Token.Punctuation, ';'), - ] - - # make sure we get the right opening tokens - assert tokens[:len(tokens_intro)] == tokens_intro - # and make sure we get the right body tokens even though the script is - # unclosed - assert tokens[len(tokens_intro):-1] == tokens_body * reps - # and of course, the newline we get for free from get_tokens - assert tokens[-1] == (Token.Text.Whitespace, "\n") - - -def test_long_unclosed_css_fragment(lexer_html): - """unclosed, long CSS fragments should parse quickly""" - - reps = 2000 - fragment = "<style>"+".ui-helper-hidden{display:none}"*reps - start_time = time.time() - tokens = list(lexer_html.get_tokens(fragment)) - assert time.time() - start_time < MAX_HL_TIME, \ - 'The HTML lexer might have an expensive error style case' - - tokens_intro = [ - (Token.Punctuation, '<'), - (Token.Name.Tag, 'style'), - (Token.Punctuation, '>'), - ] - tokens_body = [ - (Token.Punctuation, '.'), - (Token.Name.Class, 'ui-helper-hidden'), - (Token.Punctuation, '{'), - (Token.Keyword, 'display'), - (Token.Punctuation, ':'), - (Token.Keyword.Constant, 'none'), - (Token.Punctuation, '}'), - ] - - # make sure we get the right opening tokens - assert tokens[:len(tokens_intro)] == tokens_intro - # and make sure we get the right body tokens even though the style block is - # unclosed - assert tokens[len(tokens_intro):-1] == tokens_body * reps - # and of course, the newline we get for free from get_tokens - assert tokens[-1] == (Token.Text.Whitespace, "\n") - - -def test_unclosed_fragment_with_newline_recovery(lexer_html): - """unclosed Javascript fragments should recover on the next line""" - - fragment = "<script type=\"text/javascript\">"+"alert(\"hi\");"*20+"\n<div>hi</div>" - tokens = list(lexer_html.get_tokens(fragment)) - recovery_tokens = [ - (Token.Punctuation, '<'), - (Token.Name.Tag, 'div'), - (Token.Punctuation, '>'), - (Token.Text, 'hi'), - (Token.Punctuation, '<'), - (Token.Punctuation, '/'), - (Token.Name.Tag, 'div'), - (Token.Punctuation, '>'), - (Token.Text, '\n'), - ] - assert tokens[-1*len(recovery_tokens):] == recovery_tokens diff --git a/tests/test_java.py b/tests/test_java.py deleted file mode 100644 index a50e862e..00000000 --- a/tests/test_java.py +++ /dev/null @@ -1,40 +0,0 @@ -""" - Basic JavaLexer Test - ~~~~~~~~~~~~~~~~~~~~ - - :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import time - -import pytest - -from pygments.token import String -from pygments.lexers import JavaLexer - - -@pytest.fixture(scope='module') -def lexer(): - yield JavaLexer() - - -@pytest.mark.parametrize( - 'text', - ( - '""', '"abc"', '"ひらがな"', '"123"', - '"\\\\"', '"\\t"' '"\\""', - ), -) -def test_string_literals_positive_match(lexer, text): - """Test positive matches for string literals.""" - tokens = list(lexer.get_tokens_unprocessed(text)) - assert all([token is String for _, token, _ in tokens]) - assert ''.join([value for _, _, value in tokens]) == text - - -def test_string_literals_backtracking(lexer): - """Test catastrophic backtracking for string literals.""" - start_time = time.time() - list(lexer.get_tokens_unprocessed('"' + '\\' * 100)) - assert time.time() - start_time < 1, 'possible backtracking bug' diff --git a/tests/test_thingsdb.py b/tests/test_thingsdb.py deleted file mode 100644 index 25e38168..00000000 --- a/tests/test_thingsdb.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -""" - Basic ThingsDB Test - ~~~~~~~~~~~~~~~~~~~~ - - :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import unittest - -from pygments.token import Number, Text, Comment -from pygments.lexers import ThingsDBLexer - - -class ThingsDBTest(unittest.TestCase): - - def setUp(self): - self.lexer = ThingsDBLexer() - self.maxDiff = None - - def testNumber(self): - fragment = u'42' - tokens = [ - (Number.Integer, u'42'), - (Text.Whitespace, '\n') - ] - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) - - def testThingId(self): - fragment = u'#42' - tokens = [ - (Comment.Preproc, u'#42'), - (Text.Whitespace, '\n') - ] - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_usd.py b/tests/test_usd.py deleted file mode 100755 index 779e3d3d..00000000 --- a/tests/test_usd.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python - -"""Test that syntax highlighting for USD files works correctly.""" - -import unittest - -from pygments.lexers import UsdLexer -from pygments.token import Name, String, Whitespace - - -class _Common(unittest.TestCase): - """A basic class that makes it easier to write unittests.""" - - def setUp(self): - """Create a fresh USD lexer class before each test runs.""" - self.lexer = UsdLexer() - - def _get(self, code): - """Tokenize the code into its unique parts. - - :param code: The USD source code to split up. - :type code: str - - :returns: The tokenized pieces. - :rtype: list[:class:`pygments.token._TokenType`] - - """ - return list(self.lexer.get_tokens(code)) - - -class Features(_Common): - """Test that different features of USD highlight as expected.""" - - def test_asset_path(self): - """Check that a regular file path highlights correctly.""" - for path in [ - "@./some/path/to/a/file/foo.usda@", - "@/some/path/to/a/file/foo.usda@", - "@some/path/to/a/file/foo.usda@", - r"@file://SPECI__Z-_ALIZED(syntax_here)?with_arbitrary#)(%*&)\characters.tar.gz@", - ]: - expected = [ - (String.Interpol, path), - (Whitespace, "\n"), - ] - - self.assertEqual(expected, self._get(path)) - - def test_target_absolute(self): - """Check that SdfPath syntax examples work correctly.""" - for code in [ - # Absolute paths - "</some/another_one/here>", - "</some/path/here.property_name>", - "</some/path/here>", - # Relative paths - "<../some/another_one/here>", - "<../some/path/here.property_name>", - "<../some/path/here>", - ]: - self.assertEqual( - [(Name.Namespace, code), (Whitespace, "\n")], self._get(code), - ) |