From 9b4af216900704b05040f6314138c8621898c10c Mon Sep 17 00:00:00 2001 From: Jean Abou-Samra Date: Wed, 29 Mar 2023 16:26:26 +0200 Subject: Move more tests to tests/snippets instead of .py files (#2344) These are easier to update in tests/snippets/ (just run with --update-goldens). --- .../devicetree/test_fragment_out_of_root_node.txt | 72 +++++++++ .../test_binary_literal_negative_matches.txt | 27 ++++ .../test_binary_literal_positive_matches.txt | 15 ++ .../test_float_literals_negative_matches.txt | 60 +++++++ .../test_float_literals_positive_matches.txt | 52 ++++++ .../test_hexadecimal_literal_negative_matches.txt | 32 ++++ .../test_hexadecimal_literal_positive_matches.txt | 19 +++ .../test_integer_literal_negative_matches.txt | 20 +++ .../test_integer_literal_positive_matches.txt | 15 ++ .../test_octal_literal_negative_matches.txt | 40 +++++ .../test_octal_literal_positive_matches.txt | 23 +++ tests/snippets/markdown/test_code.txt | 72 +++++++++ tests/snippets/markdown/test_headings.txt | 81 ++++++++++ tests/snippets/markdown/test_invalid_code.txt | 52 ++++++ tests/snippets/markdown/test_setext_headings.txt | 121 ++++++++++++++ .../snippets/markdown/test_setext_subheadings.txt | 128 +++++++++++++++ tests/snippets/markdown/test_subheadings.txt | 115 +++++++++++++ tests/snippets/procfile/test_basic.txt | 15 ++ tests/snippets/robotframework/test_basic.txt | 40 +++++ tests/test_devicetree_lexer.py | 32 ---- tests/test_javascript.py | 84 ---------- tests/test_markdown_lexer.py | 178 --------------------- tests/test_procfile.py | 40 ----- tests/test_robotframework_lexer.py | 38 ----- 24 files changed, 999 insertions(+), 372 deletions(-) create mode 100644 tests/snippets/devicetree/test_fragment_out_of_root_node.txt create mode 100644 tests/snippets/javascript/test_binary_literal_negative_matches.txt create mode 100644 tests/snippets/javascript/test_binary_literal_positive_matches.txt create mode 100644 tests/snippets/javascript/test_float_literals_negative_matches.txt create mode 100644 tests/snippets/javascript/test_float_literals_positive_matches.txt create mode 100644 tests/snippets/javascript/test_hexadecimal_literal_negative_matches.txt create mode 100644 tests/snippets/javascript/test_hexadecimal_literal_positive_matches.txt create mode 100644 tests/snippets/javascript/test_integer_literal_negative_matches.txt create mode 100644 tests/snippets/javascript/test_integer_literal_positive_matches.txt create mode 100644 tests/snippets/javascript/test_octal_literal_negative_matches.txt create mode 100644 tests/snippets/javascript/test_octal_literal_positive_matches.txt create mode 100644 tests/snippets/markdown/test_code.txt create mode 100644 tests/snippets/markdown/test_headings.txt create mode 100644 tests/snippets/markdown/test_invalid_code.txt create mode 100644 tests/snippets/markdown/test_setext_headings.txt create mode 100644 tests/snippets/markdown/test_setext_subheadings.txt create mode 100644 tests/snippets/markdown/test_subheadings.txt create mode 100644 tests/snippets/procfile/test_basic.txt create mode 100644 tests/snippets/robotframework/test_basic.txt delete mode 100644 tests/test_devicetree_lexer.py delete mode 100644 tests/test_javascript.py delete mode 100644 tests/test_markdown_lexer.py delete mode 100644 tests/test_procfile.py delete mode 100644 tests/test_robotframework_lexer.py (limited to 'tests') diff --git a/tests/snippets/devicetree/test_fragment_out_of_root_node.txt b/tests/snippets/devicetree/test_fragment_out_of_root_node.txt new file mode 100644 index 00000000..eef534c6 --- /dev/null +++ b/tests/snippets/devicetree/test_fragment_out_of_root_node.txt @@ -0,0 +1,72 @@ +---input--- +nodelabel: node@0 { foo = "bar"; }; +nodelabel: node { foo = "bar"; }; +nodelabel0: nodelabel1: node@0 { foo = "bar"; }; + +---tokens--- +'nodelabel' Name.Label +':' Punctuation +' ' Text.Whitespace +'node' Name.Function +'@' Operator +'0' Literal.Number.Integer +' ' Comment.Multiline +'{' Punctuation +' ' Text.Whitespace +'foo' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'bar' Literal.String +'"' Literal.String +';' Punctuation +' ' Text.Whitespace +'}' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'nodelabel' Name.Label +':' Punctuation +' ' Text.Whitespace +'node' Name.Function +' ' Comment.Multiline +'{' Punctuation +' ' Text.Whitespace +'foo' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'bar' Literal.String +'"' Literal.String +';' Punctuation +' ' Text.Whitespace +'}' Punctuation +';' Punctuation +'\n' Text.Whitespace + +'nodelabel0' Name.Label +':' Punctuation +' ' Text.Whitespace +'nodelabel1' Name.Label +':' Punctuation +' ' Text.Whitespace +'node' Name.Function +'@' Operator +'0' Literal.Number.Integer +' ' Comment.Multiline +'{' Punctuation +' ' Text.Whitespace +'foo' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'bar' Literal.String +'"' Literal.String +';' Punctuation +' ' Text.Whitespace +'}' Punctuation +';' Punctuation +'\n' Text.Whitespace diff --git a/tests/snippets/javascript/test_binary_literal_negative_matches.txt b/tests/snippets/javascript/test_binary_literal_negative_matches.txt new file mode 100644 index 00000000..f36ce045 --- /dev/null +++ b/tests/snippets/javascript/test_binary_literal_negative_matches.txt @@ -0,0 +1,27 @@ +---input--- +// Test text that should **not** be tokenized as binary literals. +0b0N +0b +0bb +0b2 + +---tokens--- +'' Text +'// Test text that should **not** be tokenized as binary literals.' Comment.Single +'\n' Text.Whitespace + +'0b0' Literal.Number.Bin +'N' Name.Other +'\n' Text.Whitespace + +'0' Literal.Number.Float +'b' Name.Other +'\n' Text.Whitespace + +'0' Literal.Number.Float +'bb' Name.Other +'\n' Text.Whitespace + +'0' Literal.Number.Float +'b2' Name.Other +'\n' Text.Whitespace diff --git a/tests/snippets/javascript/test_binary_literal_positive_matches.txt b/tests/snippets/javascript/test_binary_literal_positive_matches.txt new file mode 100644 index 00000000..9773c95b --- /dev/null +++ b/tests/snippets/javascript/test_binary_literal_positive_matches.txt @@ -0,0 +1,15 @@ +---input--- +// Test literals that should be tokenized as binary literals. +0b01 +0B10n + +---tokens--- +'' Text +'// Test literals that should be tokenized as binary literals.' Comment.Single +'\n' Text.Whitespace + +'0b01' Literal.Number.Bin +'\n' Text.Whitespace + +'0B10n' Literal.Number.Bin +'\n' Text.Whitespace diff --git a/tests/snippets/javascript/test_float_literals_negative_matches.txt b/tests/snippets/javascript/test_float_literals_negative_matches.txt new file mode 100644 index 00000000..dc13b5f6 --- /dev/null +++ b/tests/snippets/javascript/test_float_literals_negative_matches.txt @@ -0,0 +1,60 @@ +---input--- +// Test text that should **not** be tokenized as float literals. +.୪ +. +1.. +1n +1ee +1e +1e- +1e--1 +1e++1 +1e1.0 + +---tokens--- +'' Text +'// Test text that should **not** be tokenized as float literals.' Comment.Single +'\n' Text.Whitespace + +'.' Punctuation +'୪' Error +'\n' Text.Whitespace + +'.' Punctuation +'\n' Text.Whitespace + +'1.' Literal.Number.Float +'.' Punctuation +'\n' Text.Whitespace + +'1n' Literal.Number.Integer +'\n' Text.Whitespace + +'1' Literal.Number.Float +'ee' Name.Other +'\n' Text.Whitespace + +'1' Literal.Number.Float +'e' Name.Other +'\n' Text.Whitespace + +'1' Literal.Number.Float +'e' Name.Other +'-' Operator +'\n' Text.Whitespace + +'1' Literal.Number.Float +'e' Name.Other +'--' Operator +'1' Literal.Number.Float +'\n' Text.Whitespace + +'1' Literal.Number.Float +'e' Name.Other +'++' Operator +'1' Literal.Number.Float +'\n' Text.Whitespace + +'1e1' Literal.Number.Float +'.0' Literal.Number.Float +'\n' Text.Whitespace diff --git a/tests/snippets/javascript/test_float_literals_positive_matches.txt b/tests/snippets/javascript/test_float_literals_positive_matches.txt new file mode 100644 index 00000000..3342112e --- /dev/null +++ b/tests/snippets/javascript/test_float_literals_positive_matches.txt @@ -0,0 +1,52 @@ +---input--- +// Test literals that should be tokenized as float literals. +1 +1. + .1 +1.1 +1e1 +1E1 +1e+1 +1E-1 +1.e1 +1e1 +0888 // octal prefix with non-octal numbers + +---tokens--- +'' Text +'// Test literals that should be tokenized as float literals.' Comment.Single +'\n' Text.Whitespace + +'1' Literal.Number.Float +'\n' Text.Whitespace + +'1.' Literal.Number.Float +'\n ' Text.Whitespace +'.1' Literal.Number.Float +'\n' Text.Whitespace + +'1.1' Literal.Number.Float +'\n' Text.Whitespace + +'1e1' Literal.Number.Float +'\n' Text.Whitespace + +'1E1' Literal.Number.Float +'\n' Text.Whitespace + +'1e+1' Literal.Number.Float +'\n' Text.Whitespace + +'1E-1' Literal.Number.Float +'\n' Text.Whitespace + +'1.e1' Literal.Number.Float +'\n' Text.Whitespace + +'1e1' Literal.Number.Float +'\n' Text.Whitespace + +'0888' Literal.Number.Float +' ' Text.Whitespace +'// octal prefix with non-octal numbers' Comment.Single +'\n' Text.Whitespace diff --git a/tests/snippets/javascript/test_hexadecimal_literal_negative_matches.txt b/tests/snippets/javascript/test_hexadecimal_literal_negative_matches.txt new file mode 100644 index 00000000..be3e606a --- /dev/null +++ b/tests/snippets/javascript/test_hexadecimal_literal_negative_matches.txt @@ -0,0 +1,32 @@ +---input--- +// Test text that should **not** be tokenized as hexadecimal literals. +0x0N +0x +0Xx +0xg +0xhn + +---tokens--- +'' Text +'// Test text that should **not** be tokenized as hexadecimal literals.' Comment.Single +'\n' Text.Whitespace + +'0x0' Literal.Number.Hex +'N' Name.Other +'\n' Text.Whitespace + +'0' Literal.Number.Float +'x' Name.Other +'\n' Text.Whitespace + +'0' Literal.Number.Float +'Xx' Name.Other +'\n' Text.Whitespace + +'0' Literal.Number.Float +'xg' Name.Other +'\n' Text.Whitespace + +'0' Literal.Number.Float +'xhn' Name.Other +'\n' Text.Whitespace diff --git a/tests/snippets/javascript/test_hexadecimal_literal_positive_matches.txt b/tests/snippets/javascript/test_hexadecimal_literal_positive_matches.txt new file mode 100644 index 00000000..32ae4b6a --- /dev/null +++ b/tests/snippets/javascript/test_hexadecimal_literal_positive_matches.txt @@ -0,0 +1,19 @@ +---input--- +// Test literals that should be tokenized as hexadecimal literals. +0x01 +0Xefn +0x0EF + +---tokens--- +'' Text +'// Test literals that should be tokenized as hexadecimal literals.' Comment.Single +'\n' Text.Whitespace + +'0x01' Literal.Number.Hex +'\n' Text.Whitespace + +'0Xefn' Literal.Number.Hex +'\n' Text.Whitespace + +'0x0EF' Literal.Number.Hex +'\n' Text.Whitespace diff --git a/tests/snippets/javascript/test_integer_literal_negative_matches.txt b/tests/snippets/javascript/test_integer_literal_negative_matches.txt new file mode 100644 index 00000000..57bc35f3 --- /dev/null +++ b/tests/snippets/javascript/test_integer_literal_negative_matches.txt @@ -0,0 +1,20 @@ +---input--- +// Test text that should **not** be tokenized as integer literals. +1N +1 +1.0 + +---tokens--- +'' Text +'// Test text that should **not** be tokenized as integer literals.' Comment.Single +'\n' Text.Whitespace + +'1' Literal.Number.Float +'N' Name.Other +'\n' Text.Whitespace + +'1' Literal.Number.Float +'\n' Text.Whitespace + +'1.0' Literal.Number.Float +'\n' Text.Whitespace diff --git a/tests/snippets/javascript/test_integer_literal_positive_matches.txt b/tests/snippets/javascript/test_integer_literal_positive_matches.txt new file mode 100644 index 00000000..c69f489e --- /dev/null +++ b/tests/snippets/javascript/test_integer_literal_positive_matches.txt @@ -0,0 +1,15 @@ +---input--- +// Test literals that should be tokenized as integer literals. +0n +123n + +---tokens--- +'' Text +'// Test literals that should be tokenized as integer literals.' Comment.Single +'\n' Text.Whitespace + +'0n' Literal.Number.Integer +'\n' Text.Whitespace + +'123n' Literal.Number.Integer +'\n' Text.Whitespace diff --git a/tests/snippets/javascript/test_octal_literal_negative_matches.txt b/tests/snippets/javascript/test_octal_literal_negative_matches.txt new file mode 100644 index 00000000..53aea59b --- /dev/null +++ b/tests/snippets/javascript/test_octal_literal_negative_matches.txt @@ -0,0 +1,40 @@ +---input--- +// Test text that should **not** be tokenized as octal literals. +01N +089 +098 +0o +0OO +0o88 +0O88n + +---tokens--- +'' Text +'// Test text that should **not** be tokenized as octal literals.' Comment.Single +'\n' Text.Whitespace + +'01' Literal.Number.Oct +'N' Name.Other +'\n' Text.Whitespace + +'089' Literal.Number.Float +'\n' Text.Whitespace + +'098' Literal.Number.Float +'\n' Text.Whitespace + +'0' Literal.Number.Float +'o' Name.Other +'\n' Text.Whitespace + +'0' Literal.Number.Float +'OO' Name.Other +'\n' Text.Whitespace + +'0' Literal.Number.Float +'o88' Name.Other +'\n' Text.Whitespace + +'0' Literal.Number.Float +'O88n' Name.Other +'\n' Text.Whitespace diff --git a/tests/snippets/javascript/test_octal_literal_positive_matches.txt b/tests/snippets/javascript/test_octal_literal_positive_matches.txt new file mode 100644 index 00000000..67c90f80 --- /dev/null +++ b/tests/snippets/javascript/test_octal_literal_positive_matches.txt @@ -0,0 +1,23 @@ +---input--- +// Test literals that should be tokenized as octal literals. +017 +071n +0o11 +0O77n + +---tokens--- +'' Text +'// Test literals that should be tokenized as octal literals.' Comment.Single +'\n' Text.Whitespace + +'017' Literal.Number.Oct +'\n' Text.Whitespace + +'071n' Literal.Number.Oct +'\n' Text.Whitespace + +'0o11' Literal.Number.Oct +'\n' Text.Whitespace + +'0O77n' Literal.Number.Oct +'\n' Text.Whitespace diff --git a/tests/snippets/markdown/test_code.txt b/tests/snippets/markdown/test_code.txt new file mode 100644 index 00000000..1ab16993 --- /dev/null +++ b/tests/snippets/markdown/test_code.txt @@ -0,0 +1,72 @@ +---input--- +Code fence: + +``` +foo +``` + +Code fence with language: + +```python +import this +``` + +Code fence with unknown language: + +```invalid-lexer +foo +``` + +---tokens--- +'Code' Text +' ' Text +'fence:' Text +'\n' Text.Whitespace + +'\n```\nfoo\n```\n' Literal.String.Backtick + +'\n' Text.Whitespace + +'Code' Text +' ' Text +'fence' Text +' ' Text +'with' Text +' ' Text +'language:' Text +'\n' Text.Whitespace + +'\n```' Literal.String.Backtick +'python' Literal.String.Backtick +'\n' Text + +'import' Keyword.Namespace +' ' Text +'this' Name.Namespace +'\n' Text.Whitespace + +'```\n' Literal.String.Backtick + +'\n' Text.Whitespace + +'Code' Text +' ' Text +'fence' Text +' ' Text +'with' Text +' ' Text +'unknown' Text +' ' Text +'language:' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'```invalid-lexer' Text +'\n' Text.Whitespace + +'foo' Text +'\n' Text.Whitespace + +'```' Text +'\n' Text.Whitespace diff --git a/tests/snippets/markdown/test_headings.txt b/tests/snippets/markdown/test_headings.txt new file mode 100644 index 00000000..5ec3cbbb --- /dev/null +++ b/tests/snippets/markdown/test_headings.txt @@ -0,0 +1,81 @@ +---input--- +Headings: + +#Heading + +# Heading + +# Another heading + +# Another # heading + +# Heading # + +These are NOT parsed as headings: + +# + +a # + +*# + +---tokens--- +'Headings:' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'#Heading' Generic.Heading +'\n' Text + +'\n' Text.Whitespace + +'# Heading' Generic.Heading +'\n' Text + +'\n' Text.Whitespace + +'# Another heading' Generic.Heading +'\n' Text + +'\n' Text.Whitespace + +'# Another # heading' Generic.Heading +'\n' Text + +'\n' Text.Whitespace + +'# Heading #' Generic.Heading +'\n' Text + +'\n' Text.Whitespace + +'These' Text +' ' Text +'are' Text +' ' Text +'NOT' Text +' ' Text +'parsed' Text +' ' Text +'as' Text +' ' Text +'headings:' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'#' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'a' Text +' ' Text +'#' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'*#' Text +'\n' Text.Whitespace diff --git a/tests/snippets/markdown/test_invalid_code.txt b/tests/snippets/markdown/test_invalid_code.txt new file mode 100644 index 00000000..bf6cb931 --- /dev/null +++ b/tests/snippets/markdown/test_invalid_code.txt @@ -0,0 +1,52 @@ +---input--- +Invalid code blocks: + +```code``` + +prefix not allowed before ``` +code block +``` + + code + +---tokens--- +'Invalid' Text +' ' Text +'code' Text +' ' Text +'blocks:' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'```code```' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'prefix' Text +' ' Text +'not' Text +' ' Text +'allowed' Text +' ' Text +'before' Text +' ' Text +'```' Text +'\n' Text.Whitespace + +'code' Text +' ' Text +'block' Text +'\n' Text.Whitespace + +'```' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +' ' Text +' ' Text +'code' Text +'\n' Text.Whitespace diff --git a/tests/snippets/markdown/test_setext_headings.txt b/tests/snippets/markdown/test_setext_headings.txt new file mode 100644 index 00000000..4d7a40f4 --- /dev/null +++ b/tests/snippets/markdown/test_setext_headings.txt @@ -0,0 +1,121 @@ +---input--- +Setext headings: + +Heading += + +Heading +======= + +Heading +============== + + +These are NOT Setext headings: + +Heading + +Heading +_ + +Heading + ===== + +Heading +a====== + + += + +Heading +=======Text + +---tokens--- +'Setext' Text +' ' Text +'headings:' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Heading' Generic.Heading +'\n' Text + +'=' Generic.Heading +'\n' Text + +'\n' Text.Whitespace + +'Heading' Generic.Heading +'\n' Text + +'=======' Generic.Heading +'\n' Text + +'\n' Text.Whitespace + +'Heading' Generic.Heading +'\n' Text + +'==============' Generic.Heading +'\n' Text + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'These' Text +' ' Text +'are' Text +' ' Text +'NOT' Text +' ' Text +'Setext' Text +' ' Text +'headings:' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Heading' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Heading' Text +'\n' Text.Whitespace + +'_' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Heading' Text +'\n' Text.Whitespace + +' ' Text +'=====' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Heading' Text +'\n' Text.Whitespace + +'a======' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'=' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Heading' Text +'\n' Text.Whitespace + +'=======Text' Text +'\n' Text.Whitespace diff --git a/tests/snippets/markdown/test_setext_subheadings.txt b/tests/snippets/markdown/test_setext_subheadings.txt new file mode 100644 index 00000000..fb5c4c46 --- /dev/null +++ b/tests/snippets/markdown/test_setext_subheadings.txt @@ -0,0 +1,128 @@ +---input--- +Setext subheadings: + +Subheading +- + +Subheading +---------- + +Subheading +----------- + + +These are NOT Setext subheadings: + + +Subheading + +Subheading +_ + +Subheading + --------- + +Subheading +a--------- + +--------- + +- + +Subheading +----------Text + +---tokens--- +'Setext' Text +' ' Text +'subheadings:' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Subheading' Generic.Subheading +'\n' Text + +'-' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'Subheading' Generic.Subheading +'\n' Text + +'----------' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'Subheading' Generic.Subheading +'\n' Text + +'-----------' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'These' Text +' ' Text +'are' Text +' ' Text +'NOT' Text +' ' Text +'Setext' Text +' ' Text +'subheadings:' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Subheading' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Subheading' Text +'\n' Text.Whitespace + +'_' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Subheading' Text +'\n' Text.Whitespace + +' ' Text +'---------' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Subheading' Text +'\n' Text.Whitespace + +'a---------' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'---------' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'-' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'Subheading' Text +'\n' Text.Whitespace + +'----------Text' Text +'\n' Text.Whitespace diff --git a/tests/snippets/markdown/test_subheadings.txt b/tests/snippets/markdown/test_subheadings.txt new file mode 100644 index 00000000..3a42fe7e --- /dev/null +++ b/tests/snippets/markdown/test_subheadings.txt @@ -0,0 +1,115 @@ +---input--- +Subheadings: + +##Subheading + +## Subheading + +### Subheading + +#### Subheading + +##### Subheading + +## Another subheading + +## Another ## subheading + +##### Subheading # + +##### Subheading ##### + + +These are NOT subheadings: + +## + +a ## + +*## + +###### too many hashes + +---tokens--- +'Subheadings:' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'##Subheading' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'## Subheading' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'### Subheading' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'#### Subheading' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'##### Subheading' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'## Another subheading' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'## Another ## subheading' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'##### Subheading #' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'##### Subheading #####' Generic.Subheading +'\n' Text + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'These' Text +' ' Text +'are' Text +' ' Text +'NOT' Text +' ' Text +'subheadings:' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'##' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'a' Text +' ' Text +'##' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'*##' Text +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'###### too many hashes' Generic.Subheading +'\n' Text diff --git a/tests/snippets/procfile/test_basic.txt b/tests/snippets/procfile/test_basic.txt new file mode 100644 index 00000000..89d413d9 --- /dev/null +++ b/tests/snippets/procfile/test_basic.txt @@ -0,0 +1,15 @@ +---input--- +task: executable --options +$XDG_SESSION_PATH + +---tokens--- +'task' Name.Label +':' Punctuation +' ' Text.Whitespace +'executable' Text +' ' Text.Whitespace +'--options' Text +'\n' Text.Whitespace + +'$XDG_SESSION_PATH' Name.Variable +'\n' Text.Whitespace diff --git a/tests/snippets/robotframework/test_basic.txt b/tests/snippets/robotframework/test_basic.txt new file mode 100644 index 00000000..acdee049 --- /dev/null +++ b/tests/snippets/robotframework/test_basic.txt @@ -0,0 +1,40 @@ +---input--- +*** Variables *** +${test}[] + +*** Variables *** +@{test}[] + +*** Variables *** +&{test}[] + +---tokens--- +'*** Variables ***' Generic.Heading +'\n' Punctuation + +'${test}[]' Error +'\n' Punctuation + +'\n' Punctuation + +'*** Variables ***' Generic.Heading +'\n' Punctuation + +'@{' Punctuation +'test' Name.Variable +'}' Punctuation +'[' Punctuation +']' Punctuation +'\n' Punctuation + +'\n' Punctuation + +'*** Variables ***' Generic.Heading +'\n' Punctuation + +'&{' Punctuation +'test' Name.Variable +'}' Punctuation +'[' Punctuation +']' Punctuation +'\n' Punctuation diff --git a/tests/test_devicetree_lexer.py b/tests/test_devicetree_lexer.py deleted file mode 100644 index e3aeb4d4..00000000 --- a/tests/test_devicetree_lexer.py +++ /dev/null @@ -1,32 +0,0 @@ -""" - Devicetree Lexer Tests - ~~~~~~~~~~~~~~~~~~~~~~ - - :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import pytest - -from pygments.lexers.devicetree import DevicetreeLexer -from pygments.token import Token - - -@pytest.fixture(scope="module") -def lexer(): - yield DevicetreeLexer() - - -@pytest.mark.parametrize( - "fragment", - ( - 'nodelabel: node@0 { foo = "bar"; };', - 'nodelabel: node { foo = "bar"; };', - 'nodelabel0: nodelabel1: node@0 { foo = "bar"; };', - ), -) -def test_fragment_out_of_root_node(lexer, fragment): - """Validate that a devicetree fragment out of a root node is parsed correctly.""" - - tokens = list(lexer.get_tokens(fragment)) - assert all(x[0] != Token.Error for x in tokens) diff --git a/tests/test_javascript.py b/tests/test_javascript.py deleted file mode 100644 index 05f74e27..00000000 --- a/tests/test_javascript.py +++ /dev/null @@ -1,84 +0,0 @@ -""" - Javascript tests - ~~~~~~~~~~~~~~~~ - - :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import pytest - -from pygments.lexers.javascript import JavascriptLexer -from pygments.token import Number - - -@pytest.fixture(scope='module') -def lexer(): - yield JavascriptLexer() - - -@pytest.mark.parametrize( - 'text', - ( - '1', '1.', '.1', '1.1', '1e1', '1E1', '1e+1', '1E-1', '1.e1', '.1e1', - '0888', # octal prefix with non-octal numbers - ) -) -def test_float_literal_positive_matches(lexer, text): - """Test literals that should be tokenized as float literals.""" - assert list(lexer.get_tokens(text))[0] == (Number.Float, text) - - -@pytest.mark.parametrize('text', ('.\u0b6a', '.', '1..', '1n', '1ee', '1e', - '1e-', '1e--1', '1e++1', '1e1.0')) -def test_float_literals_negative_matches(lexer, text): - """Test text that should **not** be tokenized as float literals.""" - assert list(lexer.get_tokens(text))[0] != (Number.Float, text) - - -@pytest.mark.parametrize('text', ('0n', '123n')) -def test_integer_literal_positive_matches(lexer, text): - """Test literals that should be tokenized as integer literals.""" - assert list(lexer.get_tokens(text))[0] == (Number.Integer, text) - - -@pytest.mark.parametrize('text', ('1N', '1', '1.0')) -def test_integer_literals_negative_matches(lexer, text): - """Test text that should **not** be tokenized as integer literals.""" - assert list(lexer.get_tokens(text))[0] != (Number.Integer, text) - - -@pytest.mark.parametrize('text', ('0b01', '0B10n')) -def test_binary_literal_positive_matches(lexer, text): - """Test literals that should be tokenized as binary literals.""" - assert list(lexer.get_tokens(text))[0] == (Number.Bin, text) - - -@pytest.mark.parametrize('text', ('0b0N', '0b', '0bb', '0b2')) -def test_binary_literals_negative_matches(lexer, text): - """Test text that should **not** be tokenized as binary literals.""" - assert list(lexer.get_tokens(text))[0] != (Number.Bin, text) - - -@pytest.mark.parametrize('text', ('017', '071n', '0o11', '0O77n')) -def test_octal_literal_positive_matches(lexer, text): - """Test literals that should be tokenized as octal literals.""" - assert list(lexer.get_tokens(text))[0] == (Number.Oct, text) - - -@pytest.mark.parametrize('text', ('01N', '089', '098n', '0o', '0OO', '0o88', '0O88n')) -def test_octal_literals_negative_matches(lexer, text): - """Test text that should **not** be tokenized as octal literals.""" - assert list(lexer.get_tokens(text))[0] != (Number.Oct, text) - - -@pytest.mark.parametrize('text', ('0x01', '0Xefn', '0x0EF')) -def test_hexadecimal_literal_positive_matches(lexer, text): - """Test literals that should be tokenized as hexadecimal literals.""" - assert list(lexer.get_tokens(text))[0] == (Number.Hex, text) - - -@pytest.mark.parametrize('text', ('0x0N', '0x', '0Xx', '0xg', '0xhn')) -def test_hexadecimal_literals_negative_matches(lexer, text): - """Test text that should **not** be tokenized as hexadecimal literals.""" - assert list(lexer.get_tokens(text))[0] != (Number.Hex, text) diff --git a/tests/test_markdown_lexer.py b/tests/test_markdown_lexer.py deleted file mode 100644 index 46b29112..00000000 --- a/tests/test_markdown_lexer.py +++ /dev/null @@ -1,178 +0,0 @@ -""" - Pygments Markdown lexer tests - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import pytest -from pygments.token import Generic, Token, String - -from pygments.lexers.markup import MarkdownLexer - - -@pytest.fixture(scope='module') -def lexer(): - yield MarkdownLexer() - - -def assert_same_text(lexer, text): - """Show that lexed markdown does not remove any content. """ - tokens = list(lexer.get_tokens_unprocessed(text)) - output = ''.join(t[2] for t in tokens) - assert text == output - - -def test_code_fence(lexer): - assert_same_text(lexer, r'```\nfoo\n```\n') - - -def test_code_fence_gsm(lexer): - assert_same_text(lexer, r'```markdown\nfoo\n```\n') - - -def test_code_fence_gsm_with_no_lexer(lexer): - assert_same_text(lexer, r'```invalid-lexer\nfoo\n```\n') - - -def test_invalid_atx_heading(lexer): - fragments = ( - '#', - 'a #', - '*#', - ) - - for fragment in fragments: - for token, _ in lexer.get_tokens(fragment): - assert token != Generic.Heading - - -def test_atx_heading(lexer): - fragments = ( - '#Heading', - '# Heading', - '# Another heading', - '# Another # heading', - '# Heading #', - ) - - for fragment in fragments: - tokens = [ - (Generic.Heading, fragment), - (Token.Text, '\n'), - ] - assert list(lexer.get_tokens(fragment)) == tokens - - -def test_invalid_atx_subheading(lexer): - fragments = ( - '##', - 'a ##', - '*##', - '####### too many hashes' - ) - - for fragment in fragments: - for token, _ in lexer.get_tokens(fragment): - assert token != Generic.Subheading - - -def test_atx_subheading(lexer): - fragments = ( - '##Subheading', - '## Subheading', - '### Subheading', - '#### Subheading', - '##### Subheading', - '###### Subheading', - '## Another subheading', - '## Another ## subheading', - '###### Subheading #', - '###### Subheading ######', - ) - - for fragment in fragments: - tokens = [ - (Generic.Subheading, fragment), - (Token.Text, '\n'), - ] - assert list(lexer.get_tokens(fragment)) == tokens - - -def test_invalid_setext_heading(lexer): - fragments = ( - 'Heading\n', - 'Heading\n_', - 'Heading\n =====', - 'Heading\na=====', - '=====', - '\n=\n', - 'Heading\n=====Text' - ) - - for fragment in fragments: - for token, _ in lexer.get_tokens(fragment): - assert token != Generic.Heading - - -def test_setext_heading(lexer): - fragments = ( - 'Heading\n=', - 'Heading\n=======', - 'Heading\n==========', - ) - - for fragment in fragments: - tokens = [ - (Generic.Heading, fragment.split('\n')[0]), - (Token.Text, '\n'), - (Generic.Heading, fragment.split('\n')[1]), - (Token.Text, '\n'), - ] - assert list(lexer.get_tokens(fragment)) == tokens - - -def test_invalid_setext_subheading(lexer): - fragments = ( - 'Subheading\n', - 'Subheading\n_', - 'Subheading\n -----', - 'Subheading\na-----', - '-----', - '\n-\n', - 'Subheading\n-----Text' - ) - - for fragment in fragments: - for token, _ in lexer.get_tokens(fragment): - assert token != Generic.Subheading - - -def test_setext_subheading(lexer): - fragments = ( - 'Subheading\n-', - 'Subheading\n----------', - 'Subheading\n-----------', - ) - - for fragment in fragments: - tokens = [ - (Generic.Subheading, fragment.split('\n')[0]), - (Token.Text, '\n'), - (Generic.Subheading, fragment.split('\n')[1]), - (Token.Text, '\n'), - ] - assert list(lexer.get_tokens(fragment)) == tokens - - -def test_invalid_code_block(lexer): - fragments = ( - '```code```', - 'prefix not allowed before ```\ncode block\n```' - ' code', - ) - - for fragment in fragments: - for token, _ in lexer.get_tokens(fragment): - assert token != String.Backtick diff --git a/tests/test_procfile.py b/tests/test_procfile.py deleted file mode 100644 index d919771a..00000000 --- a/tests/test_procfile.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -""" - Basic ProcfileLexer Test - ~~~~~~~~~~~~~~~~~~~~ - - :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import pytest - -from pygments.token import Name, Punctuation, Text -from pygments.lexers.procfile import ProcfileLexer - - -@pytest.fixture(scope='module') -def lexer(): - yield ProcfileLexer() - - -def test_basic_line(lexer): - text = 'task: executable --options' - - tokens = lexer.get_tokens(text) - - for index, token in enumerate(tokens): - if index == 0: - assert token == (Name.Label, 'task') - elif index == 1: - assert token == (Punctuation, ':') - else: - assert token[0] in (Text, Text.Whitespace) - - -def test_environment_variable(lexer): - text = '$XDG_SESSION_PATH' - - token = list(lexer.get_tokens(text))[0] - - assert token == (Name.Variable, text) diff --git a/tests/test_robotframework_lexer.py b/tests/test_robotframework_lexer.py deleted file mode 100644 index 807fbc4e..00000000 --- a/tests/test_robotframework_lexer.py +++ /dev/null @@ -1,38 +0,0 @@ -""" - Pygments Robot Framework lexer tests - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import pytest - -from pygments.lexers.robotframework import RobotFrameworkLexer - - -@pytest.fixture(scope='module') -def lexer(): - yield RobotFrameworkLexer() - - -def assert_same_text(lexer, text): - """Show that lexed text does not remove any content. """ - tokens = list(lexer.get_tokens_unprocessed(text)) - output = ''.join(t[2] for t in tokens) - assert text == output - - -def test_empty_brackets_after_scalar_variable(lexer): - assert_same_text(lexer, '*** Variables ***\n' - '${test}[]\n') - - -def test_empty_brackets_after_list_variable(lexer): - assert_same_text(lexer, '*** Variables ***\n' - '@{test}[]\n') - - -def test_empty_brackets_after_dict_variable(lexer): - assert_same_text(lexer, '*** Variables ***\n' - '&{test}[]\n') -- cgit v1.2.1