From b1f8dddc0b72923f05ec07dc9856f7fc143f6b97 Mon Sep 17 00:00:00 2001
From: Jean Abou-Samra <jean@abou-samra.fr>
Date: Thu, 23 Feb 2023 16:41:01 +0100
Subject: More tests moved to snippets/ (#2351)

---
 .../crystal/test_empty_percent_strings.txt         |  27 ++
 tests/snippets/crystal/test_numbers.txt            |  84 ++++++
 tests/snippets/crystal/test_string_escapes.txt     |  58 +++++
 tests/snippets/crystal/test_symbols.txt            |  20 ++
 tests/snippets/html/css.txt                        |  29 +++
 tests/snippets/html/css_backtracking.txt           |  93 +++++++
 tests/snippets/html/javascript.txt                 |  28 ++
 tests/snippets/html/javascript_backtracking.txt    | 285 +++++++++++++++++++++
 tests/snippets/html/javascript_unclosed.txt        |  40 +++
 tests/snippets/java/test_backtracking.txt          |  66 +++++
 tests/snippets/java/test_string_literals.txt       |  44 ++++
 tests/snippets/json/test_arrays.txt                |  15 ++
 tests/snippets/json/test_backtracking.txt          |   6 +
 tests/snippets/json/test_comments.txt              |  65 +++++
 tests/snippets/json/test_constants.txt             |  14 +
 tests/snippets/json/test_escape_sequences.txt      |  79 ++++++
 tests/snippets/json/test_floats.txt                |  42 +++
 tests/snippets/json/test_integers.txt              |  18 ++
 tests/snippets/json/test_objects.txt               |  15 ++
 tests/snippets/json/test_round_trip.txt            |  29 +++
 tests/snippets/json/test_strings.txt               |  42 +++
 tests/snippets/json/test_whitespace.txt            |  17 ++
 tests/snippets/jsonld/test_json_ld.txt             | 251 ++++++++++++++++++
 tests/snippets/thingsdb/basic.txt                  |  10 +
 tests/snippets/usd/basic.txt                       |  52 ++++
 tests/test_crystal.py                              |  80 ------
 tests/test_data.py                                 | 285 ---------------------
 tests/test_html_lexer.py                           | 131 ----------
 tests/test_java.py                                 |  40 ---
 tests/test_thingsdb.py                             |  36 ---
 tests/test_usd.py                                  |  63 -----
 31 files changed, 1429 insertions(+), 635 deletions(-)
 create mode 100644 tests/snippets/crystal/test_empty_percent_strings.txt
 create mode 100644 tests/snippets/crystal/test_numbers.txt
 create mode 100644 tests/snippets/crystal/test_string_escapes.txt
 create mode 100644 tests/snippets/crystal/test_symbols.txt
 create mode 100644 tests/snippets/html/css.txt
 create mode 100644 tests/snippets/html/css_backtracking.txt
 create mode 100644 tests/snippets/html/javascript.txt
 create mode 100644 tests/snippets/html/javascript_backtracking.txt
 create mode 100644 tests/snippets/html/javascript_unclosed.txt
 create mode 100644 tests/snippets/java/test_backtracking.txt
 create mode 100644 tests/snippets/java/test_string_literals.txt
 create mode 100644 tests/snippets/json/test_arrays.txt
 create mode 100644 tests/snippets/json/test_backtracking.txt
 create mode 100644 tests/snippets/json/test_comments.txt
 create mode 100644 tests/snippets/json/test_constants.txt
 create mode 100644 tests/snippets/json/test_escape_sequences.txt
 create mode 100644 tests/snippets/json/test_floats.txt
 create mode 100644 tests/snippets/json/test_integers.txt
 create mode 100644 tests/snippets/json/test_objects.txt
 create mode 100644 tests/snippets/json/test_round_trip.txt
 create mode 100644 tests/snippets/json/test_strings.txt
 create mode 100644 tests/snippets/json/test_whitespace.txt
 create mode 100644 tests/snippets/jsonld/test_json_ld.txt
 create mode 100644 tests/snippets/thingsdb/basic.txt
 create mode 100644 tests/snippets/usd/basic.txt
 delete mode 100644 tests/test_crystal.py
 delete mode 100644 tests/test_data.py
 delete mode 100644 tests/test_html_lexer.py
 delete mode 100644 tests/test_java.py
 delete mode 100644 tests/test_thingsdb.py
 delete mode 100755 tests/test_usd.py

(limited to 'tests')

diff --git a/tests/snippets/crystal/test_empty_percent_strings.txt b/tests/snippets/crystal/test_empty_percent_strings.txt
new file mode 100644
index 00000000..3cc0844e
--- /dev/null
+++ b/tests/snippets/crystal/test_empty_percent_strings.txt
@@ -0,0 +1,27 @@
+---input---
+%()
+%[]
+%{}
+%<>
+%||
+
+---tokens---
+'%('          Literal.String.Other
+')'           Literal.String.Other
+'\n'          Text.Whitespace
+
+'%['          Literal.String.Other
+']'           Literal.String.Other
+'\n'          Text.Whitespace
+
+'%{'          Literal.String.Other
+'}'           Literal.String.Other
+'\n'          Text.Whitespace
+
+'%<'          Literal.String.Other
+'>'           Literal.String.Other
+'\n'          Text.Whitespace
+
+'%|'          Literal.String.Other
+'|'           Literal.String.Other
+'\n'          Text.Whitespace
diff --git a/tests/snippets/crystal/test_numbers.txt b/tests/snippets/crystal/test_numbers.txt
new file mode 100644
index 00000000..0ddfec2a
--- /dev/null
+++ b/tests/snippets/crystal/test_numbers.txt
@@ -0,0 +1,84 @@
+---input---
+# Integers
+0
+1
+1_000_000
+1u8
+11231231231121312i64
+
+# Floats
+0.0
+1.0_f32
+1_f32
+0f64
+1e+4
+1e111
+1_234.567_890
+
+# Error
+01
+0b2
+0x129g2
+0o12358
+
+---tokens---
+'# Integers'  Comment.Single
+'\n'          Text.Whitespace
+
+'0'           Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'1'           Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'1_000_000'   Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'1u8'         Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'11231231231121312i64' Literal.Number.Integer
+'\n\n'        Text.Whitespace
+
+'# Floats'    Comment.Single
+'\n'          Text.Whitespace
+
+'0.0'         Literal.Number.Float
+'\n'          Text.Whitespace
+
+'1.0_f32'     Literal.Number.Float
+'\n'          Text.Whitespace
+
+'1_f32'       Literal.Number.Float
+'\n'          Text.Whitespace
+
+'0f64'        Literal.Number.Float
+'\n'          Text.Whitespace
+
+'1e+4'        Literal.Number.Float
+'\n'          Text.Whitespace
+
+'1e111'       Literal.Number.Float
+'\n'          Text.Whitespace
+
+'1_234.567_890' Literal.Number.Float
+'\n\n'        Text.Whitespace
+
+'# Error'     Comment.Single
+'\n'          Text.Whitespace
+
+'0'           Error
+'1'           Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'0'           Error
+'b2'          Name
+'\n'          Text.Whitespace
+
+'0'           Error
+'x129g2'      Name
+'\n'          Text.Whitespace
+
+'0'           Error
+'o12358'      Name
+'\n'          Text.Whitespace
diff --git a/tests/snippets/crystal/test_string_escapes.txt b/tests/snippets/crystal/test_string_escapes.txt
new file mode 100644
index 00000000..3ea12f4d
--- /dev/null
+++ b/tests/snippets/crystal/test_string_escapes.txt
@@ -0,0 +1,58 @@
+---input---
+"a\nz"
+"a\az"
+"a\xffz"
+"a\u1234z"
+"a\000z"
+"a\u{0}z"
+"a\u{10AfF9}z"
+
+---tokens---
+'"'           Literal.String.Double
+'a'           Literal.String.Double
+'\\n'         Literal.String.Escape
+'z'           Literal.String.Double
+'"'           Literal.String.Double
+'\n'          Text.Whitespace
+
+'"'           Literal.String.Double
+'a'           Literal.String.Double
+'\\a'         Literal.String.Escape
+'z'           Literal.String.Double
+'"'           Literal.String.Double
+'\n'          Text.Whitespace
+
+'"'           Literal.String.Double
+'a'           Literal.String.Double
+'\\xff'       Literal.String.Escape
+'z'           Literal.String.Double
+'"'           Literal.String.Double
+'\n'          Text.Whitespace
+
+'"'           Literal.String.Double
+'a'           Literal.String.Double
+'\\u1234'     Literal.String.Escape
+'z'           Literal.String.Double
+'"'           Literal.String.Double
+'\n'          Text.Whitespace
+
+'"'           Literal.String.Double
+'a'           Literal.String.Double
+'\\000'       Literal.String.Escape
+'z'           Literal.String.Double
+'"'           Literal.String.Double
+'\n'          Text.Whitespace
+
+'"'           Literal.String.Double
+'a'           Literal.String.Double
+'\\u{0}'      Literal.String.Escape
+'z'           Literal.String.Double
+'"'           Literal.String.Double
+'\n'          Text.Whitespace
+
+'"'           Literal.String.Double
+'a'           Literal.String.Double
+'\\u{10AfF9}' Literal.String.Escape
+'z'           Literal.String.Double
+'"'           Literal.String.Double
+'\n'          Text.Whitespace
diff --git a/tests/snippets/crystal/test_symbols.txt b/tests/snippets/crystal/test_symbols.txt
new file mode 100644
index 00000000..3d2cb0aa
--- /dev/null
+++ b/tests/snippets/crystal/test_symbols.txt
@@ -0,0 +1,20 @@
+---input---
+:sym_bol
+:あ
+:question?
+:"symbol"
+
+---tokens---
+':sym_bol'    Literal.String.Symbol
+'\n'          Text.Whitespace
+
+':あ'          Literal.String.Symbol
+'\n'          Text.Whitespace
+
+':question?'  Literal.String.Symbol
+'\n'          Text.Whitespace
+
+':"'          Literal.String.Symbol
+'symbol'      Literal.String.Symbol
+'"'           Literal.String.Symbol
+'\n'          Text.Whitespace
diff --git a/tests/snippets/html/css.txt b/tests/snippets/html/css.txt
new file mode 100644
index 00000000..08b7e726
--- /dev/null
+++ b/tests/snippets/html/css.txt
@@ -0,0 +1,29 @@
+---input---
+<style>
+.ui-helper-hidden { display: none }
+</style>
+
+---tokens---
+'<'           Punctuation
+'style'       Name.Tag
+'>'           Punctuation
+'\n'          Text.Whitespace
+
+'.'           Punctuation
+'ui-helper-hidden' Name.Class
+' '           Text.Whitespace
+'{'           Punctuation
+' '           Text.Whitespace
+'display'     Keyword
+':'           Punctuation
+' '           Text.Whitespace
+'none'        Keyword.Constant
+' '           Text.Whitespace
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'<'           Punctuation
+'/'           Punctuation
+'style'       Name.Tag
+'>'           Punctuation
+'\n'          Text
diff --git a/tests/snippets/html/css_backtracking.txt b/tests/snippets/html/css_backtracking.txt
new file mode 100644
index 00000000..7efb4c17
--- /dev/null
+++ b/tests/snippets/html/css_backtracking.txt
@@ -0,0 +1,93 @@
+---input---
+<style>
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+.ui-helper-hidden { display: none }
+
+---tokens---
+'<'           Punctuation
+'style'       Name.Tag
+'>'           Punctuation
+'\n'          Text.Whitespace
+
+'.'           Punctuation
+'ui-helper-hidden' Name.Class
+' '           Text.Whitespace
+'{'           Punctuation
+' '           Text.Whitespace
+'display'     Keyword
+':'           Punctuation
+' '           Text.Whitespace
+'none'        Keyword.Constant
+' '           Text.Whitespace
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n.ui-helper-hidden { display: none }\n' Text
diff --git a/tests/snippets/html/javascript.txt b/tests/snippets/html/javascript.txt
new file mode 100644
index 00000000..5e12df2e
--- /dev/null
+++ b/tests/snippets/html/javascript.txt
@@ -0,0 +1,28 @@
+---input---
+<script type="text/javascript">
+alert("hi");
+</script>
+
+---tokens---
+'<'           Punctuation
+'script'      Name.Tag
+' '           Text
+'type'        Name.Attribute
+'='           Operator
+'"text/javascript"' Literal.String
+'>'           Punctuation
+''            Text
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'<'           Punctuation
+'/'           Punctuation
+'script'      Name.Tag
+'>'           Punctuation
+'\n'          Text
diff --git a/tests/snippets/html/javascript_backtracking.txt b/tests/snippets/html/javascript_backtracking.txt
new file mode 100644
index 00000000..0bb8fa6f
--- /dev/null
+++ b/tests/snippets/html/javascript_backtracking.txt
@@ -0,0 +1,285 @@
+---input---
+<script type="text/javascript">
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+// Note the missing </script> tag; this should not cause catastrophic backtracking
+
+---tokens---
+'<'           Punctuation
+'script'      Name.Tag
+' '           Text
+'type'        Name.Attribute
+'='           Operator
+'"text/javascript"' Literal.String
+'>'           Punctuation
+''            Text
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'// Note the missing ' Comment.Single
+'<'           Punctuation
+'/'           Punctuation
+'script'      Name.Tag
+'>'           Punctuation
+' tag; this should not cause catastrophic backtracking\n' Text
diff --git a/tests/snippets/html/javascript_unclosed.txt b/tests/snippets/html/javascript_unclosed.txt
new file mode 100644
index 00000000..5754a4f1
--- /dev/null
+++ b/tests/snippets/html/javascript_unclosed.txt
@@ -0,0 +1,40 @@
+---input---
+<script type="text/javascript">
+alert("hi");
+alert("hi");
+alert("hi");
+alert("hi");
+<div>
+hi
+</div>
+
+---tokens---
+'<'           Punctuation
+'script'      Name.Tag
+' '           Text
+'type'        Name.Attribute
+'='           Operator
+'"text/javascript"' Literal.String
+'>'           Punctuation
+''            Text
+'\n'          Text.Whitespace
+
+'alert'       Name.Other
+'('           Punctuation
+'"hi"'        Literal.String.Double
+')'           Punctuation
+';'           Punctuation
+'\n'          Text.Whitespace
+
+'alert("hi");\nalert("hi");\nalert("hi");\n' Text
+
+'<'           Punctuation
+'div'         Name.Tag
+'>'           Punctuation
+'\nhi\n'      Text
+
+'<'           Punctuation
+'/'           Punctuation
+'div'         Name.Tag
+'>'           Punctuation
+'\n'          Text
diff --git a/tests/snippets/java/test_backtracking.txt b/tests/snippets/java/test_backtracking.txt
new file mode 100644
index 00000000..63532b81
--- /dev/null
+++ b/tests/snippets/java/test_backtracking.txt
@@ -0,0 +1,66 @@
+---input---
+"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
+
+---tokens---
+'"'           Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\\\\'        Literal.String
+'\n'          Literal.String
diff --git a/tests/snippets/java/test_string_literals.txt b/tests/snippets/java/test_string_literals.txt
new file mode 100644
index 00000000..edb2d111
--- /dev/null
+++ b/tests/snippets/java/test_string_literals.txt
@@ -0,0 +1,44 @@
+---input---
+""
+"abc"
+"ひらがな"
+"123"
+"\\"
+"\t"
+"\""
+
+---tokens---
+'"'           Literal.String
+'"'           Literal.String
+'\n'          Text.Whitespace
+
+'"'           Literal.String
+'abc'         Literal.String
+'"'           Literal.String
+'\n'          Text.Whitespace
+
+'"'           Literal.String
+'ひらがな'        Literal.String
+'"'           Literal.String
+'\n'          Text.Whitespace
+
+'"'           Literal.String
+'123'         Literal.String
+'"'           Literal.String
+'\n'          Text.Whitespace
+
+'"'           Literal.String
+'\\\\'        Literal.String
+'"'           Literal.String
+'\n'          Text.Whitespace
+
+'"'           Literal.String
+'\\'          Literal.String
+'t'           Literal.String
+'"'           Literal.String
+'\n'          Text.Whitespace
+
+'"'           Literal.String
+'\\"'         Literal.String
+'"'           Literal.String
+'\n'          Text.Whitespace
diff --git a/tests/snippets/json/test_arrays.txt b/tests/snippets/json/test_arrays.txt
new file mode 100644
index 00000000..b4e7724f
--- /dev/null
+++ b/tests/snippets/json/test_arrays.txt
@@ -0,0 +1,15 @@
+---input---
+[]
+["a", "b"]
+
+---tokens---
+'[]'          Punctuation
+'\n'          Text.Whitespace
+
+'['           Punctuation
+'"a"'         Literal.String.Double
+','           Punctuation
+' '           Text.Whitespace
+'"b"'         Literal.String.Double
+']'           Punctuation
+'\n'          Text.Whitespace
diff --git a/tests/snippets/json/test_backtracking.txt b/tests/snippets/json/test_backtracking.txt
new file mode 100644
index 00000000..5c462f52
--- /dev/null
+++ b/tests/snippets/json/test_backtracking.txt
@@ -0,0 +1,6 @@
+---input---
+{"\u00D0000\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\63CD
+
+---tokens---
+'{'           Punctuation
+'"\\u00D0000\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\63CD\n' Error
diff --git a/tests/snippets/json/test_comments.txt b/tests/snippets/json/test_comments.txt
new file mode 100644
index 00000000..ba0ed53f
--- /dev/null
+++ b/tests/snippets/json/test_comments.txt
@@ -0,0 +1,65 @@
+---input---
+{"a//b"//C1
+:123/////C2
+}
+
+/** / **/{"a /**/ b"/*
+*/: 123}
+
+// Invalid comments
+/
+1/
+/1
+""/
+
+// Unclosed
+
+""/**
+
+---tokens---
+'{'           Punctuation
+'"a//b"'      Name.Tag
+'//C1'        Comment.Single
+'\n'          Text.Whitespace
+
+':'           Punctuation
+'123'         Literal.Number.Integer
+'/////C2'     Comment.Single
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n\n'        Text.Whitespace
+
+'/** / **/'   Comment.Multiline
+'{'           Punctuation
+'"a /**/ b"'  Name.Tag
+'/*\n*/'      Comment.Multiline
+':'           Punctuation
+' '           Text.Whitespace
+'123'         Literal.Number.Integer
+'}'           Punctuation
+'\n\n'        Text.Whitespace
+
+'// Invalid comments' Comment.Single
+'\n'          Text.Whitespace
+
+'/'           Error
+'\n'          Text.Whitespace
+
+'1'           Literal.Number.Integer
+'/'           Error
+'\n'          Text.Whitespace
+
+'/'           Error
+'1'           Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'""'          Literal.String.Double
+'/'           Error
+'\n\n'        Text.Whitespace
+
+'// Unclosed' Comment.Single
+'\n\n'        Text.Whitespace
+
+'""'          Literal.String.Double
+'/**\n'       Error
diff --git a/tests/snippets/json/test_constants.txt b/tests/snippets/json/test_constants.txt
new file mode 100644
index 00000000..3d7e4d50
--- /dev/null
+++ b/tests/snippets/json/test_constants.txt
@@ -0,0 +1,14 @@
+---input---
+true
+false
+null
+
+---tokens---
+'true'        Keyword.Constant
+'\n'          Text.Whitespace
+
+'false'       Keyword.Constant
+'\n'          Text.Whitespace
+
+'null'        Keyword.Constant
+'\n'          Text.Whitespace
diff --git a/tests/snippets/json/test_escape_sequences.txt b/tests/snippets/json/test_escape_sequences.txt
new file mode 100644
index 00000000..becc0236
--- /dev/null
+++ b/tests/snippets/json/test_escape_sequences.txt
@@ -0,0 +1,79 @@
+---input---
+"\""
+"\\"
+"\/"
+"\b"
+"\f"
+"\n"
+"\r"
+"\t"
+"\u0123"
+"\u4567"
+"\u89ab"
+"\ucdef"
+"\uABCD"
+"\uEF01"
+
+// Incomplete sequences
+"\uz"
+"\u1z"
+"\u12z"
+"\u123z"
+
+---tokens---
+'"\\""'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\\\"'      Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\/"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\b"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\f"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\n"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\r"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\t"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\u0123"'   Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\u4567"'   Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\u89ab"'   Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\ucdef"'   Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\uABCD"'   Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\uEF01"'   Literal.String.Double
+'\n\n'        Text.Whitespace
+
+'// Incomplete sequences' Comment.Single
+'\n'          Text.Whitespace
+
+'"\\uz"'      Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\u1z"'     Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\u12z"'    Literal.String.Double
+'\n'          Text.Whitespace
+
+'"\\u123z"'   Literal.String.Double
+'\n'          Text.Whitespace
diff --git a/tests/snippets/json/test_floats.txt b/tests/snippets/json/test_floats.txt
new file mode 100644
index 00000000..1380578f
--- /dev/null
+++ b/tests/snippets/json/test_floats.txt
@@ -0,0 +1,42 @@
+---input---
+123456789.0123456789
+-123456789.0123456789
+1e10
+-1E10
+1e-10
+-1E+10
+1.0e10
+-1.0E10
+1.0e-10
+-1.0E+10
+
+---tokens---
+'123456789.0123456789' Literal.Number.Float
+'\n'          Text.Whitespace
+
+'-123456789.0123456789' Literal.Number.Float
+'\n'          Text.Whitespace
+
+'1e10'        Literal.Number.Float
+'\n'          Text.Whitespace
+
+'-1E10'       Literal.Number.Float
+'\n'          Text.Whitespace
+
+'1e-10'       Literal.Number.Float
+'\n'          Text.Whitespace
+
+'-1E+10'      Literal.Number.Float
+'\n'          Text.Whitespace
+
+'1.0e10'      Literal.Number.Float
+'\n'          Text.Whitespace
+
+'-1.0E10'     Literal.Number.Float
+'\n'          Text.Whitespace
+
+'1.0e-10'     Literal.Number.Float
+'\n'          Text.Whitespace
+
+'-1.0E+10'    Literal.Number.Float
+'\n'          Text.Whitespace
diff --git a/tests/snippets/json/test_integers.txt b/tests/snippets/json/test_integers.txt
new file mode 100644
index 00000000..d1125c31
--- /dev/null
+++ b/tests/snippets/json/test_integers.txt
@@ -0,0 +1,18 @@
+---input---
+0
+-1
+1234567890
+-1234567890
+
+---tokens---
+'0'           Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'-1'          Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'1234567890'  Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'-1234567890' Literal.Number.Integer
+'\n'          Text.Whitespace
diff --git a/tests/snippets/json/test_objects.txt b/tests/snippets/json/test_objects.txt
new file mode 100644
index 00000000..bb6f623c
--- /dev/null
+++ b/tests/snippets/json/test_objects.txt
@@ -0,0 +1,15 @@
+---input---
+{}
+{"a": "b"}
+
+---tokens---
+'{}'          Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"a"'         Name.Tag
+':'           Punctuation
+' '           Text.Whitespace
+'"b"'         Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
diff --git a/tests/snippets/json/test_round_trip.txt b/tests/snippets/json/test_round_trip.txt
new file mode 100644
index 00000000..510a6f9d
--- /dev/null
+++ b/tests/snippets/json/test_round_trip.txt
@@ -0,0 +1,29 @@
+---input---
++
+0
+""0
+a
+b
+""/-
+
+---tokens---
+'+'           Error
+'\n'          Text.Whitespace
+
+'0'           Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'""'          Literal.String.Double
+'0'           Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'a'           Error
+'\n'          Text.Whitespace
+
+'b'           Error
+'\n'          Text.Whitespace
+
+'""'          Literal.String.Double
+'/'           Error
+'-'           Literal.Number.Integer
+'\n'          Text.Whitespace
diff --git a/tests/snippets/json/test_strings.txt b/tests/snippets/json/test_strings.txt
new file mode 100644
index 00000000..07468e42
--- /dev/null
+++ b/tests/snippets/json/test_strings.txt
@@ -0,0 +1,42 @@
+---input---
+""
+"abc"
+"ひらがな"
+"123"
+"[]"
+"{}"
+"true"
+"false"
+"nul"
+":,"
+
+---tokens---
+'""'          Literal.String.Double
+'\n'          Text.Whitespace
+
+'"abc"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'"ひらがな"'      Literal.String.Double
+'\n'          Text.Whitespace
+
+'"123"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'"[]"'        Literal.String.Double
+'\n'          Text.Whitespace
+
+'"{}"'        Literal.String.Double
+'\n'          Text.Whitespace
+
+'"true"'      Literal.String.Double
+'\n'          Text.Whitespace
+
+'"false"'     Literal.String.Double
+'\n'          Text.Whitespace
+
+'"nul"'       Literal.String.Double
+'\n'          Text.Whitespace
+
+'":,"'        Literal.String.Double
+'\n'          Text.Whitespace
diff --git a/tests/snippets/json/test_whitespace.txt b/tests/snippets/json/test_whitespace.txt
new file mode 100644
index 00000000..c4c86588
--- /dev/null
+++ b/tests/snippets/json/test_whitespace.txt
@@ -0,0 +1,17 @@
+---input---
+{
+	"tabs": "here",
+}
+
+---tokens---
+'{'           Punctuation
+'\n\t'        Text.Whitespace
+'"tabs"'      Name.Tag
+':'           Punctuation
+' '           Text.Whitespace
+'"here"'      Literal.String.Double
+','           Punctuation
+'\n'          Text.Whitespace
+
+'}'           Punctuation
+'\n'          Text.Whitespace
diff --git a/tests/snippets/jsonld/test_json_ld.txt b/tests/snippets/jsonld/test_json_ld.txt
new file mode 100644
index 00000000..dbd29699
--- /dev/null
+++ b/tests/snippets/jsonld/test_json_ld.txt
@@ -0,0 +1,251 @@
+---input---
+// JSON-LD keywords
+{"@base": ""}
+{"@container": ""}
+{"@context": ""}
+{"@direction": ""}
+{"@graph": ""}
+{"@id": ""}
+{"@import": ""}
+{"@included": ""}
+{"@index": ""}
+{"@json": ""}
+{"@language": ""}
+{"@list": ""}
+{"@nest": ""}
+{"@none": ""}
+{"@prefix": ""}
+{"@propagate": ""}
+{"@protected": ""}
+{"@reverse": ""}
+{"@set": ""}
+{"@type": ""}
+{"@value": ""}
+{"@version": ""}
+{"@vocab": ""}
+
+// Not keywords
+{"@bogus": ""} // "@" does not guarantee a keyword match
+{"@bases": ""} // keyword is "base"
+{"container": ""} // no leading "@"
+
+---tokens---
+'// JSON-LD keywords' Comment.Single
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@base"'     Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@container"' Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@context"'  Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@direction"' Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@graph"'    Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@id"'       Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@import"'   Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@included"' Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@index"'    Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@json"'     Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@language"' Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@list"'     Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@nest"'     Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@none"'     Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@prefix"'   Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@propagate"' Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@protected"' Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@reverse"'  Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@set"'      Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@type"'     Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@value"'    Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@version"'  Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@vocab"'    Name.Decorator
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+'\n\n'        Text.Whitespace
+
+'// Not keywords' Comment.Single
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@bogus"'    Name.Tag
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+' '           Text.Whitespace
+'// "@" does not guarantee a keyword match' Comment.Single
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"@bases"'    Name.Tag
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+' '           Text.Whitespace
+'// keyword is "base"' Comment.Single
+'\n'          Text.Whitespace
+
+'{'           Punctuation
+'"container"' Name.Tag
+':'           Punctuation
+' '           Text.Whitespace
+'""'          Literal.String.Double
+'}'           Punctuation
+' '           Text.Whitespace
+'// no leading "@"' Comment.Single
+'\n'          Text.Whitespace
diff --git a/tests/snippets/thingsdb/basic.txt b/tests/snippets/thingsdb/basic.txt
new file mode 100644
index 00000000..f962cf24
--- /dev/null
+++ b/tests/snippets/thingsdb/basic.txt
@@ -0,0 +1,10 @@
+---input---
+42
+#42
+
+---tokens---
+'42'          Literal.Number.Integer
+'\n'          Text.Whitespace
+
+'#42'         Comment.Preproc
+'\n'          Text.Whitespace
diff --git a/tests/snippets/usd/basic.txt b/tests/snippets/usd/basic.txt
new file mode 100644
index 00000000..7dd68d98
--- /dev/null
+++ b/tests/snippets/usd/basic.txt
@@ -0,0 +1,52 @@
+---input---
+# Regular file paths
+
+@./some/path/to/a/file/foo.usda@
+@/some/path/to/a/file/foo.usda@
+@some/path/to/a/file/foo.usda@
+@file://SPECI__Z-_ALIZED(syntax_here)?with_arbitrary#)(%*&)\characters.tar.gz@
+
+# SdfPath syntax
+</some/another_one/here>
+</some/path/here.property_name>
+</some/path/here>
+<../some/another_one/here>
+<../some/path/here.property_name>
+<../some/path/here>
+
+---tokens---
+'# Regular file paths' Comment.Single
+'\n\n'        Text.Whitespace
+
+'@./some/path/to/a/file/foo.usda@' Literal.String.Interpol
+'\n'          Text.Whitespace
+
+'@/some/path/to/a/file/foo.usda@' Literal.String.Interpol
+'\n'          Text.Whitespace
+
+'@some/path/to/a/file/foo.usda@' Literal.String.Interpol
+'\n'          Text.Whitespace
+
+'@file://SPECI__Z-_ALIZED(syntax_here)?with_arbitrary#)(%*&)\\characters.tar.gz@' Literal.String.Interpol
+'\n\n'        Text.Whitespace
+
+'# SdfPath syntax' Comment.Single
+'\n'          Text.Whitespace
+
+'</some/another_one/here>' Name.Namespace
+'\n'          Text.Whitespace
+
+'</some/path/here.property_name>' Name.Namespace
+'\n'          Text.Whitespace
+
+'</some/path/here>' Name.Namespace
+'\n'          Text.Whitespace
+
+'<../some/another_one/here>' Name.Namespace
+'\n'          Text.Whitespace
+
+'<../some/path/here.property_name>' Name.Namespace
+'\n'          Text.Whitespace
+
+'<../some/path/here>' Name.Namespace
+'\n'          Text.Whitespace
diff --git a/tests/test_crystal.py b/tests/test_crystal.py
deleted file mode 100644
index 9bc2f11a..00000000
--- a/tests/test_crystal.py
+++ /dev/null
@@ -1,80 +0,0 @@
-"""
-    Basic CrystalLexer Test
-    ~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import pytest
-
-from pygments.token import String, Number, Punctuation, Error, Whitespace
-from pygments.lexers import CrystalLexer
-
-
-@pytest.fixture(scope='module')
-def lexer():
-    yield CrystalLexer()
-
-
-def test_numbers(lexer):
-    for kind, testset in [
-        (Number.Integer, '0  1  1_000_000  1u8  11231231231121312i64'),
-        (Number.Float, '0.0  1.0_f32  1_f32  0f64  1e+4  1e111  1_234.567_890'),
-        (Number.Bin, '0b1001_0110  0b0u8'),
-        (Number.Oct, '0o17  0o7_i32'),
-        (Number.Hex, '0xdeadBEEF'),
-    ]:
-        for fragment in testset.split():
-            assert list(lexer.get_tokens(fragment + '\n')) == \
-                [(kind, fragment), (Whitespace, '\n')]
-
-    for fragment in '01  0b2  0x129g2  0o12358'.split():
-        assert next(lexer.get_tokens(fragment + '\n'))[0] == Error
-
-
-def test_symbols(lexer):
-    for fragment in [':sym_bol', ':\u3042', ':question?']:
-        assert list(lexer.get_tokens(fragment + '\n')) == \
-            [(String.Symbol, fragment), (Whitespace, '\n')]
-
-    fragment = ':"sym bol"\n'
-    tokens = [
-        (String.Symbol, ':"'),
-        (String.Symbol, 'sym bol'),
-        (String.Symbol, '"'),
-        (Whitespace, '\n'),
-    ]
-    assert list(lexer.get_tokens(fragment)) == tokens
-
-
-def test_chars(lexer):
-    for fragment in ["'a'", "'я'", "'\\u{1234}'", "'\n'"]:
-        assert list(lexer.get_tokens(fragment + '\n')) == \
-            [(String.Char, fragment), (Whitespace, '\n')]
-    assert next(lexer.get_tokens("'abc'"))[0] == Error
-
-
-def test_string_escapes(lexer):
-    for body in ['\\n', '\\a', '\\xff', '\\u1234', '\\000', '\\u{0}', '\\u{10AfF9}']:
-        fragment = '"a' + body + 'z"\n'
-        assert list(lexer.get_tokens(fragment)) == [
-            (String.Double, '"'),
-            (String.Double, 'a'),
-            (String.Escape, body),
-            (String.Double, 'z'),
-            (String.Double, '"'),
-            (Whitespace, '\n'),
-        ]
-
-
-def test_empty_percent_strings(lexer):
-    for body in ['%()', '%[]', '%{}', '%<>', '%||']:
-        fragment = '(' + body + ')\n'
-        assert list(lexer.get_tokens(fragment)) == [
-            (Punctuation, '('),
-            (String.Other, body[:-1]),
-            (String.Other, body[-1]),
-            (Punctuation, ')'),
-            (Whitespace, '\n'),
-        ]
diff --git a/tests/test_data.py b/tests/test_data.py
deleted file mode 100644
index 719f8bfb..00000000
--- a/tests/test_data.py
+++ /dev/null
@@ -1,285 +0,0 @@
-"""
-    Data Tests
-    ~~~~~~~~~~
-
-    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import time
-
-import pytest
-
-from pygments.lexers.data import JsonLexer, JsonBareObjectLexer, JsonLdLexer
-from pygments.token import Comment, Error, Token, Punctuation, Number, String, \
-        Keyword, Name, Whitespace
-
-
-@pytest.fixture(scope='module')
-def lexer_json():
-    yield JsonLexer()
-
-
-@pytest.fixture(scope='module')
-def lexer_bare():
-    yield JsonBareObjectLexer()
-
-
-@pytest.fixture(scope='module')
-def lexer_json_ld():
-    yield JsonLdLexer()
-
-
-@pytest.mark.parametrize(
-    'text, expected_token_types',
-    (
-        # Integers
-        ('0', (Number.Integer,)),
-        ('-1', (Number.Integer,)),
-        ('1234567890', (Number.Integer,)),
-        ('-1234567890', (Number.Integer,)),
-
-        # Floats, including scientific notation
-        ('123456789.0123456789', (Number.Float,)),
-        ('-123456789.0123456789', (Number.Float,)),
-        ('1e10', (Number.Float,)),
-        ('-1E10', (Number.Float,)),
-        ('1e-10', (Number.Float,)),
-        ('-1E+10', (Number.Float,)),
-        ('1.0e10', (Number.Float,)),
-        ('-1.0E10', (Number.Float,)),
-        ('1.0e-10', (Number.Float,)),
-        ('-1.0E+10', (Number.Float,)),
-
-        # Strings (escapes are tested elsewhere)
-        ('""', (String.Double,)),
-        ('"abc"', (String.Double,)),
-        ('"ひらがな"', (String.Double,)),
-        ('"123"', (String.Double,)),
-        ('"[]"', (String.Double,)),
-        ('"{}"', (String.Double,)),
-        ('"true"', (String.Double,)),
-        ('"false"', (String.Double,)),
-        ('"null"', (String.Double,)),
-        ('":,"', (String.Double,)),
-
-        # Constants
-        ('true', (Keyword.Constant, )),
-        ('false', (Keyword.Constant, )),
-        ('null', (Keyword.Constant, )),
-
-        # Arrays
-        ('[]', (Punctuation,)),
-        ('["a", "b"]', (Punctuation, String.Double, Punctuation,
-                        Whitespace, String.Double, Punctuation)),
-
-        # Objects
-        ('{}', (Punctuation,)),
-        ('{"a": "b"}', (Punctuation, Name.Tag, Punctuation,
-                        Whitespace, String.Double, Punctuation)),
-    )
-)
-@pytest.mark.parametrize('end', ('', '\n'))
-def test_json_literals_positive_match(lexer_json, text, expected_token_types, end):
-    """Validate that syntactically-correct JSON literals are parsed correctly."""
-
-    tokens = list(lexer_json.get_tokens_unprocessed(text + end))
-    assert len(tokens) == len(expected_token_types) + bool(end)
-    assert all(token[1] is expected_token
-               for token, expected_token in zip(tokens, expected_token_types))
-    assert ''.join(token[2] for token in tokens) == text + end
-
-
-@pytest.mark.parametrize(
-    'text, expected',
-    (
-        ('\u0020', Whitespace),  # space
-        ('\u000a', Whitespace),  # newline
-        ('\u000d', Whitespace),  # carriage return
-        ('\u0009', Whitespace),  # tab
-    )
-)
-def test_json_whitespace_positive_match(lexer_json, text, expected):
-    """Validate that whitespace is parsed correctly."""
-
-    tokens = list(lexer_json.get_tokens_unprocessed(text))
-    assert tokens == [(0, expected, text)]
-
-    # Expand the whitespace and verify parsing again.
-    tokens = list(lexer_json.get_tokens_unprocessed(text * 2 + ' '))
-    assert tokens == [(0, expected, text * 2 + ' ')]
-
-
-@pytest.mark.parametrize(
-    'text',
-    (
-        '"', '\\', '/', 'b', 'f', 'n', 'r', 't',
-        'u0123', 'u4567', 'u89ab', 'ucdef', 'uABCD', 'uEF01',
-    )
-)
-def test_json_object_key_escapes_positive_match(lexer_json, text):
-    """Validate that escape sequences in JSON object keys are parsed correctly."""
-
-    tokens = list(lexer_json.get_tokens_unprocessed('{"\\%s": 1}' % text))
-    assert len(tokens) == 6
-    assert tokens[1][1] is Name.Tag
-    assert tokens[1][2] == '"\\%s"' % text
-
-
-@pytest.mark.parametrize('text', ('uz', 'u1z', 'u12z', 'u123z'))
-def test_json_string_unicode_escapes_negative_match(lexer_json, text):
-    """Validate that if unicode escape sequences end abruptly there's no problem."""
-
-    tokens = list(lexer_json.get_tokens_unprocessed('"\\%s"' % text))
-    assert len(tokens) == 1
-    assert tokens[0] == (0, String.Double, '"\\%s"' % text)
-
-
-@pytest.mark.parametrize(
-    'text',
-    (
-        '"', '\\', '/', 'b', 'f', 'n', 'r', 't',
-        'u0123', 'u4567', 'u89ab', 'ucdef', 'uABCD', 'uEF01',
-    )
-)
-def test_json_string_escapes_positive_match(lexer_json, text):
-    """Validate that escape sequences in JSON string values are parsed correctly."""
-
-    text = '"\\%s"' % text
-    tokens = list(lexer_json.get_tokens_unprocessed(text))
-    assert len(tokens) == 1
-    assert tokens[0][1] is String.Double
-    assert tokens[0][2] == text
-
-
-@pytest.mark.parametrize('text', ('+\n', '0\n', '""0\n', 'a\nb\n', '""/-'))
-def test_json_round_trip_errors(lexer_json, text):
-    """Validate that past round-trip errors never crop up again."""
-
-    tokens = list(lexer_json.get_tokens_unprocessed(text))
-    assert ''.join(t[2] for t in tokens) == text
-
-
-def test_json_comments_single_line_positive_matches(lexer_json):
-    """Verify that single-line comments are tokenized correctly."""
-
-    text = '{"a//b"//C1\n:123/////C2\n}\n// // C3'
-    tokens = list(lexer_json.get_tokens_unprocessed(text))
-    assert tokens[2] == (7, Comment.Single, "//C1")
-    assert tokens[6] == (16, Comment.Single, "/////C2")
-    assert tokens[10] == (26, Comment.Single, "// // C3")
-
-    comment_count = sum(1 for token in tokens if token[1] == Comment or token[1].parent == Comment)
-    assert comment_count == 3
-
-    parsed_text = ''.join(token[2] for token in tokens)
-    assert parsed_text == text, 'Input and output texts must match!'
-
-
-def test_json_comments_multiline_positive_matches(lexer_json):
-    """Verify that multiline comments are tokenized correctly."""
-
-    text = '/** / **/{"a /**/ b"/* \n */:123}'
-    tokens = list(lexer_json.get_tokens_unprocessed(text))
-    assert tokens[0] == (0, Comment.Multiline, "/** / **/")
-    assert tokens[3] == (20, Comment.Multiline, "/* \n */")
-
-    comment_count = sum(1 for token in tokens if token[1] == Comment or token[1].parent == Comment)
-    assert comment_count == 2
-
-    parsed_text = ''.join(token[2] for token in tokens)
-    assert parsed_text == text, 'Input and output texts must match!'
-
-
-@pytest.mark.parametrize(
-    "text, expected",
-    (
-        # Unfinished comment openers
-        ('/', (0, Error, '/')),
-        ('1/', (1, Error, '/')),
-        ('/1', (0, Error, '/')),
-        ('""/', (2, Error, '/')),
-        # Unclosed multiline comments
-        ('/*', (0, Error, '/*')),
-        ('/**', (0, Error, '/**')),
-        ('/*/', (0, Error, '/*/')),
-        ('1/*', (1, Error, '/*')),
-        ('""/*', (2, Error, '/*')),
-        ('""/**', (2, Error, '/**')),
-    )
-)
-def test_json_comments_negative_matches(lexer_json, text, expected):
-    """Verify that the unfinished or unclosed comments are parsed as errors."""
-
-    tokens = list(lexer_json.get_tokens_unprocessed(text))
-    assert expected in tokens
-
-    parsed_text = ''.join(token[2] for token in tokens)
-    assert parsed_text == text, 'Input and output texts must match!'
-
-
-def test_json_escape_backtracking(lexer_json):
-    """Confirm that there is no catastrophic backtracking in the lexer.
-
-    This no longer applies because the JSON lexer doesn't use regular expressions,
-    but the test is included to ensure no loss of functionality now or in the future.
-    """
-
-    fragment = r'{"\u00D0000\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\63CD'
-    start_time = time.time()
-    list(lexer_json.get_tokens(fragment))
-    assert time.time() - start_time < 1, 'The JSON lexer may have catastrophic backtracking'
-
-
-@pytest.mark.parametrize(
-    'keyword',
-    (
-        'base',
-        'container',
-        'context',
-        'direction',
-        'graph',
-        'id',
-        'import',
-        'included',
-        'index',
-        'json',
-        'language',
-        'list',
-        'nest',
-        'none',
-        'prefix',
-        'propagate',
-        'protected',
-        'reverse',
-        'set',
-        'type',
-        'value',
-        'version',
-        'vocab',
-    )
-)
-def test_json_ld_keywords_positive_match(lexer_json_ld, keyword):
-    """Validate that JSON-LD keywords are parsed correctly."""
-
-    tokens = list(lexer_json_ld.get_tokens_unprocessed('{"@%s": ""}' % keyword))
-    assert len(tokens) == 6
-    assert tokens[1][1] is Token.Name.Decorator
-    assert tokens[1][2] == '"@%s"' % keyword
-
-
-@pytest.mark.parametrize(
-    'keyword',
-    (
-        '@bogus',  # "@" does not guarantee a keyword match
-        '@bases',  # Begins with the keyword "@base"
-        'container',  # Matches "container" but has no leading "@"
-    )
-)
-def test_json_ld_keywords_negative_match(lexer_json_ld, keyword):
-    """Validate that JSON-LD non-keywords are parsed correctly."""
-
-    tokens = list(lexer_json_ld.get_tokens_unprocessed('{"%s": ""}' % keyword))
-    assert len(tokens) == 6
-    assert tokens[1][1] is Token.Name.Tag
-    assert tokens[1][2] == '"%s"' % keyword
diff --git a/tests/test_html_lexer.py b/tests/test_html_lexer.py
deleted file mode 100644
index fe991498..00000000
--- a/tests/test_html_lexer.py
+++ /dev/null
@@ -1,131 +0,0 @@
-"""
-    HTML Lexer Tests
-    ~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import time
-
-import pytest
-
-from pygments.lexers.html import HtmlLexer
-from pygments.token import Token
-
-MAX_HL_TIME = 10
-
-
-@pytest.fixture(scope='module')
-def lexer_html():
-    yield HtmlLexer()
-
-
-def test_happy_javascript_fragment(lexer_html):
-    """valid, even long Javascript fragments should still get parsed ok"""
-
-    fragment = "<script type=\"text/javascript\">"+"alert(\"hi\");"*2000+"</script>"
-    start_time = time.time()
-    tokens = list(lexer_html.get_tokens(fragment))
-    assert all(x[1] != Token.Error for x in tokens)
-    assert time.time() - start_time < MAX_HL_TIME, \
-        'The HTML lexer might have an expensive happy-path script case'
-
-
-def test_happy_css_fragment(lexer_html):
-    """valid, even long CSS fragments should still get parsed ok"""
-
-    fragment = "<style>"+".ui-helper-hidden{display:none}"*2000+"</style>"
-    start_time = time.time()
-    tokens = list(lexer_html.get_tokens(fragment))
-    assert all(x[1] != Token.Error for x in tokens)
-    assert time.time() - start_time < MAX_HL_TIME, \
-        'The HTML lexer might have an expensive happy-path style case'
-
-
-def test_long_unclosed_javascript_fragment(lexer_html):
-    """unclosed, long Javascript fragments should parse quickly"""
-
-    reps = 2000
-    fragment = "<script type=\"text/javascript\">"+"alert(\"hi\");"*reps
-    start_time = time.time()
-    tokens = list(lexer_html.get_tokens(fragment))
-    assert time.time() - start_time < MAX_HL_TIME, \
-        'The HTML lexer might have an expensive error script case'
-    tokens_intro = [
-        (Token.Punctuation, '<'),
-        (Token.Name.Tag, 'script'),
-        (Token.Text, ' '),
-        (Token.Name.Attribute, 'type'),
-        (Token.Operator, '='),
-        (Token.Literal.String, '"text/javascript"'),
-        (Token.Punctuation, '>'),
-    ]
-    tokens_body = [
-        (Token.Name.Other, 'alert'),
-        (Token.Punctuation, '('),
-        (Token.Literal.String.Double, '"hi"'),
-        (Token.Punctuation, ')'),
-        (Token.Punctuation, ';'),
-    ]
-
-    # make sure we get the right opening tokens
-    assert tokens[:len(tokens_intro)] == tokens_intro
-    # and make sure we get the right body tokens even though the script is
-    # unclosed
-    assert tokens[len(tokens_intro):-1] == tokens_body * reps
-    # and of course, the newline we get for free from get_tokens
-    assert tokens[-1] == (Token.Text.Whitespace, "\n")
-
-
-def test_long_unclosed_css_fragment(lexer_html):
-    """unclosed, long CSS fragments should parse quickly"""
-
-    reps = 2000
-    fragment = "<style>"+".ui-helper-hidden{display:none}"*reps
-    start_time = time.time()
-    tokens = list(lexer_html.get_tokens(fragment))
-    assert time.time() - start_time < MAX_HL_TIME, \
-        'The HTML lexer might have an expensive error style case'
-
-    tokens_intro = [
-        (Token.Punctuation, '<'),
-        (Token.Name.Tag, 'style'),
-        (Token.Punctuation, '>'),
-    ]
-    tokens_body = [
-        (Token.Punctuation, '.'),
-        (Token.Name.Class, 'ui-helper-hidden'),
-        (Token.Punctuation, '{'),
-        (Token.Keyword, 'display'),
-        (Token.Punctuation, ':'),
-        (Token.Keyword.Constant, 'none'),
-        (Token.Punctuation, '}'),
-    ]
-
-    # make sure we get the right opening tokens
-    assert tokens[:len(tokens_intro)] == tokens_intro
-    # and make sure we get the right body tokens even though the style block is
-    # unclosed
-    assert tokens[len(tokens_intro):-1] == tokens_body * reps
-    # and of course, the newline we get for free from get_tokens
-    assert tokens[-1] == (Token.Text.Whitespace, "\n")
-
-
-def test_unclosed_fragment_with_newline_recovery(lexer_html):
-    """unclosed Javascript fragments should recover on the next line"""
-
-    fragment = "<script type=\"text/javascript\">"+"alert(\"hi\");"*20+"\n<div>hi</div>"
-    tokens = list(lexer_html.get_tokens(fragment))
-    recovery_tokens = [
-        (Token.Punctuation, '<'),
-        (Token.Name.Tag, 'div'),
-        (Token.Punctuation, '>'),
-        (Token.Text, 'hi'),
-        (Token.Punctuation, '<'),
-        (Token.Punctuation, '/'),
-        (Token.Name.Tag, 'div'),
-        (Token.Punctuation, '>'),
-        (Token.Text, '\n'),
-    ]
-    assert tokens[-1*len(recovery_tokens):] == recovery_tokens
diff --git a/tests/test_java.py b/tests/test_java.py
deleted file mode 100644
index a50e862e..00000000
--- a/tests/test_java.py
+++ /dev/null
@@ -1,40 +0,0 @@
-"""
-    Basic JavaLexer Test
-    ~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import time
-
-import pytest
-
-from pygments.token import String
-from pygments.lexers import JavaLexer
-
-
-@pytest.fixture(scope='module')
-def lexer():
-    yield JavaLexer()
-
-
-@pytest.mark.parametrize(
-    'text',
-    (
-        '""', '"abc"', '"ひらがな"', '"123"',
-        '"\\\\"', '"\\t"' '"\\""',
-    ),
-)
-def test_string_literals_positive_match(lexer, text):
-    """Test positive matches for string literals."""
-    tokens = list(lexer.get_tokens_unprocessed(text))
-    assert all([token is String for _, token, _ in tokens])
-    assert ''.join([value for _, _, value in tokens]) == text
-
-
-def test_string_literals_backtracking(lexer):
-    """Test catastrophic backtracking for string literals."""
-    start_time = time.time()
-    list(lexer.get_tokens_unprocessed('"' + '\\' * 100))
-    assert time.time() - start_time < 1, 'possible backtracking bug'
diff --git a/tests/test_thingsdb.py b/tests/test_thingsdb.py
deleted file mode 100644
index 25e38168..00000000
--- a/tests/test_thingsdb.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    Basic ThingsDB Test
-    ~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import unittest
-
-from pygments.token import Number, Text, Comment
-from pygments.lexers import ThingsDBLexer
-
-
-class ThingsDBTest(unittest.TestCase):
-
-    def setUp(self):
-        self.lexer = ThingsDBLexer()
-        self.maxDiff = None
-
-    def testNumber(self):
-        fragment = u'42'
-        tokens = [
-            (Number.Integer, u'42'),
-             (Text.Whitespace, '\n')
-        ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
-
-    def testThingId(self):
-        fragment = u'#42'
-        tokens = [
-            (Comment.Preproc, u'#42'),
-             (Text.Whitespace, '\n')
-        ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
diff --git a/tests/test_usd.py b/tests/test_usd.py
deleted file mode 100755
index 779e3d3d..00000000
--- a/tests/test_usd.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-
-"""Test that syntax highlighting for USD files works correctly."""
-
-import unittest
-
-from pygments.lexers import UsdLexer
-from pygments.token import Name, String, Whitespace
-
-
-class _Common(unittest.TestCase):
-    """A basic class that makes it easier to write unittests."""
-
-    def setUp(self):
-        """Create a fresh USD lexer class before each test runs."""
-        self.lexer = UsdLexer()
-
-    def _get(self, code):
-        """Tokenize the code into its unique parts.
-
-        :param code: The USD source code to split up.
-        :type code: str
-
-        :returns: The tokenized pieces.
-        :rtype: list[:class:`pygments.token._TokenType`]
-
-        """
-        return list(self.lexer.get_tokens(code))
-
-
-class Features(_Common):
-    """Test that different features of USD highlight as expected."""
-
-    def test_asset_path(self):
-        """Check that a regular file path highlights correctly."""
-        for path in [
-            "@./some/path/to/a/file/foo.usda@",
-            "@/some/path/to/a/file/foo.usda@",
-            "@some/path/to/a/file/foo.usda@",
-            r"@file://SPECI__Z-_ALIZED(syntax_here)?with_arbitrary#)(%*&)\characters.tar.gz@",
-        ]:
-            expected = [
-                (String.Interpol, path),
-                (Whitespace, "\n"),
-            ]
-
-            self.assertEqual(expected, self._get(path))
-
-    def test_target_absolute(self):
-        """Check that SdfPath syntax examples work correctly."""
-        for code in [
-            # Absolute paths
-            "</some/another_one/here>",
-            "</some/path/here.property_name>",
-            "</some/path/here>",
-            # Relative paths
-            "<../some/another_one/here>",
-            "<../some/path/here.property_name>",
-            "<../some/path/here>",
-        ]:
-            self.assertEqual(
-                [(Name.Namespace, code), (Whitespace, "\n")], self._get(code),
-            )
-- 
cgit v1.2.1