conftest: disallow error tokens in examplefiles

They are ok in small snippets to demonstrate error cases.

Also recode all examplefiles to UTF-8.

9 files changed, 31 insertions, 23 deletions

diff --git a/tests/conftest.py b/tests/conftest.py
index ecadfd27..2b279423 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -15,15 +15,18 @@
     :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
+
 from pathlib import Path
 
 import pytest
 
 import pygments.lexers
+from pygments.token import Error
 
 
 def pytest_addoption(parser):
-    parser.addoption('--update-goldens', action='store_true', help='reset golden master benchmarks')
+    parser.addoption('--update-goldens', action='store_true',
+                     help='reset golden master benchmarks')
 
 
 class LexerTestItem(pytest.Item):
@@ -35,6 +38,8 @@ class LexerTestItem(pytest.Item):
     @classmethod
     def _prettyprint_tokens(cls, tokens):
         for tok, val in tokens:
+            if tok is Error and not cls.allow_errors:
+                raise ValueError('generated Error token at {!r}'.format(val))
             yield '{!r:<13} {}'.format(val, str(tok)[6:])
             if val.endswith('\n'):
                 yield ''
@@ -49,6 +54,9 @@ class LexerTestItem(pytest.Item):
     def _test_file_rel_path(self):
         return Path(str(self.fspath)).relative_to(Path(__file__).parent.parent)
 
+    def _prunetraceback(self, excinfo):
+        excinfo.traceback = excinfo.traceback.cut(__file__).filter()
+
     def repr_failure(self, excinfo):
         if isinstance(excinfo.value, AssertionError):
             rel_path = self._test_file_rel_path()
@@ -59,6 +67,8 @@ class LexerTestItem(pytest.Item):
             ).format(self.lexer, rel_path, Path(*rel_path.parts[:2]))
             diff = str(excinfo.value).split('\n', 1)[-1]
             return message + '\n\n' + diff
+        else:
+            return pytest.Item.repr_failure(self, excinfo)
 
     def reportinfo(self):
         return self.fspath, None, str(self._test_file_rel_path())
@@ -69,10 +79,12 @@ class LexerTestItem(pytest.Item):
 
 
 class LexerSeparateTestItem(LexerTestItem):
+    allow_errors = False
+
     def __init__(self, name, parent):
         super().__init__(name, parent)
 
-        self.input = self.fspath.read_binary()
+        self.input = self.fspath.read_text('utf-8')
         output_path = self.fspath + '.output'
         if output_path.check():
             self.expected = output_path.read_text(encoding='utf-8')
@@ -85,6 +97,8 @@ class LexerSeparateTestItem(LexerTestItem):
 
 
 class LexerInlineTestItem(LexerTestItem):
+    allow_errors = True
+
     def __init__(self, name, parent):
         super().__init__(name, parent)
 
diff --git a/tests/examplefiles/cpp/example.cpp b/tests/examplefiles/cpp/example.cpp
index 334e7ca7..c95ff58e 100644
--- a/tests/examplefiles/cpp/example.cpp
+++ b/tests/examplefiles/cpp/example.cpp
@@ -2,7 +2,7 @@
                     ansigenerator.cpp  -  description
                              -------------------
     begin                : Jul 5 2004
-    copyright            : (C) 2004 by André Simon
+    copyright            : (C) 2004 by André Simon
     email                : andre.simon1@gmx.de
  ***************************************************************************/
 
diff --git a/tests/examplefiles/delphi/example.pas b/tests/examplefiles/delphi/example.pas
index ab11ee67..9fa8307d 100644
--- a/tests/examplefiles/delphi/example.pas
+++ b/tests/examplefiles/delphi/example.pas
@@ -7,7 +7,7 @@ unit YTools;
    cYcnus.YTools 1.0.3 Beta for Delphi 4+
    by licenser and Murphy
 
-   ©2000-2003 by cYcnus
+   ©2000-2003 by cYcnus
    visit www.cYcnus.de
 
    licenser@cYcnus.de (Heinz N. Gies)
diff --git a/tests/examplefiles/html+ng2/example.ng2.output b/tests/examplefiles/html+ng2/example.ng2.output
index 0b315801..96707106 100644
--- a/tests/examplefiles/html+ng2/example.ng2.output
+++ b/tests/examplefiles/html+ng2/example.ng2.output
@@ -1,4 +1,3 @@
-'\ufeff'      Text
 '<'           Punctuation
 'div'         Name.Tag
 '>'           Punctuation
diff --git a/tests/examplefiles/html/example.xhtml b/tests/examplefiles/html/example.xhtml
index a08cf753..5b1c5823 100644
--- a/tests/examplefiles/html/example.xhtml
+++ b/tests/examplefiles/html/example.xhtml
@@ -373,4 +373,4 @@ Here is your <b>Token</b>: #{token}
   <render href="/right" />
 </Page>
 
-<!-- Copyright © 2006 Kashia Buch (kashia@vfemail.net), Fabian Buch (fabian@fabian-buch.de). All rights reserved. -->
+<!-- Copyright © 2006 Kashia Buch (kashia@vfemail.net), Fabian Buch (fabian@fabian-buch.de). All rights reserved. -->
diff --git a/tests/examplefiles/js/BOM.js.output b/tests/examplefiles/js/BOM.js.output
index 5217b483..54fd6a72 100644
--- a/tests/examplefiles/js/BOM.js.output
+++ b/tests/examplefiles/js/BOM.js.output
@@ -1,3 +1,3 @@
-'\ufeff'      Error
+''            Text
 '/* There is a BOM at the beginning of this file. */' Comment.Multiline
 '\n'          Text
diff --git a/tests/examplefiles/mupad/AlternatingGroup.mu b/tests/examplefiles/mupad/AlternatingGroup.mu
index 2cb19924..0c1f146f 100644
--- a/tests/examplefiles/mupad/AlternatingGroup.mu
+++ b/tests/examplefiles/mupad/AlternatingGroup.mu
@@ -6,7 +6,7 @@ n	   - integer >= 1
 
 Elements are represented as in Dom::PermutationGroup(n)
 
-Author:	     Nicolas M. Thiéry <nthiery@users.sourceforge.net>
+Author:	     Nicolas M. Thiéry <nthiery@users.sourceforge.net>
 License:     LGPL
 Created:     August 8th, 1999
 Last update: $Date: 2003/09/08 15:00:47 $
diff --git a/tests/examplefiles/tads3/tads3_example.t.output b/tests/examplefiles/tads3/tads3_example.t.output
index 828c155d..fb03c825 100644
--- a/tests/examplefiles/tads3/tads3_example.t.output
+++ b/tests/examplefiles/tads3/tads3_example.t.output
@@ -1,12 +1,6 @@
-'\ufeff'      Text
-'#'           Error
-''            Text
-'charset'     Name.Class
-' '           Text
-'"'           Literal.String.Double
-'utf-8'       Literal.String.Double
-'"'           Literal.String.Double
-'\n\n'        Text
+'#charset "utf-8"\n' Comment.Preproc
+
+'\n'          Text
 
 '#include <adv3.h>\n' Comment.Preproc
 
@@ -14,6 +8,7 @@
 
 '\n'          Text
 
+''            Text
 'extern'      Keyword.Reserved
 ' '           Text
 'function'    Keyword.Reserved
diff --git a/tests/examplefiles/xtend/example.xtend b/tests/examplefiles/xtend/example.xtend
index f6a51f7a..6c37d1ba 100644
--- a/tests/examplefiles/xtend/example.xtend
+++ b/tests/examplefiles/xtend/example.xtend
@@ -11,20 +11,20 @@ class BottleSong {
 	}
 	
 	def singTheSong(int all) '''
-		«FOR i : all .. 1»
-			«i.Bottles» of beer on the wall, «i.bottles» of beer.
-			Take one down and pass it around, «(i - 1).bottles» of beer on the wall.
+		«FOR i : all .. 1»
+			«i.Bottles» of beer on the wall, «i.bottles» of beer.
+			Take one down and pass it around, «(i - 1).bottles» of beer on the wall.
 			
-		«ENDFOR»
+		«ENDFOR»
 		No more bottles of beer on the wall, no more bottles of beer.
-		Go to the store and buy some more, «all.bottles» of beer on the wall.
+		Go to the store and buy some more, «all.bottles» of beer on the wall.
 	'''
 
 	def private java.lang.String bottles(int i) {
 		switch i {
 			case 0 : 'no more bottles'
 			case 1 : 'one bottle'
-			default : '''«i» bottles'''
+			default : '''«i» bottles'''
 		}.toString
 	}