Merged in SylvainCorlay/pygments-main/css-variables (pull request #814)

Allow for CSS variable in pygments stylesheets Approved-by: Anteru <bitbucket@ca.sh13.net>
author: Anteru <bitbucket@ca.sh13.net> 2019-05-09 05:06:00 +0000
committer: Anteru <bitbucket@ca.sh13.net> 2019-05-09 05:06:00 +0000
commit: dba5f50da1218be160cf0de5d07c4febf28eafe2 (patch)
tree: ecde56931e857612345e82146081d945851b6153
parent: 8f733ca887f216cdcf1b1e5f9b70954811bcfc17 (diff)
parent: 2461ec34c6aac22c0600b3e0e32a9f802765e9de (diff)
download: pygments-dba5f50da1218be160cf0de5d07c4febf28eafe2.tar.gz
12 files changed, 88 insertions, 25 deletions
diff --git a/CHANGES b/CHANGES
index db1fb485..9ebde077 100644
--- a/CHANGES
+++ b/CHANGES
@@ -8,7 +8,7 @@ pull request numbers to the requests at
 
 Version 2.4.0
 -------------
-(not released yet)
+(released May 8, 2019)
 
 - Added lexers:
 
@@ -36,18 +36,22 @@ Version 2.4.0
   * LLVM (PR#792)
   * Makefiles (PR#766)
   * PHP (#1482)
+  * Rust
   * SQL (PR#672)
   * Stan (PR#774)
   * Stata (PR#800)
   * Terraform (PR#787)
+  * YAML
 
 - Add solarized style (PR#708)
 - Add support for Markdown reference-style links (PR#753)
+- Add license information to generated HTML/CSS files (#1496)
 - Change ANSI color names (PR#777)
 - Fix catastrophic backtracking in the bash lexer (#1494)
 - Fix documentation failing to build using Sphinx 2.0 (#1501)
 - Fix incorrect links in the Lisp and R lexer documentation (PR#775)
 - Fix rare unicode errors on Python 2.7 (PR#798, #1492)
+- Fix lexers popping from an empty stack (#1506)
 - TypoScript uses ``.typoscript`` now (#1498)
 - Updated Trove classifiers and ``pip`` requirements (PR#799)
 
diff --git a/LICENSE b/LICENSE
index 21815527..13d1c74b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2006-2017 by the respective authors (see AUTHORS file).
+Copyright (c) 2006-2019 by the respective authors (see AUTHORS file).
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/pygments/__init__.py b/pygments/__init__.py
index 4dd38fee..0da0649d 100644
--- a/pygments/__init__.py
+++ b/pygments/__init__.py
@@ -29,7 +29,7 @@ import sys
 
 from pygments.util import StringIO, BytesIO
 
-__version__ = '2.3.1'
+__version__ = '2.4.0'
 __docformat__ = 'restructuredtext'
 
 __all__ = ['lex', 'format', 'highlight']
diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py
index bbddf507..d65c09ce 100644
--- a/pygments/formatters/html.py
+++ b/pygments/formatters/html.py
@@ -5,7 +5,7 @@
 
     Formatter for HTML output.
 
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -60,6 +60,11 @@ def _get_ttype_class(ttype):
 
 
 CSSFILE_TEMPLATE = '''\
+/*
+generated by Pygments <http://pygments.org>
+Copyright 2006-2019 by the Pygments team.
+Licensed under the BSD license, see LICENSE for details.
+*/
 td.linenos { background-color: #f0f0f0; padding-right: 10px; }
 span.lineno { background-color: #f0f0f0; padding: 0 5px 0 5px; }
 pre { line-height: 125%%; }
@@ -69,7 +74,11 @@ pre { line-height: 125%%; }
 DOC_HEADER = '''\
 <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
    "http://www.w3.org/TR/html4/strict.dtd">
-
+<!--
+generated by Pygments <http://pygments.org>
+Copyright 2006-2019 by the Pygments team.
+Licensed under the BSD license, see LICENSE for details.
+-->
 <html>
 <head>
   <title>%(title)s</title>
@@ -327,11 +336,17 @@ class HtmlFormatter(Formatter):
         .. versionadded:: 1.6
 
     `filename`
-        A string used to generate a filename when rendering <pre> blocks,
+        A string used to generate a filename when rendering ``<pre>`` blocks,
         for example if displaying source code.
 
         .. versionadded:: 2.1
 
+    `wrapcode`
+        Wrap the code inside ``<pre>`` blocks using ``<code>``, as recommended
+        by the HTML5 specification.
+
+        .. versionadded:: 2.4
+
 
     **Subclassing the HTML formatter**
 
@@ -400,6 +415,7 @@ class HtmlFormatter(Formatter):
         self.tagsfile = self._decodeifneeded(options.get('tagsfile', ''))
         self.tagurlformat = self._decodeifneeded(options.get('tagurlformat', ''))
         self.filename = self._decodeifneeded(options.get('filename', ''))
+        self.wrapcode = get_bool_opt(options, 'wrapcode', False)
 
         if self.tagsfile:
             if not ctags:
@@ -713,6 +729,12 @@ class HtmlFormatter(Formatter):
             yield tup
         yield 0, '</pre>'
 
+    def _wrap_code(self, inner):
+        yield 0, '<code>'
+        for tup in inner:
+            yield tup
+        yield 0, '</code>'
+
     def _format_lines(self, tokensource):
         """
         Just format the tokens, without any wrapping tags.
@@ -819,7 +841,10 @@ class HtmlFormatter(Formatter):
         individual lines, in custom generators. See docstring
         for `format`. Can be overridden.
         """
-        return self._wrap_div(self._wrap_pre(source))
+        if self.wrapcode:
+            return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
+        else:
+            return self._wrap_div(self._wrap_pre(source))
 
     def format_unencoded(self, tokensource, outfile):
         """
diff --git a/pygments/lexer.py b/pygments/lexer.py
index 90905ba5..62d66318 100644
--- a/pygments/lexer.py
+++ b/pygments/lexer.py
@@ -639,14 +639,20 @@ class RegexLexer(Lexer):
                         if isinstance(new_state, tuple):
                             for state in new_state:
                                 if state == '#pop':
-                                    statestack.pop()
+                                    if len(statestack) > 1:
+                                        statestack.pop()
                                 elif state == '#push':
                                     statestack.append(statestack[-1])
                                 else:
                                     statestack.append(state)
                         elif isinstance(new_state, int):
-                            # pop
-                            del statestack[new_state:]
+                            # pop, but keep at least one state on the stack
+                            # (random code leading to unexpected pops should
+                            # not allow exceptions)
+                            if abs(new_state) >= len(statestack):
+                                del statestack[1:]
+                            else:
+                                del statestack[new_state:]
                         elif new_state == '#push':
                             statestack.append(statestack[-1])
                         else:
@@ -724,14 +730,18 @@ class ExtendedRegexLexer(RegexLexer):
                         if isinstance(new_state, tuple):
                             for state in new_state:
                                 if state == '#pop':
-                                    ctx.stack.pop()
+                                    if len(ctx.stack) > 1:
+                                        ctx.stack.pop()
                                 elif state == '#push':
                                     ctx.stack.append(ctx.stack[-1])
                                 else:
                                     ctx.stack.append(state)
                         elif isinstance(new_state, int):
-                            # pop
-                            del ctx.stack[new_state:]
+                            # see RegexLexer for why this check is made
+                            if abs(new_state) >= len(ctx.stack):
+                                del ctx.state[1:]
+                            else:
+                                del ctx.stack[new_state:]
                         elif new_state == '#push':
                             ctx.stack.append(ctx.stack[-1])
                         else:
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index e4e8cf43..aea19b4e 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -442,7 +442,7 @@ LEXERS = {
     'UcodeLexer': ('pygments.lexers.unicon', 'ucode', ('ucode',), ('*.u', '*.u1', '*.u2'), ()),
     'UniconLexer': ('pygments.lexers.unicon', 'Unicon', ('unicon',), ('*.icn',), ('text/unicon',)),
     'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)),
-    'VBScriptLexer': ('pygments.lexers.basic', 'VBScript', (), ('*.vbs', '*.VBS'), ()),
+    'VBScriptLexer': ('pygments.lexers.basic', 'VBScript', ('vbscript',), ('*.vbs', '*.VBS'), ()),
     'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)),
     'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)),
     'VCTreeStatusLexer': ('pygments.lexers.console', 'VCTreeStatus', ('vctreestatus',), (), ()),
diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py
index 7100868c..b522450c 100644
--- a/pygments/lexers/asm.py
+++ b/pygments/lexers/asm.py
@@ -690,7 +690,7 @@ class Dasm16Lexer(RegexLexer):
 
     # Regexes yo
     char = r'[a-zA-Z$._0-9@]'
-    identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
+    identifier = r'(?:[a-zA-Z$_]' + char + r'*|\.' + char + '+)'
     number = r'[+-]?(?:0[xX][a-zA-Z0-9]+|\d+)'
     binary_number = r'0b[01_]+'
     instruction = r'(?i)(' + '|'.join(INSTRUCTIONS) + ')'
diff --git a/pygments/lexers/basic.py b/pygments/lexers/basic.py
index 1aa3274c..f93d6d52 100644
--- a/pygments/lexers/basic.py
+++ b/pygments/lexers/basic.py
@@ -510,7 +510,7 @@ class VBScriptLexer(RegexLexer):
     .. versionadded:: 2.4
     """
     name = 'VBScript'
-    aliases = []
+    aliases = ['vbscript']
     filenames = ['*.vbs', '*.VBS']
     flags = re.IGNORECASE
 
diff --git a/pygments/lexers/boa.py b/pygments/lexers/boa.py
index 41398cd4..dda31eb4 100644
--- a/pygments/lexers/boa.py
+++ b/pygments/lexers/boa.py
@@ -1,4 +1,14 @@
 # -*- coding: utf-8 -*-
+"""
+    pygments.lexers.boa
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Boa language.
+
+    :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
 import re
 
 from pygments.lexer import RegexLexer, words
@@ -11,7 +21,9 @@ line_re = re.compile('.*?\n')
 
 class BoaLexer(RegexLexer):
     """
-    http://boa.cs.iastate.edu/docs/
+    Lexer for the `Boa <http://boa.cs.iastate.edu/docs/>`_ language.
+
+    .. versionadded:: 2.4
     """
     name = 'Boa'
     aliases = ['boa']
diff --git a/pygments/lexers/rust.py b/pygments/lexers/rust.py
index 10097fba..b7b8cb7e 100644
--- a/pygments/lexers/rust.py
+++ b/pygments/lexers/rust.py
@@ -29,7 +29,7 @@ class RustLexer(RegexLexer):
 
     keyword_types = (
         words(('u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64',
-               'usize', 'isize', 'f32', 'f64', 'str', 'bool'),
+               'i128', 'u128', 'usize', 'isize', 'f32', 'f64', 'str', 'bool'),
               suffix=r'\b'),
         Keyword.Type)
 
diff --git a/setup.py b/setup.py
index 52889227..bd54b0f3 100755
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,7 @@ else:
 
 setup(
     name = 'Pygments',
-    version = '2.3.1',
+    version = '2.4.0',
     url = 'http://pygments.org/',
     license = 'BSD License',
     author = 'Georg Brandl',
diff --git a/tests/test_regexlexer.py b/tests/test_regexlexer.py
index d919a950..778f3d03 100644
--- a/tests/test_regexlexer.py
+++ b/tests/test_regexlexer.py
@@ -11,7 +11,6 @@ import unittest
 
 from pygments.token import Text
 from pygments.lexer import RegexLexer
-from pygments.lexer import bygroups
 from pygments.lexer import default
 
 
@@ -21,6 +20,8 @@ class TestLexer(RegexLexer):
         'root': [
             ('a', Text.Root, 'rag'),
             ('e', Text.Root),
+            ('#', Text.Root, '#pop'),
+            ('@', Text.Root, ('#pop', '#pop')),
             default(('beer', 'beer'))
         ],
         'beer': [
@@ -37,18 +38,29 @@ class TupleTransTest(unittest.TestCase):
     def test(self):
         lx = TestLexer()
         toks = list(lx.get_tokens_unprocessed('abcde'))
-        self.assertEqual(toks,
-           [(0, Text.Root, 'a'), (1, Text.Rag, 'b'), (2, Text.Rag, 'c'),
+        self.assertEqual(toks, [
+            (0, Text.Root, 'a'), (1, Text.Rag, 'b'), (2, Text.Rag, 'c'),
             (3, Text.Beer, 'd'), (4, Text.Root, 'e')])
 
     def test_multiline(self):
         lx = TestLexer()
         toks = list(lx.get_tokens_unprocessed('a\ne'))
-        self.assertEqual(toks,
-           [(0, Text.Root, 'a'), (1, Text, u'\n'),
-            (2, Text.Root, 'e')])
+        self.assertEqual(toks, [
+            (0, Text.Root, 'a'), (1, Text, u'\n'), (2, Text.Root, 'e')])
 
     def test_default(self):
         lx = TestLexer()
         toks = list(lx.get_tokens_unprocessed('d'))
         self.assertEqual(toks, [(0, Text.Beer, 'd')])
+
+
+class PopEmptyTest(unittest.TestCase):
+    def test_regular(self):
+        lx = TestLexer()
+        toks = list(lx.get_tokens_unprocessed('#e'))
+        self.assertEqual(toks, [(0, Text.Root, '#'), (1, Text.Root, 'e')])
+
+    def test_tuple(self):
+        lx = TestLexer()
+        toks = list(lx.get_tokens_unprocessed('@e'))
+        self.assertEqual(toks, [(0, Text.Root, '@'), (1, Text.Root, 'e')])
author	Anteru <bitbucket@ca.sh13.net>	2019-05-09 05:06:00 +0000
committer	Anteru <bitbucket@ca.sh13.net>	2019-05-09 05:06:00 +0000
commit	dba5f50da1218be160cf0de5d07c4febf28eafe2 (patch)
tree	ecde56931e857612345e82146081d945851b6153
parent	8f733ca887f216cdcf1b1e5f9b70954811bcfc17 (diff)
parent	2461ec34c6aac22c0600b3e0e32a9f802765e9de (diff)
download	pygments-dba5f50da1218be160cf0de5d07c4febf28eafe2.tar.gz