merge with stable

9 files changed, 101 insertions, 94 deletions

diff --git a/pygments/__init__.py b/pygments/__init__.py
index 82436974..1ce34b2a 100644
--- a/pygments/__init__.py
+++ b/pygments/__init__.py
@@ -26,7 +26,7 @@
     :license: BSD, see LICENSE for details.
 """
 
-__version__ = '2.0.2'
+__version__ = '2.1a0'
 __docformat__ = 'restructuredtext'
 
 __all__ = ['lex', 'format', 'highlight']
diff --git a/pygments/cmdline.py b/pygments/cmdline.py
index 9e9f02bc..f5ea5653 100644
--- a/pygments/cmdline.py
+++ b/pygments/cmdline.py
@@ -19,7 +19,7 @@ from pygments import __version__, highlight
 from pygments.util import ClassNotFound, OptionError, docstring_headline, \
     guess_decode, guess_decode_from_terminal, terminal_encoding
 from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
-    get_lexer_for_filename, find_lexer_class, TextLexer
+    get_lexer_for_filename, find_lexer_class_for_filename, TextLexer
 from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
 from pygments.formatters import get_all_formatters, get_formatter_by_name, \
     get_formatter_for_filename, find_formatter_class, \
@@ -30,7 +30,7 @@ from pygments.styles import get_all_styles, get_style_by_name
 
 USAGE = """\
 Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
-          [-O <options>] [-P <option=value>] [-s] [-o <outfile>] [<infile>]
+          [-O <options>] [-P <option=value>] [-s] [-v] [-o <outfile>] [<infile>]
 
        %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
        %s -L [<which> ...]
@@ -90,6 +90,9 @@ waiting to process the entire file.  This only works for stdin, and
 is intended for streaming input such as you get from 'tail -f'.
 Example usage: "tail -f sql.log | pygmentize -s -l sql"
 
+The -v option prints a detailed traceback on unhandled exceptions,
+which is useful for debugging and bug reports.
+
 The -h option prints this help.
 The -V option prints the package version.
 """
@@ -100,7 +103,7 @@ def _parse_options(o_strs):
     if not o_strs:
         return opts
     for o_str in o_strs:
-        if not o_str:
+        if not o_str.strip():
             continue
         o_args = o_str.split(',')
         for o_arg in o_args:
@@ -132,7 +135,7 @@ def _parse_filters(f_strs):
 def _print_help(what, name):
     try:
         if what == 'lexer':
-            cls = find_lexer_class(name)
+            cls = get_lexer_by_name(name)
             print("Help on the %s lexer:" % cls.name)
             print(dedent(cls.__doc__))
         elif what == 'formatter':
@@ -143,8 +146,10 @@ def _print_help(what, name):
             cls = find_filter_class(name)
             print("Help on the %s filter:" % name)
             print(dedent(cls.__doc__))
-    except AttributeError:
+        return 0
+    except (AttributeError, ValueError):
         print("%s not found!" % what, file=sys.stderr)
+        return 1
 
 
 def _print_list(what):
@@ -247,8 +252,7 @@ def main_inner(popts, args, usage):
             print(usage, file=sys.stderr)
             return 2
 
-        _print_help(what, name)
-        return 0
+        return _print_help(what, name)
 
     # parse -O options
     parsed_opts = _parse_options(O_opts)
@@ -271,13 +275,9 @@ def main_inner(popts, args, usage):
     # handle ``pygmentize -N``
     infn = opts.pop('-N', None)
     if infn is not None:
-        try:
-            lexer = get_lexer_for_filename(infn, **parsed_opts)
-        except ClassNotFound as err:
-            lexer = TextLexer()
-        except OptionError as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        lexer = find_lexer_class_for_filename(infn)
+        if lexer is None:
+            lexer = TextLexer
 
         print(lexer.aliases[0])
         return 0
@@ -301,12 +301,7 @@ def main_inner(popts, args, usage):
             print(err, file=sys.stderr)
             return 1
 
-        arg = a_opt or ''
-        try:
-            print(fmter.get_style_defs(arg))
-        except Exception as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        print(fmter.get_style_defs(a_opt or ''))
         return 0
 
     # if no -S is given, -a is not allowed
@@ -341,7 +336,7 @@ def main_inner(popts, args, usage):
         if '-s' in opts:
             print('Error: -s option not usable when input file specified',
                   file=sys.stderr)
-            return 1
+            return 2
 
         infn = args[0]
         try:
@@ -387,6 +382,20 @@ def main_inner(popts, args, usage):
             except ClassNotFound:
                 lexer = TextLexer(**parsed_opts)
 
+    else:  # -s option needs a lexer with -l
+        if not lexer:
+            print('Error: when using -s a lexer has to be selected with -l',
+                  file=sys.stderr)
+            return 2
+
+    # process filters
+    for fname, fopts in F_opts:
+        try:
+            lexer.add_filter(fname, **fopts)
+        except ClassNotFound as err:
+            print('Error:', err, file=sys.stderr)
+            return 1
+
     # select formatter
     outfn = opts.pop('-o', None)
     fmter = opts.pop('-f', None)
@@ -429,7 +438,7 @@ def main_inner(popts, args, usage):
 
     # provide coloring under Windows, if possible
     if not outfn and sys.platform in ('win32', 'cygwin') and \
-       fmter.name in ('Terminal', 'Terminal256'):
+       fmter.name in ('Terminal', 'Terminal256'):  # pragma: no cover
         # unfortunately colorama doesn't support binary streams on Py3
         if sys.version_info > (3,):
             from pygments.util import UnclosingTextIOWrapper
@@ -452,24 +461,12 @@ def main_inner(popts, args, usage):
         right = escapeinside[1]
         lexer = LatexEmbeddedLexer(left, right, lexer)
 
-    # process filters
-    for fname, fopts in F_opts:
-        try:
-            lexer.add_filter(fname, **fopts)
-        except ClassNotFound as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
-
     # ... and do it!
     if '-s' not in opts:
         # process whole input as per normal...
         highlight(code, lexer, fmter, outfile)
         return 0
     else:
-        if not lexer:
-            print('Error: when using -s a lexer has to be selected with -l',
-                  file=sys.stderr)
-            return 1
         # line by line processing of stdin (eg: for 'tail -f')...
         try:
             while 1:
@@ -485,7 +482,8 @@ def main_inner(popts, args, usage):
                 highlight(line, lexer, fmter, outfile)
                 if hasattr(outfile, 'flush'):
                     outfile.flush()
-        except KeyboardInterrupt:
+            return 0
+        except KeyboardInterrupt:  # pragma: no cover
             return 0
 
 
@@ -496,7 +494,7 @@ def main(args=sys.argv):
     usage = USAGE % ((args[0],) * 6)
 
     try:
-        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHgs")
+        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgs")
     except getopt.GetoptError:
         print(usage, file=sys.stderr)
         return 2
@@ -504,6 +502,18 @@ def main(args=sys.argv):
     try:
         return main_inner(popts, args, usage)
     except Exception:
+        if '-v' in dict(popts):
+            print(file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print('An unhandled exception occurred while highlighting.',
+                  file=sys.stderr)
+            print('Please report the whole traceback to the issue tracker at',
+                  file=sys.stderr)
+            print('<https://bitbucket.org/birkenfeld/pygments-main/issues>.',
+                  file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print(file=sys.stderr)
+            raise
         import traceback
         info = traceback.format_exception(*sys.exc_info())
         msg = info[-1].strip()
@@ -513,4 +523,6 @@ def main(args=sys.argv):
         print(file=sys.stderr)
         print('*** Error while highlighting:', file=sys.stderr)
         print(msg, file=sys.stderr)
+        print('*** If this is a bug you want to report, please rerun with -v.',
+              file=sys.stderr)
         return 1
diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py
index babf7ebc..55548d30 100644
--- a/pygments/formatters/html.py
+++ b/pygments/formatters/html.py
@@ -36,21 +36,11 @@ _escape_html_table = {
     ord("'"): u'&#39;',
 }
 
+
 def escape_html(text, table=_escape_html_table):
     """Escape &, <, > as well as single and double quotes for HTML."""
     return text.translate(table)
 
-def get_random_id():
-    """Return a random id for javascript fields."""
-    from random import random
-    from time import time
-    try:
-        from hashlib import sha1 as sha
-    except ImportError:
-        import sha
-        sha = sha.new
-    return sha('%s|%s' % (random(), time())).hexdigest()
-
 
 def _get_ttype_class(ttype):
     fname = STANDARD_TYPES.get(ttype)
diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py
index e6747e18..15e68e37 100644
--- a/pygments/formatters/latex.py
+++ b/pygments/formatters/latex.py
@@ -360,7 +360,7 @@ class LatexFormatter(Formatter):
                         start += value[i]
 
                     value = value[len(start):]
-                    start = escape_tex(start, self.commandprefix)
+                    start = escape_tex(start, cp)
 
                     # ... but do not escape inside comment.
                     value = start + value
@@ -370,26 +370,26 @@ class LatexFormatter(Formatter):
                     in_math = False
                     for i, part in enumerate(parts):
                         if not in_math:
-                            parts[i] = escape_tex(part, self.commandprefix)
+                            parts[i] = escape_tex(part, cp)
                         in_math = not in_math
                     value = '$'.join(parts)
                 elif self.escapeinside:
                     text = value
                     value = ''
-                    while len(text) > 0:
+                    while text:
                         a, sep1, text = text.partition(self.left)
-                        if len(sep1) > 0:
+                        if sep1:
                             b, sep2, text = text.partition(self.right)
-                            if len(sep2) > 0:
-                                value += escape_tex(a, self.commandprefix) + b
+                            if sep2:
+                                value += escape_tex(a, cp) + b
                             else:
-                                value += escape_tex(a + sep1 + b, self.commandprefix)
+                                value += escape_tex(a + sep1 + b, cp)
                         else:
-                            value = value + escape_tex(a, self.commandprefix)
+                            value += escape_tex(a, cp)
                 else:
-                    value = escape_tex(value, self.commandprefix)
+                    value = escape_tex(value, cp)
             elif ttype not in Token.Escape:
-                value = escape_tex(value, self.commandprefix)
+                value = escape_tex(value, cp)
             styles = []
             while ttype is not Token:
                 try:
@@ -423,8 +423,7 @@ class LatexFormatter(Formatter):
 
 
 class LatexEmbeddedLexer(Lexer):
-    r"""
-
+    """
     This lexer takes one lexer as argument, the lexer for the language
     being formatted, and the left and right delimiters for escaped text.
 
diff --git a/pygments/lexers/_stan_builtins.py b/pygments/lexers/_stan_builtins.py
index e276ae29..0a225eba 100644
--- a/pygments/lexers/_stan_builtins.py
+++ b/pygments/lexers/_stan_builtins.py
@@ -4,7 +4,7 @@
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     This file contains the names of functions for Stan used by
-    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.4.0.
+    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.5.0.
 
     :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
@@ -16,10 +16,12 @@ KEYWORDS = (
     'if',
     'in',
     'increment_log_prob',
+    'integrate_ode',
     'lp__',
     'print',
+    'reject',
     'return',
-    'while',
+    'while'
 )
 
 TYPES = (
@@ -36,7 +38,7 @@ TYPES = (
     'simplex',
     'unit_vector',
     'vector',
-    'void',
+    'void'
 )
 
 FUNCTIONS = (
@@ -45,6 +47,8 @@ FUNCTIONS = (
     'abs',
     'acos',
     'acosh',
+    'append_col',
+    'append_row',
     'asin',
     'asinh',
     'atan',
@@ -144,6 +148,11 @@ FUNCTIONS = (
     'fmax',
     'fmin',
     'fmod',
+    'frechet_ccdf_log',
+    'frechet_cdf',
+    'frechet_cdf_log',
+    'frechet_log',
+    'frechet_rng',
     'gamma_ccdf_log',
     'gamma_cdf',
     'gamma_cdf_log',
@@ -152,6 +161,7 @@ FUNCTIONS = (
     'gamma_q',
     'gamma_rng',
     'gaussian_dlm_obs_log',
+    'get_lp',
     'gumbel_ccdf_log',
     'gumbel_cdf',
     'gumbel_cdf_log',
@@ -182,6 +192,8 @@ FUNCTIONS = (
     'inv_wishart_rng',
     'inverse',
     'inverse_spd',
+    'is_inf',
+    'is_nan',
     'lbeta',
     'lgamma',
     'lkj_corr_cholesky_log',
@@ -252,6 +264,7 @@ FUNCTIONS = (
     'normal_log',
     'normal_rng',
     'not_a_number',
+    'num_elements',
     'ordered_logistic_log',
     'ordered_logistic_rng',
     'owens_t',
@@ -260,6 +273,11 @@ FUNCTIONS = (
     'pareto_cdf_log',
     'pareto_log',
     'pareto_rng',
+    'pareto_type_2_ccdf_log',
+    'pareto_type_2_cdf',
+    'pareto_type_2_cdf_log',
+    'pareto_type_2_log',
+    'pareto_type_2_rng',
     'pi',
     'poisson_ccdf_log',
     'poisson_cdf',
@@ -354,7 +372,7 @@ FUNCTIONS = (
     'weibull_log',
     'weibull_rng',
     'wishart_log',
-    'wishart_rng',
+    'wishart_rng'
 )
 
 DISTRIBUTIONS = (
@@ -372,6 +390,7 @@ DISTRIBUTIONS = (
     'double_exponential',
     'exp_mod_normal',
     'exponential',
+    'frechet',
     'gamma',
     'gaussian_dlm_obs',
     'gumbel',
@@ -396,6 +415,7 @@ DISTRIBUTIONS = (
     'normal',
     'ordered_logistic',
     'pareto',
+    'pareto_type_2',
     'poisson',
     'poisson_log',
     'rayleigh',
@@ -405,7 +425,7 @@ DISTRIBUTIONS = (
     'uniform',
     'von_mises',
     'weibull',
-    'wishart',
+    'wishart'
 )
 
 RESERVED = (
@@ -494,5 +514,6 @@ RESERVED = (
     'volatile',
     'wchar_t',
     'xor',
-    'xor_eq',
+    'xor_eq'
 )
+
diff --git a/pygments/lexers/modeling.py b/pygments/lexers/modeling.py
index 1d8bd342..43194436 100644
--- a/pygments/lexers/modeling.py
+++ b/pygments/lexers/modeling.py
@@ -284,8 +284,8 @@ class StanLexer(RegexLexer):
     """Pygments Lexer for Stan models.
 
     The Stan modeling language is specified in the *Stan Modeling Language
-    User's Guide and Reference Manual, v2.4.0*,
-    `pdf <https://github.com/stan-dev/stan/releases/download/v2.4.0/stan-reference-2.4.0.pdf>`__.
+    User's Guide and Reference Manual, v2.5.0*,
+    `pdf <https://github.com/stan-dev/stan/releases/download/v2.5.0/stan-reference-2.5.0.pdf>`__.
 
     .. versionadded:: 1.6
     """
diff --git a/pygments/lexers/textfmts.py b/pygments/lexers/textfmts.py
index 8a6872ca..43b16f8c 100644
--- a/pygments/lexers/textfmts.py
+++ b/pygments/lexers/textfmts.py
@@ -102,7 +102,7 @@ class GettextLexer(RegexLexer):
             (r'^(")([A-Za-z-]+:)(.*")$',
              bygroups(String, Name.Property, String)),
             (r'^".*"$', String),
-            (r'^(msgid|msgid_plural|msgstr)(\s+)(".*")$',
+            (r'^(msgid|msgid_plural|msgstr|msgctxt)(\s+)(".*")$',
              bygroups(Name.Variable, Text, String)),
             (r'^(msgstr\[)(\d)(\])(\s+)(".*")$',
              bygroups(Name.Variable, Number.Integer, Name.Variable, Text, String)),
diff --git a/pygments/lexers/theorem.py b/pygments/lexers/theorem.py
index 0b688b03..9898b05d 100644
--- a/pygments/lexers/theorem.py
+++ b/pygments/lexers/theorem.py
@@ -395,11 +395,13 @@ class LeanLexer(RegexLexer):
                  'options', 'precedence', 'postfix', 'prefix', 'calc_trans', 'calc_subst', 'calc_refl',
                  'infix', 'infixl', 'infixr', 'notation', 'eval', 'check', 'exit', 'coercion', 'end',
                  'private', 'using', 'namespace', 'including', 'instance', 'section', 'context',
-                 'protected', 'expose', 'export', 'set_option', 'add_rewrite', 'extends')
+                 'protected', 'expose', 'export', 'set_option', 'add_rewrite', 'extends',
+                 'open', 'example', 'constant', 'constants', 'print', 'opaque', 'reducible', 'irreducible'
+    )
 
     keywords2 = (
-        'forall', 'exists', 'fun', 'Pi', 'obtain', 'from', 'have', 'show', 'assume', 'take',
-        'let', 'if', 'else', 'then', 'by', 'in', 'with', 'begin', 'proof', 'qed', 'calc'
+        'forall', 'fun', 'Pi', 'obtain', 'from', 'have', 'show', 'assume', 'take',
+        'let', 'if', 'else', 'then', 'by', 'in', 'with', 'begin', 'proof', 'qed', 'calc', 'match'
     )
 
     keywords3 = (
@@ -407,46 +409,29 @@ class LeanLexer(RegexLexer):
         'Type', 'Prop',
     )
 
-    keywords4 = (
-        # Tactics
-        'apply', 'and_then', 'or_else', 'append', 'interleave', 'par', 'fixpoint', 'repeat',
-        'at_most', 'discard', 'focus_at', 'rotate', 'try_for', 'now', 'assumption', 'eassumption',
-        'state', 'intro', 'generalize', 'exact', 'unfold', 'beta', 'trace', 'focus', 'repeat1',
-        'determ', 'destruct', 'try', 'auto', 'intros'
-    )
-
     operators = (
-        '!=', '#', '&', '&&', '*', '+', '-', '/', '@',
+        '!=', '#', '&', '&&', '*', '+', '-', '/', '@', '!', '`',
         '-.', '->', '.', '..', '...', '::', ':>', ';', ';;', '<',
         '<-', '=', '==', '>', '_', '`', '|', '||', '~', '=>', '<=', '>=',
         '/\\', '\\/', u'∀', u'Π', u'λ', u'↔', u'∧', u'∨', u'≠', u'≤', u'≥',
-        u'¬', u'⁻¹', u'⬝', u'▸', u'→', u'∃', u'ℕ', u'ℤ', u'≈'
+        u'¬', u'⁻¹', u'⬝', u'▸', u'→', u'∃', u'ℕ', u'ℤ', u'≈', u'×', u'⌞', u'⌟', u'≡'
     )
 
-    word_operators = ('and', 'or', 'not', 'iff', 'eq')
-
     punctuation = ('(', ')', ':', '{', '}', '[', ']', u'⦃', u'⦄', ':=', ',')
 
-    primitives = ('unit', 'int', 'bool', 'string', 'char', 'list',
-                  'array', 'prod', 'sum', 'pair', 'real', 'nat', 'num', 'path')
-
     tokens = {
         'root': [
             (r'\s+', Text),
-            (r'\b(false|true)\b|\(\)|\[\]', Name.Builtin.Pseudo),
             (r'/-', Comment, 'comment'),
             (r'--.*?$', Comment.Single),
             (words(keywords1, prefix=r'\b', suffix=r'\b'), Keyword.Namespace),
             (words(keywords2, prefix=r'\b', suffix=r'\b'), Keyword),
             (words(keywords3, prefix=r'\b', suffix=r'\b'), Keyword.Type),
-            (words(keywords4, prefix=r'\b', suffix=r'\b'), Keyword),
             (words(operators), Name.Builtin.Pseudo),
-            (words(word_operators, prefix=r'\b', suffix=r'\b'), Name.Builtin.Pseudo),
             (words(punctuation), Operator),
-            (words(primitives, prefix=r'\b', suffix=r'\b'), Keyword.Type),
             (u"[A-Za-z_\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2100-\u214f]"
              u"[A-Za-z_'\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2070-\u2079"
-             u"\u207f-\u2089\u2090-\u209c\u2100-\u214f]*", Name),
+             u"\u207f-\u2089\u2090-\u209c\u2100-\u214f0-9]*", Name),
             (r'\d+', Number.Integer),
             (r'"', String.Double, 'string'),
             (r'[~?][a-z][\w\']*:', Name.Variable)
diff --git a/pygments/sphinxext.py b/pygments/sphinxext.py
index 7409759d..e63d3d35 100644
--- a/pygments/sphinxext.py
+++ b/pygments/sphinxext.py
@@ -133,8 +133,8 @@ class PygmentsDoc(Directive):
             if isinstance(docstring, bytes):
                 docstring = docstring.decode('utf8')
             heading = cls.__name__
-            out.append(FMTERDOC % (heading, ', '.join(data[1]) or 'None',
-                                   ', '.join(data[2]).replace('*', '\\*') or 'None',
+            out.append(FMTERDOC % (heading, ', '.join(data[2]) or 'None',
+                                   ', '.join(data[3]).replace('*', '\\*') or 'None',
                                    docstring))
         return ''.join(out)