merge with stable

15 files changed, 342 insertions, 174 deletions

diff --git a/CHANGES b/CHANGES
index 57f03a2b..fd7e2ac7 100644
--- a/CHANGES
+++ b/CHANGES
@@ -2,9 +2,9 @@ Pygments changelog
 ==================
 
 Issue numbers refer to the tracker at
-<http://bitbucket.org/birkenfeld/pygments-main/issues>,
+<https://bitbucket.org/birkenfeld/pygments-main/issues>,
 pull request numbers to the requests at
-<http://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>.
+<https://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>.
 
 
 Version 2.0.3
diff --git a/pygments/__init__.py b/pygments/__init__.py
index 82436974..1ce34b2a 100644
--- a/pygments/__init__.py
+++ b/pygments/__init__.py
@@ -26,7 +26,7 @@
     :license: BSD, see LICENSE for details.
 """
 
-__version__ = '2.0.2'
+__version__ = '2.1a0'
 __docformat__ = 'restructuredtext'
 
 __all__ = ['lex', 'format', 'highlight']
diff --git a/pygments/cmdline.py b/pygments/cmdline.py
index 9e9f02bc..f5ea5653 100644
--- a/pygments/cmdline.py
+++ b/pygments/cmdline.py
@@ -19,7 +19,7 @@ from pygments import __version__, highlight
 from pygments.util import ClassNotFound, OptionError, docstring_headline, \
     guess_decode, guess_decode_from_terminal, terminal_encoding
 from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
-    get_lexer_for_filename, find_lexer_class, TextLexer
+    get_lexer_for_filename, find_lexer_class_for_filename, TextLexer
 from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
 from pygments.formatters import get_all_formatters, get_formatter_by_name, \
     get_formatter_for_filename, find_formatter_class, \
@@ -30,7 +30,7 @@ from pygments.styles import get_all_styles, get_style_by_name
 
 USAGE = """\
 Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
-          [-O <options>] [-P <option=value>] [-s] [-o <outfile>] [<infile>]
+          [-O <options>] [-P <option=value>] [-s] [-v] [-o <outfile>] [<infile>]
 
        %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
        %s -L [<which> ...]
@@ -90,6 +90,9 @@ waiting to process the entire file.  This only works for stdin, and
 is intended for streaming input such as you get from 'tail -f'.
 Example usage: "tail -f sql.log | pygmentize -s -l sql"
 
+The -v option prints a detailed traceback on unhandled exceptions,
+which is useful for debugging and bug reports.
+
 The -h option prints this help.
 The -V option prints the package version.
 """
@@ -100,7 +103,7 @@ def _parse_options(o_strs):
     if not o_strs:
         return opts
     for o_str in o_strs:
-        if not o_str:
+        if not o_str.strip():
             continue
         o_args = o_str.split(',')
         for o_arg in o_args:
@@ -132,7 +135,7 @@ def _parse_filters(f_strs):
 def _print_help(what, name):
     try:
         if what == 'lexer':
-            cls = find_lexer_class(name)
+            cls = get_lexer_by_name(name)
             print("Help on the %s lexer:" % cls.name)
             print(dedent(cls.__doc__))
         elif what == 'formatter':
@@ -143,8 +146,10 @@ def _print_help(what, name):
             cls = find_filter_class(name)
             print("Help on the %s filter:" % name)
             print(dedent(cls.__doc__))
-    except AttributeError:
+        return 0
+    except (AttributeError, ValueError):
         print("%s not found!" % what, file=sys.stderr)
+        return 1
 
 
 def _print_list(what):
@@ -247,8 +252,7 @@ def main_inner(popts, args, usage):
             print(usage, file=sys.stderr)
             return 2
 
-        _print_help(what, name)
-        return 0
+        return _print_help(what, name)
 
     # parse -O options
     parsed_opts = _parse_options(O_opts)
@@ -271,13 +275,9 @@ def main_inner(popts, args, usage):
     # handle ``pygmentize -N``
     infn = opts.pop('-N', None)
     if infn is not None:
-        try:
-            lexer = get_lexer_for_filename(infn, **parsed_opts)
-        except ClassNotFound as err:
-            lexer = TextLexer()
-        except OptionError as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        lexer = find_lexer_class_for_filename(infn)
+        if lexer is None:
+            lexer = TextLexer
 
         print(lexer.aliases[0])
         return 0
@@ -301,12 +301,7 @@ def main_inner(popts, args, usage):
             print(err, file=sys.stderr)
             return 1
 
-        arg = a_opt or ''
-        try:
-            print(fmter.get_style_defs(arg))
-        except Exception as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        print(fmter.get_style_defs(a_opt or ''))
         return 0
 
     # if no -S is given, -a is not allowed
@@ -341,7 +336,7 @@ def main_inner(popts, args, usage):
         if '-s' in opts:
             print('Error: -s option not usable when input file specified',
                   file=sys.stderr)
-            return 1
+            return 2
 
         infn = args[0]
         try:
@@ -387,6 +382,20 @@ def main_inner(popts, args, usage):
             except ClassNotFound:
                 lexer = TextLexer(**parsed_opts)
 
+    else:  # -s option needs a lexer with -l
+        if not lexer:
+            print('Error: when using -s a lexer has to be selected with -l',
+                  file=sys.stderr)
+            return 2
+
+    # process filters
+    for fname, fopts in F_opts:
+        try:
+            lexer.add_filter(fname, **fopts)
+        except ClassNotFound as err:
+            print('Error:', err, file=sys.stderr)
+            return 1
+
     # select formatter
     outfn = opts.pop('-o', None)
     fmter = opts.pop('-f', None)
@@ -429,7 +438,7 @@ def main_inner(popts, args, usage):
 
     # provide coloring under Windows, if possible
     if not outfn and sys.platform in ('win32', 'cygwin') and \
-       fmter.name in ('Terminal', 'Terminal256'):
+       fmter.name in ('Terminal', 'Terminal256'):  # pragma: no cover
         # unfortunately colorama doesn't support binary streams on Py3
         if sys.version_info > (3,):
             from pygments.util import UnclosingTextIOWrapper
@@ -452,24 +461,12 @@ def main_inner(popts, args, usage):
         right = escapeinside[1]
         lexer = LatexEmbeddedLexer(left, right, lexer)
 
-    # process filters
-    for fname, fopts in F_opts:
-        try:
-            lexer.add_filter(fname, **fopts)
-        except ClassNotFound as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
-
     # ... and do it!
     if '-s' not in opts:
         # process whole input as per normal...
         highlight(code, lexer, fmter, outfile)
         return 0
     else:
-        if not lexer:
-            print('Error: when using -s a lexer has to be selected with -l',
-                  file=sys.stderr)
-            return 1
         # line by line processing of stdin (eg: for 'tail -f')...
         try:
             while 1:
@@ -485,7 +482,8 @@ def main_inner(popts, args, usage):
                 highlight(line, lexer, fmter, outfile)
                 if hasattr(outfile, 'flush'):
                     outfile.flush()
-        except KeyboardInterrupt:
+            return 0
+        except KeyboardInterrupt:  # pragma: no cover
             return 0
 
 
@@ -496,7 +494,7 @@ def main(args=sys.argv):
     usage = USAGE % ((args[0],) * 6)
 
     try:
-        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHgs")
+        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgs")
     except getopt.GetoptError:
         print(usage, file=sys.stderr)
         return 2
@@ -504,6 +502,18 @@ def main(args=sys.argv):
     try:
         return main_inner(popts, args, usage)
     except Exception:
+        if '-v' in dict(popts):
+            print(file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print('An unhandled exception occurred while highlighting.',
+                  file=sys.stderr)
+            print('Please report the whole traceback to the issue tracker at',
+                  file=sys.stderr)
+            print('<https://bitbucket.org/birkenfeld/pygments-main/issues>.',
+                  file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print(file=sys.stderr)
+            raise
         import traceback
         info = traceback.format_exception(*sys.exc_info())
         msg = info[-1].strip()
@@ -513,4 +523,6 @@ def main(args=sys.argv):
         print(file=sys.stderr)
         print('*** Error while highlighting:', file=sys.stderr)
         print(msg, file=sys.stderr)
+        print('*** If this is a bug you want to report, please rerun with -v.',
+              file=sys.stderr)
         return 1
diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py
index babf7ebc..55548d30 100644
--- a/pygments/formatters/html.py
+++ b/pygments/formatters/html.py
@@ -36,21 +36,11 @@ _escape_html_table = {
     ord("'"): u'&#39;',
 }
 
+
 def escape_html(text, table=_escape_html_table):
     """Escape &, <, > as well as single and double quotes for HTML."""
     return text.translate(table)
 
-def get_random_id():
-    """Return a random id for javascript fields."""
-    from random import random
-    from time import time
-    try:
-        from hashlib import sha1 as sha
-    except ImportError:
-        import sha
-        sha = sha.new
-    return sha('%s|%s' % (random(), time())).hexdigest()
-
 
 def _get_ttype_class(ttype):
     fname = STANDARD_TYPES.get(ttype)
diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py
index e6747e18..15e68e37 100644
--- a/pygments/formatters/latex.py
+++ b/pygments/formatters/latex.py
@@ -360,7 +360,7 @@ class LatexFormatter(Formatter):
                         start += value[i]
 
                     value = value[len(start):]
-                    start = escape_tex(start, self.commandprefix)
+                    start = escape_tex(start, cp)
 
                     # ... but do not escape inside comment.
                     value = start + value
@@ -370,26 +370,26 @@ class LatexFormatter(Formatter):
                     in_math = False
                     for i, part in enumerate(parts):
                         if not in_math:
-                            parts[i] = escape_tex(part, self.commandprefix)
+                            parts[i] = escape_tex(part, cp)
                         in_math = not in_math
                     value = '$'.join(parts)
                 elif self.escapeinside:
                     text = value
                     value = ''
-                    while len(text) > 0:
+                    while text:
                         a, sep1, text = text.partition(self.left)
-                        if len(sep1) > 0:
+                        if sep1:
                             b, sep2, text = text.partition(self.right)
-                            if len(sep2) > 0:
-                                value += escape_tex(a, self.commandprefix) + b
+                            if sep2:
+                                value += escape_tex(a, cp) + b
                             else:
-                                value += escape_tex(a + sep1 + b, self.commandprefix)
+                                value += escape_tex(a + sep1 + b, cp)
                         else:
-                            value = value + escape_tex(a, self.commandprefix)
+                            value += escape_tex(a, cp)
                 else:
-                    value = escape_tex(value, self.commandprefix)
+                    value = escape_tex(value, cp)
             elif ttype not in Token.Escape:
-                value = escape_tex(value, self.commandprefix)
+                value = escape_tex(value, cp)
             styles = []
             while ttype is not Token:
                 try:
@@ -423,8 +423,7 @@ class LatexFormatter(Formatter):
 
 
 class LatexEmbeddedLexer(Lexer):
-    r"""
-
+    """
     This lexer takes one lexer as argument, the lexer for the language
     being formatted, and the left and right delimiters for escaped text.
 
diff --git a/pygments/lexers/_stan_builtins.py b/pygments/lexers/_stan_builtins.py
index e276ae29..0a225eba 100644
--- a/pygments/lexers/_stan_builtins.py
+++ b/pygments/lexers/_stan_builtins.py
@@ -4,7 +4,7 @@
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     This file contains the names of functions for Stan used by
-    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.4.0.
+    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.5.0.
 
     :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
@@ -16,10 +16,12 @@ KEYWORDS = (
     'if',
     'in',
     'increment_log_prob',
+    'integrate_ode',
     'lp__',
     'print',
+    'reject',
     'return',
-    'while',
+    'while'
 )
 
 TYPES = (
@@ -36,7 +38,7 @@ TYPES = (
     'simplex',
     'unit_vector',
     'vector',
-    'void',
+    'void'
 )
 
 FUNCTIONS = (
@@ -45,6 +47,8 @@ FUNCTIONS = (
     'abs',
     'acos',
     'acosh',
+    'append_col',
+    'append_row',
     'asin',
     'asinh',
     'atan',
@@ -144,6 +148,11 @@ FUNCTIONS = (
     'fmax',
     'fmin',
     'fmod',
+    'frechet_ccdf_log',
+    'frechet_cdf',
+    'frechet_cdf_log',
+    'frechet_log',
+    'frechet_rng',
     'gamma_ccdf_log',
     'gamma_cdf',
     'gamma_cdf_log',
@@ -152,6 +161,7 @@ FUNCTIONS = (
     'gamma_q',
     'gamma_rng',
     'gaussian_dlm_obs_log',
+    'get_lp',
     'gumbel_ccdf_log',
     'gumbel_cdf',
     'gumbel_cdf_log',
@@ -182,6 +192,8 @@ FUNCTIONS = (
     'inv_wishart_rng',
     'inverse',
     'inverse_spd',
+    'is_inf',
+    'is_nan',
     'lbeta',
     'lgamma',
     'lkj_corr_cholesky_log',
@@ -252,6 +264,7 @@ FUNCTIONS = (
     'normal_log',
     'normal_rng',
     'not_a_number',
+    'num_elements',
     'ordered_logistic_log',
     'ordered_logistic_rng',
     'owens_t',
@@ -260,6 +273,11 @@ FUNCTIONS = (
     'pareto_cdf_log',
     'pareto_log',
     'pareto_rng',
+    'pareto_type_2_ccdf_log',
+    'pareto_type_2_cdf',
+    'pareto_type_2_cdf_log',
+    'pareto_type_2_log',
+    'pareto_type_2_rng',
     'pi',
     'poisson_ccdf_log',
     'poisson_cdf',
@@ -354,7 +372,7 @@ FUNCTIONS = (
     'weibull_log',
     'weibull_rng',
     'wishart_log',
-    'wishart_rng',
+    'wishart_rng'
 )
 
 DISTRIBUTIONS = (
@@ -372,6 +390,7 @@ DISTRIBUTIONS = (
     'double_exponential',
     'exp_mod_normal',
     'exponential',
+    'frechet',
     'gamma',
     'gaussian_dlm_obs',
     'gumbel',
@@ -396,6 +415,7 @@ DISTRIBUTIONS = (
     'normal',
     'ordered_logistic',
     'pareto',
+    'pareto_type_2',
     'poisson',
     'poisson_log',
     'rayleigh',
@@ -405,7 +425,7 @@ DISTRIBUTIONS = (
     'uniform',
     'von_mises',
     'weibull',
-    'wishart',
+    'wishart'
 )
 
 RESERVED = (
@@ -494,5 +514,6 @@ RESERVED = (
     'volatile',
     'wchar_t',
     'xor',
-    'xor_eq',
+    'xor_eq'
 )
+
diff --git a/pygments/lexers/modeling.py b/pygments/lexers/modeling.py
index 1d8bd342..43194436 100644
--- a/pygments/lexers/modeling.py
+++ b/pygments/lexers/modeling.py
@@ -284,8 +284,8 @@ class StanLexer(RegexLexer):
     """Pygments Lexer for Stan models.
 
     The Stan modeling language is specified in the *Stan Modeling Language
-    User's Guide and Reference Manual, v2.4.0*,
-    `pdf <https://github.com/stan-dev/stan/releases/download/v2.4.0/stan-reference-2.4.0.pdf>`__.
+    User's Guide and Reference Manual, v2.5.0*,
+    `pdf <https://github.com/stan-dev/stan/releases/download/v2.5.0/stan-reference-2.5.0.pdf>`__.
 
     .. versionadded:: 1.6
     """
diff --git a/pygments/lexers/textfmts.py b/pygments/lexers/textfmts.py
index 8a6872ca..43b16f8c 100644
--- a/pygments/lexers/textfmts.py
+++ b/pygments/lexers/textfmts.py
@@ -102,7 +102,7 @@ class GettextLexer(RegexLexer):
             (r'^(")([A-Za-z-]+:)(.*")$',
              bygroups(String, Name.Property, String)),
             (r'^".*"$', String),
-            (r'^(msgid|msgid_plural|msgstr)(\s+)(".*")$',
+            (r'^(msgid|msgid_plural|msgstr|msgctxt)(\s+)(".*")$',
              bygroups(Name.Variable, Text, String)),
             (r'^(msgstr\[)(\d)(\])(\s+)(".*")$',
              bygroups(Name.Variable, Number.Integer, Name.Variable, Text, String)),
diff --git a/pygments/lexers/theorem.py b/pygments/lexers/theorem.py
index 0b688b03..9898b05d 100644
--- a/pygments/lexers/theorem.py
+++ b/pygments/lexers/theorem.py
@@ -395,11 +395,13 @@ class LeanLexer(RegexLexer):
                  'options', 'precedence', 'postfix', 'prefix', 'calc_trans', 'calc_subst', 'calc_refl',
                  'infix', 'infixl', 'infixr', 'notation', 'eval', 'check', 'exit', 'coercion', 'end',
                  'private', 'using', 'namespace', 'including', 'instance', 'section', 'context',
-                 'protected', 'expose', 'export', 'set_option', 'add_rewrite', 'extends')
+                 'protected', 'expose', 'export', 'set_option', 'add_rewrite', 'extends',
+                 'open', 'example', 'constant', 'constants', 'print', 'opaque', 'reducible', 'irreducible'
+    )
 
     keywords2 = (
-        'forall', 'exists', 'fun', 'Pi', 'obtain', 'from', 'have', 'show', 'assume', 'take',
-        'let', 'if', 'else', 'then', 'by', 'in', 'with', 'begin', 'proof', 'qed', 'calc'
+        'forall', 'fun', 'Pi', 'obtain', 'from', 'have', 'show', 'assume', 'take',
+        'let', 'if', 'else', 'then', 'by', 'in', 'with', 'begin', 'proof', 'qed', 'calc', 'match'
     )
 
     keywords3 = (
@@ -407,46 +409,29 @@ class LeanLexer(RegexLexer):
         'Type', 'Prop',
     )
 
-    keywords4 = (
-        # Tactics
-        'apply', 'and_then', 'or_else', 'append', 'interleave', 'par', 'fixpoint', 'repeat',
-        'at_most', 'discard', 'focus_at', 'rotate', 'try_for', 'now', 'assumption', 'eassumption',
-        'state', 'intro', 'generalize', 'exact', 'unfold', 'beta', 'trace', 'focus', 'repeat1',
-        'determ', 'destruct', 'try', 'auto', 'intros'
-    )
-
     operators = (
-        '!=', '#', '&', '&&', '*', '+', '-', '/', '@',
+        '!=', '#', '&', '&&', '*', '+', '-', '/', '@', '!', '`',
         '-.', '->', '.', '..', '...', '::', ':>', ';', ';;', '<',
         '<-', '=', '==', '>', '_', '`', '|', '||', '~', '=>', '<=', '>=',
         '/\\', '\\/', u'∀', u'Π', u'λ', u'↔', u'∧', u'∨', u'≠', u'≤', u'≥',
-        u'¬', u'⁻¹', u'⬝', u'▸', u'→', u'∃', u'ℕ', u'ℤ', u'≈'
+        u'¬', u'⁻¹', u'⬝', u'▸', u'→', u'∃', u'ℕ', u'ℤ', u'≈', u'×', u'⌞', u'⌟', u'≡'
     )
 
-    word_operators = ('and', 'or', 'not', 'iff', 'eq')
-
     punctuation = ('(', ')', ':', '{', '}', '[', ']', u'⦃', u'⦄', ':=', ',')
 
-    primitives = ('unit', 'int', 'bool', 'string', 'char', 'list',
-                  'array', 'prod', 'sum', 'pair', 'real', 'nat', 'num', 'path')
-
     tokens = {
         'root': [
             (r'\s+', Text),
-            (r'\b(false|true)\b|\(\)|\[\]', Name.Builtin.Pseudo),
             (r'/-', Comment, 'comment'),
             (r'--.*?$', Comment.Single),
             (words(keywords1, prefix=r'\b', suffix=r'\b'), Keyword.Namespace),
             (words(keywords2, prefix=r'\b', suffix=r'\b'), Keyword),
             (words(keywords3, prefix=r'\b', suffix=r'\b'), Keyword.Type),
-            (words(keywords4, prefix=r'\b', suffix=r'\b'), Keyword),
             (words(operators), Name.Builtin.Pseudo),
-            (words(word_operators, prefix=r'\b', suffix=r'\b'), Name.Builtin.Pseudo),
             (words(punctuation), Operator),
-            (words(primitives, prefix=r'\b', suffix=r'\b'), Keyword.Type),
             (u"[A-Za-z_\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2100-\u214f]"
              u"[A-Za-z_'\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2070-\u2079"
-             u"\u207f-\u2089\u2090-\u209c\u2100-\u214f]*", Name),
+             u"\u207f-\u2089\u2090-\u209c\u2100-\u214f0-9]*", Name),
             (r'\d+', Number.Integer),
             (r'"', String.Double, 'string'),
             (r'[~?][a-z][\w\']*:', Name.Variable)
diff --git a/pygments/sphinxext.py b/pygments/sphinxext.py
index 7409759d..e63d3d35 100644
--- a/pygments/sphinxext.py
+++ b/pygments/sphinxext.py
@@ -133,8 +133,8 @@ class PygmentsDoc(Directive):
             if isinstance(docstring, bytes):
                 docstring = docstring.decode('utf8')
             heading = cls.__name__
-            out.append(FMTERDOC % (heading, ', '.join(data[1]) or 'None',
-                                   ', '.join(data[2]).replace('*', '\\*') or 'None',
+            out.append(FMTERDOC % (heading, ', '.join(data[2]) or 'None',
+                                   ', '.join(data[3]).replace('*', '\\*') or 'None',
                                    docstring))
         return ''.join(out)
 
diff --git a/setup.py b/setup.py
index a1842d79..951404e5 100755
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ else:
 
 setup(
     name = 'Pygments',
-    version = '2.0.2',
+    version = '2.1a0',
     url = 'http://pygments.org/',
     license = 'BSD License',
     author = 'Georg Brandl',
diff --git a/tests/examplefiles/example.stan b/tests/examplefiles/example.stan
index 716b4d12..69c9ac70 100644
--- a/tests/examplefiles/example.stan
+++ b/tests/examplefiles/example.stan
@@ -111,6 +111,9 @@ model {
   print("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_~@#$%^&*`'-+={}[].,;: ");
   print("Hello, world!");
   print("");
+
+  // reject statement
+  reject("I just don't like it");
   
 }
 generated quantities {
diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py
index e1aa123d..c1e83077 100644
--- a/tests/test_cmdline.py
+++ b/tests/test_cmdline.py
@@ -11,32 +11,44 @@ from __future__ import print_function
 
 import io
 import os
+import re
 import sys
 import tempfile
 import unittest
 
-from pygments import highlight, cmdline
-from pygments.util import StringIO, BytesIO
-
 import support
+from pygments import cmdline, highlight
+from pygments.util import BytesIO, StringIO
+
 
 TESTFILE, TESTDIR = support.location(__file__)
+TESTCODE = '''\
+def func(args):
+    pass
+'''
 
 
-def run_cmdline(*args):
+def run_cmdline(*args, **kwds):
+    saved_stdin = sys.stdin
     saved_stdout = sys.stdout
     saved_stderr = sys.stderr
     if sys.version_info > (3,):
+        stdin_buffer = BytesIO()
         stdout_buffer = BytesIO()
         stderr_buffer = BytesIO()
+        new_stdin = sys.stdin = io.TextIOWrapper(stdin_buffer, 'utf-8')
         new_stdout = sys.stdout = io.TextIOWrapper(stdout_buffer, 'utf-8')
         new_stderr = sys.stderr = io.TextIOWrapper(stderr_buffer, 'utf-8')
     else:
+        stdin_buffer = new_stdin = sys.stdin = StringIO()
         stdout_buffer = new_stdout = sys.stdout = StringIO()
         stderr_buffer = new_stderr = sys.stderr = StringIO()
+    new_stdin.write(kwds.get('stdin', ''))
+    new_stdin.seek(0, 0)
     try:
-        ret = cmdline.main(["pygmentize"] + list(args))
+        ret = cmdline.main(['pygmentize'] + list(args))
     finally:
+        sys.stdin = saved_stdin
         sys.stdout = saved_stdout
         sys.stderr = saved_stderr
     new_stdout.flush()
@@ -48,54 +60,18 @@ def run_cmdline(*args):
 
 class CmdLineTest(unittest.TestCase):
 
-    def test_L_opt(self):
-        c, o, e = run_cmdline("-L")
-        self.assertEqual(c, 0)
-        self.assertTrue("Lexers" in o and "Formatters" in o and
-                        "Filters" in o and "Styles" in o)
-        c, o, e = run_cmdline("-L", "lexer")
-        self.assertEqual(c, 0)
-        self.assertTrue("Lexers" in o and "Formatters" not in o)
-        c, o, e = run_cmdline("-L", "lexers")
-        self.assertEqual(c, 0)
-
-    def test_O_opt(self):
-        filename = TESTFILE
-        c, o, e = run_cmdline("-Ofull=1,linenos=true,foo=bar",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
-        self.assertTrue("<html" in o)
-        self.assertTrue('class="linenos"' in o)
+    def check_success(self, *cmdline, **kwds):
+        code, out, err = run_cmdline(*cmdline, **kwds)
+        self.assertEqual(code, 0)
+        self.assertEqual(err, '')
+        return out
 
-    def test_P_opt(self):
-        filename = TESTFILE
-        c, o, e = run_cmdline("-Pfull", "-Ptitle=foo, bar=baz=,",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
-        self.assertTrue("<title>foo, bar=baz=,</title>" in o)
-
-    def test_F_opt(self):
-        filename = TESTFILE
-        c, o, e = run_cmdline("-Fhighlight:tokentype=Name.Blubb,"
-                              "names=TESTFILE filename",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
-        self.assertTrue('<span class="n-Blubb' in o)
-
-    def test_H_opt(self):
-        c, o, e = run_cmdline("-H", "formatter", "html")
-        self.assertEqual(c, 0)
-        self.assertTrue('HTML' in o)
-
-    def test_S_opt(self):
-        c, o, e = run_cmdline("-S", "default", "-f", "html", "-O", "linenos=1")
-        self.assertEqual(c, 0)
-
-    def test_invalid_opts(self):
-        for opts in [("-L", "-lpy"), ("-L", "-fhtml"), ("-L", "-Ox"),
-                     ("-a",), ("-Sst", "-lpy"), ("-H",),
-                     ("-H", "formatter")]:
-            self.assertTrue(run_cmdline(*opts)[0] == 2)
+    def check_failure(self, *cmdline, **kwds):
+        expected_code = kwds.pop('code', 1)
+        code, out, err = run_cmdline(*cmdline, **kwds)
+        self.assertEqual(code, expected_code)
+        self.assertEqual(out, '')
+        return err
 
     def test_normal(self):
         # test that cmdline gives the same output as library api
@@ -107,11 +83,20 @@ class CmdLineTest(unittest.TestCase):
 
         output = highlight(code, PythonLexer(), HtmlFormatter())
 
-        c, o, e = run_cmdline("-lpython", "-fhtml", filename)
-
+        o = self.check_success('-lpython', '-fhtml', filename)
         self.assertEqual(o, output)
-        self.assertEqual(e, "")
-        self.assertEqual(c, 0)
+
+    def test_stdin(self):
+        o = self.check_success('-lpython', '-fhtml', stdin=TESTCODE)
+        o = re.sub('<[^>]*>', '', o)
+        # rstrip is necessary since HTML inserts a \n after the last </div>
+        self.assertEqual(o.rstrip(), TESTCODE.rstrip())
+
+        # guess if no lexer given
+        o = self.check_success('-fhtml', stdin=TESTCODE)
+        o = re.sub('<[^>]*>', '', o)
+        # rstrip is necessary since HTML inserts a \n after the last </div>
+        self.assertEqual(o.rstrip(), TESTCODE.rstrip())
 
     def test_outfile(self):
         # test that output file works with and without encoding
@@ -121,15 +106,100 @@ class CmdLineTest(unittest.TestCase):
                      ['-flatex', '-o', name, TESTFILE],
                      ['-fhtml', '-o', name, '-O', 'encoding=utf-8', TESTFILE]]:
             try:
-                self.assertEqual(run_cmdline(*opts)[0], 0)
+                self.check_success(*opts)
             finally:
                 os.unlink(name)
 
-    def check_failure(self, *cmdline):
-        c, o, e = run_cmdline(*cmdline)
-        self.assertEqual(c, 1)
-        self.assertEqual(o, '')
-        return e
+    def test_stream_opt(self):
+        o = self.check_success('-lpython', '-s', '-fterminal', stdin=TESTCODE)
+        o = re.sub(r'\x1b\[.*?m', '', o)
+        self.assertEqual(o.replace('\r\n', '\n'), TESTCODE)
+
+    def test_h_opt(self):
+        o = self.check_success('-h')
+        self.assertTrue('Usage:' in o)
+
+    def test_L_opt(self):
+        o = self.check_success('-L')
+        self.assertTrue('Lexers' in o and 'Formatters' in o and
+                        'Filters' in o and 'Styles' in o)
+        o = self.check_success('-L', 'lexer')
+        self.assertTrue('Lexers' in o and 'Formatters' not in o)
+        self.check_success('-L', 'lexers')
+
+    def test_O_opt(self):
+        filename = TESTFILE
+        o = self.check_success('-Ofull=1,linenos=true,foo=bar',
+                               '-fhtml', filename)
+        self.assertTrue('<html' in o)
+        self.assertTrue('class="linenos"' in o)
+
+        # "foobar" is invalid for a bool option
+        e = self.check_failure('-Ostripnl=foobar', TESTFILE)
+        self.assertTrue('Error: Invalid value' in e)
+        e = self.check_failure('-Ostripnl=foobar', '-lpy')
+        self.assertTrue('Error: Invalid value' in e)
+
+    def test_P_opt(self):
+        filename = TESTFILE
+        o = self.check_success('-Pfull', '-Ptitle=foo, bar=baz=,',
+                               '-fhtml', filename)
+        self.assertTrue('<title>foo, bar=baz=,</title>' in o)
+
+    def test_F_opt(self):
+        filename = TESTFILE
+        o = self.check_success('-Fhighlight:tokentype=Name.Blubb,'
+                               'names=TESTFILE filename',
+                               '-fhtml', filename)
+        self.assertTrue('<span class="n-Blubb' in o)
+
+    def test_H_opt(self):
+        o = self.check_success('-H', 'formatter', 'html')
+        self.assertTrue('HTML' in o)
+        o = self.check_success('-H', 'lexer', 'python')
+        self.assertTrue('Python' in o)
+        o = self.check_success('-H', 'filter', 'raiseonerror')
+        self.assertTrue('raiseonerror', o)
+        e = self.check_failure('-H', 'lexer', 'foobar')
+        self.assertTrue('not found' in e)
+
+    def test_S_opt(self):
+        o = self.check_success('-S', 'default', '-f', 'html', '-O', 'linenos=1')
+        lines = o.splitlines()
+        for line in lines:
+            # every line is for a token class
+            parts = line.split()
+            self.assertTrue(parts[0].startswith('.'))
+            self.assertTrue(parts[1] == '{')
+            if parts[0] != '.hll':
+                self.assertTrue(parts[-4] == '}')
+                self.assertTrue(parts[-3] == '/*')
+                self.assertTrue(parts[-1] == '*/')
+        self.check_failure('-S', 'default', '-f', 'foobar')
+
+    def test_N_opt(self):
+        o = self.check_success('-N', 'test.py')
+        self.assertEqual('python', o.strip())
+        o = self.check_success('-N', 'test.unknown')
+        self.assertEqual('text', o.strip())
+
+    def test_invalid_opts(self):
+        for opts in [
+            ('-X',),
+            ('-L', '-lpy'),
+            ('-L', '-fhtml'),
+            ('-L', '-Ox'),
+            ('-S', 'default', '-l', 'py', '-f', 'html'),
+            ('-S', 'default'),
+            ('-a', 'arg'),
+            ('-H',),
+            (TESTFILE, TESTFILE),
+            ('-H', 'formatter'),
+            ('-H', 'foo', 'bar'),
+            ('-s',),
+            ('-s', TESTFILE),
+        ]:
+            self.check_failure(*opts, code=2)
 
     def test_errors(self):
         # input file not found
@@ -145,6 +215,10 @@ class CmdLineTest(unittest.TestCase):
         e = self.check_failure('-lpython', '-ffoo', TESTFILE)
         self.assertTrue('Error: no formatter found for name' in e)
 
+        # formatter for outfile not found
+        e = self.check_failure('-ofoo.foo', TESTFILE)
+        self.assertTrue('Error: no formatter found for file name' in e)
+
         # output file not writable
         e = self.check_failure('-o', os.path.join('nonexistent', 'dir', 'out.html'),
                                '-lpython', TESTFILE)
@@ -156,10 +230,23 @@ class CmdLineTest(unittest.TestCase):
         self.assertTrue('Error: filter \'foo\' not found' in e)
 
     def test_exception(self):
-        # unexpected exception while highlighting
-        cmdline.highlight = None  # override callable
+        cmdline.highlight = None  # override callable to provoke TypeError
         try:
+            # unexpected exception while highlighting
             e = self.check_failure('-lpython', TESTFILE)
+            self.assertTrue('*** Error while highlighting:' in e)
+            self.assertTrue('TypeError' in e)
+
+            # same with -v: should reraise the exception
+            try:
+                self.check_failure('-lpython', '-v', TESTFILE)
+            except Exception:
+                pass
+            else:
+                self.fail('exception not reraised')
         finally:
             cmdline.highlight = highlight
-        self.assertTrue('*** Error while highlighting:' in e)
+
+    def test_parse_opts(self):
+        self.assertEqual(cmdline._parse_options(['  ', 'keyonly,key = value ']),
+                         {'keyonly': True, 'key': 'value'})
diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py
index 79bf5072..a82aaaf7 100644
--- a/tests/test_html_formatter.py
+++ b/tests/test_html_formatter.py
@@ -68,15 +68,32 @@ class HtmlFormatterTest(unittest.TestCase):
             pass
 
     def test_all_options(self):
-        for optdict in [dict(nowrap=True),
-                        dict(linenos=True),
-                        dict(linenos=True, full=True),
-                        dict(linenos=True, full=True, noclasses=True)]:
-
+        def check(optdict):
             outfile = StringIO()
             fmt = HtmlFormatter(**optdict)
             fmt.format(tokensource, outfile)
 
+        for optdict in [
+            dict(nowrap=True),
+            dict(linenos=True, full=True),
+            dict(linenos=True, linespans='L'),
+            dict(hl_lines=[1, 5, 10, 'xxx']),
+            dict(hl_lines=[1, 5, 10], noclasses=True),
+        ]:
+            check(optdict)
+
+        for linenos in [False, 'table', 'inline']:
+            for noclasses in [False, True]:
+                for linenospecial in [0, 5]:
+                    for anchorlinenos in [False, True]:
+                        optdict = dict(
+                            linenos=linenos,
+                            noclasses=noclasses,
+                            linenospecial=linenospecial,
+                            anchorlinenos=anchorlinenos,
+                        )
+                        check(optdict)
+
     def test_linenos(self):
         optdict = dict(linenos=True)
         outfile = StringIO()
diff --git a/tests/test_util.py b/tests/test_util.py
index 092a9474..720b384a 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -10,12 +10,12 @@
 import re
 import unittest
 
-from pygments import util
+from pygments import util, console
 
 
 class FakeLexer(object):
     def analyse(text):
-        return float(text)
+        return text
     analyse = util.make_analysator(analyse)
 
 
@@ -40,6 +40,10 @@ class UtilTest(unittest.TestCase):
         equals(util.get_list_opt({}, 'a', '1 2'), ['1', '2'])
         raises(util.OptionError, util.get_list_opt, {}, 'a', 1)
 
+        equals(util.get_choice_opt({}, 'a', ['foo', 'bar'], 'bar'), 'bar')
+        equals(util.get_choice_opt({}, 'a', ['foo', 'bar'], 'Bar', True), 'bar')
+        raises(util.OptionError, util.get_choice_opt, {}, 'a',
+               ['foo', 'bar'], 'baz')
 
     def test_docstring_headline(self):
         def f1():
@@ -55,9 +59,12 @@ class UtilTest(unittest.TestCase):
 
             other text
             """
+        def f3():
+            pass
 
-        self.assertEqual(util.docstring_headline(f1), "docstring headline")
-        self.assertEqual(util.docstring_headline(f2), "docstring headline")
+        self.assertEqual(util.docstring_headline(f1), 'docstring headline')
+        self.assertEqual(util.docstring_headline(f2), 'docstring headline')
+        self.assertEqual(util.docstring_headline(f3), '')
 
     def test_analysator_returns_float(self):
         # If an analysator wrapped by make_analysator returns a floating point
@@ -88,10 +95,10 @@ class UtilTest(unittest.TestCase):
     def test_analysator_type_error(self):
         # When converting the analysator's return value to a float a
         # TypeError may occur.  If that happens 0.0 is returned instead.
-        self.assertEqual(FakeLexer.analyse(None), 0.0)
+        self.assertEqual(FakeLexer.analyse('xxx'), 0.0)
 
     def test_shebang_matches(self):
-        self.assertTrue(util.shebang_matches('#!/usr/bin/env python', r'python(2\.\d)?'))
+        self.assertTrue(util.shebang_matches('#!/usr/bin/env python\n', r'python(2\.\d)?'))
         self.assertTrue(util.shebang_matches('#!/usr/bin/python2.4', r'python(2\.\d)?'))
         self.assertTrue(util.shebang_matches('#!/usr/bin/startsomethingwith python',
                                              r'python(2\.\d)?'))
@@ -106,7 +113,7 @@ class UtilTest(unittest.TestCase):
 
     def test_doctype_matches(self):
         self.assertTrue(util.doctype_matches(
-            '<!DOCTYPE html PUBLIC "a"> <html>', 'html.*'))
+            '<!DOCTYPE html> <html>', 'html.*'))
         self.assertFalse(util.doctype_matches(
             '<?xml ?> <DOCTYPE html PUBLIC "a"> <html>', 'html.*'))
         self.assertTrue(util.html_doctype_matches(
@@ -157,3 +164,50 @@ class UtilTest(unittest.TestCase):
         # keeps first
         x = util.duplicates_removed(('a', 'b', 'a'))
         self.assertEqual(['a', 'b'], x)
+
+    def test_guess_decode(self):
+        # UTF-8 should be decoded as UTF-8
+        s = util.guess_decode(u'\xff'.encode('utf-8'))
+        self.assertEqual(s, (u'\xff', 'utf-8'))
+
+        # otherwise, it could be latin1 or the locale encoding...
+        import locale
+        s = util.guess_decode(b'\xff')
+        self.assertTrue(s[1] in ('latin1', locale.getpreferredencoding()))
+
+    def test_guess_decode_from_terminal(self):
+        class Term:
+            encoding = 'utf-7'
+
+        s = util.guess_decode_from_terminal(u'\xff'.encode('utf-7'), Term)
+        self.assertEqual(s, (u'\xff', 'utf-7'))
+
+        s = util.guess_decode_from_terminal(u'\xff'.encode('utf-8'), Term)
+        self.assertEqual(s, (u'\xff', 'utf-8'))
+
+    def test_add_metaclass(self):
+        class Meta(type):
+            pass
+
+        @util.add_metaclass(Meta)
+        class Cls:
+            pass
+
+        self.assertEqual(type(Cls), Meta)
+
+
+class ConsoleTest(unittest.TestCase):
+
+    def test_ansiformat(self):
+        f = console.ansiformat
+        c = console.codes
+        all_attrs = f('+*_blue_*+', 'text')
+        self.assertTrue(c['blue'] in all_attrs and c['blink'] in all_attrs
+                        and c['bold'] in all_attrs and c['underline'] in all_attrs
+                        and c['reset'] in all_attrs)
+        self.assertRaises(KeyError, f, '*mauve*', 'text')
+
+    def test_functions(self):
+        self.assertEqual(console.reset_color(), console.codes['reset'])
+        self.assertEqual(console.colorize('blue', 'text'),
+                         console.codes['blue'] + 'text' + console.codes['reset'])