Merged in adibis/pygments-main (pull request #252)

Added $finish and $random to systemverilog lexer.

17 files changed, 604 insertions, 383 deletions

diff --git a/AUTHORS b/AUTHORS
index 1c4a9992..1925e147 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -137,6 +137,7 @@ Other contributors, listed alphabetically, are:
 * Jeremy Thurgood -- Erlang, Squid config lexers
 * Brian Tiffin -- OpenCOBOL lexer
 * Erick Tryzelaar -- Felix lexer
+* Alexander Udalov -- Kotlin lexer improvements
 * Daniele Varrazzo -- PostgreSQL lexers
 * Abe Voelker -- OpenEdge ABL lexer
 * Pepijn de Vos -- HTML formatter CTags support
diff --git a/CHANGES b/CHANGES
index 9288b306..0d52642b 100644
--- a/CHANGES
+++ b/CHANGES
@@ -39,7 +39,8 @@ Version 1.7
 
 - Objective C/C++ lexers: allow "@" prefixing any expression (#871).
 
-- Ruby lexer: fix lexing of Name::Space tokens (#860).
+- Ruby lexer: fix lexing of Name::Space tokens (#860) and of symbols
+  in hashes (#873).
 
 - Stan lexer: update for version 1.3.0 of the language (PR#162).
 
@@ -55,6 +56,16 @@ Version 1.7
 
 - F# lexer: rewrite with newest spec for F# 3.0 (#842).
 
+- Kotlin lexer: general update (PR#271).
+
+- Rebol lexer: fix comment detection and analyse_text (PR#261).
+
+- Stan lexer: update to v2.0.1 (PR#255).
+
+- LLVM lexer: update keywords to v3.4 (PR#258).
+
+- PHP lexer: add new keywords and binary literals (PR#222).
+
 
 Version 1.6
 -----------
diff --git a/ez_setup.py b/ez_setup.py
index e33744ba..9dc2c872 100755..100644
--- a/ez_setup.py
+++ b/ez_setup.py
@@ -13,264 +13,370 @@ the appropriate options to ``use_setuptools()``.
 
 This file can also be run as a script to install or upgrade setuptools.
 """
+import os
+import shutil
 import sys
-DEFAULT_VERSION = "0.6c9"
-DEFAULT_URL     = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3]
-
-md5_data = {
-    'setuptools-0.6b1-py2.3.egg': '8822caf901250d848b996b7f25c6e6ca',
-    'setuptools-0.6b1-py2.4.egg': 'b79a8a403e4502fbb85ee3f1941735cb',
-    'setuptools-0.6b2-py2.3.egg': '5657759d8a6d8fc44070a9d07272d99b',
-    'setuptools-0.6b2-py2.4.egg': '4996a8d169d2be661fa32a6e52e4f82a',
-    'setuptools-0.6b3-py2.3.egg': 'bb31c0fc7399a63579975cad9f5a0618',
-    'setuptools-0.6b3-py2.4.egg': '38a8c6b3d6ecd22247f179f7da669fac',
-    'setuptools-0.6b4-py2.3.egg': '62045a24ed4e1ebc77fe039aa4e6f7e5',
-    'setuptools-0.6b4-py2.4.egg': '4cb2a185d228dacffb2d17f103b3b1c4',
-    'setuptools-0.6c1-py2.3.egg': 'b3f2b5539d65cb7f74ad79127f1a908c',
-    'setuptools-0.6c1-py2.4.egg': 'b45adeda0667d2d2ffe14009364f2a4b',
-    'setuptools-0.6c2-py2.3.egg': 'f0064bf6aa2b7d0f3ba0b43f20817c27',
-    'setuptools-0.6c2-py2.4.egg': '616192eec35f47e8ea16cd6a122b7277',
-    'setuptools-0.6c3-py2.3.egg': 'f181fa125dfe85a259c9cd6f1d7b78fa',
-    'setuptools-0.6c3-py2.4.egg': 'e0ed74682c998bfb73bf803a50e7b71e',
-    'setuptools-0.6c3-py2.5.egg': 'abef16fdd61955514841c7c6bd98965e',
-    'setuptools-0.6c4-py2.3.egg': 'b0b9131acab32022bfac7f44c5d7971f',
-    'setuptools-0.6c4-py2.4.egg': '2a1f9656d4fbf3c97bf946c0a124e6e2',
-    'setuptools-0.6c4-py2.5.egg': '8f5a052e32cdb9c72bcf4b5526f28afc',
-    'setuptools-0.6c5-py2.3.egg': 'ee9fd80965da04f2f3e6b3576e9d8167',
-    'setuptools-0.6c5-py2.4.egg': 'afe2adf1c01701ee841761f5bcd8aa64',
-    'setuptools-0.6c5-py2.5.egg': 'a8d3f61494ccaa8714dfed37bccd3d5d',
-    'setuptools-0.6c6-py2.3.egg': '35686b78116a668847237b69d549ec20',
-    'setuptools-0.6c6-py2.4.egg': '3c56af57be3225019260a644430065ab',
-    'setuptools-0.6c6-py2.5.egg': 'b2f8a7520709a5b34f80946de5f02f53',
-    'setuptools-0.6c7-py2.3.egg': '209fdf9adc3a615e5115b725658e13e2',
-    'setuptools-0.6c7-py2.4.egg': '5a8f954807d46a0fb67cf1f26c55a82e',
-    'setuptools-0.6c7-py2.5.egg': '45d2ad28f9750e7434111fde831e8372',
-    'setuptools-0.6c8-py2.3.egg': '50759d29b349db8cfd807ba8303f1902',
-    'setuptools-0.6c8-py2.4.egg': 'cba38d74f7d483c06e9daa6070cce6de',
-    'setuptools-0.6c8-py2.5.egg': '1721747ee329dc150590a58b3e1ac95b',
-    'setuptools-0.6c9-py2.3.egg': 'a83c4020414807b496e4cfbe08507c03',
-    'setuptools-0.6c9-py2.4.egg': '260a2be2e5388d66bdaee06abec6342a',
-    'setuptools-0.6c9-py2.5.egg': 'fe67c3e5a17b12c0e7c541b7ea43a8e6',
-    'setuptools-0.6c9-py2.6.egg': 'ca37b1ff16fa2ede6e19383e7b59245a',
-}
-
-import sys, os
-try: from hashlib import md5
-except ImportError: from md5 import md5
-
-def _validate_md5(egg_name, data):
-    if egg_name in md5_data:
-        digest = md5(data).hexdigest()
-        if digest != md5_data[egg_name]:
-            print >>sys.stderr, (
-                "md5 validation of %s failed!  (Possible download problem?)"
-                % egg_name
-            )
-            sys.exit(2)
-    return data
-
-def use_setuptools(
-    version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
-    download_delay=15
-):
-    """Automatically find/download setuptools and make it available on sys.path
-
-    `version` should be a valid setuptools version number that is available
-    as an egg for download under the `download_base` URL (which should end with
-    a '/').  `to_dir` is the directory where setuptools will be downloaded, if
-    it is not already available.  If `download_delay` is specified, it should
-    be the number of seconds that will be paused before initiating a download,
-    should one be required.  If an older version of setuptools is installed,
-    this routine will print a message to ``sys.stderr`` and raise SystemExit in
-    an attempt to abort the calling script.
-    """
-    was_imported = 'pkg_resources' in sys.modules or 'setuptools' in sys.modules
-    def do_download():
-        egg = download_setuptools(version, download_base, to_dir, download_delay)
-        sys.path.insert(0, egg)
-        import setuptools; setuptools.bootstrap_install_from = egg
+import tempfile
+import tarfile
+import optparse
+import subprocess
+import platform
+
+from distutils import log
+
+try:
+    from site import USER_SITE
+except ImportError:
+    USER_SITE = None
+
+DEFAULT_VERSION = "1.4.2"
+DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/"
+
+def _python_cmd(*args):
+    args = (sys.executable,) + args
+    return subprocess.call(args) == 0
+
+def _check_call_py24(cmd, *args, **kwargs):
+    res = subprocess.call(cmd, *args, **kwargs)
+    class CalledProcessError(Exception):
+        pass
+    if not res == 0:
+        msg = "Command '%s' return non-zero exit status %d" % (cmd, res)
+        raise CalledProcessError(msg)
+vars(subprocess).setdefault('check_call', _check_call_py24)
+
+def _install(tarball, install_args=()):
+    # extracting the tarball
+    tmpdir = tempfile.mkdtemp()
+    log.warn('Extracting in %s', tmpdir)
+    old_wd = os.getcwd()
+    try:
+        os.chdir(tmpdir)
+        tar = tarfile.open(tarball)
+        _extractall(tar)
+        tar.close()
+
+        # going in the directory
+        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
+        os.chdir(subdir)
+        log.warn('Now working in %s', subdir)
+
+        # installing
+        log.warn('Installing Setuptools')
+        if not _python_cmd('setup.py', 'install', *install_args):
+            log.warn('Something went wrong during the installation.')
+            log.warn('See the error message above.')
+            # exitcode will be 2
+            return 2
+    finally:
+        os.chdir(old_wd)
+        shutil.rmtree(tmpdir)
+
+
+def _build_egg(egg, tarball, to_dir):
+    # extracting the tarball
+    tmpdir = tempfile.mkdtemp()
+    log.warn('Extracting in %s', tmpdir)
+    old_wd = os.getcwd()
+    try:
+        os.chdir(tmpdir)
+        tar = tarfile.open(tarball)
+        _extractall(tar)
+        tar.close()
+
+        # going in the directory
+        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
+        os.chdir(subdir)
+        log.warn('Now working in %s', subdir)
+
+        # building an egg
+        log.warn('Building a Setuptools egg in %s', to_dir)
+        _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
+
+    finally:
+        os.chdir(old_wd)
+        shutil.rmtree(tmpdir)
+    # returning the result
+    log.warn(egg)
+    if not os.path.exists(egg):
+        raise IOError('Could not build the egg.')
+
+
+def _do_download(version, download_base, to_dir, download_delay):
+    egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg'
+                       % (version, sys.version_info[0], sys.version_info[1]))
+    if not os.path.exists(egg):
+        tarball = download_setuptools(version, download_base,
+                                      to_dir, download_delay)
+        _build_egg(egg, tarball, to_dir)
+    sys.path.insert(0, egg)
+
+    # Remove previously-imported pkg_resources if present (see
+    # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details).
+    if 'pkg_resources' in sys.modules:
+        del sys.modules['pkg_resources']
+
+    import setuptools
+    setuptools.bootstrap_install_from = egg
+
+
+def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
+                   to_dir=os.curdir, download_delay=15):
+    # making sure we use the absolute path
+    to_dir = os.path.abspath(to_dir)
+    was_imported = 'pkg_resources' in sys.modules or \
+        'setuptools' in sys.modules
     try:
         import pkg_resources
     except ImportError:
-        return do_download()
+        return _do_download(version, download_base, to_dir, download_delay)
     try:
-        pkg_resources.require("setuptools>="+version); return
-    except pkg_resources.VersionConflict, e:
+        pkg_resources.require("setuptools>=" + version)
+        return
+    except pkg_resources.VersionConflict:
+        e = sys.exc_info()[1]
         if was_imported:
-            print >>sys.stderr, (
-            "The required version of setuptools (>=%s) is not available, and\n"
-            "can't be installed while this script is running. Please install\n"
-            " a more recent version first, using 'easy_install -U setuptools'."
-            "\n\n(Currently using %r)"
-            ) % (version, e.args[0])
+            sys.stderr.write(
+            "The required version of setuptools (>=%s) is not available,\n"
+            "and can't be installed while this script is running. Please\n"
+            "install a more recent version first, using\n"
+            "'easy_install -U setuptools'."
+            "\n\n(Currently using %r)\n" % (version, e.args[0]))
             sys.exit(2)
         else:
             del pkg_resources, sys.modules['pkg_resources']    # reload ok
-            return do_download()
+            return _do_download(version, download_base, to_dir,
+                                download_delay)
     except pkg_resources.DistributionNotFound:
-        return do_download()
+        return _do_download(version, download_base, to_dir,
+                            download_delay)
 
-def download_setuptools(
-    version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
-    delay = 15
-):
-    """Download setuptools from a specified location and return its filename
+def _clean_check(cmd, target):
+    """
+    Run the command to download target. If the command fails, clean up before
+    re-raising the error.
+    """
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError:
+        if os.access(target, os.F_OK):
+            os.unlink(target)
+        raise
 
-    `version` should be a valid setuptools version number that is available
-    as an egg for download under the `download_base` URL (which should end
-    with a '/'). `to_dir` is the directory where the egg will be downloaded.
-    `delay` is the number of seconds to pause before an actual download attempt.
+def download_file_powershell(url, target):
     """
-    import urllib2, shutil
-    egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3])
-    url = download_base + egg_name
-    saveto = os.path.join(to_dir, egg_name)
-    src = dst = None
-    if not os.path.exists(saveto):  # Avoid repeated downloads
+    Download the file at url to target using Powershell (which will validate
+    trust). Raise an exception if the command cannot complete.
+    """
+    target = os.path.abspath(target)
+    cmd = [
+        'powershell',
+        '-Command',
+        "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars(),
+    ]
+    _clean_check(cmd, target)
+
+def has_powershell():
+    if platform.system() != 'Windows':
+        return False
+    cmd = ['powershell', '-Command', 'echo test']
+    devnull = open(os.path.devnull, 'wb')
+    try:
         try:
-            from distutils import log
-            if delay:
-                log.warn("""
----------------------------------------------------------------------------
-This script requires setuptools version %s to run (even to display
-help).  I will attempt to download it for you (from
-%s), but
-you may need to enable firewall access for this script first.
-I will start the download in %d seconds.
-
-(Note: if this machine does not have network access, please obtain the file
-
-   %s
-
-and place it in this directory before rerunning this script.)
----------------------------------------------------------------------------""",
-                    version, download_base, delay, url
-                ); from time import sleep; sleep(delay)
-            log.warn("Downloading %s", url)
-            src = urllib2.urlopen(url)
-            # Read/write all in one block, so we don't create a corrupt file
-            # if the download is interrupted.
-            data = _validate_md5(egg_name, src.read())
-            dst = open(saveto,"wb"); dst.write(data)
-        finally:
-            if src: src.close()
-            if dst: dst.close()
-    return os.path.realpath(saveto)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-def main(argv, version=DEFAULT_VERSION):
-    """Install or upgrade setuptools and EasyInstall"""
+            subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
+        except:
+            return False
+    finally:
+        devnull.close()
+    return True
+
+download_file_powershell.viable = has_powershell
+
+def download_file_curl(url, target):
+    cmd = ['curl', url, '--silent', '--output', target]
+    _clean_check(cmd, target)
+
+def has_curl():
+    cmd = ['curl', '--version']
+    devnull = open(os.path.devnull, 'wb')
     try:
-        import setuptools
-    except ImportError:
-        egg = None
         try:
-            egg = download_setuptools(version, delay=0)
-            sys.path.insert(0,egg)
-            from setuptools.command.easy_install import main
-            return main(list(argv)+[egg])   # we're done here
-        finally:
-            if egg and os.path.exists(egg):
-                os.unlink(egg)
-    else:
-        if setuptools.__version__ == '0.0.1':
-            print >>sys.stderr, (
-            "You have an obsolete version of setuptools installed.  Please\n"
-            "remove it from your system entirely before rerunning this script."
-            )
-            sys.exit(2)
-
-    req = "setuptools>="+version
-    import pkg_resources
+            subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
+        except:
+            return False
+    finally:
+        devnull.close()
+    return True
+
+download_file_curl.viable = has_curl
+
+def download_file_wget(url, target):
+    cmd = ['wget', url, '--quiet', '--output-document', target]
+    _clean_check(cmd, target)
+
+def has_wget():
+    cmd = ['wget', '--version']
+    devnull = open(os.path.devnull, 'wb')
     try:
-        pkg_resources.require(req)
-    except pkg_resources.VersionConflict:
         try:
-            from setuptools.command.easy_install import main
-        except ImportError:
-            from easy_install import main
-        main(list(argv)+[download_setuptools(delay=0)])
-        sys.exit(0) # try to force an exit
-    else:
-        if argv:
-            from setuptools.command.easy_install import main
-            main(argv)
-        else:
-            print "Setuptools version",version,"or greater has been installed."
-            print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)'
+            subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
+        except:
+            return False
+    finally:
+        devnull.close()
+    return True
 
-def update_md5(filenames):
-    """Update our built-in md5 registry"""
+download_file_wget.viable = has_wget
 
-    import re
-
-    for name in filenames:
-        base = os.path.basename(name)
-        f = open(name,'rb')
-        md5_data[base] = md5(f.read()).hexdigest()
-        f.close()
-
-    data = ["    %r: %r,\n" % it for it in md5_data.items()]
-    data.sort()
-    repl = "".join(data)
-
-    import inspect
-    srcfile = inspect.getsourcefile(sys.modules[__name__])
-    f = open(srcfile, 'rb'); src = f.read(); f.close()
+def download_file_insecure(url, target):
+    """
+    Use Python to download the file, even though it cannot authenticate the
+    connection.
+    """
+    try:
+        from urllib.request import urlopen
+    except ImportError:
+        from urllib2 import urlopen
+    src = dst = None
+    try:
+        src = urlopen(url)
+        # Read/write all in one block, so we don't create a corrupt file
+        # if the download is interrupted.
+        data = src.read()
+        dst = open(target, "wb")
+        dst.write(data)
+    finally:
+        if src:
+            src.close()
+        if dst:
+            dst.close()
+
+download_file_insecure.viable = lambda: True
+
+def get_best_downloader():
+    downloaders = [
+        download_file_powershell,
+        download_file_curl,
+        download_file_wget,
+        download_file_insecure,
+    ]
+
+    for dl in downloaders:
+        if dl.viable():
+            return dl
+
+def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
+                        to_dir=os.curdir, delay=15,
+                        downloader_factory=get_best_downloader):
+    """Download setuptools from a specified location and return its filename
 
-    match = re.search("\nmd5_data = {\n([^}]+)}", src)
-    if not match:
-        print >>sys.stderr, "Internal error!"
-        sys.exit(2)
+    `version` should be a valid setuptools version number that is available
+    as an egg for download under the `download_base` URL (which should end
+    with a '/'). `to_dir` is the directory where the egg will be downloaded.
+    `delay` is the number of seconds to pause before an actual download
+    attempt.
 
-    src = src[:match.start(1)] + repl + src[match.end(1):]
-    f = open(srcfile,'w')
-    f.write(src)
-    f.close()
+    ``downloader_factory`` should be a function taking no arguments and
+    returning a function for downloading a URL to a target.
+    """
+    # making sure we use the absolute path
+    to_dir = os.path.abspath(to_dir)
+    tgz_name = "setuptools-%s.tar.gz" % version
+    url = download_base + tgz_name
+    saveto = os.path.join(to_dir, tgz_name)
+    if not os.path.exists(saveto):  # Avoid repeated downloads
+        log.warn("Downloading %s", url)
+        downloader = downloader_factory()
+        downloader(url, saveto)
+    return os.path.realpath(saveto)
 
 
-if __name__=='__main__':
-    if len(sys.argv)>2 and sys.argv[1]=='--md5update':
-        update_md5(sys.argv[2:])
+def _extractall(self, path=".", members=None):
+    """Extract all members from the archive to the current working
+       directory and set owner, modification time and permissions on
+       directories afterwards. `path' specifies a different directory
+       to extract to. `members' is optional and must be a subset of the
+       list returned by getmembers().
+    """
+    import copy
+    import operator
+    from tarfile import ExtractError
+    directories = []
+
+    if members is None:
+        members = self
+
+    for tarinfo in members:
+        if tarinfo.isdir():
+            # Extract directories with a safe mode.
+            directories.append(tarinfo)
+            tarinfo = copy.copy(tarinfo)
+            tarinfo.mode = 448  # decimal for oct 0700
+        self.extract(tarinfo, path)
+
+    # Reverse sort directories.
+    if sys.version_info < (2, 4):
+        def sorter(dir1, dir2):
+            return cmp(dir1.name, dir2.name)
+        directories.sort(sorter)
+        directories.reverse()
     else:
-        main(sys.argv[1:])
-
-
-
-
+        directories.sort(key=operator.attrgetter('name'), reverse=True)
 
+    # Set correct owner, mtime and filemode on directories.
+    for tarinfo in directories:
+        dirpath = os.path.join(path, tarinfo.name)
+        try:
+            self.chown(tarinfo, dirpath)
+            self.utime(tarinfo, dirpath)
+            self.chmod(tarinfo, dirpath)
+        except ExtractError:
+            e = sys.exc_info()[1]
+            if self.errorlevel > 1:
+                raise
+            else:
+                self._dbg(1, "tarfile: %s" % e)
+
+
+def _build_install_args(options):
+    """
+    Build the arguments to 'python setup.py install' on the setuptools package
+    """
+    install_args = []
+    if options.user_install:
+        if sys.version_info < (2, 6):
+            log.warn("--user requires Python 2.6 or later")
+            raise SystemExit(1)
+        install_args.append('--user')
+    return install_args
+
+def _parse_args():
+    """
+    Parse the command line for options
+    """
+    parser = optparse.OptionParser()
+    parser.add_option(
+        '--user', dest='user_install', action='store_true', default=False,
+        help='install in user site package (requires Python 2.6 or later)')
+    parser.add_option(
+        '--download-base', dest='download_base', metavar="URL",
+        default=DEFAULT_URL,
+        help='alternative URL from where to download the setuptools package')
+    parser.add_option(
+        '--insecure', dest='downloader_factory', action='store_const',
+        const=lambda: download_file_insecure, default=get_best_downloader,
+        help='Use internal, non-validating downloader'
+    )
+    options, args = parser.parse_args()
+    # positional arguments are ignored
+    return options
+
+def main(version=DEFAULT_VERSION):
+    """Install or upgrade setuptools and EasyInstall"""
+    options = _parse_args()
+    tarball = download_setuptools(download_base=options.download_base,
+        downloader_factory=options.downloader_factory)
+    return _install(tarball, _build_install_args(options))
 
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/pygments/cmdline.py b/pygments/cmdline.py
index c25204bf..f260f76a 100644
--- a/pygments/cmdline.py
+++ b/pygments/cmdline.py
@@ -92,7 +92,7 @@ def _parse_options(o_strs):
         for o_arg in o_args:
             o_arg = o_arg.strip()
             try:
-                o_key, o_val = o_arg.split('=')
+                o_key, o_val = o_arg.split('=', 1)
                 o_key = o_key.strip()
                 o_val = o_val.strip()
             except ValueError:
diff --git a/pygments/lexer.py b/pygments/lexer.py
index 662c059e..8538fe7c 100644
--- a/pygments/lexer.py
+++ b/pygments/lexer.py
@@ -192,7 +192,9 @@ class Lexer(object):
 
     def get_tokens_unprocessed(self, text):
         """
-        Return an iterable of (tokentype, value) pairs.
+        Return an iterable of (index, tokentype, value) pairs where "index"
+        is the starting position of the token within the input text.
+
         In subclasses, implement this method as a generator to
         maximize effectiveness.
         """
diff --git a/pygments/lexers/_stan_builtins.py b/pygments/lexers/_stan_builtins.py
index 637072e4..1cea6ba4 100644
--- a/pygments/lexers/_stan_builtins.py
+++ b/pygments/lexers/_stan_builtins.py
@@ -10,9 +10,10 @@ This file contains the names of functions for Stan used by
 :license: BSD, see LICENSE for details.
 """
 
-KEYWORDS = ['else', 'for', 'if', 'in', 'lower', 'lp__', 'print', 'upper', 'while']
+KEYWORDS = ['else', 'for', 'if', 'in', 'lp__', 'print', 'while']
 
-TYPES = [   'corr_matrix',
+TYPES = [   'cholesky_factor_cov',
+    'corr_matrix',
     'cov_matrix',
     'int',
     'matrix',
@@ -34,35 +35,53 @@ FUNCTIONS = [   'Phi',
     'atan',
     'atan2',
     'atanh',
+    'bernoulli_ccdf_log',
     'bernoulli_cdf',
+    'bernoulli_cdf_log',
     'bernoulli_log',
     'bernoulli_logit_log',
     'bernoulli_rng',
+    'bessel_first_kind',
+    'bessel_second_kind',
+    'beta_binomial_ccdf_log',
     'beta_binomial_cdf',
+    'beta_binomial_cdf_log',
     'beta_binomial_log',
     'beta_binomial_rng',
+    'beta_ccdf_log',
     'beta_cdf',
+    'beta_cdf_log',
     'beta_log',
     'beta_rng',
     'binary_log_loss',
+    'binomial_ccdf_log',
     'binomial_cdf',
+    'binomial_cdf_log',
     'binomial_coefficient_log',
     'binomial_log',
     'binomial_logit_log',
     'binomial_rng',
     'block',
     'categorical_log',
+    'categorical_logit_log',
     'categorical_rng',
+    'cauchy_ccdf_log',
     'cauchy_cdf',
+    'cauchy_cdf_log',
     'cauchy_log',
     'cauchy_rng',
     'cbrt',
     'ceil',
+    'chi_square_ccdf_log',
+    'chi_square_cdf',
+    'chi_square_cdf_log',
     'chi_square_log',
     'chi_square_rng',
     'cholesky_decompose',
     'col',
     'cols',
+    'columns_dot_product',
+    'columns_dot_self',
     'cos',
     'cosh',
     'crossprod',
@@ -77,55 +96,78 @@ FUNCTIONS = [   'Phi',
     'dirichlet_rng',
     'dot_product',
     'dot_self',
+    'double_exponential_ccdf_log',
+    'double_exponential_cdf',
+    'double_exponential_cdf_log',
     'double_exponential_log',
     'double_exponential_rng',
     'e',
     'eigenvalues_sym',
     'eigenvectors_sym',
-    'epsilon',
     'erf',
     'erfc',
     'exp',
     'exp2',
+    'exp_mod_normal_ccdf_log',
     'exp_mod_normal_cdf',
+    'exp_mod_normal_cdf_log',
     'exp_mod_normal_log',
     'exp_mod_normal_rng',
     'expm1',
+    'exponential_ccdf_log',
     'exponential_cdf',
+    'exponential_cdf_log',
     'exponential_log',
     'exponential_rng',
     'fabs',
+    'falling_factorial',
     'fdim',
     'floor',
     'fma',
     'fmax',
     'fmin',
     'fmod',
+    'gamma_ccdf_log',
+    'gamma_cdf',
+    'gamma_cdf_log',
     'gamma_log',
+    'gamma_p',
+    'gamma_q',
     'gamma_rng',
+    'gaussian_dlm_obs_log',
+    'gumbel_ccdf_log',
     'gumbel_cdf',
+    'gumbel_cdf_log',
     'gumbel_log',
     'gumbel_rng',
+    'head',
     'hypergeometric_log',
     'hypergeometric_rng',
     'hypot',
     'if_else',
+    'increment_log_prob',
     'int_step',
+    'inv',
+    'inv_chi_square_ccdf_log',
     'inv_chi_square_cdf',
+    'inv_chi_square_cdf_log',
     'inv_chi_square_log',
     'inv_chi_square_rng',
     'inv_cloglog',
+    'inv_gamma_ccdf_log',
     'inv_gamma_cdf',
+    'inv_gamma_cdf_log',
     'inv_gamma_log',
     'inv_gamma_rng',
     'inv_logit',
+    'inv_sqrt',
+    'inv_square',
     'inv_wishart_log',
     'inv_wishart_rng',
     'inverse',
+    'inverse_spd',
     'lbeta',
     'lgamma',
-    'lkj_corr_cholesky_log',
-    'lkj_corr_cholesky_rng',
     'lkj_corr_log',
     'lkj_corr_rng',
     'lkj_cov_log',
@@ -133,96 +175,145 @@ FUNCTIONS = [   'Phi',
     'log',
     'log10',
     'log1m',
+    'log1m_exp',
     'log1m_inv_logit',
     'log1p',
     'log1p_exp',
     'log2',
     'log_determinant',
+    'log_diff_exp',
+    'log_falling_factorial',
     'log_inv_logit',
+    'log_rising_factorial',
+    'log_softmax',
     'log_sum_exp',
+    'logistic_ccdf_log',
     'logistic_cdf',
+    'logistic_cdf_log',
     'logistic_log',
     'logistic_rng',
     'logit',
+    'lognormal_ccdf_log',
     'lognormal_cdf',
+    'lognormal_cdf_log',
     'lognormal_log',
     'lognormal_rng',
+    'machine_precision',
     'max',
     'mdivide_left_tri_low',
     'mdivide_right_tri_low',
     'mean',
     'min',
+    'modified_bessel_first_kind',
+    'modified_bessel_second_kind',
     'multi_normal_cholesky_log',
     'multi_normal_log',
     'multi_normal_prec_log',
     'multi_normal_rng',
     'multi_student_t_log',
     'multi_student_t_rng',
-    'multinomial_cdf',
     'multinomial_log',
     'multinomial_rng',
     'multiply_log',
     'multiply_lower_tri_self_transpose',
+    'neg_binomial_ccdf_log',
     'neg_binomial_cdf',
+    'neg_binomial_cdf_log',
     'neg_binomial_log',
     'neg_binomial_rng',
-    'negative_epsilon',
     'negative_infinity',
+    'normal_ccdf_log',
     'normal_cdf',
+    'normal_cdf_log',
     'normal_log',
     'normal_rng',
     'not_a_number',
     'ordered_logistic_log',
     'ordered_logistic_rng',
     'owens_t',
+    'pareto_ccdf_log',
     'pareto_cdf',
+    'pareto_cdf_log',
     'pareto_log',
     'pareto_rng',
     'pi',
+    'poisson_ccdf_log',
     'poisson_cdf',
+    'poisson_cdf_log',
     'poisson_log',
     'poisson_log_log',
     'poisson_rng',
     'positive_infinity',
     'pow',
     'prod',
+    'quad_form',
+    'rank',
+    'rayleigh_ccdf_log',
+    'rayleigh_cdf',
+    'rayleigh_cdf_log',
+    'rayleigh_log',
+    'rayleigh_rng',
     'rep_array',
     'rep_matrix',
     'rep_row_vector',
     'rep_vector',
+    'rising_factorial',
     'round',
     'row',
     'rows',
+    'rows_dot_product',
+    'rows_dot_self',
+    'scaled_inv_chi_square_ccdf_log',
     'scaled_inv_chi_square_cdf',
+    'scaled_inv_chi_square_cdf_log',
     'scaled_inv_chi_square_log',
     'scaled_inv_chi_square_rng',
     'sd',
+    'segment',
     'sin',
     'singular_values',
     'sinh',
     'size',
+    'skew_normal_ccdf_log',
     'skew_normal_cdf',
+    'skew_normal_cdf_log',
     'skew_normal_log',
     'skew_normal_rng',
     'softmax',
+    'sort_asc',
+    'sort_desc',
     'sqrt',
     'sqrt2',
     'square',
     'step',
+    'student_t_ccdf_log',
     'student_t_cdf',
+    'student_t_cdf_log',
     'student_t_log',
     'student_t_rng',
+    'sub_col',
+    'sub_row',
     'sum',
+    'tail',
     'tan',
     'tanh',
     'tcrossprod',
     'tgamma',
+    'to_vector',
     'trace',
+    'trace_gen_quad_form',
+    'trace_quad_form',
     'trunc',
+    'uniform_ccdf_log',
+    'uniform_cdf',
+    'uniform_cdf_log',
     'uniform_log',
     'uniform_rng',
     'variance',
+    'von_mises_log',
+    'weibull_ccdf_log',
     'weibull_cdf',
+    'weibull_cdf_log',
     'weibull_log',
     'weibull_rng',
     'wishart_log',
@@ -236,6 +327,7 @@ DISTRIBUTIONS = [   'bernoulli',
     'binomial_coefficient',
     'binomial_logit',
     'categorical',
+    'categorical_logit',
     'cauchy',
     'chi_square',
     'dirichlet',
@@ -243,13 +335,13 @@ DISTRIBUTIONS = [   'bernoulli',
     'exp_mod_normal',
     'exponential',
     'gamma',
+    'gaussian_dlm_obs',
     'gumbel',
     'hypergeometric',
     'inv_chi_square',
     'inv_gamma',
     'inv_wishart',
     'lkj_corr',
-    'lkj_corr_cholesky',
     'lkj_cov',
     'logistic',
     'lognormal',
@@ -265,10 +357,12 @@ DISTRIBUTIONS = [   'bernoulli',
     'pareto',
     'poisson',
     'poisson_log',
+    'rayleigh',
     'scaled_inv_chi_square',
     'skew_normal',
     'student_t',
     'uniform',
+    'von_mises',
     'weibull',
     'wishart']
 
@@ -357,4 +451,3 @@ RESERVED = [   'alignas',
     'wchar_t',
     'xor',
     'xor_eq']
-
diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py
index 1f81365e..23404320 100644
--- a/pygments/lexers/agile.py
+++ b/pygments/lexers/agile.py
@@ -532,7 +532,7 @@ class RubyLexer(ExtendedRegexLexer):
             (r":'(\\\\|\\'|[^'])*'", String.Symbol),
             (r"'(\\\\|\\'|[^'])*'", String.Single),
             (r':"', String.Symbol, 'simple-sym'),
-            (r'([a-zA-Z_][a-zA-Z0-9]*)(:)(?!:)',
+            (r'([a-zA-Z_][a-zA-Z0-9_]*)(:)(?!:)',
              bygroups(String.Symbol, Punctuation)),  # Since Ruby 1.9
             (r'"', String.Double, 'simple-string'),
             (r'(?<!\.)`', String.Backtick, 'simple-backtick'),
@@ -848,7 +848,7 @@ class PerlLexer(RegexLexer):
 
     name = 'Perl'
     aliases = ['perl', 'pl']
-    filenames = ['*.pl', '*.pm']
+    filenames = ['*.pl', '*.pm', '*.t']
     mimetypes = ['text/x-perl', 'application/x-perl']
 
     flags = re.DOTALL | re.MULTILINE
@@ -1937,7 +1937,7 @@ class Perl6Lexer(ExtendedRegexLexer):
     name      = 'Perl6'
     aliases   = ['perl6', 'pl6']
     filenames = ['*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6',
-                 '*.6pm', '*.p6m', '*.pm6']
+                 '*.6pm', '*.p6m', '*.pm6', '*.t']
     mimetypes = ['text/x-perl6', 'application/x-perl6']
     flags     = re.MULTILINE | re.DOTALL | re.UNICODE
 
diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py
index 3f67862c..c599deff 100644
--- a/pygments/lexers/asm.py
+++ b/pygments/lexers/asm.py
@@ -216,6 +216,7 @@ class LlvmLexer(RegexLexer):
             (r'@' + identifier, Name.Variable.Global),#Name.Identifier.Global),
             (r'%\d+', Name.Variable.Anonymous),#Name.Identifier.Anonymous),
             (r'@\d+', Name.Variable.Global),#Name.Identifier.Anonymous),
+            (r'#\d+', Name.Variable.Global),#Name.Identifier.Global),
             (r'!' + identifier, Name.Variable),
             (r'!\d+', Name.Variable.Anonymous),
             (r'c?' + string, String),
@@ -242,17 +243,24 @@ class LlvmLexer(RegexLexer):
              r'|thread_local|zeroinitializer|undef|null|to|tail|target|triple'
              r'|datalayout|volatile|nuw|nsw|nnan|ninf|nsz|arcp|fast|exact|inbounds'
              r'|align|addrspace|section|alias|module|asm|sideeffect|gc|dbg'
+             r'|linker_private_weak'
+             r'|attributes|blockaddress|initialexec|localdynamic|localexec'
+             r'|prefix|unnamed_addr'
 
              r'|ccc|fastcc|coldcc|x86_stdcallcc|x86_fastcallcc|arm_apcscc'
              r'|arm_aapcscc|arm_aapcs_vfpcc|ptx_device|ptx_kernel'
+             r'|intel_ocl_bicc|msp430_intrcc|spir_func|spir_kernel'
+             r'|x86_64_sysvcc|x86_64_win64cc|x86_thiscallcc'
 
              r'|cc|c'
 
              r'|signext|zeroext|inreg|sret|nounwind|noreturn|noalias|nocapture'
              r'|byval|nest|readnone|readonly'
-
              r'|inlinehint|noinline|alwaysinline|optsize|ssp|sspreq|noredzone'
              r'|noimplicitfloat|naked'
+             r'|builtin|cold|nobuiltin|noduplicate|nonlazybind|optnone'
+             r'|returns_twice|sanitize_address|sanitize_memory|sanitize_thread'
+             r'|sspstrong|uwtable|returned'
 
              r'|type|opaque'
 
@@ -261,24 +269,30 @@ class LlvmLexer(RegexLexer):
              r'|oeq|one|olt|ogt|ole'
              r'|oge|ord|uno|ueq|une'
              r'|x'
+             r'|acq_rel|acquire|alignstack|atomic|catch|cleanup|filter'
+             r'|inteldialect|max|min|monotonic|nand|personality|release'
+             r'|seq_cst|singlethread|umax|umin|unordered|xchg'
 
              # instructions
              r'|add|fadd|sub|fsub|mul|fmul|udiv|sdiv|fdiv|urem|srem|frem|shl'
              r'|lshr|ashr|and|or|xor|icmp|fcmp'
 
              r'|phi|call|trunc|zext|sext|fptrunc|fpext|uitofp|sitofp|fptoui'
-             r'fptosi|inttoptr|ptrtoint|bitcast|select|va_arg|ret|br|switch'
+             r'|fptosi|inttoptr|ptrtoint|bitcast|select|va_arg|ret|br|switch'
              r'|invoke|unwind|unreachable'
+             r'|indirectbr|landingpad|resume'
 
              r'|malloc|alloca|free|load|store|getelementptr'
 
              r'|extractelement|insertelement|shufflevector|getresult'
              r'|extractvalue|insertvalue'
+             
+             r'|atomicrmw|cmpxchg|fence'
 
              r')\b', Keyword),
 
             # Types
-            (r'void|float|double|x86_fp80|fp128|ppc_fp128|label|metadata',
+            (r'void|half|float|double|x86_fp80|fp128|ppc_fp128|label|metadata',
              Keyword.Type),
 
             # Integer types
diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py
index 8b5d5964..e07656a4 100644
--- a/pygments/lexers/jvm.py
+++ b/pygments/lexers/jvm.py
@@ -15,7 +15,6 @@ from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \
      this
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
      Number, Punctuation
-from pygments.util import get_choice_opt
 from pygments import unistring as uni
 
 
@@ -937,114 +936,65 @@ class CeylonLexer(RegexLexer):
 
 class KotlinLexer(RegexLexer):
     """
-    For `Kotlin <http://confluence.jetbrains.net/display/Kotlin/>`_
+    For `Kotlin <http://kotlin.jetbrains.org/>`_
     source code.
 
-    Additional options accepted:
-
-    `unicodelevel`
-      Determines which Unicode characters this lexer allows for identifiers.
-      The possible values are:
-
-      * ``none`` -- only the ASCII letters and numbers are allowed. This
-        is the fastest selection.
-      * ``basic`` -- all Unicode characters from the specification except
-        category ``Lo`` are allowed.
-      * ``full`` -- all Unicode characters as specified in the C# specs
-        are allowed.  Note that this means a considerable slowdown since the
-        ``Lo`` category has more than 40,000 characters in it!
-
-      The default value is ``basic``.
-
     *New in Pygments 1.5.*
     """
 
     name = 'Kotlin'
     aliases = ['kotlin']
     filenames = ['*.kt']
-    mimetypes = ['text/x-kotlin'] # inferred
+    mimetypes = ['text/x-kotlin']
 
     flags = re.MULTILINE | re.DOTALL | re.UNICODE
 
-    # for the range of allowed unicode characters in identifiers,
-    # see http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf
-
-    levels = {
-        'none': '@?[_a-zA-Z][a-zA-Z0-9_]*',
-        'basic': ('@?[_' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + ']' +
-                  '[' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl +
-                  uni.Nd + uni.Pc + uni.Cf + uni.Mn + uni.Mc + ']*'),
-        'full': ('@?(?:_|[^' +
-                 uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl') + '])'
-                 + '[^' + uni.allexcept('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl',
-                                        'Nd', 'Pc', 'Cf', 'Mn', 'Mc') + ']*'),
-    }
+    kt_name = ('@?[_' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + ']' +
+               '[' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + uni.Nd +
+               uni.Pc + uni.Cf + uni.Mn + uni.Mc + ']*')
+    kt_id = '(' + kt_name + '|`' + kt_name + '`)'
 
-    tokens = {}
-    token_variants = True
-
-    for levelname, cs_ident in levels.items():
-        tokens[levelname] = {
-            'root': [
-                # method names
-                (r'^([ \t]*(?:' + cs_ident + r'(?:\[\])?\s+)+?)' # return type
-                 r'(' + cs_ident + ')'                           # method name
-                 r'(\s*)(\()',                               # signature start
-                 bygroups(using(this), Name.Function, Text, Punctuation)),
-                (r'^\s*\[.*?\]', Name.Attribute),
-                (r'[^\S\n]+', Text),
-                (r'\\\n', Text), # line continuation
-                (r'//.*?\n', Comment.Single),
-                (r'/[*](.|\n)*?[*]/', Comment.Multiline),
-                (r'\n', Text),
-                (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation),
-                (r'[{}]', Punctuation),
-                (r'@"(""|[^"])*"', String),
-                (r'"(\\\\|\\"|[^"\n])*["\n]', String),
-                (r"'\\.'|'[^\\]'", String.Char),
-                (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?"
-                 r"[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?", Number),
-                (r'#[ \t]*(if|endif|else|elif|define|undef|'
-                 r'line|error|warning|region|endregion|pragma)\b.*?\n',
-                 Comment.Preproc),
-                (r'\b(extern)(\s+)(alias)\b', bygroups(Keyword, Text,
-                 Keyword)),
-                (r'(abstract|as|break|catch|'
-                 r'fun|continue|default|delegate|'
-                 r'do|else|enum|extern|false|finally|'
-                 r'fixed|for|goto|if|implicit|in|interface|'
-                 r'internal|is|lock|null|'
-                 r'out|override|private|protected|public|readonly|'
-                 r'ref|return|sealed|sizeof|'
-                 r'when|this|throw|true|try|typeof|'
-                 r'unchecked|unsafe|virtual|void|while|'
-                 r'get|set|new|partial|yield|val|var)\b', Keyword),
-                (r'(global)(::)', bygroups(Keyword, Punctuation)),
-                (r'(bool|byte|char|decimal|double|dynamic|float|int|long|'
-                 r'short)\b\??', Keyword.Type),
-                (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'class'),
-                (r'(package|using)(\s+)', bygroups(Keyword, Text), 'package'),
-                (cs_ident, Name),
-            ],
-            'class': [
-                (cs_ident, Name.Class, '#pop')
-            ],
-            'package': [
-                (r'(?=\()', Text, '#pop'), # using (resource)
-                ('(' + cs_ident + r'|\.)+', Name.Namespace, '#pop')
-            ]
-        }
-
-    def __init__(self, **options):
-        level = get_choice_opt(options, 'unicodelevel', self.tokens.keys(),
-                               'basic')
-        if level not in self._all_tokens:
-            # compile the regexes now
-            self._tokens = self.__class__.process_tokendef(level)
-        else:
-            self._tokens = self._all_tokens[level]
-
-        RegexLexer.__init__(self, **options)
+    tokens = {
+        'root': [
+            (r'^\s*\[.*?\]', Name.Attribute),
+            (r'[^\S\n]+', Text),
+            (r'\\\n', Text), # line continuation
+            (r'//.*?\n', Comment.Single),
+            (r'/[*].*?[*]/', Comment.Multiline),
+            (r'\n', Text),
+            (r'::|!!|\?[:.]', Operator),
+            (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation),
+            (r'[{}]', Punctuation),
+            (r'@"(""|[^"])*"', String),
+            (r'"(\\\\|\\"|[^"\n])*["\n]', String),
+            (r"'\\.'|'[^\\]'", String.Char),
+            (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?[flFL]?|"
+             r"0[xX][0-9a-fA-F]+[Ll]?", Number),
+            (r'(class)(\s+)(object)', bygroups(Keyword, Text, Keyword)),
+            (r'(class|trait|object)(\s+)', bygroups(Keyword, Text), 'class'),
+            (r'(package|import)(\s+)', bygroups(Keyword, Text), 'package'),
+            (r'(val|var)(\s+)', bygroups(Keyword, Text), 'property'),
+            (r'(fun)(\s+)', bygroups(Keyword, Text), 'function'),
+            (r'(abstract|annotation|as|break|by|catch|class|continue|do|else|'
+             r'enum|false|final|finally|for|fun|get|if|import|in|inner|'
+             r'internal|is|null|object|open|out|override|package|private|'
+             r'protected|public|reified|return|set|super|this|throw|trait|'
+             r'true|try|type|val|var|vararg|when|where|while|This)\b', Keyword),
+            (kt_id, Name),
+        ],
+        'package': [
+            (r'\S+', Name.Namespace, '#pop')
+        ],
+        'class': [
+            (kt_id, Name.Class, '#pop')
+        ],
+        'property': [
+            (kt_id, Name.Property, '#pop')
+        ],
+        'function': [
+            (kt_id, Name.Function, '#pop')
+        ],
+    }
 
 
 class XtendLexer(RegexLexer):
diff --git a/pygments/lexers/math.py b/pygments/lexers/math.py
index f0e49fef..1cdcfb43 100644
--- a/pygments/lexers/math.py
+++ b/pygments/lexers/math.py
@@ -167,8 +167,8 @@ class JuliaConsoleLexer(Lexer):
 
             if line.startswith('julia>'):
                 insertions.append((len(curcode),
-                                   [(0, Generic.Prompt, line[:3])]))
-                curcode += line[3:]
+                                   [(0, Generic.Prompt, line[:6])]))
+                curcode += line[6:]
 
             elif line.startswith('      '):
 
@@ -1308,9 +1308,9 @@ class JagsLexer(RegexLexer):
 class StanLexer(RegexLexer):
     """Pygments Lexer for Stan models.
 
-    The Stan modeling language is specified in the *Stan 1.3.0
+    The Stan modeling language is specified in the *Stan 2.0.1
     Modeling Language Manual* `pdf
-    <http://code.google.com/p/stan/downloads/detail?name=stan-reference-1.3.0.pdf>`_.
+    <https://github.com/stan-dev/stan/releases/download/v2.0.1/stan-reference-2.0.1.pdf>`__
 
     *New in Pygments 1.6.*
     """
diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py
index 10598fb4..44d7404e 100644
--- a/pygments/lexers/other.py
+++ b/pygments/lexers/other.py
@@ -1217,15 +1217,16 @@ class ModelicaLexer(RegexLexer):
         ],
         'statements': [
             (r'"', String, 'string'),
+            (r'\'', Name, 'quoted_ident'),
             (r'(\d+\.\d*|\.\d+|\d+|\d.)[eE][+-]?\d+[lL]?', Number.Float),
             (r'(\d+\.\d*|\.\d+)', Number.Float),
             (r'\d+[Ll]?', Number.Integer),
             (r'[~!%^&*+=|?:<>/-]', Operator),
             (r'[()\[\]{},.;]', Punctuation),
             (r'(true|false|NULL|Real|Integer|Boolean)\b', Name.Builtin),
-            (r"([a-zA-Z_][\w]*|'[a-zA-Z_\+\-\*\/\^][\w]*')"
-             r"(\.([a-zA-Z_][\w]*|'[a-zA-Z_\+\-\*\/\^][\w]*'))+", Name.Class),
-            (r"('[\w\+\-\*\/\^]+'|\w+)", Name),
+            (r'([a-zA-Z_][\w\[\]]*|\'[a-zA-Z_\+\-\*\/\^][\w]*\')'
+             r'(\.([a-zA-Z_\][\w\[\]]*|\'[a-zA-Z_\+\-\*\/\^][\w]*\'))+', Name.Class),
+            (r'(\'[\w\+\-\*\/\^]+\'|\w+)', Name),
         ],
         'root': [
             include('whitespace'),
@@ -1239,7 +1240,7 @@ class ModelicaLexer(RegexLexer):
         'keywords': [
             (r'(algorithm|annotation|break|connect|constant|constrainedby|'
             r'discrete|each|else|elseif|elsewhen|encapsulated|enumeration|'
-            r'end|equation|exit|expandable|extends|'
+            r'equation|exit|expandable|extends|'
             r'external|false|final|flow|for|if|import|impure|in|initial\sequation|'
             r'inner|input|loop|nondiscrete|outer|output|parameter|partial|'
             r'protected|public|pure|redeclare|replaceable|stream|time|then|true|'
@@ -1258,9 +1259,13 @@ class ModelicaLexer(RegexLexer):
              r'terminate)\b', Name.Builtin),
         ],
         'classes': [
-            (r'(block|class|connector|function|model|package|'
-             r'record|type)(\s+)([A-Za-z_]+)',
-             bygroups(Keyword, Text, Name.Class))
+            (r'(block|class|connector|end|function|model|package|'
+             r'record|type)(\s+)((?!if|when|while)[A-Za-z_]\w*|[\'][^\']+[\'])([;]?)',
+             bygroups(Keyword, Text, Name.Class, Text))
+        ],
+        'quoted_ident': [
+            (r'\'', Name, '#pop'),
+            (r'[^\']+', Name), # all other characters
         ],
         'string': [
             (r'"', String, '#pop'),
@@ -1271,7 +1276,7 @@ class ModelicaLexer(RegexLexer):
             (r'\\', String), # stray backslash
         ],
         'html-content': [
-            (r'<\s*/\s*html\s*>', Name.Tag, '#pop'),
+            (r'<\s*/\s*html\s*>"', Name.Tag, '#pop'),
             (r'.+?(?=<\s*/\s*html\s*>)', using(HtmlLexer)),
         ]
     }
@@ -1381,9 +1386,9 @@ class RebolLexer(RegexLexer):
 
     tokens = {
         'root': [
-            (r'REBOL', Generic.Strong, 'script'),
-            (r'R', Comment),
             (r'[^R]+', Comment),
+            (r'REBOL\s+\[', Generic.Strong, 'script'),
+            (r'R', Comment)
         ],
         'script': [
             (r'\s+', Text),
@@ -1400,8 +1405,8 @@ class RebolLexer(RegexLexer):
             (r'%[^(\^{^")\s\[\]]+', Name.Decorator),
             (r'[+-]?([a-zA-Z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money
             (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other), # time
-            (r'\d+\-[0-9a-zA-Z]+\-\d+(\/\d+\:\d+(\:\d+)?'
-             r'([\.\d+]?([+-]?\d+:\d+)?)?)?', String.Other), # date
+            (r'\d+[\-\/][0-9a-zA-Z]+[\-\/]\d+(\/\d+\:\d+((\:\d+)?'
+             r'([\.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other), # date
             (r'\d+(\.\d+)+\.\d+', Keyword.Constant), # tuple
             (r'\d+[xX]\d+', Keyword.Constant), # pair
             (r'[+-]?\d+(\'\d+)?([\.,]\d*)?[eE][+-]?\d+', Number.Float),
@@ -1493,6 +1498,16 @@ class RebolLexer(RegexLexer):
             (r'[^(\[\])]+', Comment),
         ],
     }
+    def analyse_text(text):
+        """
+        Check if code contains REBOL header and so it probably not R code
+        """
+        if re.match(r'^\s*REBOL\s*\[', text, re.IGNORECASE):
+            # The code starts with REBOL header
+            return 1.0
+        elif re.search(r'\s*REBOL\s*[', text, re.IGNORECASE):
+            # The code contains REBOL header but also some text before it
+            return 0.5
 
 
 class ABAPLexer(RegexLexer):
@@ -1680,6 +1695,7 @@ class ABAPLexer(RegexLexer):
             # because < and > are part of field symbols.
             (r'[?*<>=\-+]', Operator),
             (r"'(''|[^'])*'", String.Single),
+            (r"`([^`])*`", String.Single),
             (r'[/;:()\[\],\.]', Punctuation)
         ],
     }
@@ -1780,6 +1796,7 @@ class GherkinLexer(RegexLexer):
         'examples_table_header': [
             (r"\s+\|\s*$", Keyword, "#pop:2"),
             include('comments'),
+            (r"\\\|", Name.Variable),
             (r"\s*\|", Keyword),
             (r"[^\|]", Name.Variable),
           ],
@@ -1822,6 +1839,7 @@ class GherkinLexer(RegexLexer):
           'table_content': [
             (r"\s+\|\s*$", Keyword, "#pop"),
             include('comments'),
+            (r"\\\|", String),
             (r"\s*\|", Keyword),
             include('string'),
           ],
diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py
index 142fef57..b0c5d4b7 100644
--- a/pygments/lexers/web.py
+++ b/pygments/lexers/web.py
@@ -833,7 +833,8 @@ class PhpLexer(RegexLexer):
              r'endif|list|__LINE__|endswitch|new|__sleep|endwhile|not|'
              r'array|__wakeup|E_ALL|NULL|final|php_user_filter|interface|'
              r'implements|public|private|protected|abstract|clone|try|'
-             r'catch|throw|this|use|namespace|trait)\b', Keyword),
+             r'catch|throw|this|use|namespace|trait|yield|'
+             r'finally)\b', Keyword),
             (r'(true|false|null)\b', Keyword.Constant),
             (r'\$\{\$+[a-zA-Z_][a-zA-Z0-9_]*\}', Name.Variable),
             (r'\$+[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable),
@@ -843,6 +844,7 @@ class PhpLexer(RegexLexer):
             (r'0[0-7]+', Number.Oct),
             (r'0[xX][a-fA-F0-9]+', Number.Hex),
             (r'\d+', Number.Integer),
+            (r'0b[01]+', Number.Binary),
             (r"'([^'\\]*(?:\\.[^'\\]*)*)'", String.Single),
             (r'`([^`\\]*(?:\\.[^`\\]*)*)`', String.Backtick),
             (r'"', String.Double, 'string'),
@@ -2442,10 +2444,10 @@ class CoffeeScriptLexer(RegexLexer):
             #(r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
             include('commentsandwhitespace'),
             (r'\+\+|~|&&|\band\b|\bor\b|\bis\b|\bisnt\b|\bnot\b|\?|:|'
-             r'\|\||\\(?=\n)|(<<|>>>?|==?|!=?|'
-             r'=(?!>)|-(?!>)|[<>+*`%&\|\^/])=?',
+             r'\|\||\\(?=\n)|'
+             r'(<<|>>>?|==?(?!>)|!=?|=(?!>)|-(?!>)|[<>+*`%&\|\^/])=?',
              Operator, 'slashstartsregex'),
-            (r'(?:\([^()]+\))?\s*[=-]>', Name.Function),
+            (r'(?:\([^()]*\))?\s*[=-]>', Name.Function),
             (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
             (r'[})\].]', Punctuation),
             (r'(?<![\.\$])(for|own|in|of|while|until|'
diff --git a/pygments/styles/monokai.py b/pygments/styles/monokai.py
index 31dc83b2..7b41b3ec 100644
--- a/pygments/styles/monokai.py
+++ b/pygments/styles/monokai.py
@@ -93,14 +93,14 @@ class MonokaiStyle(Style):
         String.Symbol:             "",        # class: 'ss'
 
         Generic:                   "",        # class: 'g'
-        Generic.Deleted:           "",        # class: 'gd',
+        Generic.Deleted:           "#f92672", # class: 'gd',
         Generic.Emph:              "italic",  # class: 'ge'
         Generic.Error:             "",        # class: 'gr'
         Generic.Heading:           "",        # class: 'gh'
-        Generic.Inserted:          "",        # class: 'gi'
+        Generic.Inserted:          "#a6e22e", # class: 'gi'
         Generic.Output:            "",        # class: 'go'
         Generic.Prompt:            "",        # class: 'gp'
         Generic.Strong:            "bold",    # class: 'gs'
-        Generic.Subheading:        "",        # class: 'gu'
+        Generic.Subheading:        "#75715e", # class: 'gu'
         Generic.Traceback:         "",        # class: 'gt'
     }
diff --git a/setup.py b/setup.py
index 17bbf814..0513738f 100755
--- a/setup.py
+++ b/setup.py
@@ -31,16 +31,22 @@ try:
     from setuptools import setup, find_packages
     have_setuptools = True
 except ImportError:
-    from distutils.core import setup
-    def find_packages():
-        return [
-            'pygments',
-            'pygments.lexers',
-            'pygments.formatters',
-            'pygments.styles',
-            'pygments.filters',
-        ]
-    have_setuptools = False
+    try:
+        import ez_setup
+        ez_setup.use_setuptools()
+        from setuptools import setup, find_packages
+        have_setuptools = True
+    except ImportError:
+        from distutils.core import setup
+        def find_packages(*args, **kwargs):
+            return [
+                'pygments',
+                'pygments.lexers',
+                'pygments.formatters',
+                'pygments.styles',
+                'pygments.filters',
+            ]
+        have_setuptools = False
 
 try:
     from distutils.command.build_py import build_py_2to3 as build_py
@@ -68,7 +74,7 @@ setup(
     description = 'Pygments is a syntax highlighting package written in Python.',
     long_description = __doc__,
     keywords = 'syntax highlighting',
-    packages = find_packages(),
+    packages = find_packages(exclude=['ez_setup']),
     platforms = 'any',
     zip_safe = False,
     include_package_data = True,
diff --git a/tests/examplefiles/example.stan b/tests/examplefiles/example.stan
index e936f54a..7eb6fdfc 100644
--- a/tests/examplefiles/example.stan
+++ b/tests/examplefiles/example.stan
@@ -19,6 +19,7 @@ data {
   positive_ordered[3] wibble;
   corr_matrix[3] grault;
   cov_matrix[3] garply;
+  cholesky_factor_cov[3] waldo;
   
   real<lower=-1,upper=1> foo1;
   real<lower=0> foo2;
@@ -94,6 +95,7 @@ model {
   // lp__ should be highlighted
   // normal_log as a function
   lp__ <- lp__ + normal_log(plugh, 0, 1);
+  increment_log_prob(normal_log(plugh, 0, 1));
   
   // print statement and string literal
   print("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_~@#$%^&*`'-+={}[].,;: ");
diff --git a/tests/examplefiles/function_arrows.coffee b/tests/examplefiles/function_arrows.coffee
new file mode 100644
index 00000000..cd1ef1e8
--- /dev/null
+++ b/tests/examplefiles/function_arrows.coffee
@@ -0,0 +1,11 @@
+methodA:-> 'A'
+methodB:=> 'B'
+methodC:()=> 'C'
+methodD:()-> 'D'
+methodE:(a,b)-> 'E'
+methodF:(c,d)-> 'F'
+-> 'G'
+=> 'H'
+
+(-> 'I')
+(=> 'J')
diff --git a/tests/examplefiles/hash_syntax.rb b/tests/examplefiles/hash_syntax.rb
new file mode 100644
index 00000000..35b27723
--- /dev/null
+++ b/tests/examplefiles/hash_syntax.rb
@@ -0,0 +1,5 @@
+{ :old_syntax => 'ok' }
+{ 'stings as key' => 'should be ok' }
+{ new_syntax: 'broken until now' }
+{ withoutunderscore: 'should be ok' }
+{ _underscoreinfront: 'might be ok, if I understand the pygments code correct' }