Merged in EricFromCanada/pygments-main (pull request #163)

Lasso lexer: better method highlighting + fix

12 files changed, 218 insertions, 28 deletions

diff --git a/AUTHORS b/AUTHORS
index 9447bd0f..34b40db4 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -64,6 +64,7 @@ Other contributors, listed alphabetically, are:
 * Igor Kalnitsky -- vhdl lexer
 * Pekka Klärck -- Robot Framework lexer
 * Eric Knibbe -- Lasso lexer
+* Stepan Koltsov -- Clay lexer
 * Adam Koprowski -- Opa lexer
 * Benjamin Kowarsch -- Modula-2 lexer
 * Alexander Kriegisch -- Kconfig and AspectJ lexers
diff --git a/CHANGES b/CHANGES
index c2a3528f..5def718f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -6,6 +6,22 @@ Issue numbers refer to the tracker at
 pull request numbers to the requests at
 <http://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>.
 
+Version 1.7
+-----------
+(under development)
+
+- Lexers added:
+
+  * Clay (PR#184)
+
+- Python 3 lexer: add new exceptions from PEP 3151.
+
+- Opa lexer: add new keywords (PR#170).
+
+- Julia lexer: add keywords and underscore-separated number
+  literals (PR#176).
+
+
 Version 1.6
 -----------
 (released Feb 3, 2013)
diff --git a/docs/src/lexerdevelopment.txt b/docs/src/lexerdevelopment.txt
index 6ffc4b72..730a08b2 100644
--- a/docs/src/lexerdevelopment.txt
+++ b/docs/src/lexerdevelopment.txt
@@ -83,6 +83,58 @@ If no rule matches at the current position, the current char is emitted as an
 1.
 
 
+Adding and testing a new lexer
+==============================
+
+To make pygments aware of your new lexer, you have to perform the following
+steps:
+
+First, change to the current directory containing the pygments source code:
+
+.. sourcecode:: console
+
+    $ cd .../pygments-main
+
+Next, make sure the lexer is known from outside of the module. All modules in
+the ``pygments.lexers`` specify ``__all__``. For example, ``other.py`` sets:
+
+.. sourcecode:: python
+
+    __all__ = ['BrainfuckLexer', 'BefungeLexer', ...]
+
+Simply add the name of your lexer class to this list.
+
+Finally the lexer can be made publically known by rebuilding the lexer
+mapping:
+
+.. sourcecode:: console
+
+    $ make mapfiles
+
+To test the new lexer, store an example file with the proper extension in
+``tests/examplefiles``. For example, to test your ``DiffLexer``, add a
+``tests/examplefiles/example.diff`` containing a sample diff output.
+
+Now you can use pygmentize to render your example to HTML:
+
+.. sourcecode:: console
+
+    $ ./pygmentize -O full -f html -o /tmp/example.html tests/examplefiles/example.diff
+
+Note that this explicitely calls the ``pygmentize`` in the current directory
+by preceding it with ``./``. This ensures your modifications are used.
+Otherwise a possibly already installed, unmodified version without your new
+lexer would have been called from the system search path (``$PATH``).
+
+To view the result, open ``/tmp/example.html`` in your browser.
+
+Once the example renders as expected, you should run the complete test suite:
+
+.. sourcecode:: console
+
+    $ make test
+
+
 Regex Flags
 ===========
 
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 53e09176..b45c56cd 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -58,6 +58,7 @@ LEXERS = {
     'CheetahJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Cheetah', ('js+cheetah', 'javascript+cheetah', 'js+spitfire', 'javascript+spitfire'), (), ('application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', 'application/x-javascript+spitfire', 'text/x-javascript+spitfire', 'text/javascript+spitfire')),
     'CheetahLexer': ('pygments.lexers.templates', 'Cheetah', ('cheetah', 'spitfire'), ('*.tmpl', '*.spt'), ('application/x-cheetah', 'application/x-spitfire')),
     'CheetahXmlLexer': ('pygments.lexers.templates', 'XML+Cheetah', ('xml+cheetah', 'xml+spitfire'), (), ('application/xml+cheetah', 'application/xml+spitfire')),
+    'ClayLexer': ('pygments.lexers.compiled', 'Clay', ('clay',), ('*.clay',), ('text/x-clay',)),
     'ClojureLexer': ('pygments.lexers.jvm', 'Clojure', ('clojure', 'clj'), ('*.clj',), ('text/x-clojure', 'application/x-clojure')),
     'CobolFreeformatLexer': ('pygments.lexers.compiled', 'COBOLFree', ('cobolfree',), ('*.cbl', '*.CBL'), ()),
     'CobolLexer': ('pygments.lexers.compiled', 'COBOL', ('cobol',), ('*.cob', '*.COB', '*.cpy', '*.CPY'), ('text/x-cobol',)),
diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py
index 8bcb1d46..3c1525d0 100644
--- a/pygments/lexers/agile.py
+++ b/pygments/lexers/agile.py
@@ -234,7 +234,14 @@ class Python3Lexer(RegexLexer):
          r'TypeError|UnboundLocalError|UnicodeDecodeError|'
          r'UnicodeEncodeError|UnicodeError|UnicodeTranslateError|'
          r'UnicodeWarning|UserWarning|ValueError|VMSError|Warning|'
-         r'WindowsError|ZeroDivisionError)\b', Name.Exception),
+         r'WindowsError|ZeroDivisionError|'
+         # new builtin exceptions from PEP 3151
+         r'BlockingIOError|ChildProcessError|ConnectionError|'
+         r'BrokenPipeError|ConnectionAbortedError|ConnectionRefusedError|'
+         r'ConnectionResetError|FileExistsError|FileNotFoundError|'
+         r'InterruptedError|IsADirectoryError|NotADirectoryError|'
+         r'PermissionError|ProcessLookupError|TimeoutError)\b',
+         Name.Exception),
     ]
     tokens['numbers'] = [
         (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py
index 7ff64bcc..f080327b 100644
--- a/pygments/lexers/asm.py
+++ b/pygments/lexers/asm.py
@@ -244,7 +244,7 @@ class LlvmLexer(RegexLexer):
              r'|align|addrspace|section|alias|module|asm|sideeffect|gc|dbg'
 
              r'|ccc|fastcc|coldcc|x86_stdcallcc|x86_fastcallcc|arm_apcscc'
-             r'|arm_aapcscc|arm_aapcs_vfpcc'
+             r'|arm_aapcscc|arm_aapcs_vfpcc|ptx_device|ptx_kernel'
 
              r'|cc|c'
 
diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py
index 7513a4e1..d44ab6f6 100644
--- a/pygments/lexers/compiled.py
+++ b/pygments/lexers/compiled.py
@@ -29,7 +29,7 @@ __all__ = ['CLexer', 'CppLexer', 'DLexer', 'DelphiLexer', 'ECLexer', 'DylanLexer
            'FelixLexer', 'AdaLexer', 'Modula2Lexer', 'BlitzMaxLexer',
            'NimrodLexer', 'FantomLexer', 'RustLexer', 'CudaLexer', 'MonkeyLexer',
            'DylanLidLexer', 'DylanConsoleLexer', 'CobolLexer',
-           'CobolFreeformatLexer', 'LogosLexer']
+           'CobolFreeformatLexer', 'LogosLexer', 'ClayLexer']
 
 
 class CFamilyLexer(RegexLexer):
@@ -266,6 +266,57 @@ class ECLexer(CLexer):
     }
 
 
+class ClayLexer(RegexLexer):
+    """
+    For `Clay <http://claylabs.com/clay/>`_ source.
+
+    *New in Pygments 1.7.*
+    """
+    name = 'Clay'
+    filenames = ['*.clay']
+    aliases = ['clay']
+    mimetypes = ['text/x-clay']
+    tokens = {
+        'root': [
+            (r'\s', Text),
+            (r'//.*?$', Comment.Singleline),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+            (r'\b(public|private|import|as|record|variant|instance'
+             r'|define|overload|default|external|alias'
+             r'|rvalue|ref|forward|inline|noinline|forceinline'
+             r'|enum|var|and|or|not|if|else|goto|return|while'
+             r'|switch|case|break|continue|for|in|true|false|try|catch|throw'
+             r'|finally|onerror|staticassert|eval|when|newtype'
+             r'|__FILE__|__LINE__|__COLUMN__|__ARG__'
+             r')\b', Keyword),
+            (r'[~!%^&*+=|:<>/-]', Operator),
+            (r'[#(){}\[\],;.]', Punctuation),
+            (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
+            (r'\d+[LlUu]*', Number.Integer),
+            (r'\b(true|false)\b', Name.Builtin),
+            (r'(?i)[a-z_?][a-z_?0-9]*', Name),
+            (r'"""', String, 'tdqs'),
+            (r'"', String, 'dqs'),
+        ],
+        'strings': [
+            (r'(?i)\\(x[0-9a-f]{2}|.)', String.Escape),
+            (r'.', String),
+        ],
+        'nl': [
+            (r'\n', String),
+        ],
+        'dqs': [
+            (r'"', String, '#pop'),
+            include('strings'),
+        ],
+        'tdqs': [
+            (r'"""', String, '#pop'),
+            include('strings'),
+            include('nl'),
+        ],
+    }
+
+
 class DLexer(RegexLexer):
     """
     For D source.
diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py
index a082811b..889e7ec6 100644
--- a/pygments/lexers/functional.py
+++ b/pygments/lexers/functional.py
@@ -1663,9 +1663,10 @@ class OpaLexer(RegexLexer):
     # but if you color only real keywords, you might just
     # as well not color anything
     keywords = [
-        'and', 'as', 'begin', 'css', 'database', 'db', 'do', 'else', 'end',
-        'external', 'forall', 'if', 'import', 'match', 'package', 'parser',
-        'rec', 'server', 'then', 'type', 'val', 'with', 'xml_parser',
+        'and', 'as', 'begin', 'case', 'client', 'css', 'database', 'db', 'do',
+        'else', 'end', 'external', 'forall', 'function', 'if', 'import',
+        'match', 'module', 'or', 'package', 'parser', 'rec', 'server', 'then',
+        'type', 'val', 'with', 'xml_parser',
     ]
 
     # matches both stuff and `stuff`
diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py
index 717621e9..ed4d257c 100644
--- a/pygments/lexers/jvm.py
+++ b/pygments/lexers/jvm.py
@@ -888,11 +888,11 @@ class CeylonLexer(RegexLexer):
             (r'[^\S\n]+', Text),
             (r'//.*?\n', Comment.Single),
             (r'/\*.*?\*/', Comment.Multiline),
-            (r'(variable|shared|abstract|doc|by|formal|actual)',
+            (r'(variable|shared|abstract|doc|by|formal|actual|late|native)',
              Name.Decorator),
             (r'(break|case|catch|continue|default|else|finally|for|in|'
-             r'variable|if|return|switch|this|throw|try|while|is|exists|'
-             r'nonempty|then|outer)\b', Keyword),
+             r'variable|if|return|switch|this|throw|try|while|is|exists|dynamic|'
+             r'nonempty|then|outer|assert)\b', Keyword),
             (r'(abstracts|extends|satisfies|adapts|'
              r'super|given|of|out|assign|'
              r'transient|volatile)\b', Keyword.Declaration),
@@ -900,16 +900,16 @@ class CeylonLexer(RegexLexer):
              Keyword.Type),
             (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)),
             (r'(true|false|null)\b', Keyword.Constant),
-            (r'(class|interface|object)(\s+)',
+            (r'(class|interface|object|alias)(\s+)',
              bygroups(Keyword.Declaration, Text), 'class'),
             (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
             (r'"(\\\\|\\"|[^"])*"', String),
-            (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Quoted),
-            (r"`\\.`|`[^\\]`|`\\u[0-9a-fA-F]{4}`", String.Char),
-            (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)',
+            (r"'\\.'|'[^\\]'|'\\\{#[0-9a-fA-F]{4}\}'", String.Char),
+            (r'".*``.*``.*"', String.Interpol),
+            (r'(\.)([a-z_][a-zA-Z0-9_]*)',
              bygroups(Operator, Name.Attribute)),
             (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label),
-            (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name),
+            (r'[a-zA-Z_][a-zA-Z0-9_]*', Name),
             (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator),
             (r'\d{1,3}(_\d{3})+\.\d{1,3}(_\d{3})+[kMGTPmunpf]?', Number.Float),
             (r'\d{1,3}(_\d{3})+\.[0-9]+([eE][+-]?[0-9]+)?[kMGTPmunpf]?',
@@ -917,16 +917,19 @@ class CeylonLexer(RegexLexer):
             (r'[0-9][0-9]*\.\d{1,3}(_\d{3})+[kMGTPmunpf]?', Number.Float),
             (r'[0-9][0-9]*\.[0-9]+([eE][+-]?[0-9]+)?[kMGTPmunpf]?',
              Number.Float),
-            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'#([0-9a-fA-F]{4})(_[0-9a-fA-F]{4})+', Number.Hex),
+            (r'#[0-9a-fA-F]+', Number.Hex),
+            (r'\$([01]{4})(_[01]{4})+', Number.Integer),
+            (r'\$[01]+', Number.Integer),
             (r'\d{1,3}(_\d{3})+[kMGTP]?', Number.Integer),
             (r'[0-9]+[kMGTP]?', Number.Integer),
             (r'\n', Text)
         ],
         'class': [
-            (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop')
+            (r'[A-Za-z_][a-zA-Z0-9_]*', Name.Class, '#pop')
         ],
         'import': [
-            (r'[a-zA-Z0-9_.]+\w+ \{([a-zA-Z,]+|\.\.\.)\}',
+            (r'[a-z][a-zA-Z0-9_.]*',
              Name.Namespace, '#pop')
         ],
     }
diff --git a/pygments/lexers/math.py b/pygments/lexers/math.py
index 537c6d0e..4dda7372 100644
--- a/pygments/lexers/math.py
+++ b/pygments/lexers/math.py
@@ -59,7 +59,7 @@ class JuliaLexer(RegexLexer):
             (r'(begin|while|for|in|return|break|continue|'
              r'macro|quote|let|if|elseif|else|try|catch|end|'
              r'bitstype|ccall|do|using|module|import|export|'
-             r'importall|baremodule)\b', Keyword),
+             r'importall|baremodule|immutable)\b', Keyword),
             (r'(local|global|const)\b', Keyword.Declaration),
             (r'(Bool|Int|Int8|Int16|Int32|Int64|Uint|Uint8|Uint16|Uint32|Uint64'
              r'|Float32|Float64|Complex64|Complex128|Any|Nothing|None)\b',
@@ -99,11 +99,17 @@ class JuliaLexer(RegexLexer):
             (r'[a-zA-Z_][a-zA-Z0-9_]*', Name),
 
             # numbers
+            (r'(\d+(_\d+)+\.\d*|\d*\.\d+(_\d+)+)([eEf][+-]?[0-9]+)?', Number.Float),
             (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float),
+            (r'\d+(_\d+)+[eEf][+-]?[0-9]+', Number.Float),
             (r'\d+[eEf][+-]?[0-9]+', Number.Float),
+            (r'0b[01]+(_[01]+)+', Number.Binary),
             (r'0b[01]+', Number.Binary),
+            (r'0o[0-7]+(_[0-7]+)+', Number.Oct),
             (r'0o[0-7]+', Number.Oct),
+            (r'0x[a-fA-F0-9]+(_[a-fA-F0-9]+)+', Number.Hex),
             (r'0x[a-fA-F0-9]+', Number.Hex),
+            (r'\d+(_\d+)+', Number.Integer),
             (r'\d+', Number.Integer)
         ],
 
diff --git a/tests/examplefiles/example.ceylon b/tests/examplefiles/example.ceylon
index b136b995..04223c56 100644
--- a/tests/examplefiles/example.ceylon
+++ b/tests/examplefiles/example.ceylon
@@ -1,33 +1,52 @@
+import ceylon.language { parseInteger }
+
 doc "A top-level function,
      with multi-line documentation."
-void topLevel(String? a, Integer b=5, String... seqs) {
+void topLevel(String? a, Integer b=5, String* seqs) {
     function nested(String s) {
         print(s[1..2]);
         return true;
     }
-    for (s in seqs.filter((String x) x.size > 2)) {
+    for (s in seqs.filter((String x) => x.size > 2)) {
         nested(s);
     }
-    value uppers = seqs.sequence[].uppercased;
-    String|Nothing z = a;
-    Sequence<Integer> ints = { 1, 2, 3, 4, 5 };
+    value uppers = seqs.map((String x) {
+        return x.uppercased;
+    });
+    String|Null z = a;
+    {Integer+} ints = { 1, 2, 3, 4, 5 };
+    value numbers = [ 1, #ffff, #ffff_ffff, $10101010, $1010_1010_1010_1010,
+        123_456_789 ];
+    value chars = ['a', '\{#ffff}' ];
 }
 
-shared class Example<Element>(name, element) satisfies Comparable<Example<Element>>
+shared class Example_1<Element>(name, element) satisfies Comparable<Example_1<Element>>
         given Element satisfies Comparable<Element> {
     shared String name;
     shared Element element;
+    shared [Integer,String] tuple = [1, "2"];
+    shared late String lastName;
+    variable Integer cnt = 0;
+
+    shared Integer count => cnt;
+    assign count {
+        assert(count >= cnt);
+        cnt = count;
+    }
 
-    shared actual Comparison compare(Example<Element> other) {
+    shared actual Comparison compare(Example_1<Element> other) {
         return element <=> other.element;
     }
 
     shared actual String string {
-        return "Example with " + element.string;
+        return "Example with ``element.string``";
     }
 }
 
-Example<Integer> instance = Example {
-    name = "Named args call";
+Example_1<Integer> instance = Example_1 {
     element = 5;
+    name = "Named args call \{#0060}";
 };
+
+object example1 extends Example_1<Integer>("object", 5) {
+}
\ No newline at end of file
diff --git a/tests/examplefiles/example.clay b/tests/examplefiles/example.clay
new file mode 100644
index 00000000..784752c6
--- /dev/null
+++ b/tests/examplefiles/example.clay
@@ -0,0 +1,33 @@
+
+/// @section  StringLiteralRef 
+
+record StringLiteralRef (
+    sizep : Pointer[SizeT],
+);
+
+
+/// @section  predicates 
+
+overload ContiguousSequence?(#StringLiteralRef) : Bool = true;
+[s when StringLiteral?(s)]
+overload ContiguousSequence?(#Static[s]) : Bool = true;
+
+
+
+/// @section  size, begin, end, index 
+
+forceinline overload size(a:StringLiteralRef) = a.sizep^;
+
+forceinline overload begin(a:StringLiteralRef) : Pointer[Char] = Pointer[Char](a.sizep + 1);
+forceinline overload end(a:StringLiteralRef) = begin(a) + size(a);
+
+[I when Integer?(I)]
+forceinline overload index(a:StringLiteralRef, i:I) : ByRef[Char] {
+    assert["boundsChecks"](i >= 0 and i < size(a), "StringLiteralRef index out of bounds");
+    return ref (begin(a) + i)^;
+}
+
+foo() = """
+long\tlong
+story
+"""