Merged in ChrisMorgan/pygments-main (pull request #423)

Generally improve Rust highlighting

1 files changed, 68 insertions, 45 deletions

diff --git a/pygments/lexers/rust.py b/pygments/lexers/rust.py
index 2ca860d6..d8939678 100644
--- a/pygments/lexers/rust.py
+++ b/pygments/lexers/rust.py
@@ -10,7 +10,7 @@
 """
 
 from pygments.lexer import RegexLexer, include, bygroups, words, default
-from pygments.token import Comment, Operator, Keyword, Name, String, \
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation, Whitespace
 
 __all__ = ['RustLexer']
@@ -18,33 +18,40 @@ __all__ = ['RustLexer']
 
 class RustLexer(RegexLexer):
     """
-    Lexer for the Rust programming language (version 0.9).
+    Lexer for the Rust programming language (version 1.0).
 
     .. versionadded:: 1.6
     """
     name = 'Rust'
     filenames = ['*.rs']
     aliases = ['rust']
-    mimetypes = ['text/x-rustsrc']
+    mimetypes = ['text/rust']
 
     tokens = {
         'root': [
+            # rust allows a file to start with a shebang, but if the first line
+            # starts with #![ then it’s not a shebang but a crate attribute.
+            (r'#![^[\r\n].*$', Comment.Preproc),
+            default('base'),
+        ],
+        'base': [
             # Whitespace and Comments
             (r'\n', Whitespace),
             (r'\s+', Whitespace),
-            (r'//[/!](.*?)\n', Comment.Doc),
+            (r'//!.*?\n', String.Doc),
+            (r'///(\n|[^/].*?\n)', String.Doc),
             (r'//(.*?)\n', Comment.Single),
+            (r'/\*\*(\n|[^/*])', String.Doc, 'doccomment'),
+            (r'/\*!', String.Doc, 'doccomment'),
             (r'/\*', Comment.Multiline, 'comment'),
 
-            # Lifetime
-            (r"""'[a-zA-Z_]\w*""", Name.Label),
             # Macro parameters
             (r"""\$([a-zA-Z_]\w*|\(,?|\),?|,?)""", Comment.Preproc),
             # Keywords
             (words((
-                'as', 'box', 'break', 'continue', 'do', 'else', 'enum', 'extern',
+                'as', 'box', 'crate', 'do', 'else', 'enum', 'extern',  # break and continue are in labels
                 'fn', 'for', 'if', 'impl', 'in', 'loop', 'match', 'mut', 'priv',
-                'proc', 'pub', 'ref', 'return', 'static', '\'static', 'struct',
+                'proc', 'pub', 'ref', 'return', 'static', 'struct',
                 'trait', 'true', 'type', 'unsafe', 'while'), suffix=r'\b'),
              Keyword),
             (words(('alignof', 'be', 'const', 'offsetof', 'pure', 'sizeof',
@@ -53,44 +60,45 @@ class RustLexer(RegexLexer):
             (r'(mod|use)\b', Keyword.Namespace),
             (r'(true|false)\b', Keyword.Constant),
             (r'let\b', Keyword.Declaration),
-            (words(('u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64', 'uint',
-                    'int', 'f32', 'f64', 'str', 'bool'), suffix=r'\b'),
+            (words(('u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64', 'usize',
+                    'isize', 'f32', 'f64', 'str', 'bool'), suffix=r'\b'),
              Keyword.Type),
             (r'self\b', Name.Builtin.Pseudo),
-            # Prelude
+            # Prelude (taken from Rust’s src/libstd/prelude.rs)
             (words((
-                'Freeze', 'Pod', 'Send', 'Sized', 'Add', 'Sub', 'Mul', 'Div', 'Rem', 'Neg', 'Not', 'BitAnd',
-                'BitOr', 'BitXor', 'Drop', 'Shl', 'Shr', 'Index', 'Option', 'Some', 'None', 'Result',
-                'Ok', 'Err', 'from_str', 'range', 'print', 'println', 'Any', 'AnyOwnExt', 'AnyRefExt',
-                'AnyMutRefExt', 'Ascii', 'AsciiCast', 'OnwedAsciiCast', 'AsciiStr',
-                'IntoBytes', 'Bool', 'ToCStr', 'Char', 'Clone', 'DeepClone', 'Eq', 'ApproxEq',
-                'Ord', 'TotalEq', 'Ordering', 'Less', 'Equal', 'Greater', 'Equiv', 'Container',
-                'Mutable', 'Map', 'MutableMap', 'Set', 'MutableSet', 'Default', 'FromStr',
-                'Hash', 'FromIterator', 'Extendable', 'Iterator', 'DoubleEndedIterator',
-                'RandomAccessIterator', 'CloneableIterator', 'OrdIterator',
-                'MutableDoubleEndedIterator', 'ExactSize', 'Times', 'Algebraic',
-                'Trigonometric', 'Exponential', 'Hyperbolic', 'Bitwise', 'BitCount',
-                'Bounded', 'Integer', 'Fractional', 'Real', 'RealExt', 'Num', 'NumCast',
-                'CheckedAdd', 'CheckedSub', 'CheckedMul', 'Orderable', 'Signed',
-                'Unsigned', 'Round', 'Primitive', 'Int', 'Float', 'ToStrRadix',
-                'ToPrimitive', 'FromPrimitive', 'GenericPath', 'Path', 'PosixPath',
-                'WindowsPath', 'RawPtr', 'Buffer', 'Writer', 'Reader', 'Seek',
-                'SendStr', 'SendStrOwned', 'SendStrStatic', 'IntoSendStr', 'Str',
-                'StrVector', 'StrSlice', 'OwnedStr', 'IterBytes', 'ToStr', 'IntoStr',
-                'CopyableTuple', 'ImmutableTuple', 'ImmutableEqVector', 'ImmutableTotalOrdVector',
-                'ImmutableCopyableVector', 'OwnedVector', 'OwnedCopyableVector',
-                'OwnedEqVector', 'MutableVector', 'MutableTotalOrdVector',
-                'Vector', 'VectorVector', 'CopyableVector', 'ImmutableVector',
-                'Port', 'Chan', 'SharedChan', 'spawn', 'drop'), suffix=r'\b'),
+                # Reexported core operators
+                'Copy', 'Send', 'Sized', 'Sync',
+                'Drop', 'Fn', 'FnMut', 'FnOnce',
+
+                # Reexported functions
+                'drop',
+
+                # Reexported types and traits
+                'Box',
+                'ToOwned',
+                'Clone',
+                'PartialEq', 'PartialOrd', 'Eq', 'Ord',
+                'AsRef', 'AsMut', 'Into', 'From',
+                'Default',
+                'Iterator', 'Extend', 'IntoIterator',
+                'DoubleEndedIterator', 'ExactSizeIterator',
+                'Option',
+                'Some', 'None',
+                'Result',
+                'Ok', 'Err',
+                'SliceConcatExt',
+                'String', 'ToString',
+                'Vec',
+                ), suffix=r'\b'),
              Name.Builtin),
-            (r'(ImmutableTuple\d+|Tuple\d+)\b', Name.Builtin),
-            # Borrowed pointer
-            (r'(&)(\'[A-Za-z_]\w*)?', bygroups(Operator, Name)),
             # Labels
-            (r'\'[A-Za-z_]\w*:', Name.Label),
+            (r'(break|continue)(\s*)(\'[A-Za-z_]\w*)?', bygroups(Keyword, Text.Whitespace, Name.Label)),
             # Character Literal
-            (r"""'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}"""
-             r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|.)'""",
+            (r"""'(\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""",
+             String.Char),
+            (r"""b'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""",
              String.Char),
             # Binary Literal
             (r'0b[01_]+', Number.Bin, 'number_lit'),
@@ -100,11 +108,16 @@ class RustLexer(RegexLexer):
             (r'0[xX][0-9a-fA-F_]+', Number.Hex, 'number_lit'),
             # Decimal Literal
             (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|'
-             r'\.[0-9_]*|[eE][+\-]?[0-9_]+)', Number.Float, 'number_lit'),
+             r'\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)', Number.Float, 'number_lit'),
             (r'[0-9][0-9_]*', Number.Integer, 'number_lit'),
             # String Literal
+            (r'b"', String, 'bytestring'),
             (r'"', String, 'string'),
-            (r'r(#*)".*?"\1', String.Raw),
+            (r'b?r(#*)".*?"\1', String),
+
+            # Lifetime
+            (r"""'static""", Name.Builtin),
+            (r"""'[a-zA-Z_]\w*""", Name.Attribute),
 
             # Operators and Punctuation
             (r'[{}()\[\],.;]', Punctuation),
@@ -129,18 +142,28 @@ class RustLexer(RegexLexer):
             (r'\*/', Comment.Multiline, '#pop'),
             (r'[*/]', Comment.Multiline),
         ],
+        'doccomment': [
+            (r'[^*/]+', String.Doc),
+            (r'/\*', String.Doc, '#push'),
+            (r'\*/', String.Doc, '#pop'),
+            (r'[*/]', String.Doc),
+        ],
         'number_lit': [
-            (r'[ui](8|16|32|64)', Keyword, '#pop'),
+            (r'[ui](8|16|32|64|size)', Keyword, '#pop'),
             (r'f(32|64)', Keyword, '#pop'),
             default('#pop'),
         ],
         'string': [
             (r'"', String, '#pop'),
-            (r"""\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}"""
-             r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}""", String.Escape),
+            (r"""\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}""", String.Escape),
             (r'[^\\"]+', String),
             (r'\\', String),
         ],
+        'bytestring': [
+            (r"""\\x[89a-fA-F][0-9a-fA-F]""", String.Escape),
+            include('string'),
+        ],
         'macro{': [
             (r'\{', Operator, '#push'),
             (r'\}', Operator, '#pop'),