From fd085802a08b820874c371399ebaf14b145fd10a Mon Sep 17 00:00:00 2001 From: Stephen Date: Thu, 23 May 2019 07:06:13 +0900 Subject: Added lexer for Zig programming language and test file. --- pygments/lexers/_mapping.py | 1 + pygments/lexers/zig.py | 126 ++++++++++++++++++++ tests/examplefiles/example.zig | 263 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 390 insertions(+) create mode 100644 pygments/lexers/zig.py create mode 100644 tests/examplefiles/example.zig diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index ce1b6dfc..adb1ec83 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -472,6 +472,7 @@ LEXERS = { 'YamlJinjaLexer': ('pygments.lexers.templates', 'YAML+Jinja', ('yaml+jinja', 'salt', 'sls'), ('*.sls',), ('text/x-yaml+jinja', 'text/x-sls')), 'YamlLexer': ('pygments.lexers.data', 'YAML', ('yaml',), ('*.yaml', '*.yml'), ('text/x-yaml',)), 'ZephirLexer': ('pygments.lexers.php', 'Zephir', ('zephir',), ('*.zep',), ()), + 'ZigLexer': ('pygments.lexers.zig', 'Zig', ('zig',), ('*.zig',), ('text/zig',)), } if __name__ == '__main__': # pragma: no cover diff --git a/pygments/lexers/zig.py b/pygments/lexers/zig.py new file mode 100644 index 00000000..fc7a5a48 --- /dev/null +++ b/pygments/lexers/zig.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.zig + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for Zig. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re +from pygments.lexer import RegexLexer, bygroups, include, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Error, Whitespace + +__all__ = ['ZigLexer'] + +class ZigLexer(RegexLexer): + """ + For `Zig `_ source code. + + grammar: https://ziglang.org/documentation/master/#Grammar + """ + name = 'Zig' + aliases = ['zig'] + filenames = ['*.zig'] + mimetypes = ['text/zig'] + + type_keywords = ( + words(('bool', 'f16', 'f32', 'f64', 'f128', 'void', 'noreturn', 'type', 'anyerror', 'promise', + 'i0', 'u0', 'isize', 'usize', 'comptime_int', 'comptime_float', + 'c_short', 'c_ushort', 'c_int', 'c_uint', 'c_long', 'c_ulong', 'c_longlong', 'c_ulonglong', 'c_longdouble', 'c_void' + 'i8', 'u8', 'i16', 'u16', 'i32', 'u32', 'i64', 'u64', 'i128', 'u128' + ), suffix=r'\b'), + Keyword.Type) + + storage_keywords = ( + words(('const', 'var', 'extern', 'packed', 'export', 'pub', 'noalias', + 'inline', 'comptime', 'nakedcc', 'stdcallcc', 'volatile', 'allowzero', + 'align', 'linksection', 'threadlocal'), suffix=r'\b'), + Keyword.Reserved) + + structure_keywords = ( + words(('struct', 'enum', 'union', 'error'), suffix=r'\b'), + Keyword) + + statement_keywords = ( + words(('break', 'return', 'continue', 'asm', 'defer', 'errdefer', + 'unreachable', 'try', 'catch', 'async', 'await', 'suspend', + 'resume', 'cancel'), suffix=r'\b'), + Keyword) + + conditional_keywords = ( + words(('if', 'else', 'switch', 'and', 'or', 'orelse'), suffix=r'\b'), + Keyword) + + repeat_keywords = ( + words(('while', 'for'), suffix=r'\b'), + Keyword) + + other_keywords = ( + words(('fn', 'use', 'test'), suffix=r'\b'), + Keyword) + + constant_keywords = ( + words(('true', 'false', 'null', 'undefined'), suffix=r'\b'), + Keyword.Constant) + + tokens = { + 'root': [ + (r'\n', Whitespace), + (r'\s+', Whitespace), + (r'//.*?\n', Comment.Single), + + # Keywords + statement_keywords, + storage_keywords, + structure_keywords, + repeat_keywords, + type_keywords, + constant_keywords, + conditional_keywords, + other_keywords, + + # Floats + (r'0x[0-9a-fA-F]+\.[0-9a-fA-F]+([pP][\-+]?[0-9a-fA-F]+)?', Number.Float), + (r'0x[0-9a-fA-F]+\.?[pP][\-+]?[0-9a-fA-F]+', Number.Float), + (r'[0-9]+\.[0-9]+([eE][-+]?[0-9]+)?', Number.Float), + (r'[0-9]+\.?[eE][-+]?[0-9]+', Number.Float), + + # Integers + (r'0b[01]+', Number.Bin), + (r'0o[0-7]+', Number.Oct), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + + # Identifier + (r'@[a-zA-Z_]\w*',Name.Builtin), + (r'[a-zA-Z_]\w*', Name), + + # Characters + (r'\'\\\'\'', String.Escape), + (r'\'\\(|x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])\'', String.Escape), + (r'\'[^\\\']\'', String), + + # Strings + (r'\\\\[^\n]*', String.Heredoc), + (r'c\\\\[^\n]*', String.Heredoc), + (r'c?"',String, 'string'), + + # Operators, Punctuation + (r'[+%=><|^!?/\-*&~:]', Operator), + (r'[{}()\[\],.;]', Punctuation) + ], + 'string': [ + (r'\\(x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])', String.Escape), + (r'[^\\"\n]+', String), + (r'"', String, '#pop') + ] + } + + def get_tokens_unprocessed(self, text): + for index, token, value in \ + RegexLexer.get_tokens_unprocessed(self, text): + yield index, token, value diff --git a/tests/examplefiles/example.zig b/tests/examplefiles/example.zig new file mode 100644 index 00000000..32e72849 --- /dev/null +++ b/tests/examplefiles/example.zig @@ -0,0 +1,263 @@ +const std = @import("std"); +const Allocator = mem.Allocator; +const mem = std.mem; +const ast = std.zig.ast; +const Visib = @import("visib.zig").Visib; +const event = std.event; +const Value = @import("value.zig").Value; +const Token = std.zig.Token; +const errmsg = @import("errmsg.zig"); +const Scope = @import("scope.zig").Scope; +const Compilation = @import("compilation.zig").Compilation; + +pub const Decl = struct { + id: Id, + name: []const u8, + visib: Visib, + resolution: event.Future(Compilation.BuildError!void), + parent_scope: *Scope, + + // TODO when we destroy the decl, deref the tree scope + tree_scope: *Scope.AstTree, + + pub const Table = std.HashMap([]const u8, *Decl, mem.hash_slice_u8, mem.eql_slice_u8); + + pub fn cast(base: *Decl, comptime T: type) ?*T { + if (base.id != @field(Id, @typeName(T))) return null; + return @fieldParentPtr(T, "base", base); + } + + pub fn isExported(base: *const Decl, tree: *ast.Tree) bool { + switch (base.id) { + Id.Fn => { + const fn_decl = @fieldParentPtr(Fn, "base", base); + return fn_decl.isExported(tree); + }, + else => return false, + } + } + + pub fn getSpan(base: *const Decl) errmsg.Span { + switch (base.id) { + Id.Fn => { + const fn_decl = @fieldParentPtr(Fn, "base", base); + const fn_proto = fn_decl.fn_proto; + const start = fn_proto.fn_token; + const end = fn_proto.name_token orelse start; + return errmsg.Span{ + .first = start, + .last = end + 1, + }; + }, + else => @panic("TODO"), + } + } + + pub fn findRootScope(base: *const Decl) *Scope.Root { + return base.parent_scope.findRoot(); + } + + pub const Id = enum { + Var, + Fn, + CompTime, + }; + + pub const Var = struct { + base: Decl, + }; + + pub const Fn = struct { + base: Decl, + value: Val, + fn_proto: *ast.Node.FnProto, + + // TODO https://github.com/ziglang/zig/issues/683 and then make this anonymous + pub const Val = union(enum) { + Unresolved: void, + Fn: *Value.Fn, + FnProto: *Value.FnProto, + }; + + pub fn externLibName(self: Fn, tree: *ast.Tree) ?[]const u8 { + return if (self.fn_proto.extern_export_inline_token) |tok_index| x: { + const token = tree.tokens.at(tok_index); + break :x switch (token.id) { + Token.Id.Extern => tree.tokenSlicePtr(token), + else => null, + }; + } else null; + } + + pub fn isExported(self: Fn, tree: *ast.Tree) bool { + if (self.fn_proto.extern_export_inline_token) |tok_index| { + const token = tree.tokens.at(tok_index); + return token.id == Token.Id.Keyword_export; + } else { + return false; + } + } + }; + + pub const CompTime = struct { + base: Decl, + }; +}; + +pub const info_zen = + \\ + \\ * Communicate intent precisely. + \\ * Edge cases matter. + \\ * Favor reading code over writing code. + \\ * Only one obvious way to do things. + \\ * Runtime crashes are better than bugs. + \\ * Compile errors are better than runtime crashes. + \\ * Incremental improvements. + \\ * Avoid local maximums. + \\ * Reduce the amount one must remember. + \\ * Minimize energy spent on coding style. + \\ * Together we serve end users. + \\ + \\ +; + +fn cmdZen(allocator: *Allocator, args: []const []const u8) !void { + try stdout.write(info_zen); +} + +const usage_internal = + \\usage: zig internal [subcommand] + \\ + \\Sub-Commands: + \\ build-info Print static compiler build-info + \\ + \\ +; + +fn cmdInternal(allocator: *Allocator, args: []const []const u8) !void { + if (args.len == 0) { + try stderr.write(usage_internal); + os.exit(1); + } + + const sub_commands = []Command{Command{ + .name = "build-info", + .exec = cmdInternalBuildInfo, + }}; + + for (sub_commands) |sub_command| { + if (mem.eql(u8, sub_command.name, args[0])) { + try sub_command.exec(allocator, args[1..]); + return; + } + } + + try stderr.print("unknown sub command: {}\n\n", args[0]); + try stderr.write(usage_internal); +} + +fn cmdInternalBuildInfo(allocator: *Allocator, args: []const []const u8) !void { + try stdout.print( + \\ZIG_CMAKE_BINARY_DIR {} + \\ZIG_CXX_COMPILER {} + \\ZIG_LLVM_CONFIG_EXE {} + \\ZIG_LLD_INCLUDE_PATH {} + \\ZIG_LLD_LIBRARIES {} + \\ZIG_STD_FILES {} + \\ZIG_C_HEADER_FILES {} + \\ZIG_DIA_GUIDS_LIB {} + \\ + , + std.cstr.toSliceConst(c.ZIG_CMAKE_BINARY_DIR), + std.cstr.toSliceConst(c.ZIG_CXX_COMPILER), + std.cstr.toSliceConst(c.ZIG_LLVM_CONFIG_EXE), + std.cstr.toSliceConst(c.ZIG_LLD_INCLUDE_PATH), + std.cstr.toSliceConst(c.ZIG_LLD_LIBRARIES), + std.cstr.toSliceConst(c.ZIG_STD_FILES), + std.cstr.toSliceConst(c.ZIG_C_HEADER_FILES), + std.cstr.toSliceConst(c.ZIG_DIA_GUIDS_LIB), + ); +} + +fn test__floatuntisf(a: u128, expected: f32) void { + const x = __floatuntisf(a); + testing.expect(x == expected); +} + +test "floatuntisf" { + test__floatuntisf(0, 0.0); + + test__floatuntisf(1, 1.0); + test__floatuntisf(2, 2.0); + test__floatuntisf(20, 20.0); + + test__floatuntisf(0x7FFFFF8000000000, 0x1.FFFFFEp+62); + test__floatuntisf(0x7FFFFF0000000000, 0x1.FFFFFCp+62); + + test__floatuntisf(make_ti(0x8000008000000000, 0), 0x1.000001p+127); + test__floatuntisf(make_ti(0x8000000000000800, 0), 0x1.0p+127); + test__floatuntisf(make_ti(0x8000010000000000, 0), 0x1.000002p+127); + + test__floatuntisf(make_ti(0x8000000000000000, 0), 0x1.000000p+127); + + test__floatuntisf(0x0007FB72E8000000, 0x1.FEDCBAp+50); + + test__floatuntisf(0x0007FB72EA000000, 0x1.FEDCBA8p+50); + test__floatuntisf(0x0007FB72EB000000, 0x1.FEDCBACp+50); + + test__floatuntisf(0x0007FB72EC000000, 0x1.FEDCBBp+50); + + test__floatuntisf(0x0007FB72E6000000, 0x1.FEDCB98p+50); + test__floatuntisf(0x0007FB72E7000000, 0x1.FEDCB9Cp+50); + test__floatuntisf(0x0007FB72E4000000, 0x1.FEDCB9p+50); + + test__floatuntisf(0xFFFFFFFFFFFFFFFE, 0x1p+64); + test__floatuntisf(0xFFFFFFFFFFFFFFFF, 0x1p+64); + + test__floatuntisf(0x0007FB72E8000000, 0x1.FEDCBAp+50); + + test__floatuntisf(0x0007FB72EA000000, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72EB000000, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72EBFFFFFF, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72EC000000, 0x1.FEDCBCp+50); + test__floatuntisf(0x0007FB72E8000001, 0x1.FEDCBAp+50); + + test__floatuntisf(0x0007FB72E6000000, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72E7000000, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72E7FFFFFF, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72E4000001, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72E4000000, 0x1.FEDCB8p+50); + + test__floatuntisf(make_ti(0x0000000000001FED, 0xCB90000000000001), 0x1.FEDCBAp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBA0000000000000), 0x1.FEDCBAp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBAFFFFFFFFFFFFF), 0x1.FEDCBAp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBB0000000000000), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBB0000000000001), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBBFFFFFFFFFFFFF), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBC0000000000000), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBC0000000000001), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBD0000000000000), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBD0000000000001), 0x1.FEDCBEp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBDFFFFFFFFFFFFF), 0x1.FEDCBEp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBE0000000000000), 0x1.FEDCBEp+76); +} + +fn trimStart(slice: []const u8, ch: u8) []const u8 { + var i: usize = 0; + const test_string = "test\"string"; + for (slice) |b| { + if (b == '\xa3') break; + if (b == '\ua3d3') break; + if (b == '\Ua3d3d3') break; + if (b == '\t') break; + if (b == '\n') break; + if (b == '\\') break; + if (b == '\'') break; + if (b == '"') break; + if (b != 'n') break; + if (b != '-') break; + i += 1; + } + + return slice[i..]; +} -- cgit v1.2.1 From c94d9a50fab15ed58f71ccd981bf6046669286ba Mon Sep 17 00:00:00 2001 From: Stephen Date: Tue, 9 Jul 2019 11:20:55 +0900 Subject: use keyword no longer valid, replaced with usingnamespace --- pygments/lexers/zig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/zig.py b/pygments/lexers/zig.py index fc7a5a48..7850fdf0 100644 --- a/pygments/lexers/zig.py +++ b/pygments/lexers/zig.py @@ -60,7 +60,7 @@ class ZigLexer(RegexLexer): Keyword) other_keywords = ( - words(('fn', 'use', 'test'), suffix=r'\b'), + words(('fn', 'usingnamespace', 'test'), suffix=r'\b'), Keyword) constant_keywords = ( -- cgit v1.2.1