From 501df7613c67bcd743eaa00071f0dbb179607394 Mon Sep 17 00:00:00 2001 From: Quintus Date: Sun, 22 Apr 2012 22:10:42 +0200 Subject: Lua scanner for CodeRay. Meta-commit. This commit is a super-commit containing all the subcommits for implementing the Lua scanner. --- lib/coderay/scanners/lua.rb | 267 ++++++++++++++++++++++++++++++++++++++++++++ lib/coderay/styles/alpha.rb | 3 + lib/coderay/token_kinds.rb | 1 + 3 files changed, 271 insertions(+) create mode 100644 lib/coderay/scanners/lua.rb diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb new file mode 100644 index 0000000..e7706fc --- /dev/null +++ b/lib/coderay/scanners/lua.rb @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- + +# Scanner for the Lua[http://lua.org] programming lanuage. +# +# The language’s complete syntax is defined in +# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], +# which is what this scanner tries to conform to. +class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + + register_for :lua + file_extension "lua" + title "Lua" + + # Keywords used in Lua. + KEYWORDS = %w[and break do else elseif end + for function goto if in + local not or repeat return + then until while + ] + + # Constants set by the Lua core. + PREDEFINED_CONSTANTS = %w[false true nil] + + # The expressions contained in this array are parts of Lua’s `basic' + # library. Although it’s not entirely necessary to load that library, + # it is highly recommended and one would have to provide own implementations + # of some of these expressions if one does not do so. They however aren’t + # keywords, neither are they constants, but nearly predefined, so they + # get tagged as `predefined' rather than anything else. + # + # This list excludes values of form `_UPPERCASE' because the Lua manual + # requires such identifiers to be reserved by Lua anyway and they are + # highlighted directly accordingly, without the need for specific + # identifiers to be listed here. + PREDEFINED_EXPRESSIONS = %w[ + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall + ] + + # Automatic token kind selection for normal words. + IDENT_KIND = CodeRay::WordList.new(:ident) + .add(KEYWORDS, :keyword) + .add(PREDEFINED_CONSTANTS, :predefined_constant) + .add(PREDEFINED_EXPRESSIONS, :predefined) + + protected + + # Scanner initialization. + def setup + @state = :initial + @brace_depth = 0 + end + + # CodeRay entry hook. Starts parsing. + def scan_tokens(encoder, options) + @encoder = encoder + @options = options + + send(:"handle_state_#@state") until eos? + + @encoder + end + + def handle_state_initial + if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] + @num_equals = match.count("=") # Number must match for comment end + @encoder.begin_group(:comment) + @encoder.text_token(match, :delimiter) + @state = :long_comment + + elsif match = scan(/--.*?$/) # --Lua comment + @encoder.text_token(match, :comment) + + elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] + @num_equals = match.count("=") # Number must match for comment end + @encoder.begin_group(:string) + @encoder.text_token(match, :delimiter) + @state = :long_string + + elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: + @encoder.text_token(match, :label) + + elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua + @encoder.text_token(match, :predefined) + + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) + kind = IDENT_KIND[match] + + # Extra highlighting for entities following certain keywords + if kind == :keyword and match == "function" + @state = :function_expected + elsif kind == :keyword and match == "goto" + @state = :goto_label_expected + elsif kind == :keyword and match == "local" + @state = :local_var_expected + end + + @encoder.text_token(match, kind) + + elsif match = scan(/{/) # Opening table brace { + @encoder.begin_group(:table) + @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) + @brace_depth += 1 + @state = :table + + elsif match = scan(/}/) # Closing table brace } + if @brace_depth == 1 + @brace_depth = 0 + @encoder.text_token(match, :delimiter) + elsif @brace_depth == 0 # Mismatched brace + @encoder.text_token(match, :error) + else + @brace_depth -= 1 + @encoder.text_token(match, :inline_delimiter) + @state = :table + end + @encoder.end_group(:table) + + elsif match = scan(/["']/) # String delimiters " and ' + @encoder.begin_group(:string) + @encoder.text_token(match, :delimiter) + @start_delim = match + @state = :string + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + @encoder.text_token(match, :float) + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + @encoder.text_token(match, :integer) + + elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators + @encoder.text_token(match, :operator) + + elsif match = scan(/\s+/) # Space + @encoder.text_token(match, :space) + + else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. + @encoder.text_token(getch, :error) + end + + # It may be that we’re scanning a full-blown subexpression of a table + # (tables can contain full expressions in parts). + # If this is the case, return to :table scanning state. + @state = :table if @state == :initial && @brace_depth >= 1 + end + + def handle_state_function_expected + if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name + @encoder.text_token(match, :operator) + @state = :initial + elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator + @encoder.text_token(match, :ident) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() + @encoder.text_token(match, :function) + @state = :initial + elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + @state = :initial + end + end + + def handle_state_goto_label_expected + if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + @encoder.text_token(match, :label) + @state = :initial + elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_local_var_expected + if match = scan(/function/) # local function ... + @encoder.text_token(match, :keyword) + @state = :function_expected + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + @encoder.text_token(match, :local_variable) + elsif match = scan(/,/) + @encoder.text_token(match, :operator) + elsif match = scan(/=/) + @encoder.text_token(match, :operator) + # After encountering the equal sign, arbitrary expressions are + # allowed again, so just return to the main state for further + # parsing. + @state = :initial + elsif match = scan(/\n/) + @encoder.text_token(match, :space) + @state = :initial + elsif match = scan(/\s+/) + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_long_comment + if match = scan(/.*?(?=\]={#@num_equals}\])/m) + @encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + @encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + @encoder.text_token(rest, :error) + terminate + end + @encoder.end_group(:comment) + @state = :initial + end + + def handle_state_long_string + if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + @encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + @encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + @encoder.text_token(rest, :error) + terminate + end + @encoder.end_group(:string) + @state = :initial + end + + def handle_state_string + if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + @encoder.text_token(match, :content) + elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) + @encoder.text_token(match, :char) + elsif match = scan(Regexp.compile(@start_delim)) + @encoder.text_token(match, :delimiter) + @encoder.end_group(:string) + @state = :initial + elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings + @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings + @encoder.end_group(:string) + @state = :initial + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_table + if match = scan(/[,;]/) + @encoder.text_token(match, :operator) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) + @encoder.text_token(match, :key) + @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) + @encoder.text_token(scan(/=/), :operator) + @state = :initial + elsif match = scan(/\s+/m) + @encoder.text_token(match, :space) + else + # Note this clause doesn’t advance the scan pointer, it’s a kind of + # "retry with other options" (the :initial state then of course + # advances the pointer). + @state = :initial + end + end + +end diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 8506d10..257083e 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -116,6 +116,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } .symbol { color:#A60 } +.table .content { color:#808 } +.table .delimiter { color:#40A} +.table { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070 } .type { color:#339; font-weight:bold } .value { color: #088; } diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 3b8d07e..e245623 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -63,6 +63,7 @@ module CodeRay :shell => 'shell', :string => 'string', :symbol => 'symbol', + :table => 'table', :tag => 'tag', :type => 'type', :value => 'value', -- cgit v1.2.1 From 8979cc621431248fded86c341e2102a67c5344bb Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 19 Jun 2012 17:45:49 +0200 Subject: use case+when instead of send and methods --- lib/coderay/scanners/lua.rb | 64 ++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e7706fc..e712061 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -40,10 +40,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ] # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident) - .add(KEYWORDS, :keyword) - .add(PREDEFINED_CONSTANTS, :predefined_constant) - .add(PREDEFINED_EXPRESSIONS, :predefined) + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) protected @@ -57,13 +57,11 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner def scan_tokens(encoder, options) @encoder = encoder @options = options - - send(:"handle_state_#@state") until eos? - - @encoder - end - - def handle_state_initial + + until eos? + case state + + when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] @num_equals = match.count("=") # Number must match for comment end @encoder.begin_group(:comment) @@ -99,13 +97,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, kind) - elsif match = scan(/{/) # Opening table brace { + elsif match = scan(/\{/) # Opening table brace { @encoder.begin_group(:table) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 @state = :table - elsif match = scan(/}/) # Closing table brace } + elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @brace_depth = 0 @encoder.text_token(match, :delimiter) @@ -146,9 +144,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # (tables can contain full expressions in parts). # If this is the case, return to :table scanning state. @state = :table if @state == :initial && @brace_depth >= 1 - end - - def handle_state_function_expected + + when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @encoder.text_token(match, :operator) @state = :initial @@ -163,9 +160,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(getch, :error) @state = :initial end - end - def handle_state_goto_label_expected + when :goto_label_expected if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) @encoder.text_token(match, :label) @state = :initial @@ -174,9 +170,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - end - - def handle_state_local_var_expected + + when :local_var_expected if match = scan(/function/) # local function ... @encoder.text_token(match, :keyword) @state = :function_expected @@ -198,9 +193,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - end - - def handle_state_long_comment + + when :long_comment if match = scan(/.*?(?=\]={#@num_equals}\])/m) @encoder.text_token(match, :content) @@ -212,9 +206,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:comment) @state = :initial - end - - def handle_state_long_string + + when :long_string if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences @encoder.text_token(match, :content) @@ -226,9 +219,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:string) @state = :initial - end - - def handle_state_string + + when :string if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) @encoder.text_token(match, :content) elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) @@ -244,9 +236,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - end - - def handle_state_table + + when :table if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) @@ -262,6 +253,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # advances the pointer). @state = :initial end + else + raise + end + + end + + @encoder end end -- cgit v1.2.1 From 2b16d115f7d6caf21864934df763556e126d6357 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 19 Jun 2012 18:05:04 +0200 Subject: fix for Ruby 1.8, escape { and } in regexps --- lib/coderay/scanners/lua.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e7706fc..8464ef7 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -40,10 +40,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ] # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident) - .add(KEYWORDS, :keyword) - .add(PREDEFINED_CONSTANTS, :predefined_constant) - .add(PREDEFINED_EXPRESSIONS, :predefined) + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) protected @@ -99,13 +99,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, kind) - elsif match = scan(/{/) # Opening table brace { + elsif match = scan(/\{/) # Opening table brace { @encoder.begin_group(:table) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 @state = :table - elsif match = scan(/}/) # Closing table brace } + elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @brace_depth = 0 @encoder.text_token(match, :delimiter) -- cgit v1.2.1 From abb92f30b12e11781afa76f43a344627520b5b34 Mon Sep 17 00:00:00 2001 From: Eric Guo Date: Sun, 8 Jul 2012 14:32:28 +0800 Subject: New: *Go Encoder* Draft version, copy from c --- lib/coderay/helpers/file_type.rb | 37 ++++---- lib/coderay/scanners/go.rb | 195 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 18 deletions(-) create mode 100644 lib/coderay/scanners/go.rb diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 637001b..5159054 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ module CodeRay name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ module CodeRay TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ module CodeRay if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ module CodeRay raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ module CodeRay end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -86,6 +86,7 @@ module CodeRay 'dpr' => :delphi, 'erb' => :erb, 'gemspec' => :ruby, + 'go' => :go, 'groovy' => :groovy, 'gvy' => :groovy, 'h' => :c, @@ -128,16 +129,16 @@ module CodeRay for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, } - + end - + end diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb new file mode 100644 index 0000000..4431ef2 --- /dev/null +++ b/lib/coderay/scanners/go.rb @@ -0,0 +1,195 @@ +module CodeRay +module Scanners + + # Scanner for Go, copy from c + class Go < Scanner + + register_for :go + file_extension 'go' + + # http://golang.org/ref/spec#Keywords + KEYWORDS = [ + 'break', 'default', 'func', 'interface', 'select', + 'case', 'defer', 'go', 'map', 'struct', + 'chan', 'else', 'goto', 'package', 'switch', + 'const', 'fallthrough', 'if', 'range', 'type', + 'continue', 'for', 'import', 'return', 'var', + ] # :nodoc: + + # http://golang.org/ref/spec#Types + PREDEFINED_TYPES = [ + 'bool', + 'uint8', 'uint16', 'uint32', 'uint64', + 'int8', 'int16', 'int32', 'int64', + 'float32', 'float64', + 'complex64', 'complex128', + 'byte', 'rune', + 'uint', 'int', 'uintptr', + ] # :nodoc: + + PREDEFINED_CONSTANTS = [ + 'nil', 'iota', + 'true', 'false', + ] # :nodoc: + + DIRECTIVES = [ + 'go_no_directive', # Seems no directive concept in Go? + ] # :nodoc: + + IDENT_KIND = WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_TYPES, :predefined_type). + add(DIRECTIVES, :directive). + add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: + + ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + + protected + + def scan_tokens encoder, options + + state = :initial + label_expected = true + case_expected = false + label_expected_before_preproc_line = nil + in_preproc_line = false + + until eos? + + case state + + when :initial + + if match = scan(/ \s+ | \\\n /x) + if in_preproc_line && match != "\\\n" && match.index(?\n) + in_preproc_line = false + label_expected = label_expected_before_preproc_line + end + encoder.text_token match, :space + + elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) + encoder.text_token match, :comment + + elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) + label_expected = match =~ /[;\{\}]/ + if case_expected + label_expected = true if match == ':' + case_expected = false + end + encoder.text_token match, :operator + + elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) + kind = IDENT_KIND[match] + if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) + kind = :label + match << matched + else + label_expected = false + if kind == :keyword + case match + when 'case', 'default' + case_expected = true + end + end + end + encoder.text_token match, kind + + elsif match = scan(/L?"/) + encoder.begin_group :string + if match[0] == ?L + encoder.text_token 'L', :modifier + match = '"' + end + encoder.text_token match, :delimiter + state = :string + + elsif match = scan(/ \# \s* if \s* 0 /x) + match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? + encoder.text_token match, :comment + + elsif match = scan(/#[ \t]*(\w*)/) + encoder.text_token match, :preprocessor + in_preproc_line = true + label_expected_before_preproc_line = label_expected + state = :include_expected if self[1] == 'include' + + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) + label_expected = false + encoder.text_token match, :char + + elsif match = scan(/\$/) + encoder.text_token match, :ident + + elsif match = scan(/0[xX][0-9A-Fa-f]+/) + label_expected = false + encoder.text_token match, :hex + + elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) + label_expected = false + encoder.text_token match, :octal + + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + label_expected = false + encoder.text_token match, :integer + + elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) + label_expected = false + encoder.text_token match, :float + + else + encoder.text_token getch, :error + + end + + when :string + if match = scan(/[^\\\n"]+/) + encoder.text_token match, :content + elsif match = scan(/"/) + encoder.text_token match, :delimiter + encoder.end_group :string + state = :initial + label_expected = false + elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) + encoder.text_token match, :char + elsif match = scan(/ \\ | $ /x) + encoder.end_group :string + encoder.text_token match, :error + state = :initial + label_expected = false + else + raise_inspect "else case \" reached; %p not handled." % peek(1), encoder + end + + when :include_expected + if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/) + encoder.text_token match, :include + state = :initial + + elsif match = scan(/\s+/) + encoder.text_token match, :space + state = :initial if match.index ?\n + + else + state = :initial + + end + + else + raise_inspect 'Unknown state', encoder + + end + + end + + if state == :string + encoder.end_group :string + end + + encoder + end + + end + +end +end -- cgit v1.2.1 From ec39f80865cac97a49a6fec138dedc22e5971bad Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 28 Jul 2012 13:12:37 +0200 Subject: fix whitespace --- lib/coderay/helpers/file_type.rb | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 5159054..5cc7446 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ module CodeRay name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ module CodeRay TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ module CodeRay if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ module CodeRay raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ module CodeRay end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -129,16 +129,16 @@ module CodeRay for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, } - + end - + end -- cgit v1.2.1 From b081c1f3ab670301199005c04ecbc11614593285 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Tue, 11 Sep 2012 15:04:45 -0600 Subject: escape = in regex (my editor gets confused:-) --- lib/coderay/scanners/lua.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e712061..e640397 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -57,10 +57,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner def scan_tokens(encoder, options) @encoder = encoder @options = options - + until eos? case state - + when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] @num_equals = match.count("=") # Number must match for comment end @@ -144,7 +144,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # (tables can contain full expressions in parts). # If this is the case, return to :table scanning state. @state = :table if @state == :initial && @brace_depth >= 1 - + when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @encoder.text_token(match, :operator) @@ -170,7 +170,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - + when :local_var_expected if match = scan(/function/) # local function ... @encoder.text_token(match, :keyword) @@ -179,7 +179,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, :local_variable) elsif match = scan(/,/) @encoder.text_token(match, :operator) - elsif match = scan(/=/) + elsif match = scan(/\=/) @encoder.text_token(match, :operator) # After encountering the equal sign, arbitrary expressions are # allowed again, so just return to the main state for further @@ -193,7 +193,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - + when :long_comment if match = scan(/.*?(?=\]={#@num_equals}\])/m) @encoder.text_token(match, :content) @@ -206,7 +206,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:comment) @state = :initial - + when :long_string if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences @encoder.text_token(match, :content) @@ -219,7 +219,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:string) @state = :initial - + when :string if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) @encoder.text_token(match, :content) @@ -236,14 +236,14 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - + when :table if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) @encoder.text_token(match, :key) @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) - @encoder.text_token(scan(/=/), :operator) + @encoder.text_token(scan(/\=/), :operator) @state = :initial elsif match = scan(/\s+/m) @encoder.text_token(match, :space) -- cgit v1.2.1 From 279348d3c2159df4ce6ac5949ada0177aa7c7159 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sat, 27 Oct 2012 23:41:52 -0600 Subject: :map token kind Use :map instead of :table. It's more generic, and won't be confused with the :table rendering style. --- lib/coderay/scanners/lua.rb | 14 +++++++------- lib/coderay/styles/alpha.rb | 8 ++++---- lib/coderay/token_kinds.rb | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e640397..64763dc 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -98,10 +98,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, kind) elsif match = scan(/\{/) # Opening table brace { - @encoder.begin_group(:table) + @encoder.begin_group(:map) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 - @state = :table + @state = :map elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @@ -112,9 +112,9 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @brace_depth -= 1 @encoder.text_token(match, :inline_delimiter) - @state = :table + @state = :map end - @encoder.end_group(:table) + @encoder.end_group(:map) elsif match = scan(/["']/) # String delimiters " and ' @encoder.begin_group(:string) @@ -142,8 +142,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # It may be that we’re scanning a full-blown subexpression of a table # (tables can contain full expressions in parts). - # If this is the case, return to :table scanning state. - @state = :table if @state == :initial && @brace_depth >= 1 + # If this is the case, return to :map scanning state. + @state = :map if @state == :initial && @brace_depth >= 1 when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @@ -237,7 +237,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(getch, :error) end - when :table + when :map if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 257083e..a60725b 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -1,6 +1,6 @@ module CodeRay module Styles - + # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style @@ -116,9 +116,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } .symbol { color:#A60 } -.table .content { color:#808 } -.table .delimiter { color:#40A} -.table { background-color:hsla(200,100%,50%,0.06); } +.map .content { color:#808 } +.map .delimiter { color:#40A} +.map { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070 } .type { color:#339; font-weight:bold } .value { color: #088; } diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index e245623..a0ad5e4 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,14 +1,14 @@ module CodeRay - + # A Hash of all known token kinds and their associated CSS classes. TokenKinds = Hash.new do |h, k| warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG false end - + # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity - + TokenKinds.update( # :nodoc: :annotation => 'annotation', :attribute_name => 'attribute-name', @@ -50,6 +50,7 @@ module CodeRay :keyword => 'keyword', :label => 'label', :local_variable => 'local-variable', + :map => 'map', :modifier => 'modifier', :namespace => 'namespace', :octal => 'octal', @@ -63,29 +64,28 @@ module CodeRay :shell => 'shell', :string => 'string', :symbol => 'symbol', - :table => 'table', :tag => 'tag', :type => 'type', :value => 'value', :variable => 'variable', - + :change => 'change', :delete => 'delete', :head => 'head', :insert => 'insert', - + :eyecatcher => 'eyecatcher', - + :ident => false, :operator => false, - + :space => false, :plain => false ) - + TokenKinds[:method] = TokenKinds[:function] TokenKinds[:escape] = TokenKinds[:delimiter] TokenKinds[:docstring] = TokenKinds[:comment] - + TokenKinds.freeze end -- cgit v1.2.1 From d9d6dd5f4a73363ea5d353ecda142f77ed4eba5a Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Thu, 6 Jun 2013 10:43:19 -0600 Subject: Support CDATA blocks in HTML/XML --- lib/coderay/scanners/html.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 3ba3b79..fcf249a 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -99,7 +99,17 @@ module Scanners case state when :initial - if match = scan(/|.*)/m) + if match = scan(//m) + encoder.text_token match[0..-4], :content + encoder.text_token ']]>', :delimiter + else + encoder.text_token scan(/.*/m), :error + end + encoder.end_group :string + elsif match = scan(/|.*)/m) encoder.text_token match, :comment elsif match = scan(/|.*)|\]>/m) encoder.text_token match, :doctype -- cgit v1.2.1 From d0d1b129f09dbae8fc25c093c09f72c6396b122a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Jun 2013 23:53:32 +0200 Subject: whitespace --- lib/coderay/styles/alpha.rb | 2 +- lib/coderay/token_kinds.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index c05ffd5..c231423 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -1,6 +1,6 @@ module CodeRay module Styles - + # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index f1696df..9137a49 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,14 +1,14 @@ module CodeRay - + # A Hash of all known token kinds and their associated CSS classes. TokenKinds = Hash.new do |h, k| warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG false end - + # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity - + TokenKinds.update( # :nodoc: :debug => 'debug', # highlight for debugging (white on blue background) -- cgit v1.2.1 From 90adb394e8b4da3a733941fc764c308b34482293 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 00:15:59 +0200 Subject: add lua file type --- lib/coderay/helpers/file_type.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 19f27ac..6d4fa92 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -96,6 +96,7 @@ module CodeRay 'java' => :java, 'js' => :java_script, 'json' => :json, + 'lua' => :lua, 'mab' => :ruby, 'pas' => :delphi, 'patch' => :diff, -- cgit v1.2.1 From fbaf55d69c20897655c80e9723973e2f9490484d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:04:53 +0200 Subject: more helpful error messages from DebugLint --- lib/coderay/encoders/debug_lint.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index eeb2a92..0ac89ef 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -35,7 +35,7 @@ module Encoders end def end_group kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind super end @@ -45,7 +45,7 @@ module Encoders end def end_line kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind super end -- cgit v1.2.1 From c0dc9f2b82b84e6a7511ff9bfe3e80aaa1fef1d1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:05:41 +0200 Subject: move .map styles before .string --- lib/coderay/styles/alpha.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index c231423..a869d9e 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -99,6 +99,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } .local-variable { color:#963 } +.map .content { color:#808 } +.map .delimiter { color:#40A} +.map { background-color:hsla(200,100%,50%,0.06); } .namespace { color:#707; font-weight:bold } .octal { color:#40E } .operator { } @@ -123,9 +126,6 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol { color:#A60 } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } -.map .content { color:#808 } -.map .delimiter { color:#40A} -.map { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070; font-weight:bold } .type { color:#339; font-weight:bold } .value { color: #088 } -- cgit v1.2.1 From da1425058e9b9e04bd988ddf606331d9cd0c827b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:06:33 +0200 Subject: cleanup Lua scanner, fix end_group(:map) issue - use local instead of instance variables --- lib/coderay/scanners/lua.rb | 524 ++++++++++++++++++++++---------------------- 1 file changed, 267 insertions(+), 257 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 64763dc..3bee275 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -1,265 +1,275 @@ -# -*- coding: utf-8 -*- +# encoding: utf-8 -# Scanner for the Lua[http://lua.org] programming lanuage. -# -# The language’s complete syntax is defined in -# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], -# which is what this scanner tries to conform to. -class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner +module CodeRay +module Scanners - register_for :lua - file_extension "lua" - title "Lua" - - # Keywords used in Lua. - KEYWORDS = %w[and break do else elseif end - for function goto if in - local not or repeat return - then until while - ] - - # Constants set by the Lua core. - PREDEFINED_CONSTANTS = %w[false true nil] - - # The expressions contained in this array are parts of Lua’s `basic' - # library. Although it’s not entirely necessary to load that library, - # it is highly recommended and one would have to provide own implementations - # of some of these expressions if one does not do so. They however aren’t - # keywords, neither are they constants, but nearly predefined, so they - # get tagged as `predefined' rather than anything else. + # Scanner for the Lua[http://lua.org] programming lanuage. # - # This list excludes values of form `_UPPERCASE' because the Lua manual - # requires such identifiers to be reserved by Lua anyway and they are - # highlighted directly accordingly, without the need for specific - # identifiers to be listed here. - PREDEFINED_EXPRESSIONS = %w[ - assert collectgarbage dofile error getmetatable - ipairs load loadfile next pairs pcall print - rawequal rawget rawlen rawset select setmetatable - tonumber tostring type xpcall - ] - - # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident). - add(KEYWORDS, :keyword). - add(PREDEFINED_CONSTANTS, :predefined_constant). - add(PREDEFINED_EXPRESSIONS, :predefined) - - protected - - # Scanner initialization. - def setup - @state = :initial - @brace_depth = 0 - end - - # CodeRay entry hook. Starts parsing. - def scan_tokens(encoder, options) - @encoder = encoder - @options = options - - until eos? - case state - - when :initial - if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] - @num_equals = match.count("=") # Number must match for comment end - @encoder.begin_group(:comment) - @encoder.text_token(match, :delimiter) - @state = :long_comment - - elsif match = scan(/--.*?$/) # --Lua comment - @encoder.text_token(match, :comment) - - elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] - @num_equals = match.count("=") # Number must match for comment end - @encoder.begin_group(:string) - @encoder.text_token(match, :delimiter) - @state = :long_string - - elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: - @encoder.text_token(match, :label) - - elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua - @encoder.text_token(match, :predefined) - - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) - kind = IDENT_KIND[match] - - # Extra highlighting for entities following certain keywords - if kind == :keyword and match == "function" - @state = :function_expected - elsif kind == :keyword and match == "goto" - @state = :goto_label_expected - elsif kind == :keyword and match == "local" - @state = :local_var_expected + # The language’s complete syntax is defined in + # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], + # which is what this scanner tries to conform to. + class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + + register_for :lua + file_extension "lua" + title "Lua" + + # Keywords used in Lua. + KEYWORDS = %w[and break do else elseif end + for function goto if in + local not or repeat return + then until while + ] + + # Constants set by the Lua core. + PREDEFINED_CONSTANTS = %w[false true nil] + + # The expressions contained in this array are parts of Lua’s `basic' + # library. Although it’s not entirely necessary to load that library, + # it is highly recommended and one would have to provide own implementations + # of some of these expressions if one does not do so. They however aren’t + # keywords, neither are they constants, but nearly predefined, so they + # get tagged as `predefined' rather than anything else. + # + # This list excludes values of form `_UPPERCASE' because the Lua manual + # requires such identifiers to be reserved by Lua anyway and they are + # highlighted directly accordingly, without the need for specific + # identifiers to be listed here. + PREDEFINED_EXPRESSIONS = %w[ + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall + ] + + # Automatic token kind selection for normal words. + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) + + protected + + # Scanner initialization. + def setup + @state = :initial + @brace_depth = 0 + end + + # CodeRay entry hook. Starts parsing. + def scan_tokens(encoder, options) + state = options[:state] || @state + + until eos? + case state + + when :initial + if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] + @num_equals = match.count("=") # Number must match for comment end + encoder.begin_group(:comment) + encoder.text_token(match, :delimiter) + state = :long_comment + + elsif match = scan(/--.*$/) # --Lua comment + encoder.text_token(match, :comment) + + elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] + @num_equals = match.count("=") # Number must match for comment end + encoder.begin_group(:string) + encoder.text_token(match, :delimiter) + state = :long_string + + elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: + encoder.text_token(match, :label) + + elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua + encoder.text_token(match, :predefined) + + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) + kind = IDENT_KIND[match] + + # Extra highlighting for entities following certain keywords + if kind == :keyword and match == "function" + state = :function_expected + elsif kind == :keyword and match == "goto" + state = :goto_label_expected + elsif kind == :keyword and match == "local" + state = :local_var_expected + end + + encoder.text_token(match, kind) + + elsif match = scan(/\{/) # Opening table brace { + encoder.begin_group(:map) + encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) + @brace_depth += 1 + state = :map + + elsif match = scan(/\}/) # Closing table brace } + if @brace_depth == 1 + @brace_depth = 0 + encoder.text_token(match, :delimiter) + encoder.end_group(:map) + elsif @brace_depth == 0 # Mismatched brace + encoder.text_token(match, :error) + else + @brace_depth -= 1 + encoder.text_token(match, :inline_delimiter) + encoder.end_group(:map) + state = :map + end + + elsif match = scan(/["']/) # String delimiters " and ' + encoder.begin_group(:string) + encoder.text_token(match, :delimiter) + @start_delim = match + state = :string + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + encoder.text_token(match, :float) + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + encoder.text_token(match, :integer) + + elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators + encoder.text_token(match, :operator) + + elsif match = scan(/\s+/) # Space + encoder.text_token(match, :space) + + else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. + encoder.text_token(getch, :error) + end + + # It may be that we’re scanning a full-blown subexpression of a table + # (tables can contain full expressions in parts). + # If this is the case, return to :map scanning state. + state = :map if state == :initial && @brace_depth >= 1 + + when :function_expected + if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name + encoder.text_token(match, :operator) + state = :initial + elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator + encoder.text_token(match, :ident) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() + encoder.text_token(match, :function) + state = :initial + elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + state = :initial + end + + when :goto_label_expected + if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + encoder.text_token(match, :label) + state = :initial + elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + end + + when :local_var_expected + if match = scan(/function/) # local function ... + encoder.text_token(match, :keyword) + state = :function_expected + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + encoder.text_token(match, :local_variable) + elsif match = scan(/,/) + encoder.text_token(match, :operator) + elsif match = scan(/\=/) + encoder.text_token(match, :operator) + # After encountering the equal sign, arbitrary expressions are + # allowed again, so just return to the main state for further + # parsing. + state = :initial + elsif match = scan(/\n/) + encoder.text_token(match, :space) + state = :initial + elsif match = scan(/\s+/) + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + end + + when :long_comment + if match = scan(/.*?(?=\]={#@num_equals}\])/m) + encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + encoder.text_token(rest, :error) + terminate + end + encoder.end_group(:comment) + state = :initial + + when :long_string + if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + encoder.text_token(rest, :error) + terminate + end + encoder.end_group(:string) + state = :initial + + when :string + if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + encoder.text_token(match, :content) + elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) + encoder.text_token(match, :char) + elsif match = scan(Regexp.compile(@start_delim)) + encoder.text_token(match, :delimiter) + encoder.end_group(:string) + state = :initial + elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings + encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings + encoder.end_group(:string) + state = :initial + else + encoder.text_token(getch, :error) + end + + when :map + if match = scan(/[,;]/) + encoder.text_token(match, :operator) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) + encoder.text_token(match, :key) + encoder.text_token(scan(/\s+/), :space) if check(/\s+/) + encoder.text_token(scan(/\=/), :operator) + state = :initial + elsif match = scan(/\s+/m) + encoder.text_token(match, :space) + else + # Note this clause doesn’t advance the scan pointer, it’s a kind of + # "retry with other options" (the :initial state then of course + # advances the pointer). + state = :initial + end + else + raise + end + end - - @encoder.text_token(match, kind) - - elsif match = scan(/\{/) # Opening table brace { - @encoder.begin_group(:map) - @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) - @brace_depth += 1 - @state = :map - - elsif match = scan(/\}/) # Closing table brace } - if @brace_depth == 1 - @brace_depth = 0 - @encoder.text_token(match, :delimiter) - elsif @brace_depth == 0 # Mismatched brace - @encoder.text_token(match, :error) - else - @brace_depth -= 1 - @encoder.text_token(match, :inline_delimiter) - @state = :map + + if options[:keep_state] + @state = state end - @encoder.end_group(:map) - - elsif match = scan(/["']/) # String delimiters " and ' - @encoder.begin_group(:string) - @encoder.text_token(match, :delimiter) - @start_delim = match - @state = :string - - # ↓Prefix hex number ←|→ decimal number - elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power - @encoder.text_token(match, :float) - - # ↓Prefix hex number ←|→ decimal number - elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power - @encoder.text_token(match, :integer) - - elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators - @encoder.text_token(match, :operator) - - elsif match = scan(/\s+/) # Space - @encoder.text_token(match, :space) - - else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. - @encoder.text_token(getch, :error) - end - - # It may be that we’re scanning a full-blown subexpression of a table - # (tables can contain full expressions in parts). - # If this is the case, return to :map scanning state. - @state = :map if @state == :initial && @brace_depth >= 1 - - when :function_expected - if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name - @encoder.text_token(match, :operator) - @state = :initial - elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator - @encoder.text_token(match, :ident) - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() - @encoder.text_token(match, :function) - @state = :initial - elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - @state = :initial - end - - when :goto_label_expected - if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) - @encoder.text_token(match, :label) - @state = :initial - elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - end - - when :local_var_expected - if match = scan(/function/) # local function ... - @encoder.text_token(match, :keyword) - @state = :function_expected - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) - @encoder.text_token(match, :local_variable) - elsif match = scan(/,/) - @encoder.text_token(match, :operator) - elsif match = scan(/\=/) - @encoder.text_token(match, :operator) - # After encountering the equal sign, arbitrary expressions are - # allowed again, so just return to the main state for further - # parsing. - @state = :initial - elsif match = scan(/\n/) - @encoder.text_token(match, :space) - @state = :initial - elsif match = scan(/\s+/) - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - end - - when :long_comment - if match = scan(/.*?(?=\]={#@num_equals}\])/m) - @encoder.text_token(match, :content) - - delim = scan(/\]={#@num_equals}\]/) - @encoder.text_token(delim, :delimiter) - else # No terminator found till EOF - @encoder.text_token(rest, :error) - terminate - end - @encoder.end_group(:comment) - @state = :initial - - when :long_string - if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences - @encoder.text_token(match, :content) - - delim = scan(/\]={#@num_equals}\]/) - @encoder.text_token(delim, :delimiter) - else # No terminator found till EOF - @encoder.text_token(rest, :error) - terminate - end - @encoder.end_group(:string) - @state = :initial - - when :string - if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) - @encoder.text_token(match, :content) - elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) - @encoder.text_token(match, :char) - elsif match = scan(Regexp.compile(@start_delim)) - @encoder.text_token(match, :delimiter) - @encoder.end_group(:string) - @state = :initial - elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings - @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings - @encoder.end_group(:string) - @state = :initial - else - @encoder.text_token(getch, :error) - end - - when :map - if match = scan(/[,;]/) - @encoder.text_token(match, :operator) - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) - @encoder.text_token(match, :key) - @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) - @encoder.text_token(scan(/\=/), :operator) - @state = :initial - elsif match = scan(/\s+/m) - @encoder.text_token(match, :space) - else - # Note this clause doesn’t advance the scan pointer, it’s a kind of - # "retry with other options" (the :initial state then of course - # advances the pointer). - @state = :initial + + encoder end - else - raise + end - - end - - @encoder - end - + +end end -- cgit v1.2.1 From 546b489e4b3856d361a480de5f1f02cb55c7f002 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:09:14 +0200 Subject: fix Tokens documentation --- lib/coderay/tokens.rb | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 54358d4..e7bffce 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -3,17 +3,16 @@ module CodeRay # The Tokens class represents a list of tokens returned from # a Scanner. It's actually just an Array with a few helper methods. # - # A token itself is not a special object, just a two-element Array - # consisting of + # A token itself is not a special object, just two elements in an Array: # * the _token_ _text_ (the original source of the token in a String) or # a _token_ _action_ (begin_group, end_group, begin_line, end_line) # * the _token_ _kind_ (a Symbol representing the type of the token) # # It looks like this: # - # ['# It looks like this', :comment] - # ['3.1415926', :float] - # ['$^', :error] + # ..., '# It looks like this', :comment, ... + # ..., '3.1415926', :float, ... + # ..., '$^', :error, ... # # Some scanners also yield sub-tokens, represented by special # token actions, for example :begin_group and :end_group. @@ -21,11 +20,11 @@ module CodeRay # The Ruby scanner, for example, splits "a string" into: # # [ - # [:begin_group, :string], - # ['"', :delimiter], - # ['a string', :content], - # ['"', :delimiter], - # [:end_group, :string] + # :begin_group, :string, + # '"', :delimiter, + # 'a string', :content, + # '"', :delimiter, + # :end_group, :string # ] # # Tokens can be used to save the output of a Scanners in a simple -- cgit v1.2.1 From 06cf0bab5975bc9069f6859cf58eed117fd9f474 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:16:30 +0200 Subject: changelog --- Changes.textile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes.textile b/Changes.textile index fcee29b..930fdbc 100644 --- a/Changes.textile +++ b/Changes.textile @@ -4,6 +4,7 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release h2. Changes in 1.1 +* New scanner: Lua [#21, #22, thanks to Quintus] * New scanner: Sass [#93] * New scanner: Taskpaper [#39, thanks to shimomura] * Diff scanner: Highlight inline changes in multi-line changes [#99] @@ -17,6 +18,7 @@ h2. Changes in 1.1 * @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner] * New token type @:id@ for CSS/Sass [#27] * New token type @:done@ for Taskpaper [#39] +* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany] * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] * Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05] * Fixed @:docstring@ token type style -- cgit v1.2.1 From 90c401c9001e0a670360a55d1189e9f814ae7592 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:17:49 +0200 Subject: some more code cleanups before merge --- lib/coderay/scanners/lua.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 3bee275..25bebbe 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -8,17 +8,17 @@ module Scanners # The language’s complete syntax is defined in # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], # which is what this scanner tries to conform to. - class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + class Lua < Scanner register_for :lua - file_extension "lua" - title "Lua" + file_extension 'lua' + title 'Lua' # Keywords used in Lua. KEYWORDS = %w[and break do else elseif end - for function goto if in - local not or repeat return - then until while + for function goto if in + local not or repeat return + then until while ] # Constants set by the Lua core. @@ -36,10 +36,10 @@ module Scanners # highlighted directly accordingly, without the need for specific # identifiers to be listed here. PREDEFINED_EXPRESSIONS = %w[ - assert collectgarbage dofile error getmetatable - ipairs load loadfile next pairs pcall print - rawequal rawget rawlen rawset select setmetatable - tonumber tostring type xpcall + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall ] # Automatic token kind selection for normal words. @@ -52,7 +52,7 @@ module Scanners # Scanner initialization. def setup - @state = :initial + @state = :initial @brace_depth = 0 end -- cgit v1.2.1 From fa9848b6dbd95a4a97a3e63cb99ad2d4c26516df Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:37:06 +0200 Subject: avoid empty tokens in CSS --- lib/coderay/scanners/css.rb | 2 +- lib/coderay/scanners/sass.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 732f9c5..6c52c8a 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -148,7 +148,7 @@ module Scanners encoder.text_token match[start.size..-2], :content encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content + encoder.text_token match[start.size..-1], :content if start.size < match.size end encoder.end_group :function diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 167051d..65d40b0 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -176,7 +176,7 @@ module Scanners encoder.text_token match[start.size..-2], :content encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content + encoder.text_token match[start.size..-1], :content if start.size < match.size end encoder.end_group :function -- cgit v1.2.1 From bedd4e4fb5a18270a742e71b65f88fb8e3182050 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:40:56 +0200 Subject: avoid empty tokens in Raydebug --- lib/coderay/scanners/raydebug.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index 7a21354..d39d962 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -1,11 +1,11 @@ module CodeRay module Scanners - + # = Debug Scanner # # Parses the output of the Encoders::Debug encoder. class Raydebug < Scanner - + register_for :raydebug file_extension 'raydebug' title 'CodeRay Token Dump' @@ -13,11 +13,11 @@ module Scanners protected def scan_tokens encoder, options - + opened_tokens = [] - + until eos? - + if match = scan(/\s+/) encoder.text_token match, :space @@ -26,7 +26,7 @@ module Scanners encoder.text_token kind, :class encoder.text_token '(', :operator match = self[2] - encoder.text_token match, kind.to_sym + encoder.text_token match, kind.to_sym unless match.empty? encoder.text_token match, :operator if match = scan(/\)/) elsif match = scan(/ (\w+) ([<\[]) /x) @@ -59,8 +59,8 @@ module Scanners encoder end - + end - + end end -- cgit v1.2.1 From 39c074f171eeca1c91aa89757a3f3a12fc78323b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:42:32 +0200 Subject: avoid empty tokens in YAML --- lib/coderay/scanners/yaml.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb index 96f4e93..358a3f1 100644 --- a/lib/coderay/scanners/yaml.rb +++ b/lib/coderay/scanners/yaml.rb @@ -47,7 +47,7 @@ module Scanners when !check(/(?:"[^"]*")(?=: |:$)/) && match = scan(/"/) encoder.begin_group :string encoder.text_token match, :delimiter - encoder.text_token match, :content if match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx) + encoder.text_token match, :content if (match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)) && !match.empty? encoder.text_token match, :delimiter if match = scan(/"/) encoder.end_group :string next -- cgit v1.2.1 From e7df328cb49fd09a4e5d0f02e877e3270a8703ed Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:53:53 +0200 Subject: DebugLint check open tokens at the end --- lib/coderay/encoders/debug_lint.rb | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 0ac89ef..17a0795 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -19,11 +19,6 @@ module Encoders EmptyToken = Class.new InvalidTokenStream IncorrectTokenGroupNesting = Class.new InvalidTokenStream - def initialize options = {} - super - @opened = [] - end - def text_token text, kind raise EmptyToken, 'empty token' if text.empty? super @@ -35,7 +30,8 @@ module Encoders end def end_group kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop super end @@ -45,7 +41,20 @@ module Encoders end def end_line kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + super + end + + protected + + def setup options + super + @opened = [] + end + + def finish options + raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? super end -- cgit v1.2.1 From d6cc5767c9a9bfb4c08bcd5127145b38cb1d29de Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:54:31 +0200 Subject: HTML encoder shouldn't warn about open tokens --- lib/coderay/encoders/html.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index b897f5e..20f2409 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -193,7 +193,6 @@ module Encoders def finish options unless @opened.empty? - warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG @out << '' while @opened.pop @last_opened = nil end -- cgit v1.2.1 From b059a63e7ff3cd7c26dd5664bf048aa42338fbe1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:54:56 +0200 Subject: close open tokens in Groovy scanner --- lib/coderay/scanners/groovy.rb | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index cf55daf..d04a535 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -36,9 +36,12 @@ module Scanners protected + def setup + @state = :initial + end + def scan_tokens encoder, options - - state = :initial + state = options[:state] || @state inline_block_stack = [] inline_block_paren_depth = nil string_delimiter = nil @@ -246,6 +249,16 @@ module Scanners encoder.end_group state end + if options[:keep_state] + @state = state + end + + until inline_block_stack.empty? + state, = *inline_block_stack.pop + encoder.end_group :inline + encoder.end_group state + end + encoder end -- cgit v1.2.1 From 3c3060acaf2422c23784539b270e365decc9cb2e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:55:23 +0200 Subject: close open tokens in PHP scanner --- lib/coderay/scanners/php.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb index 6c68834..40037f9 100644 --- a/lib/coderay/scanners/php.rb +++ b/lib/coderay/scanners/php.rb @@ -500,6 +500,10 @@ module Scanners end + while state = states.pop + encoder.end_group :string if [:sqstring, :dqstring].include? state + end + encoder end -- cgit v1.2.1 From 7f45168efd1abe679b261a7a91f3641984091d12 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:06:16 +0200 Subject: remove instance variables from Lua scanner --- lib/coderay/scanners/lua.rb | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 25bebbe..185c133 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -59,13 +59,15 @@ module Scanners # CodeRay entry hook. Starts parsing. def scan_tokens(encoder, options) state = options[:state] || @state + brace_depth = @brace_depth + num_equals = nil until eos? case state when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] - @num_equals = match.count("=") # Number must match for comment end + num_equals = match.count("=") # Number must match for comment end encoder.begin_group(:comment) encoder.text_token(match, :delimiter) state = :long_comment @@ -74,7 +76,7 @@ module Scanners encoder.text_token(match, :comment) elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] - @num_equals = match.count("=") # Number must match for comment end + num_equals = match.count("=") # Number must match for comment end encoder.begin_group(:string) encoder.text_token(match, :delimiter) state = :long_string @@ -101,19 +103,19 @@ module Scanners elsif match = scan(/\{/) # Opening table brace { encoder.begin_group(:map) - encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) - @brace_depth += 1 + encoder.text_token(match, brace_depth >= 1 ? :inline_delimiter : :delimiter) + brace_depth += 1 state = :map elsif match = scan(/\}/) # Closing table brace } - if @brace_depth == 1 - @brace_depth = 0 + if brace_depth == 1 + brace_depth = 0 encoder.text_token(match, :delimiter) encoder.end_group(:map) - elsif @brace_depth == 0 # Mismatched brace + elsif brace_depth == 0 # Mismatched brace encoder.text_token(match, :error) else - @brace_depth -= 1 + brace_depth -= 1 encoder.text_token(match, :inline_delimiter) encoder.end_group(:map) state = :map @@ -122,7 +124,7 @@ module Scanners elsif match = scan(/["']/) # String delimiters " and ' encoder.begin_group(:string) encoder.text_token(match, :delimiter) - @start_delim = match + start_delim = match state = :string # ↓Prefix hex number ←|→ decimal number @@ -146,7 +148,7 @@ module Scanners # It may be that we’re scanning a full-blown subexpression of a table # (tables can contain full expressions in parts). # If this is the case, return to :map scanning state. - state = :map if state == :initial && @brace_depth >= 1 + state = :map if state == :initial && brace_depth >= 1 when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @@ -198,10 +200,10 @@ module Scanners end when :long_comment - if match = scan(/.*?(?=\]={#@num_equals}\])/m) + if match = scan(/.*?(?=\]={#{num_equals}}\])/m) encoder.text_token(match, :content) - delim = scan(/\]={#@num_equals}\]/) + delim = scan(/\]={#{num_equals}}\]/) encoder.text_token(delim, :delimiter) else # No terminator found till EOF encoder.text_token(rest, :error) @@ -211,10 +213,10 @@ module Scanners state = :initial when :long_string - if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + if match = scan(/.*?(?=\]={#{num_equals}}\])/m) # Long strings do not interpret any escape sequences encoder.text_token(match, :content) - delim = scan(/\]={#@num_equals}\]/) + delim = scan(/\]={#{num_equals}}\]/) encoder.text_token(delim, :delimiter) else # No terminator found till EOF encoder.text_token(rest, :error) @@ -224,11 +226,11 @@ module Scanners state = :initial when :string - if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + if match = scan(/[^\\#{start_delim}\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) encoder.text_token(match, :content) elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) encoder.text_token(match, :char) - elsif match = scan(Regexp.compile(@start_delim)) + elsif match = scan(Regexp.compile(start_delim)) encoder.text_token(match, :delimiter) encoder.end_group(:string) state = :initial -- cgit v1.2.1 From 45bb0c576b67b7a3c0dead02078b3a16b5583154 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:06:52 +0200 Subject: close open token groups in Lua scanner --- lib/coderay/scanners/lua.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 185c133..fb1e45a 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -268,6 +268,9 @@ module Scanners @state = state end + encoder.end_group :string if [:string].include? state + brace_depth.times { encoder.end_group :map } + encoder end -- cgit v1.2.1 From dc6071129cdc1bcd15129147bbc4d92ba870f007 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:13:32 +0200 Subject: close open token groups in Sass scanner --- lib/coderay/scanners/sass.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 65d40b0..0ba383f 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -218,6 +218,14 @@ module Scanners @state = states end + while state = states.pop + if state == :sass_inline + encoder.end_group :inline + elsif state == :string + encoder.end_group :string + end + end + encoder end -- cgit v1.2.1 From 4327fcbe932ac913f7eb8e92497f97913fb398c5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:15:55 +0200 Subject: fix Sass regexp modifier --- lib/coderay/scanners/sass.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 0ba383f..e20bebe 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -195,7 +195,7 @@ module Scanners elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/) encoder.text_token match, :color - elsif match = scan(/@else if\b|#{RE::AtKeyword}/) + elsif match = scan(/@else if\b|#{RE::AtKeyword}/o) encoder.text_token match, :directive value_expected = true -- cgit v1.2.1 From 755904f7046d8a8d6208fe8367aaa3b9a19ecd65 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:30:24 +0200 Subject: avoid empty tokens in Diff scanner --- lib/coderay/scanners/diff.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index af0f755..fd1aed6 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -69,7 +69,7 @@ module Scanners state = :added elsif match = scan(/\\ .*/) encoder.text_token match, :comment - elsif match = scan(/@@(?>[^@\n]*)@@/) + elsif match = scan(/@@(?>[^@\n]+)@@/) content_scanner.state = :initial unless match?(/\n\+/) content_scanner_entry_state = nil if check(/\n|$/) -- cgit v1.2.1 From e1d5e111d968639fa03a6074cf90535ecc90d0dd Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:30:44 +0200 Subject: close correct token groups in Groovy scanner --- lib/coderay/scanners/groovy.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index d04a535..c64454f 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -226,7 +226,7 @@ module Scanners encoder.text_token match, :content # TODO: Shouldn't this be :error? elsif match = scan(/ \\ | \n /x) - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string encoder.text_token match, :error after_def = value_expected = false state = :initial @@ -246,7 +246,7 @@ module Scanners end if [:multiline_string, :string, :regexp].include? state - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string end if options[:keep_state] @@ -256,7 +256,7 @@ module Scanners until inline_block_stack.empty? state, = *inline_block_stack.pop encoder.end_group :inline - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string end encoder -- cgit v1.2.1 From 603ff7d0b14521cfd0408aa68e2e1cb6ea9086bc Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:36:46 +0200 Subject: avoid empty tokens in YAML scanner --- lib/coderay/scanners/yaml.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb index 358a3f1..32c8e2c 100644 --- a/lib/coderay/scanners/yaml.rb +++ b/lib/coderay/scanners/yaml.rb @@ -84,7 +84,7 @@ module Scanners when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/) encoder.begin_group :key encoder.text_token match[0,1], :delimiter - encoder.text_token match[1..-2], :content + encoder.text_token match[1..-2], :content if match.size > 2 encoder.text_token match[-1,1], :delimiter encoder.end_group :key key_indent = column(pos - match.size) - 1 -- cgit v1.2.1 From 253a616924000cc5d522a2d4b8030f5693c98e43 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:39:30 +0200 Subject: close open string token groups in SQL scanner --- lib/coderay/scanners/sql.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb index b757278..93aeaf3 100644 --- a/lib/coderay/scanners/sql.rb +++ b/lib/coderay/scanners/sql.rb @@ -1,8 +1,9 @@ -module CodeRay module Scanners +module CodeRay +module Scanners # by Josh Goebel class SQL < Scanner - + register_for :sql KEYWORDS = %w( @@ -149,6 +150,7 @@ module CodeRay module Scanners string_content = '' end encoder.text_token match, :error unless match.empty? + encoder.end_group :string state = :initial else raise "else case \" reached; %p not handled." % peek(1), encoder @@ -171,4 +173,5 @@ module CodeRay module Scanners end -end end \ No newline at end of file +end +end -- cgit v1.2.1 From ad5efeb8c96c976804fcfcd3f71e0d02cf73cc10 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 13:00:13 +0200 Subject: changelog --- Changes.textile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index 930fdbc..581e485 100644 --- a/Changes.textile +++ b/Changes.textile @@ -12,6 +12,8 @@ h2. Changes in 1.1 * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] * Remove double-click toggle handler from HTML table output * Fixes to CSS scanner (floats, pseudoclasses) +* Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] +* Added @:keep_state@ functionality to more scanners [#116] * CSS scanner uses @:id@ and @:tag@ now [#27] * Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right? * Add .xaml file type [#121, thanks to Kozman Bálint] @@ -20,10 +22,13 @@ h2. Changes in 1.1 * New token type @:done@ for Taskpaper [#39] * New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany] * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] -* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05] +* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05] * Fixed @:docstring@ token type style * @Plugin@ does not warn about fallback when default is defined +* @HTML@ encoder will not warn about unclosed token groups at the end of the stream * @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now +* @Debug@ encoder will not warn about errors in the token stream +* New @DebugLint@ encoder that checks for empty tokens and correct nesting h2. Changes in 1.0.9 -- cgit v1.2.1 From c8751fbc09d4a8f43b0b037e3a7dc7ddbfbcacb1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 13:15:10 +0200 Subject: fix unclosed token group in Ruby scanner --- lib/coderay/scanners/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index c282f31..80165ca 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -269,7 +269,7 @@ module Scanners end if last_state - state = last_state + state = last_state unless state.is_a?(StringState) # otherwise, a simple 'def"' results in unclosed tokens last_state = nil end -- cgit v1.2.1 From c34ad739bc7342aa40830945a5e19e90e9814e04 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 13:27:43 +0200 Subject: fix token nesting in PHP scanner --- lib/coderay/scanners/php.rb | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb index 40037f9..7a8d75d 100644 --- a/lib/coderay/scanners/php.rb +++ b/lib/coderay/scanners/php.rb @@ -265,7 +265,7 @@ module Scanners @html_scanner.tokenize match unless match.empty? end - when :php + when :php, :php_inline if match = scan(/\s+/) encoder.text_token match, :space @@ -332,7 +332,7 @@ module Scanners if states.size == 1 encoder.text_token match, :error else - states.pop + state = states.pop if states.last.is_a?(::Array) delimiter = states.last[1] states[-1] = states.last[0] @@ -340,6 +340,7 @@ module Scanners encoder.end_group :inline else encoder.text_token match, :operator + encoder.end_group :inline if state == :php_inline label_expected = true end end @@ -350,7 +351,14 @@ module Scanners elsif match = scan(RE::PHP_END) encoder.text_token match, :inline_delimiter - states = [:initial] + while state = states.pop + encoder.end_group :string if [:sqstring, :dqstring].include? state + if state.is_a? Array + encoder.end_group :inline + encoder.end_group :string if [:sqstring, :dqstring].include? state.first + end + end + states << :initial elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o) encoder.begin_group :string @@ -400,6 +408,7 @@ module Scanners elsif match = scan(/\\/) encoder.text_token match, :error else + encoder.end_group :string states.pop end @@ -459,7 +468,7 @@ module Scanners encoder.begin_group :inline states[-1] = [states.last, delimiter] delimiter = nil - states.push :php + states.push :php_inline encoder.text_token match, :delimiter else encoder.text_token match, :content @@ -469,6 +478,7 @@ module Scanners elsif match = scan(/\$/) encoder.text_token match, :content else + encoder.end_group :string states.pop end @@ -502,6 +512,10 @@ module Scanners while state = states.pop encoder.end_group :string if [:sqstring, :dqstring].include? state + if state.is_a? Array + encoder.end_group :inline + encoder.end_group :string if [:sqstring, :dqstring].include? state.first + end end encoder -- cgit v1.2.1 From 14f3a4f28341237503a51bfa0764c922bb02bdfa Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:13:15 +0200 Subject: tweak HTML CDATA token kinds --- lib/coderay/scanners/html.rb | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index fcf249a..530ce44 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -1,13 +1,13 @@ module CodeRay module Scanners - + # HTML Scanner # # Alias: +xhtml+ # # See also: Scanners::XML class HTML < Scanner - + register_for :html KINDS_NOT_LOC = [ @@ -100,15 +100,13 @@ module Scanners when :initial if match = scan(//m) - encoder.text_token match[0..-4], :content - encoder.text_token ']]>', :delimiter - else - encoder.text_token scan(/.*/m), :error + encoder.text_token match[0..-4], :plain + encoder.text_token ']]>', :inline_delimiter + elsif match = scan(/.+/) + encoder.text_token match, :error end - encoder.end_group :string elsif match = scan(/|.*)/m) encoder.text_token match, :comment elsif match = scan(/|.*)|\]>/m) -- cgit v1.2.1 From 5b0dc0f35090949d7512a85dadc8cf2871b91aac Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:15:01 +0200 Subject: scan_css in HTML scanner (tags) --- lib/coderay/scanners/html.rb | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 3ba3b79..06728e4 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -75,9 +75,14 @@ module Scanners def scan_java_script encoder, code if code && !code.empty? @java_script_scanner ||= Scanners::JavaScript.new '', :keep_tokens => true - # encoder.begin_group :inline @java_script_scanner.tokenize code, :tokens => encoder - # encoder.end_group :inline + end + end + + def scan_css encoder, code + if code && !code.empty? + @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true + @css_scanner.tokenize code, :tokens => encoder end end @@ -110,7 +115,7 @@ module Scanners elsif match = scan(/<\/[-\w.:]*>?/m) in_tag = nil encoder.text_token match, :tag - elsif match = scan(/<(?:(script)|[-\w.:]+)(>)?/m) + elsif match = scan(/<(?:(script|style)|[-\w.:]+)(>)?/m) encoder.text_token match, :tag in_tag = self[1] if self[2] @@ -206,19 +211,23 @@ module Scanners when :in_special_tag case in_tag - when 'script' + when 'script', 'style' encoder.text_token match, :space if match = scan(/[ \t]*\n/) if scan(/(\s*)|(.*))/m) code = self[2] || self[4] closing = self[3] encoder.text_token self[1], :comment else - code = scan_until(/(?=(?:\n\s*)?<\/script>)|\z/) + code = scan_until(/(?=(?:\n\s*)?<\/#{in_tag}>)|\z/) closing = false end unless code.empty? encoder.begin_group :inline - scan_java_script encoder, code + if in_tag == 'script' + scan_java_script encoder, code + else + scan_css encoder, code + end encoder.end_group :inline end encoder.text_token closing, :comment if closing -- cgit v1.2.1 From 7ada74c8f0f77815393e887db83e1d2c36ce7235 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:32:14 +0200 Subject: scan_css in HTML scanner (arguments), change token kind from :inline to :string --- Changes.textile | 1 + lib/coderay/scanners/html.rb | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Changes.textile b/Changes.textile index 930fdbc..9509b15 100644 --- a/Changes.textile +++ b/Changes.textile @@ -10,6 +10,7 @@ h2. Changes in 1.1 * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] +* HTML scanner displays style tags and attributes now * Remove double-click toggle handler from HTML table output * Fixes to CSS scanner (floats, pseudoclasses) * CSS scanner uses @:id@ and @:tag@ now [#27] diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 06728e4..f1dfba0 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -33,7 +33,8 @@ module Scanners ) IN_ATTRIBUTE = WordList::CaseIgnoring.new(nil). - add(EVENT_ATTRIBUTES, :script) + add(EVENT_ATTRIBUTES, :script). + add(['style'], :style) ATTR_NAME = /[\w.:-]+/ # :nodoc: TAG_END = /\/?>/ # :nodoc: @@ -79,10 +80,10 @@ module Scanners end end - def scan_css encoder, code + def scan_css encoder, code, state = [:initial] if code && !code.empty? @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true - @css_scanner.tokenize code, :tokens => encoder + @css_scanner.tokenize code, :tokens => encoder, :state => state end end @@ -166,17 +167,21 @@ module Scanners encoder.text_token match, :attribute_value state = :attribute elsif match = scan(/["']/) - if in_attribute == :script - encoder.begin_group :inline - encoder.text_token match, :inline_delimiter + if in_attribute == :script || in_attribute == :style + encoder.begin_group :string + encoder.text_token match, :delimiter if scan(/javascript:[ \t]*/) encoder.text_token matched, :comment end code = scan_until(match == '"' ? /(?="|\z)/ : /(?='|\z)/) - scan_java_script encoder, code + if in_attribute == :script + scan_java_script encoder, code + else + scan_css encoder, code, [:block] + end match = scan(/["']/) - encoder.text_token match, :inline_delimiter if match - encoder.end_group :inline + encoder.text_token match, :delimiter if match + encoder.end_group :string state = :attribute in_attribute = nil else -- cgit v1.2.1 From 56f4163e99689912c3797e5d3b2b97244ce65782 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:33:22 +0200 Subject: fix another CSS empty token issue --- lib/coderay/scanners/css.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 6c52c8a..9ed4618 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -145,10 +145,10 @@ module Scanners start = match[/^\w+\(/] encoder.text_token start, :delimiter if match[-1] == ?) - encoder.text_token match[start.size..-2], :content + encoder.text_token match[start.size..-2], :content if match.size > start.size + 1 encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content if start.size < match.size + encoder.text_token match[start.size..-1], :content if match.size > start.size end encoder.end_group :function -- cgit v1.2.1 From 6a2f1a2ec7d6136f1ac9dd5723bad84541dafb26 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:38:37 +0200 Subject: changelog --- Changes.textile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index 53f4109..80f1d10 100644 --- a/Changes.textile +++ b/Changes.textile @@ -10,7 +10,7 @@ h2. Changes in 1.1 * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] -* HTML scanner displays style tags and attributes now +* HTML scanner displays style tags and attributes now [#145] * Remove double-click toggle handler from HTML table output * Fixes to CSS scanner (floats, pseudoclasses) * Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] -- cgit v1.2.1 From 9fb2fd9fa3ba4b92a440f271adbb9584b236c34e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:45:31 +0200 Subject: tweak :local_variable token color --- lib/coderay/styles/alpha.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index a869d9e..b05ae53 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -98,7 +98,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .key .delimiter { color: #404 } .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } -.local-variable { color:#963 } +.local-variable { color:#950 } .map .content { color:#808 } .map .delimiter { color:#40A} .map { background-color:hsla(200,100%,50%,0.06); } -- cgit v1.2.1 From 844975b53233b4697d1385548c1295ce18a7be72 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:46:45 +0200 Subject: whitespace --- lib/coderay/styles/alpha.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index b05ae53..ff85ecc 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -3,14 +3,14 @@ module Styles # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style - + register_for :alpha - + code_background = 'hsl(0,0%,95%)' numbers_background = 'hsl(180,65%,90%)' border_color = 'silver' normal_color = 'black' - + CSS_MAIN_STYLES = <<-MAIN # :nodoc: .CodeRay { background-color: #{code_background}; @@ -145,8 +145,8 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .change .change { color: #88f } .head .head { color: #f4f } TOKENS - + end - + end end -- cgit v1.2.1 From b89caf96d1cfc304c2114d8734ebe8b91337c799 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:53:14 +0200 Subject: fix diff/json token nesting issue --- lib/coderay/scanners/json.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb index 3754a9b..b09970c 100644 --- a/lib/coderay/scanners/json.rb +++ b/lib/coderay/scanners/json.rb @@ -26,6 +26,10 @@ module Scanners def scan_tokens encoder, options state = options[:state] || @state + if [:string, :key].include? state + encoder.begin_group state + end + until eos? case state -- cgit v1.2.1 From 965b8f4d1be88e1ae75ef11e6270a7fbfb2d86bd Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:03:57 +0200 Subject: use -w in executable tests --- test/executable/suite.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/executable/suite.rb b/test/executable/suite.rb index 997405c..ac0ff1d 100644 --- a/test/executable/suite.rb +++ b/test/executable/suite.rb @@ -18,9 +18,9 @@ class TestCodeRayExecutable < Test::Unit::TestCase EXE_COMMAND = if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success? # use Nailgun - "#{RUBY_COMMAND}--ng -I%s %s" + "#{RUBY_COMMAND}--ng -w -I%s %s" else - "#{RUBY_COMMAND} -I%s %s" + "#{RUBY_COMMAND} -w -I%s %s" end % [ROOT_DIR + 'lib', EXECUTABLE] def coderay args, options = {} -- cgit v1.2.1 From a3e79c794f0998034b26ff6baa10bc801918be9d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:07:57 +0200 Subject: trying to fix TravisCI error --- lib/coderay/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4b4f085..4115d09 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' + VERSION = '1.1.0' unless defined? VERSION end -- cgit v1.2.1 From 416243353e0b4b1c449476b269a50b0f60c13bda Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:15:24 +0200 Subject: trying different ways to load CodeRay version --- lib/coderay.rb | 2 +- lib/coderay/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay.rb b/lib/coderay.rb index 24ae5a2..0c66f49 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -134,7 +134,7 @@ module CodeRay File.join CODERAY_PATH, *path end - require coderay_path('version') + require 'coderay/version' # helpers autoload :FileType, coderay_path('helpers', 'file_type') diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4115d09..4b4f085 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' unless defined? VERSION + VERSION = '1.1.0' end -- cgit v1.2.1 From a112fca72ce802b4b5fd25fdc7d76b6360cdc718 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:22:47 +0200 Subject: allow [j]ruby-head failures in Travis --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 59bb791..b3b8cf7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,8 @@ branches: - master matrix: allow_failures: - - rvm: rbx-18mode + - rvm: ruby-head + - rvm: jruby-head + - rvm: rbx-19mode - rvm: rbx-19mode script: "rake test" # test:scanners" -- cgit v1.2.1 From 64ca2ae8ad5130bdcf652aa7aa08298de00f20f4 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:27:29 +0200 Subject: how did this happen? --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b3b8cf7..6d926f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,6 @@ matrix: allow_failures: - rvm: ruby-head - rvm: jruby-head - - rvm: rbx-19mode + - rvm: rbx-18mode - rvm: rbx-19mode script: "rake test" # test:scanners" -- cgit v1.2.1 From e1e6a6af97ef710cfc02a1380180b772f1c34672 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 09:23:54 -0600 Subject: additional Ruby files types Ruby seems to have no shortage of these specially named files. --- lib/coderay/helpers/file_type.rb | 41 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 6d4fa92..6d928d6 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ module CodeRay name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ module CodeRay TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ module CodeRay if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ module CodeRay raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ module CodeRay end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -116,7 +116,7 @@ module CodeRay 'rhtml' => :erb, 'rjs' => :ruby, 'rpdf' => :ruby, - 'ru' => :ruby, + 'ru' => :ruby, # config.ru 'rxml' => :ruby, 'sass' => :sass, 'sql' => :sql, @@ -132,16 +132,19 @@ module CodeRay for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, + 'Guardfile' => :ruby, + 'Vagrantfile' => :ruby, + 'Appraisals' => :ruby } - + end - + end -- cgit v1.2.1 From fd8c81f5c338e4f1448007c9100d15d912fadd27 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 09:43:54 -0600 Subject: additional types: string, error http://golang.org/ref/spec#Predeclared_identifiers --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 4431ef2..a66a5e3 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -23,7 +23,7 @@ module Scanners 'int8', 'int16', 'int32', 'int64', 'float32', 'float64', 'complex64', 'complex128', - 'byte', 'rune', + 'byte', 'rune', 'string', 'error', 'uint', 'int', 'uintptr', ] # :nodoc: -- cgit v1.2.1 From 004d0c83222113ff732101a3e25785ce0625bfa2 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:14:29 +0200 Subject: whitespace --- lib/coderay/scanners/go.rb | 82 +++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 4431ef2..d4a3598 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -1,12 +1,12 @@ module CodeRay module Scanners - + # Scanner for Go, copy from c class Go < Scanner - + register_for :go file_extension 'go' - + # http://golang.org/ref/spec#Keywords KEYWORDS = [ 'break', 'default', 'func', 'interface', 'select', @@ -15,7 +15,7 @@ module Scanners 'const', 'fallthrough', 'if', 'range', 'type', 'continue', 'for', 'import', 'return', 'var', ] # :nodoc: - + # http://golang.org/ref/spec#Types PREDEFINED_TYPES = [ 'bool', @@ -26,51 +26,51 @@ module Scanners 'byte', 'rune', 'uint', 'int', 'uintptr', ] # :nodoc: - + PREDEFINED_CONSTANTS = [ 'nil', 'iota', 'true', 'false', ] # :nodoc: - + DIRECTIVES = [ 'go_no_directive', # Seems no directive concept in Go? ] # :nodoc: - + IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). add(DIRECTIVES, :directive). add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: - + ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: - - protected - + + protected + def scan_tokens encoder, options - + state = :initial label_expected = true case_expected = false label_expected_before_preproc_line = nil in_preproc_line = false - + until eos? - + case state - + when :initial - + if match = scan(/ \s+ | \\\n /x) if in_preproc_line && match != "\\\n" && match.index(?\n) in_preproc_line = false label_expected = label_expected_before_preproc_line end encoder.text_token match, :space - + elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) encoder.text_token match, :comment - + elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) label_expected = match =~ /[;\{\}]/ if case_expected @@ -78,7 +78,7 @@ module Scanners case_expected = false end encoder.text_token match, :operator - + elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) kind = IDENT_KIND[match] if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) @@ -94,7 +94,7 @@ module Scanners end end encoder.text_token match, kind - + elsif match = scan(/L?"/) encoder.begin_group :string if match[0] == ?L @@ -107,41 +107,41 @@ module Scanners elsif match = scan(/ \# \s* if \s* 0 /x) match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? encoder.text_token match, :comment - + elsif match = scan(/#[ \t]*(\w*)/) encoder.text_token match, :preprocessor in_preproc_line = true label_expected_before_preproc_line = label_expected state = :include_expected if self[1] == 'include' - + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) label_expected = false encoder.text_token match, :char - + elsif match = scan(/\$/) encoder.text_token match, :ident - + elsif match = scan(/0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex - + elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) label_expected = false encoder.text_token match, :octal - + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) label_expected = false encoder.text_token match, :integer - + elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) label_expected = false encoder.text_token match, :float - + else encoder.text_token getch, :error - + end - + when :string if match = scan(/[^\\\n"]+/) encoder.text_token match, :content @@ -160,36 +160,36 @@ module Scanners else raise_inspect "else case \" reached; %p not handled." % peek(1), encoder end - + when :include_expected if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/) encoder.text_token match, :include state = :initial - + elsif match = scan(/\s+/) encoder.text_token match, :space state = :initial if match.index ?\n - + else state = :initial - + end - + else raise_inspect 'Unknown state', encoder - + end - + end - + if state == :string encoder.end_group :string end - + encoder end - + end - + end end -- cgit v1.2.1 From afa2be73c03d033056024354787abd66f2ff7fa0 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:05 +0200 Subject: add string as predefined type --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index d4a3598..dbf9595 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -23,7 +23,7 @@ module Scanners 'int8', 'int16', 'int32', 'int64', 'float32', 'float64', 'complex64', 'complex128', - 'byte', 'rune', + 'byte', 'rune', 'string', 'uint', 'int', 'uintptr', ] # :nodoc: -- cgit v1.2.1 From 7ef6f7783d788bf656a3c59600cd45cd5b694376 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:22 +0200 Subject: add support for raw strings in Go --- lib/coderay/scanners/go.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index dbf9595..c0c602d 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -103,7 +103,14 @@ module Scanners end encoder.text_token match, :delimiter state = :string - + + elsif match = scan(/ ` ([^`]+)? (`)? /x) + encoder.begin_group :shell + encoder.text_token '`', :delimiter + encoder.text_token self[1], :content if self[1] + encoder.text_token self[2], :delimiter if self[2] + encoder.end_group :shell + elsif match = scan(/ \# \s* if \s* 0 /x) match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? encoder.text_token match, :comment -- cgit v1.2.1 From a24c207a7e547408de0f6157d495b0dd2e8d5a40 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:36 +0200 Subject: fix empty token in Go scanner --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index c0c602d..76a0820 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -161,7 +161,7 @@ module Scanners encoder.text_token match, :char elsif match = scan(/ \\ | $ /x) encoder.end_group :string - encoder.text_token match, :error + encoder.text_token match, :error unless match.empty? state = :initial label_expected = false else -- cgit v1.2.1 From 4669b7086136647f2eece41a122fa204eb16e1e9 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:16:47 +0200 Subject: fix label_expected (test case?) --- lib/coderay/scanners/go.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 76a0820..23cbfe6 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -64,6 +64,7 @@ module Scanners if match = scan(/ \s+ | \\\n /x) if in_preproc_line && match != "\\\n" && match.index(?\n) in_preproc_line = false + case_expected = false label_expected = label_expected_before_preproc_line end encoder.text_token match, :space @@ -72,7 +73,6 @@ module Scanners encoder.text_token match, :comment elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) - label_expected = match =~ /[;\{\}]/ if case_expected label_expected = true if match == ':' case_expected = false @@ -83,6 +83,7 @@ module Scanners kind = IDENT_KIND[match] if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) kind = :label + label_expected = false match << matched else label_expected = false -- cgit v1.2.1 From c0d028010b081020352444078a685ee60bffe209 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:17:14 +0200 Subject: make predefined-type a bit more bright/blue --- lib/coderay/styles/alpha.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index ff85ecc..f4e9d7d 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -107,7 +107,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .operator { } .predefined { color:#369; font-weight:bold } .predefined-constant { color:#069 } -.predefined-type { color:#0a5; font-weight:bold } +.predefined-type { color:#0a8; font-weight:bold } .preprocessor { color:#579 } .pseudo-class { color:#00C; font-weight:bold } .regexp { background-color:hsla(300,100%,50%,0.06); } -- cgit v1.2.1 From 17946d7146390ee296264e66e6de31b6421e5231 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:29:02 +0200 Subject: changelog --- Changes.textile | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes.textile b/Changes.textile index 80f1d10..89b66c8 100644 --- a/Changes.textile +++ b/Changes.textile @@ -6,6 +6,7 @@ h2. Changes in 1.1 * New scanner: Lua [#21, #22, thanks to Quintus] * New scanner: Sass [#93] +* New scanner: Go [#28, thanks to Eric Guo] * New scanner: Taskpaper [#39, thanks to shimomura] * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly -- cgit v1.2.1 From 85275cf21e7d15459abc11fd76606bd7d38fb8b5 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:11:49 -0600 Subject: Go doesn't have a "f" suffix for floats like C. --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 49d24c2..afcfca5 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -141,7 +141,7 @@ module Scanners label_expected = false encoder.text_token match, :integer - elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) + elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float -- cgit v1.2.1 From cad9a00e28d61781d5a12a0556be7126eb790725 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:35:26 -0600 Subject: add imaginary numbers to Go scanner --- lib/coderay/scanners/go.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index afcfca5..04504ab 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -128,7 +128,11 @@ module Scanners elsif match = scan(/\$/) encoder.text_token match, :ident - + + elsif match = scan(/\d*(\.\d*)?([eE][+-]?\d+)?i/) + label_expected = false + encoder.text_token match, :imaginary + elsif match = scan(/0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex @@ -137,13 +141,13 @@ module Scanners label_expected = false encoder.text_token match, :octal - elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) - label_expected = false - encoder.text_token match, :integer - elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float + + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + label_expected = false + encoder.text_token match, :integer else encoder.text_token getch, :error -- cgit v1.2.1 From 4c877bf0eb655cf2c04a0efb24b4e5f83b97e160 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:43:50 -0600 Subject: predeclared identifiers http://golang.org/ref/spec#Predeclared_identifiers --- lib/coderay/scanners/go.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 04504ab..ae50aa4 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -31,6 +31,11 @@ module Scanners 'nil', 'iota', 'true', 'false', ] # :nodoc: + + PREDEFINED_FUNCTIONS = %w[ + append cap close complex copy delete imag len + make new panic print println real recover + ] # :nodoc: DIRECTIVES = [ 'go_no_directive', # Seems no directive concept in Go? @@ -40,7 +45,8 @@ module Scanners add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). add(DIRECTIVES, :directive). - add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: -- cgit v1.2.1 From dd9ec43a3ea4cfce8eb1e8dd38504a9d8f24df54 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:46:15 -0600 Subject: yup, no C-style directives (auto extern static) --- lib/coderay/scanners/go.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index ae50aa4..eb06fb0 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -37,14 +37,9 @@ module Scanners make new panic print println real recover ] # :nodoc: - DIRECTIVES = [ - 'go_no_directive', # Seems no directive concept in Go? - ] # :nodoc: - IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). - add(DIRECTIVES, :directive). add(PREDEFINED_CONSTANTS, :predefined_constant). add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: -- cgit v1.2.1 From ae1969b1809365c87a9cb485b671364c502236a0 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 13:00:17 -0600 Subject: revert trimming spaces --- lib/coderay/helpers/file_type.rb | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 6d928d6..a0f9fd6 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ module CodeRay name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ module CodeRay TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ module CodeRay if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ module CodeRay raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ module CodeRay end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -132,9 +132,9 @@ module CodeRay for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, @@ -144,7 +144,7 @@ module CodeRay 'Vagrantfile' => :ruby, 'Appraisals' => :ruby } - + end - + end -- cgit v1.2.1 From e75fecef1998836a4e209db4974b9cb9ec470e95 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 03:50:23 +0200 Subject: don't use -w flag in executable tests --- test/executable/suite.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/executable/suite.rb b/test/executable/suite.rb index ac0ff1d..997405c 100644 --- a/test/executable/suite.rb +++ b/test/executable/suite.rb @@ -18,9 +18,9 @@ class TestCodeRayExecutable < Test::Unit::TestCase EXE_COMMAND = if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success? # use Nailgun - "#{RUBY_COMMAND}--ng -w -I%s %s" + "#{RUBY_COMMAND}--ng -I%s %s" else - "#{RUBY_COMMAND} -w -I%s %s" + "#{RUBY_COMMAND} -I%s %s" end % [ROOT_DIR + 'lib', EXECUTABLE] def coderay args, options = {} -- cgit v1.2.1 From fb8c0dbfd065e19419b30213efb8176a1e968945 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 05:57:34 +0200 Subject: cleanup benchmark script (finally!) --- .gitignore | 1 - bench/bench.rb | 158 +++++++++------------------------------------- rake_tasks/benchmark.rake | 3 +- 3 files changed, 32 insertions(+), 130 deletions(-) diff --git a/.gitignore b/.gitignore index dd001c8..deed1a2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,4 @@ Gemfile.lock test/executable/source.rb.html test/executable/source.rb.json test/scanners -bench/test.div.html old-stuff diff --git a/bench/bench.rb b/bench/bench.rb index 1958c73..92f9d07 100644 --- a/bench/bench.rb +++ b/bench/bench.rb @@ -1,142 +1,46 @@ -# The most ugly test script I've ever written! -# Shame on me! - -require 'pathname' -require 'profile' if ARGV.include? '-p' - -MYDIR = File.dirname(__FILE__) -LIBDIR = Pathname.new(MYDIR).join('..', 'lib').cleanpath.to_s -$:.unshift MYDIR, LIBDIR +require 'benchmark' +$: << File.expand_path('../../lib', __FILE__) require 'coderay' -@size = ARGV.fetch(2, 100).to_i * 1000 - -lang = ARGV.fetch(0) do - puts <<-HELP -Usage: - ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream] - - SIZE defaults to 100 kB (= 100,000 bytes). - SIZE = 0 means the whole input. - --p generates a profile (slow! use with SIZE = 1) --o shows the output -stream enabled streaming mode - -Sorry for the strange interface. I will improve it in the next release. - HELP +if ARGV.include? '-h' + puts DATA.read exit end -format = ARGV.fetch(1, 'html').downcase - -$stream = ARGV.include? 'stream' -$optimize = ARGV.include? 'opt' -$style = ARGV.include? 'style' - -require 'benchmark' -require 'fileutils' +lang = ARGV.fetch(0, 'ruby') +data = nil +File.open(File.expand_path("../example.#{lang}", __FILE__), 'rb') { |f| data = f.read } +raise 'Example file is empty.' if data.empty? -if format == 'comp' - format = 'page' - begin - require 'syntax' - require 'syntax/convertors/html.rb' - rescue LoadError - puts 'Syntax no found!! (Try % gem install syntax)' - end -end +format = ARGV.fetch(1, 'html').downcase +encoder = CodeRay.encoder(format) -def here fn = nil - return MYDIR unless fn - File.join here, fn +size = ARGV.fetch(2, 1000).to_i * 1000 +unless size.zero? + data += data until data.size >= size + data = data[0, size] end +size = data.size +puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round] -n = ARGV.find { |a| a[/^N/] } -N = if n then n[/\d+/].to_i else 1 end -$filename = ARGV.include?('strange') ? 'strange' : 'example' - -Benchmark.bm(20) do |bm| -N.times do - - data = nil - File.open(here("#$filename." + lang), 'rb') { |f| data = f.read } - raise 'Example file is empty.' if data.empty? - unless @size.zero? - data += data until data.size >= @size - data = data[0, @size] - end - @size = data.size - - options = { - :tab_width => 2, - # :line_numbers => :inline, - :css => $style ? :style : :class, - } - $hl = CodeRay.encoder(format, options) - time = bm.report('CodeRay') do - if $stream || true - $o = $hl.encode(data, lang, options) - else - tokens = CodeRay.scan(data, lang) - tokens.optimize! if $optimize - $o = tokens.encode($hl) - end - end - $file_created = here('test.' + $hl.file_extension) - File.open($file_created, 'wb') do |f| - # f.write $o - end - - time_real = time.real - - puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000] - puts $o if ARGV.include? '-o' - -end +n = ARGV.fetch(3, 5).to_s[/\d+/].to_i +require 'profile' if ARGV.include? '-p' +n.times do |i| + time = Benchmark.realtime { encoder.encode(data, lang) } + puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0] end -puts "Files created: #$file_created" -STDIN.gets if ARGV.include? 'wait' +STDIN.gets if ARGV.include? '-w' __END__ -.ruby .normal {} -.ruby .comment { color: #005; font-style: italic; } -.ruby .keyword { color: #A00; font-weight: bold; } -.ruby .method { color: #077; } -.ruby .class { color: #074; } -.ruby .module { color: #050; } -.ruby .punct { color: #447; font-weight: bold; } -.ruby .symbol { color: #099; } -.ruby .string { color: #944; background: #FFE; } -.ruby .char { color: #F07; } -.ruby .ident { color: #004; } -.ruby .constant { color: #07F; } -.ruby .regex { color: #B66; background: #FEF; } -.ruby .number { color: #F99; } -.ruby .attribute { color: #7BB; } -.ruby .global { color: #7FB; } -.ruby .expr { color: #227; } -.ruby .escape { color: #277; } +Usage: + ruby bench.rb [lang] [format] [size in kB] [number of runs] -.xml .normal {} -.xml .namespace { color: #B66; font-weight: bold; } -.xml .tag { color: #F88; } -.xml .comment { color: #005; font-style: italic; } -.xml .punct { color: #447; font-weight: bold; } -.xml .string { color: #944; } -.xml .number { color: #F99; } -.xml .attribute { color: #BB7; } + - lang defaults to ruby. + - format defaults to html. + - size defaults to 1000 kB (= 1,000,000 bytes). 0 uses the whole example input. + - number of runs defaults to 5. -.yaml .normal {} -.yaml .document { font-weight: bold; color: #07F; } -.yaml .type { font-weight: bold; color: #05C; } -.yaml .key { color: #F88; } -.yaml .comment { color: #005; font-style: italic; } -.yaml .punct { color: #447; font-weight: bold; } -.yaml .string { color: #944; } -.yaml .number { color: #F99; } -.yaml .time { color: #F99; } -.yaml .date { color: #F99; } -.yaml .ref { color: #944; } -.yaml .anchor { color: #944; } +-h prints this help +-p generates a profile (slow, use with SIZE = 1) +-w waits after the benchmark (for debugging memory usw) diff --git a/rake_tasks/benchmark.rake b/rake_tasks/benchmark.rake index 040951b..2e38b57 100644 --- a/rake_tasks/benchmark.rake +++ b/rake_tasks/benchmark.rake @@ -1,7 +1,6 @@ desc 'Do a benchmark' task :benchmark do - ruby "-v" - ruby "-wIlib bench/bench.rb ruby div 3000 N5" + ruby 'bench/bench.rb ruby html 3000' end task :bench => :benchmark -- cgit v1.2.1 From dc57601571af8024700991b6a80129285a980e9e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 05:57:38 +0200 Subject: add Lint encoder; do we still need DebugLint? --- lib/coderay/encoders/debug_lint.rb | 12 ++++---- lib/coderay/encoders/lint.rb | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 lib/coderay/encoders/lint.rb diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 17a0795..2c14186 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -1,6 +1,8 @@ module CodeRay module Encoders + load :lint + # = Debug Lint Encoder # # Debug encoder with additional checks for: @@ -15,12 +17,8 @@ module Encoders register_for :debug_lint - InvalidTokenStream = Class.new StandardError - EmptyToken = Class.new InvalidTokenStream - IncorrectTokenGroupNesting = Class.new InvalidTokenStream - def text_token text, kind - raise EmptyToken, 'empty token' if text.empty? + raise Lint::EmptyToken, 'empty token' if text.empty? super end @@ -30,7 +28,7 @@ module Encoders end def end_group kind - raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind @opened.pop super end @@ -41,7 +39,7 @@ module Encoders end def end_line kind - raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind @opened.pop super end diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb new file mode 100644 index 0000000..4601e90 --- /dev/null +++ b/lib/coderay/encoders/lint.rb @@ -0,0 +1,57 @@ +module CodeRay +module Encoders + + # = Lint Encoder + # + # Checks for: + # + # - empty tokens + # - incorrect nesting + # + # It will raise an InvalidTokenStream exception when any of the above occurs. + # + # See also: Encoders::DebugLint + class Lint < Debug + + register_for :lint + + InvalidTokenStream = Class.new StandardError + EmptyToken = Class.new InvalidTokenStream + IncorrectTokenGroupNesting = Class.new InvalidTokenStream + + def text_token text, kind + raise EmptyToken, 'empty token' if text.empty? + end + + def begin_group kind + @opened << kind + end + + def end_group kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + def begin_line kind + @opened << kind + end + + def end_line kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + protected + + def setup options + @opened = [] + end + + def finish options + raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? + end + + end + +end +end -- cgit v1.2.1 From ab1bb26c29cc69a93da2d808ae8cd7b3cad6ea25 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 16:55:15 +0200 Subject: use File.expand_path instead of File.join --- lib/coderay.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay.rb b/lib/coderay.rb index 0c66f49..f759ed6 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -127,7 +127,7 @@ module CodeRay $CODERAY_DEBUG ||= false - CODERAY_PATH = File.join File.dirname(__FILE__), 'coderay' + CODERAY_PATH = File.expand_path('../coderay', __FILE__) # Assuming the path is a subpath of lib/coderay/ def self.coderay_path *path -- cgit v1.2.1 From 94e4bb3366537def28ed257fcf2b70a634711c9f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 12 Jul 2013 14:57:46 +0200 Subject: don't change value of objects you don't own --- lib/coderay/scanners/css.rb | 2 +- lib/coderay/scanners/sass.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 9ed4618..5977b9c 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -53,7 +53,7 @@ module Scanners end def scan_tokens encoder, options - states = Array(options[:state] || @state) + states = Array(options[:state] || @state).dup value_expected = @value_expected until eos? diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index e20bebe..7ba9bf5 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -19,7 +19,7 @@ module Scanners end def scan_tokens encoder, options - states = Array(options[:state] || @state) + states = Array(options[:state] || @state).dup string_delimiter = nil until eos? @@ -119,7 +119,7 @@ module Scanners else #:nocov: - raise_inspect 'Unknown state', encoder + raise_inspect 'Unknown state: %p' % [states.last], encoder #:nocov: end @@ -215,7 +215,7 @@ module Scanners end if options[:keep_state] - @state = states + @state = states.dup end while state = states.pop -- cgit v1.2.1 From 990ed25fd1bf350dabae4bed031e07ee485beb79 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:20:59 +0200 Subject: split '" string states in Sass scanner (edge case) --- lib/coderay/scanners/sass.rb | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 7ba9bf5..85b4711 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -7,11 +7,6 @@ module Scanners register_for :sass file_extension 'sass' - STRING_CONTENT_PATTERN = { - "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, - '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, - } - protected def setup @@ -20,7 +15,8 @@ module Scanners def scan_tokens encoder, options states = Array(options[:state] || @state).dup - string_delimiter = nil + + encoder.begin_group :string if states.last == :sqstring || states.last == :dqstring until eos? @@ -91,24 +87,23 @@ module Scanners next end - when :string - if match = scan(STRING_CONTENT_PATTERN[string_delimiter]) + when :sqstring, :dqstring + if match = scan(states.last == :sqstring ? /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o : /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o) encoder.text_token match, :content elsif match = scan(/['"]/) encoder.text_token match, :delimiter encoder.end_group :string - string_delimiter = nil states.pop elsif match = scan(/#\{/) encoder.begin_group :inline encoder.text_token match, :inline_delimiter states.push :sass_inline elsif match = scan(/ \\ | $ /x) - encoder.end_group :string + encoder.end_group states.last encoder.text_token match, :error unless match.empty? states.pop else - raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder + raise_inspect "else case #{states.last} reached; %p not handled." % peek(1), encoder end when :include @@ -157,15 +152,15 @@ module Scanners elsif match = scan(/['"]/) encoder.begin_group :string - string_delimiter = match encoder.text_token match, :delimiter if states.include? :sass_inline - content = scan_until(/(?=#{string_delimiter}|\}|\z)/) + # no nesting, just scan the string until delimiter + content = scan_until(/(?=#{match}|\}|\z)/) encoder.text_token content, :content unless content.empty? - encoder.text_token string_delimiter, :delimiter if scan(/#{string_delimiter}/) + encoder.text_token match, :delimiter if scan(/#{match}/) encoder.end_group :string else - states.push :string + states.push match == "'" ? :sqstring : :dqstring end elsif match = scan(/#{RE::Function}/o) @@ -221,7 +216,7 @@ module Scanners while state = states.pop if state == :sass_inline encoder.end_group :inline - elsif state == :string + elsif state == :sqstring || state == :dqstring encoder.end_group :string end end -- cgit v1.2.1 From def7e09db1963368e20bfd53c72532d6a631e0e8 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:21:42 +0200 Subject: fix #139: don't scan for :include after eos --- lib/coderay/scanners/sass.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 85b4711..1bbd534 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -209,6 +209,8 @@ module Scanners end + states.pop if states.last == :include + if options[:keep_state] @state = states.dup end -- cgit v1.2.1 From 62a0be9509f8814902a4a97df4ad84913728d059 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:34:17 +0200 Subject: fix #143 (Sass scanner key vs tag heuristic) --- lib/coderay/scanners/sass.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 1bbd534..e3296b9 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -44,7 +44,7 @@ module Scanners elsif case states.last when :initial, :media, :sass_inline if match = scan(/(?>#{RE::Ident})(?!\()/ox) - encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag) + encoder.text_token match, value_expected ? :value : (check(/.*:(?![a-z])/) ? :key : :tag) next elsif !value_expected && (match = scan(/\*/)) encoder.text_token match, :tag -- cgit v1.2.1 From 70c9ba896e1bba5ac727fb6fdfc3ba94510e652d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:34:41 +0200 Subject: fix CSS scanner for things like "nth-child(2n)" --- lib/coderay/scanners/css.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 5977b9c..55d5239 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -25,7 +25,7 @@ module Scanners HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/ - Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)/ + Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)n?/ Name = /#{NMChar}+/ Ident = /-?#{NMStart}#{NMChar}*/ AtKeyword = /@#{Ident}/ -- cgit v1.2.1 From 028ea15c5ee983bfa1b045081e68c2c178d5f126 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:36:06 +0200 Subject: changelog --- Changes.textile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index 80f1d10..7ea4bfe 100644 --- a/Changes.textile +++ b/Changes.textile @@ -12,7 +12,7 @@ h2. Changes in 1.1 * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] * HTML scanner displays style tags and attributes now [#145] * Remove double-click toggle handler from HTML table output -* Fixes to CSS scanner (floats, pseudoclasses) +* Fixes to CSS scanner (floats, pseudoclasses, nth-child) [#143] * Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] * Added @:keep_state@ functionality to more scanners [#116] * CSS scanner uses @:id@ and @:tag@ now [#27] -- cgit v1.2.1 From ee72fe95fcfca2848c3cff51b13ee78b662ba50f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:56:51 +0200 Subject: use bundler rake tasks --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index ba6c34e..5577009 100644 --- a/Rakefile +++ b/Rakefile @@ -1,3 +1,5 @@ +require 'bundler/gem_tasks' + $:.unshift File.dirname(__FILE__) unless $:.include? '.' ROOT = '.' -- cgit v1.2.1 From 4c2486353a2e5e7e393ac97e556e4e29cac6bcc3 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:29:49 +0200 Subject: create nathany for Go scanner, too --- Changes.textile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index cc401e9..58427aa 100644 --- a/Changes.textile +++ b/Changes.textile @@ -6,7 +6,7 @@ h2. Changes in 1.1 * New scanner: Lua [#21, #22, thanks to Quintus] * New scanner: Sass [#93] -* New scanner: Go [#28, thanks to Eric Guo] +* New scanner: Go [#28, thanks to Eric Guo and Nathan Youngman] * New scanner: Taskpaper [#39, thanks to shimomura] * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly -- cgit v1.2.1 From bbe4d72ba785f1bd6fd703d63b096a907da1b09f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:31:34 +0200 Subject: tweak numeral tokens handling (#147) --- lib/coderay/scanners/go.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index eb06fb0..938da9d 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -31,7 +31,7 @@ module Scanners 'nil', 'iota', 'true', 'false', ] # :nodoc: - + PREDEFINED_FUNCTIONS = %w[ append cap close complex copy delete imag len make new panic print println real recover @@ -73,7 +73,7 @@ module Scanners elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) encoder.text_token match, :comment - elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) + elsif match = scan(/ ?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) if case_expected label_expected = true if match == ':' case_expected = false @@ -129,24 +129,24 @@ module Scanners elsif match = scan(/\$/) encoder.text_token match, :ident - - elsif match = scan(/\d*(\.\d*)?([eE][+-]?\d+)?i/) + + elsif match = scan(/-?\d*(\.\d*)?([eE][+-]?\d+)?i/) label_expected = false encoder.text_token match, :imaginary - - elsif match = scan(/0[xX][0-9A-Fa-f]+/) + + elsif match = scan(/-?0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex - elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) + elsif match = scan(/-?(?:0[0-7]+)(?![89.eEfF])/) label_expected = false encoder.text_token match, :octal - elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) + elsif match = scan(/-?(?:\d*\.\d+|\d+\.)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float - - elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + + elsif match = scan(/-?(?:\d+)(?![.eEfF])L?L?/) label_expected = false encoder.text_token match, :integer -- cgit v1.2.1 From 6dd14ef018ee4417771504117819121bd4b8520d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:34:10 +0200 Subject: be a bit more graceful with buggy Go strings --- lib/coderay/scanners/go.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 938da9d..5034adc 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -165,9 +165,10 @@ module Scanners label_expected = false elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) encoder.text_token match, :char - elsif match = scan(/ \\ | $ /x) + elsif match = scan(/ \\ /x) + encoder.text_token match, :error + elsif match = scan(/$/) encoder.end_group :string - encoder.text_token match, :error unless match.empty? state = :initial label_expected = false else -- cgit v1.2.1 From 82864efabda9dc17fa6a28b0740ffc8f58706126 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:36:27 +0200 Subject: allow unicode characters in char literals --- lib/coderay/scanners/go.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 5034adc..59473f6 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -44,7 +44,7 @@ module Scanners add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: protected @@ -123,7 +123,7 @@ module Scanners label_expected_before_preproc_line = label_expected state = :include_expected if self[1] == 'include' - elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) )? '? /ox) label_expected = false encoder.text_token match, :char -- cgit v1.2.1