diff options
Diffstat (limited to 'lib/coderay')
-rw-r--r-- | lib/coderay/encoders/debug_lint.rb | 3 | ||||
-rw-r--r-- | lib/coderay/encoders/html.rb | 24 | ||||
-rw-r--r-- | lib/coderay/encoders/lint.rb | 4 | ||||
-rw-r--r-- | lib/coderay/helpers/file_type.rb | 4 | ||||
-rw-r--r-- | lib/coderay/helpers/plugin.rb | 22 | ||||
-rw-r--r-- | lib/coderay/scanners/debug.rb | 26 | ||||
-rw-r--r-- | lib/coderay/scanners/diff.rb | 2 | ||||
-rw-r--r-- | lib/coderay/scanners/go.rb | 1 | ||||
-rw-r--r-- | lib/coderay/scanners/raydebug.rb | 31 | ||||
-rw-r--r-- | lib/coderay/scanners/ruby.rb | 19 | ||||
-rw-r--r-- | lib/coderay/scanners/ruby/patterns.rb | 7 | ||||
-rw-r--r-- | lib/coderay/scanners/ruby/string_state.rb | 22 | ||||
-rw-r--r-- | lib/coderay/scanners/sql.rb | 42 | ||||
-rw-r--r-- | lib/coderay/styles/alpha.rb | 2 | ||||
-rwxr-xr-x | lib/coderay/token_kinds.rb | 8 | ||||
-rw-r--r-- | lib/coderay/version.rb | 2 |
16 files changed, 127 insertions, 92 deletions
diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 2c14186..a4eba2c 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -18,7 +18,8 @@ module Encoders register_for :debug_lint def text_token text, kind - raise Lint::EmptyToken, 'empty token' if text.empty? + raise Lint::EmptyToken, 'empty token for %p' % [kind] if text.empty? + raise Lint::UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind super end diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 1040d20..b6db051 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -25,7 +25,8 @@ module Encoders # == Options # # === :tab_width - # Convert \t characters to +n+ spaces (a number.) + # Convert \t characters to +n+ spaces (a number or false.) + # false will keep tab characters untouched. # # Default: 8 # @@ -180,7 +181,7 @@ module Encoders @break_lines = (options[:break_lines] == true) - @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width]) + @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => options[:tab_width] ? ' ' * options[:tab_width] : "\t") @opened = [] @last_opened = nil @@ -197,11 +198,13 @@ module Encoders @last_opened = nil end - if options[:line_numbers] - Numbering.number! @out, options[:line_numbers], options + if @out.respond_to? :to_str + if options[:line_numbers] + Numbering.number! @out, options[:line_numbers], options + end + @out = Output.wrap_string_in @out, options[:wrap], @css if options[:wrap] + @out = @out.sub(/(<title>)(<\/title>)/) { $1 + options[:title] + $2 } if options[:title] end - @out = Output.wrap_string_in @out, options[:wrap], @css if options[:wrap] - @out = @out.sub(/(<title>)(<\/title>)/) { $1 + options[:title] + $2 } if options[:title] if defined?(@real_out) && @real_out @real_out << @out @@ -287,7 +290,7 @@ module Encoders def make_span_for_kinds method, hint Hash.new do |h, kinds| - h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin + begin css_class = css_class_for_kinds(kinds) title = HTML.token_path_to_hint hint, kinds if hint @@ -299,6 +302,9 @@ module Encoders "<span#{title}#{" class=\"#{css_class}\"" if css_class}>" end end + end.tap do |span| + h.clear if h.size >= 100 + h[kinds] = span end end end @@ -311,8 +317,8 @@ module Encoders def break_lines text, style reopen = '' - @opened.each_with_index do |k, index| - reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>') + @opened.each_with_index do |kind, index| + reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '<span>') end text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}") end diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb index 4601e90..88c8bd1 100644 --- a/lib/coderay/encoders/lint.rb +++ b/lib/coderay/encoders/lint.rb @@ -17,10 +17,12 @@ module Encoders InvalidTokenStream = Class.new StandardError EmptyToken = Class.new InvalidTokenStream + UnknownTokenKind = Class.new InvalidTokenStream IncorrectTokenGroupNesting = Class.new InvalidTokenStream def text_token text, kind - raise EmptyToken, 'empty token' if text.empty? + raise EmptyToken, 'empty token for %p' % [kind] if text.empty? + raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind end def begin_group kind diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 5e3a1e7..7de34d5 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -38,7 +38,7 @@ module CodeRay (TypeFromExt[ext2.downcase] if ext2) || TypeFromName[name] || TypeFromName[name.downcase] - type ||= shebang(filename) if read_shebang + type ||= type_from_shebang(filename) if read_shebang type end @@ -63,7 +63,7 @@ module CodeRay protected - def shebang filename + def type_from_shebang filename return unless File.exist? filename File.open filename, 'r' do |f| if first_line = f.gets diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb index d14c5a9..9a724ff 100644 --- a/lib/coderay/helpers/plugin.rb +++ b/lib/coderay/helpers/plugin.rb @@ -30,7 +30,7 @@ module CodeRay # * a file could not be found # * the requested Plugin is not registered PluginNotFound = Class.new LoadError - HostNotFound = Class.new LoadError + HostNotFound = Class.new LoadError PLUGIN_HOSTS = [] PLUGIN_HOSTS_BY_ID = {} # dummy hash @@ -49,8 +49,8 @@ module CodeRay def [] id, *args, &blk plugin = validate_id(id) begin - plugin = plugin_hash.[] plugin, *args, &blk - end while plugin.is_a? Symbol + plugin = plugin_hash.[](plugin, *args, &blk) + end while plugin.is_a? String plugin end @@ -95,7 +95,7 @@ module CodeRay def map hash for from, to in hash from = validate_id from - to = validate_id to + to = validate_id to plugin_hash[from] = to unless plugin_hash.has_key? from end end @@ -197,22 +197,22 @@ module CodeRay File.join plugin_path, "#{plugin_id}.rb" end - # Converts +id+ to a Symbol if it is a String, - # or returns +id+ if it already is a Symbol. + # Converts +id+ to a valid plugin ID String, or returns +nil+. # # Raises +ArgumentError+ for all other objects, or if the # given String includes non-alphanumeric characters (\W). def validate_id id - if id.is_a? Symbol or id.nil? - id - elsif id.is_a? String + case id + when Symbol + id.to_s + when String if id[/\w+/] == id - id.downcase.to_sym + id.downcase else raise ArgumentError, "Invalid id given: #{id}" end else - raise ArgumentError, "String or Symbol expected, but #{id.class} given." + raise ArgumentError, "Symbol or String expected, but #{id.class} given." end end diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb index 566bfa7..83ede9a 100644 --- a/lib/coderay/scanners/debug.rb +++ b/lib/coderay/scanners/debug.rb @@ -1,9 +1,11 @@ +require 'set' + module CodeRay module Scanners # = Debug Scanner # - # Interprets the output of the Encoders::Debug encoder. + # Interprets the output of the Encoders::Debug encoder (basically the inverse function). class Debug < Scanner register_for :debug @@ -11,6 +13,11 @@ module Scanners protected + def setup + super + @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set + end + def scan_tokens encoder, options opened_tokens = [] @@ -21,16 +28,19 @@ module Scanners encoder.text_token match, :space elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x) - kind = self[1].to_sym - match = self[2].gsub(/\\(.)/m, '\1') - unless TokenKinds.has_key? kind - kind = :error - match = matched + if @known_token_kinds.include? self[1] + encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym + else + encoder.text_token matched, :unknown end - encoder.text_token match, kind elsif match = scan(/ (\w+) ([<\[]) /x) - kind = self[1].to_sym + if @known_token_kinds.include? self[1] + kind = self[1].to_sym + else + kind = :unknown + end + opened_tokens << kind case self[2] when '<' diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index fd1aed6..74a6c27 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -100,7 +100,7 @@ module Scanners next elsif match = scan(/-/) deleted_lines_count += 1 - if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/) + if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && changed_lines_count <= 100_000 && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/) deleted_lines = Array.new(changed_lines_count) { |i| skip(/\n\-/) if i > 0; scan(/.*/) } inserted_lines = Array.new(changed_lines_count) { |i| skip(/\n\+/) ; scan(/.*/) } diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 59473f6..99fdd63 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -1,7 +1,6 @@ module CodeRay module Scanners - # Scanner for Go, copy from c class Go < Scanner register_for :go diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index d39d962..1effdc8 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -1,9 +1,11 @@ +require 'set' + module CodeRay module Scanners - # = Debug Scanner + # = Raydebug Scanner # - # Parses the output of the Encoders::Debug encoder. + # Highlights the output of the Encoders::Debug encoder. class Raydebug < Scanner register_for :raydebug @@ -12,6 +14,11 @@ module Scanners protected + def setup + super + @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set + end + def scan_tokens encoder, options opened_tokens = [] @@ -26,20 +33,22 @@ module Scanners encoder.text_token kind, :class encoder.text_token '(', :operator match = self[2] - encoder.text_token match, kind.to_sym unless match.empty? + unless match.empty? + if @known_token_kinds.include? kind + encoder.text_token match, kind.to_sym + else + encoder.text_token match, :plain + end + end encoder.text_token match, :operator if match = scan(/\)/) elsif match = scan(/ (\w+) ([<\[]) /x) - kind = self[1] - case self[2] - when '<' - encoder.text_token kind, :class - when '[' - encoder.text_token kind, :class + encoder.text_token self[1], :class + if @known_token_kinds.include? self[1] + kind = self[1].to_sym else - raise 'CodeRay bug: This case should not be reached.' + kind = :unknown end - kind = kind.to_sym opened_tokens << kind encoder.begin_group kind encoder.text_token self[2], :operator diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index 80165ca..5b8de42 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -164,15 +164,19 @@ module Scanners end elsif match = scan(/ ' (?:(?>[^'\\]*) ')? | " (?:(?>[^"\\\#]*) ")? /mx) - encoder.begin_group :string if match.size == 1 + kind = check(self.class::StringState.simple_key_pattern(match)) ? :key : :string + encoder.begin_group kind encoder.text_token match, :delimiter - state = self.class::StringState.new :string, match == '"', match # important for streaming + state = self.class::StringState.new kind, match == '"', match # important for streaming else + kind = value_expected == true && scan(/:/) ? :key : :string + encoder.begin_group kind encoder.text_token match[0,1], :delimiter encoder.text_token match[1..-2], :content if match.size > 2 encoder.text_token match[-1,1], :delimiter - encoder.end_group :string + encoder.end_group kind + encoder.text_token ':', :operator if kind == :key value_expected = false end @@ -191,11 +195,14 @@ module Scanners encoder.text_token match, :error method_call_expected = false else - encoder.text_token match, self[1] ? :float : :integer # TODO: send :hex/:octal/:binary + kind = self[1] ? :float : :integer # TODO: send :hex/:octal/:binary + match << 'r' if match !~ /e/i && scan(/r/) + match << 'i' if scan(/i/) + encoder.text_token match, kind end value_expected = false - elsif match = scan(/ [-+!~^\/]=? | [:;] | [*|&]{1,2}=? | >>? /x) + elsif match = scan(/ [-+!~^\/]=? | [:;] | &\. | [*|&]{1,2}=? | >>? /x) value_expected = true encoder.text_token match, :operator @@ -208,7 +215,7 @@ module Scanners encoder.end_group kind heredocs ||= [] # create heredocs if empty heredocs << self.class::StringState.new(kind, quote != "'", delim, - self[1] == '-' ? :indented : :linestart) + self[1] ? :indented : :linestart) value_expected = false elsif value_expected && match = scan(/#{patterns::FANCY_STRING_START}/o) diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb index ed071d2..e5a156d 100644 --- a/lib/coderay/scanners/ruby/patterns.rb +++ b/lib/coderay/scanners/ruby/patterns.rb @@ -114,7 +114,7 @@ module Scanners # NOTE: This is not completely correct, but # nobody needs heredoc delimiters ending with \n. HEREDOC_OPEN = / - << (-)? # $1 = float + << ([-~])? # $1 = float (?: ( [A-Za-z_0-9]+ ) # $2 = delim | @@ -157,13 +157,16 @@ module Scanners yield ]) - FANCY_STRING_START = / % ( [QqrsWwx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x + FANCY_STRING_START = / % ( [iIqQrswWx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x FANCY_STRING_KIND = Hash.new(:string).merge({ + 'i' => :symbol, + 'I' => :symbol, 'r' => :regexp, 's' => :symbol, 'x' => :shell, }) FANCY_STRING_INTERPRETED = Hash.new(true).merge({ + 'i' => false, 'q' => false, 's' => false, 'w' => false, diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb index 2f398d1..95f1e83 100644 --- a/lib/coderay/scanners/ruby/string_state.rb +++ b/lib/coderay/scanners/ruby/string_state.rb @@ -16,7 +16,6 @@ module Scanners STRING_PATTERN = Hash.new do |h, k| delim, interpreted = *k - # delim = delim.dup # workaround for old Ruby delim_pattern = Regexp.escape(delim) if closing_paren = CLOSING_PAREN[delim] delim_pattern << Regexp.escape(closing_paren) @@ -29,12 +28,21 @@ module Scanners # '| [|?*+(){}\[\].^$]' # end - h[k] = - if interpreted && delim != '#' - / (?= [#{delim_pattern}] | \# [{$@] ) /mx - else - / (?= [#{delim_pattern}] ) /mx - end + if interpreted && delim != '#' + / (?= [#{delim_pattern}] | \# [{$@] ) /mx + else + / (?= [#{delim_pattern}] ) /mx + end.tap do |pattern| + h[k] = pattern if (delim.respond_to?(:ord) ? delim.ord : delim[0]) < 256 + end + end + + def self.simple_key_pattern delim + if delim == "'" + / (?> (?: [^\\']+ | \\. )* ) ' : /mx + else + / (?> (?: [^\\"\#]+ | \\. | \#\$[\\"] | \#\{[^\{\}]+\} | \#(?!\{) )* ) " : /mx + end end def initialize kind, interpreted, delim, heredoc = false diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb index 93aeaf3..7d57f77 100644 --- a/lib/coderay/scanners/sql.rb +++ b/lib/coderay/scanners/sql.rb @@ -57,6 +57,12 @@ module Scanners STRING_PREFIXES = /[xnb]|_\w+/i + STRING_CONTENT_PATTERN = { + '"' => / (?: [^\\"] | "" )+ /x, + "'" => / (?: [^\\'] | '' )+ /x, + '`' => / (?: [^\\`] | `` )+ /x, + } + def scan_tokens encoder, options state = :initial @@ -90,7 +96,7 @@ module Scanners state = :string encoder.text_token match, :delimiter - elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x) + elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9\$]* /x) encoder.text_token match, name_expected ? :ident : (match[0] == ?@ ? :variable : IDENT_KIND[match]) name_expected = false @@ -115,40 +121,26 @@ module Scanners end elsif state == :string - if match = scan(/[^\\"'`]+/) - string_content << match - next + if match = scan(STRING_CONTENT_PATTERN[string_type]) + encoder.text_token match, :content elsif match = scan(/["'`]/) if string_type == match if peek(1) == string_type # doubling means escape - string_content << string_type << getch - next - end - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' + encoder.text_token match + getch, :content + else + encoder.text_token match, :delimiter + encoder.end_group :string + state = :initial + string_type = nil end - encoder.text_token match, :delimiter - encoder.end_group :string - state = :initial - string_type = nil else - string_content << match + encoder.text_token match, :content end elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' - end encoder.text_token match, :char elsif match = scan(/ \\ . /mox) - string_content << match - next + encoder.text_token match, :content elsif match = scan(/ \\ | $ /x) - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' - end encoder.text_token match, :error unless match.empty? encoder.end_group :string state = :initial diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index f4e9d7d..d304dc4 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -125,7 +125,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .string .modifier { color: #E40 } .symbol { color:#A60 } .symbol .content { color:#A60 } -.symbol .delimiter { color:#630 } +.symbol .delimiter { color:#740 } .tag { color:#070; font-weight:bold } .type { color:#339; font-weight:bold } .value { color: #088 } diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 9137a49..f911862 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,10 +1,7 @@ module CodeRay # A Hash of all known token kinds and their associated CSS classes. - TokenKinds = Hash.new do |h, k| - warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG - false - end + TokenKinds = Hash.new(false) # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity @@ -83,5 +80,6 @@ module CodeRay :plain => false # almost all scanners ) - TokenKinds[:method] = TokenKinds[:function] + TokenKinds[:method] = TokenKinds[:function] + TokenKinds[:unknown] = TokenKinds[:plain] end diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4b4f085..7ea3f70 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' + VERSION = '1.1.1' end |