Merge branch 'master' into ruby-skip

15 files changed, 100 insertions, 67 deletions

diff --git a/Changes.textile b/Changes.textile
index 58427aa..cd4443a 100644
--- a/Changes.textile
+++ b/Changes.textile
@@ -10,6 +10,7 @@ h2. Changes in 1.1
 * New scanner: Taskpaper [#39, thanks to shimomura]
 * Diff scanner: Highlight inline changes in multi-line changes [#99]
 * JavaScript scanner: Highlight multi-line comments in diff correctly
+* Ruby scanner: Accept %i and %I symbol lists (Ruby 2.0) [thanks to Nathan Youngman]
 * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126]
 * HTML scanner displays style tags and attributes now [#145]
 * Remove double-click toggle handler from HTML table output
@@ -22,11 +23,17 @@ h2. Changes in 1.1
 * @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner]
 * New token type @:id@ for CSS/Sass [#27]
 * New token type @:done@ for Taskpaper [#39]
-* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany]
+* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman]
+* New token type @:unknown@ for Debug scanner
 * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
 * Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05]
 * Fixed @:docstring@ token type style
+* Fixed several problems related to Hash caches and dynamic Symbol creation that might have been exploited by an attacker [#148]
+* @PluginHost@ now works with Strings instead of Symbols internally (to avoid using @#to_sym@)
+* The @Debug@ scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input)
+* The @Raydebug@ scanner highlights unknown token kinds as @:plain@
 * @Plugin@ does not warn about fallback when default is defined
+* Fixed @HTML@ encoder when output is a StringIO (eg. when using @-HTML@)
 * @HTML@ encoder will not warn about unclosed token groups at the end of the stream
 * @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now
 * @Debug@ encoder will not warn about errors in the token stream
diff --git a/Rakefile b/Rakefile
index 5577009..c9b1e8a 100644
--- a/Rakefile
+++ b/Rakefile
@@ -34,4 +34,4 @@ else
     rd.rdoc_dir = 'doc'
   end
   
-end
\ No newline at end of file
+end
diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb
index 2c14186..a4eba2c 100644
--- a/lib/coderay/encoders/debug_lint.rb
+++ b/lib/coderay/encoders/debug_lint.rb
@@ -18,7 +18,8 @@ module Encoders
     register_for :debug_lint
     
     def text_token text, kind
-      raise Lint::EmptyToken, 'empty token' if text.empty?
+      raise Lint::EmptyToken,       'empty token for %p' % [kind] if text.empty?
+      raise Lint::UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind
       super
     end
     
diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index 20f2409..d2ebb5a 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -197,13 +197,15 @@ module Encoders
         @last_opened = nil
       end
       
-      @out.extend Output
-      @out.css = @css
-      if options[:line_numbers]
-        Numbering.number! @out, options[:line_numbers], options
+      if @out.respond_to? :to_str
+        @out.extend Output
+        @out.css = @css
+        if options[:line_numbers]
+          Numbering.number! @out, options[:line_numbers], options
+        end
+        @out.wrap! options[:wrap]
+        @out.apply_title! options[:title]
       end
-      @out.wrap! options[:wrap]
-      @out.apply_title! options[:title]
       
       if defined?(@real_out) && @real_out
         @real_out << @out
@@ -285,7 +287,7 @@ module Encoders
     
     def make_span_for_kinds method, hint
       Hash.new do |h, kinds|
-        h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin
+        begin
           css_class = css_class_for_kinds(kinds)
           title     = HTML.token_path_to_hint hint, kinds if hint
           
@@ -297,6 +299,9 @@ module Encoders
               "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
             end
           end
+        end.tap do |span|
+          h.clear if h.size >= 100
+          h[kinds] = span
         end
       end
     end
@@ -309,8 +314,8 @@ module Encoders
     
     def break_lines text, style
       reopen = ''
-      @opened.each_with_index do |k, index|
-        reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
+      @opened.each_with_index do |kind, index|
+        reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '<span>')
       end
       text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
     end
diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb
index 4601e90..88c8bd1 100644
--- a/lib/coderay/encoders/lint.rb
+++ b/lib/coderay/encoders/lint.rb
@@ -17,10 +17,12 @@ module Encoders
     
     InvalidTokenStream         = Class.new StandardError
     EmptyToken                 = Class.new InvalidTokenStream
+    UnknownTokenKind           = Class.new InvalidTokenStream
     IncorrectTokenGroupNesting = Class.new InvalidTokenStream
     
     def text_token text, kind
-      raise EmptyToken, 'empty token' if text.empty?
+      raise EmptyToken,       'empty token for %p' % [kind] if text.empty?
+      raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind
     end
     
     def begin_group kind
diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb
index 5e3a1e7..7de34d5 100644
--- a/lib/coderay/helpers/file_type.rb
+++ b/lib/coderay/helpers/file_type.rb
@@ -38,7 +38,7 @@ module CodeRay
           (TypeFromExt[ext2.downcase] if ext2) ||
           TypeFromName[name] ||
           TypeFromName[name.downcase]
-        type ||= shebang(filename) if read_shebang
+        type ||= type_from_shebang(filename) if read_shebang
         
         type
       end
@@ -63,7 +63,7 @@ module CodeRay
       
     protected
       
-      def shebang filename
+      def type_from_shebang filename
         return unless File.exist? filename
         File.open filename, 'r' do |f|
           if first_line = f.gets
diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb
index d14c5a9..9a724ff 100644
--- a/lib/coderay/helpers/plugin.rb
+++ b/lib/coderay/helpers/plugin.rb
@@ -30,7 +30,7 @@ module CodeRay
     # * a file could not be found
     # * the requested Plugin is not registered
     PluginNotFound = Class.new LoadError
-    HostNotFound = Class.new LoadError
+    HostNotFound   = Class.new LoadError
     
     PLUGIN_HOSTS = []
     PLUGIN_HOSTS_BY_ID = {}  # dummy hash
@@ -49,8 +49,8 @@ module CodeRay
     def [] id, *args, &blk
       plugin = validate_id(id)
       begin
-        plugin = plugin_hash.[] plugin, *args, &blk
-      end while plugin.is_a? Symbol
+        plugin = plugin_hash.[](plugin, *args, &blk)
+      end while plugin.is_a? String
       plugin
     end
     
@@ -95,7 +95,7 @@ module CodeRay
     def map hash
       for from, to in hash
         from = validate_id from
-        to = validate_id to
+        to   = validate_id to
         plugin_hash[from] = to unless plugin_hash.has_key? from
       end
     end
@@ -197,22 +197,22 @@ module CodeRay
       File.join plugin_path, "#{plugin_id}.rb"
     end
     
-    # Converts +id+ to a Symbol if it is a String,
-    # or returns +id+ if it already is a Symbol.
+    # Converts +id+ to a valid plugin ID String, or returns +nil+.
     #
     # Raises +ArgumentError+ for all other objects, or if the
     # given String includes non-alphanumeric characters (\W).
     def validate_id id
-      if id.is_a? Symbol or id.nil?
-        id
-      elsif id.is_a? String
+      case id
+      when Symbol
+        id.to_s
+      when String
         if id[/\w+/] == id
-          id.downcase.to_sym
+          id.downcase
         else
           raise ArgumentError, "Invalid id given: #{id}"
         end
       else
-        raise ArgumentError, "String or Symbol expected, but #{id.class} given."
+        raise ArgumentError, "Symbol or String expected, but #{id.class} given."
       end
     end
     
diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb
index 566bfa7..83ede9a 100644
--- a/lib/coderay/scanners/debug.rb
+++ b/lib/coderay/scanners/debug.rb
@@ -1,9 +1,11 @@
+require 'set'
+
 module CodeRay
 module Scanners
   
   # = Debug Scanner
   # 
-  # Interprets the output of the Encoders::Debug encoder.
+  # Interprets the output of the Encoders::Debug encoder (basically the inverse function).
   class Debug < Scanner
     
     register_for :debug
@@ -11,6 +13,11 @@ module Scanners
     
   protected
     
+    def setup
+      super
+      @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
+    end
+    
     def scan_tokens encoder, options
       
       opened_tokens = []
@@ -21,16 +28,19 @@ module Scanners
           encoder.text_token match, :space
           
         elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
-          kind = self[1].to_sym
-          match = self[2].gsub(/\\(.)/m, '\1')
-          unless TokenKinds.has_key? kind
-            kind = :error
-            match = matched
+          if @known_token_kinds.include? self[1]
+            encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym
+          else
+            encoder.text_token matched, :unknown
           end
-          encoder.text_token match, kind
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
-          kind = self[1].to_sym
+          if @known_token_kinds.include? self[1]
+            kind = self[1].to_sym
+          else
+            kind = :unknown
+          end
+          
           opened_tokens << kind
           case self[2]
           when '<'
diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb
index d39d962..1effdc8 100644
--- a/lib/coderay/scanners/raydebug.rb
+++ b/lib/coderay/scanners/raydebug.rb
@@ -1,9 +1,11 @@
+require 'set'
+
 module CodeRay
 module Scanners
   
-  # = Debug Scanner
+  # = Raydebug Scanner
   # 
-  # Parses the output of the Encoders::Debug encoder.
+  # Highlights the output of the Encoders::Debug encoder.
   class Raydebug < Scanner
     
     register_for :raydebug
@@ -12,6 +14,11 @@ module Scanners
     
   protected
     
+    def setup
+      super
+      @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
+    end
+    
     def scan_tokens encoder, options
       
       opened_tokens = []
@@ -26,20 +33,22 @@ module Scanners
           encoder.text_token kind, :class
           encoder.text_token '(', :operator
           match = self[2]
-          encoder.text_token match, kind.to_sym unless match.empty?
+          unless match.empty?
+            if @known_token_kinds.include? kind
+              encoder.text_token match, kind.to_sym
+            else
+              encoder.text_token match, :plain
+            end
+          end
           encoder.text_token match, :operator if match = scan(/\)/)
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
-          kind = self[1]
-          case self[2]
-          when '<'
-            encoder.text_token kind, :class
-          when '['
-            encoder.text_token kind, :class
+          encoder.text_token self[1], :class
+          if @known_token_kinds.include? self[1]
+            kind = self[1].to_sym
           else
-            raise 'CodeRay bug: This case should not be reached.'
+            kind = :unknown
           end
-          kind = kind.to_sym
           opened_tokens << kind
           encoder.begin_group kind
           encoder.text_token self[2], :operator
diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb
index ed071d2..0b36e13 100644
--- a/lib/coderay/scanners/ruby/patterns.rb
+++ b/lib/coderay/scanners/ruby/patterns.rb
@@ -157,13 +157,16 @@ module Scanners
       yield
     ])
     
-    FANCY_STRING_START = / % ( [QqrsWwx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
+    FANCY_STRING_START = / % ( [iIqQrswWx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
     FANCY_STRING_KIND = Hash.new(:string).merge({
+      'i' => :symbol,
+      'I' => :symbol,
       'r' => :regexp,
       's' => :symbol,
       'x' => :shell,
     })
     FANCY_STRING_INTERPRETED = Hash.new(true).merge({
+      'i' => false,
       'q' => false,
       's' => false,
       'w' => false,
diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb
index 2f398d1..28ddd6c 100644
--- a/lib/coderay/scanners/ruby/string_state.rb
+++ b/lib/coderay/scanners/ruby/string_state.rb
@@ -16,7 +16,6 @@ module Scanners
       
       STRING_PATTERN = Hash.new do |h, k|
         delim, interpreted = *k
-        # delim = delim.dup  # workaround for old Ruby
         delim_pattern = Regexp.escape(delim)
         if closing_paren = CLOSING_PAREN[delim]
           delim_pattern << Regexp.escape(closing_paren)
@@ -29,12 +28,13 @@ module Scanners
         #     '| [|?*+(){}\[\].^$]'
         #   end
         
-        h[k] =
-          if interpreted && delim != '#'
-            / (?= [#{delim_pattern}] | \# [{$@] ) /mx
-          else
-            / (?= [#{delim_pattern}] ) /mx
-          end
+        if interpreted && delim != '#'
+          / (?= [#{delim_pattern}] | \# [{$@] ) /mx
+        else
+          / (?= [#{delim_pattern}] ) /mx
+        end.tap do |pattern|
+          h[k] = pattern if (delim.respond_to?(:ord) ? delim.ord : delim[0]) < 256
+        end
       end
       
       def initialize kind, interpreted, delim, heredoc = false
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index f4e9d7d..d304dc4 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -125,7 +125,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .string .modifier { color: #E40 }
 .symbol { color:#A60 }
 .symbol .content { color:#A60 }
-.symbol .delimiter { color:#630 }
+.symbol .delimiter { color:#740 }
 .tag { color:#070; font-weight:bold }
 .type { color:#339; font-weight:bold }
 .value { color: #088 }
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index 9137a49..f911862 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -1,10 +1,7 @@
 module CodeRay
   
   # A Hash of all known token kinds and their associated CSS classes.
-  TokenKinds = Hash.new do |h, k|
-    warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG
-    false
-  end
+  TokenKinds = Hash.new(false)
   
   # speedup
   TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
@@ -83,5 +80,6 @@ module CodeRay
     :plain               => false                 # almost all scanners
   )
   
-  TokenKinds[:method] = TokenKinds[:function]
+  TokenKinds[:method]  = TokenKinds[:function]
+  TokenKinds[:unknown] = TokenKinds[:plain]
 end
diff --git a/test/functional/basic.rb b/test/functional/basic.rb
index 3053b54..752d4ba 100755
--- a/test/functional/basic.rb
+++ b/test/functional/basic.rb
@@ -164,9 +164,7 @@ more code  # and another comment, in-line.
       end
     end
     assert_equal 'reserved', CodeRay::TokenKinds[:reserved]
-    assert_warning 'Undefined Token kind: :shibboleet' do
-      assert_equal false, CodeRay::TokenKinds[:shibboleet]
-    end
+    assert_equal false,      CodeRay::TokenKinds[:shibboleet]
   end
   
   class Milk < CodeRay::Encoders::Encoder
diff --git a/test/unit/debug.rb b/test/unit/debug.rb
index f2b80bd..88baf56 100644
--- a/test/unit/debug.rb
+++ b/test/unit/debug.rb
@@ -18,15 +18,15 @@ class DebugEncoderTest < Test::Unit::TestCase
     [:begin_group, :string],
     ['test', :content],
     [:end_group, :string],
-    [:begin_line, :test],
+    [:begin_line, :head],
     ["\n", :space],
     ["\n  \t", :space],
     ["   \n", :space],
     ["[]", :method],
-    [:end_line, :test],
+    [:end_line, :head],
   ].flatten
   TEST_OUTPUT = <<-'DEBUG'.chomp
-integer(10)operator((\\\))string<content(test)>test[
+integer(10)operator((\\\))string<content(test)>head[
 
   	   
 method([])]
@@ -62,10 +62,10 @@ method([])]
     [:begin_group, :string],
     ['test', :content],
     [:end_group, :string],
-    [:begin_line, :test],
+    [:begin_line, :unknown],
     ["\n\n  \t   \n", :space],
     ["[]", :method],
-    [:end_line, :test],
+    [:end_line, :unknown],
   ].flatten
   
   def test_filtering_text_tokens