tweak Debug scanners again, introduce :unknown token kind

4 files changed, 25 insertions, 26 deletions

diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb
index ac12c16..83ede9a 100644
--- a/lib/coderay/scanners/debug.rb
+++ b/lib/coderay/scanners/debug.rb
@@ -5,7 +5,7 @@ module Scanners
   
   # = Debug Scanner
   # 
-  # Interprets the output of the Encoders::Debug encoder.
+  # Interprets the output of the Encoders::Debug encoder (basically the inverse function).
   class Debug < Scanner
     
     register_for :debug
@@ -31,21 +31,24 @@ module Scanners
           if @known_token_kinds.include? self[1]
             encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym
           else
-            encoder.text_token matched, :error
+            encoder.text_token matched, :unknown
           end
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
           if @known_token_kinds.include? self[1]
             kind = self[1].to_sym
-            opened_tokens << kind
-            case self[2]
-            when '<'
-              encoder.begin_group kind
-            when '['
-              encoder.begin_line kind
-            else
-              raise 'CodeRay bug: This case should not be reached.'
-            end
+          else
+            kind = :unknown
+          end
+          
+          opened_tokens << kind
+          case self[2]
+          when '<'
+            encoder.begin_group kind
+          when '['
+            encoder.begin_line kind
+          else
+            raise 'CodeRay bug: This case should not be reached.'
           end
           
         elsif !opened_tokens.empty? && match = scan(/ > /x)
diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb
index 6c1c10f..1effdc8 100644
--- a/lib/coderay/scanners/raydebug.rb
+++ b/lib/coderay/scanners/raydebug.rb
@@ -3,9 +3,9 @@ require 'set'
 module CodeRay
 module Scanners
   
-  # = Debug Scanner
+  # = Raydebug Scanner
   # 
-  # Parses the output of the Encoders::Debug encoder.
+  # Highlights the output of the Encoders::Debug encoder.
   class Raydebug < Scanner
     
     register_for :raydebug
@@ -43,17 +43,12 @@ module Scanners
           encoder.text_token match, :operator if match = scan(/\)/)
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
-          kind = self[1]
-          case self[2]
-          when '<'
-            encoder.text_token kind, :class
-          when '['
-            encoder.text_token kind, :class
+          encoder.text_token self[1], :class
+          if @known_token_kinds.include? self[1]
+            kind = self[1].to_sym
           else
-            raise 'CodeRay bug: This case should not be reached.'
+            kind = :unknown
           end
-          # FIXME: cache attack
-          kind = kind.to_sym
           opened_tokens << kind
           encoder.begin_group kind
           encoder.text_token self[2], :operator
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index 5f49d77..f911862 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -80,5 +80,6 @@ module CodeRay
     :plain               => false                 # almost all scanners
   )
   
-  TokenKinds[:method] = TokenKinds[:function]
+  TokenKinds[:method]  = TokenKinds[:function]
+  TokenKinds[:unknown] = TokenKinds[:plain]
 end
diff --git a/test/unit/debug.rb b/test/unit/debug.rb
index 616cda5..88baf56 100644
--- a/test/unit/debug.rb
+++ b/test/unit/debug.rb
@@ -51,7 +51,7 @@ class DebugScannerTest < Test::Unit::TestCase
   end
   
   TEST_INPUT = <<-'DEBUG'.chomp
-integer(10)operator((\\\))string<content(test)>head[
+integer(10)operator((\\\))string<content(test)>test[
 
   	   
 method([])]
@@ -62,10 +62,10 @@ method([])]
     [:begin_group, :string],
     ['test', :content],
     [:end_group, :string],
-    [:begin_line, :head],
+    [:begin_line, :unknown],
     ["\n\n  \t   \n", :space],
     ["[]", :method],
-    [:end_line, :head],
+    [:end_line, :unknown],
   ].flatten
   
   def test_filtering_text_tokens