Merge branch 'master' into cleanup-output

27 files changed, 769 insertions, 229 deletions

diff --git a/.gitignore b/.gitignore
index dd001c8..deed1a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,5 +11,4 @@ Gemfile.lock
 test/executable/source.rb.html
 test/executable/source.rb.json
 test/scanners
-bench/test.div.html
 old-stuff
diff --git a/.travis.yml b/.travis.yml
index 59bb791..6d926f3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,6 +14,8 @@ branches:
     - master
 matrix:
   allow_failures:
+    - rvm: ruby-head
+    - rvm: jruby-head
     - rvm: rbx-18mode
     - rvm: rbx-19mode
 script: "rake test" # test:scanners"
diff --git a/Changes.textile b/Changes.textile
index fcee29b..58427aa 100644
--- a/Changes.textile
+++ b/Changes.textile
@@ -4,24 +4,33 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release
 
 h2. Changes in 1.1
 
+* New scanner: Lua [#21, #22, thanks to Quintus]
 * New scanner: Sass [#93]
+* New scanner: Go [#28, thanks to Eric Guo and Nathan Youngman]
 * New scanner: Taskpaper [#39, thanks to shimomura]
 * Diff scanner: Highlight inline changes in multi-line changes [#99]
 * JavaScript scanner: Highlight multi-line comments in diff correctly
 * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126]
+* HTML scanner displays style tags and attributes now [#145]
 * Remove double-click toggle handler from HTML table output
-* Fixes to CSS scanner (floats, pseudoclasses)
+* Fixes to CSS scanner (floats, pseudoclasses, nth-child) [#143]
+* Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144]
+* Added @:keep_state@ functionality to more scanners [#116]
 * CSS scanner uses @:id@ and @:tag@ now [#27]
 * Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right?
 * Add .xaml file type [#121, thanks to Kozman Bálint]
 * @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner]
 * New token type @:id@ for CSS/Sass [#27]
 * New token type @:done@ for Taskpaper [#39]
+* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany]
 * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
-* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05]
+* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05]
 * Fixed @:docstring@ token type style
 * @Plugin@ does not warn about fallback when default is defined
+* @HTML@ encoder will not warn about unclosed token groups at the end of the stream
 * @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now
+* @Debug@ encoder will not warn about errors in the token stream
+* New @DebugLint@ encoder that checks for empty tokens and correct nesting
 
 h2. Changes in 1.0.9
 
diff --git a/Rakefile b/Rakefile
index ba6c34e..5577009 100644
--- a/Rakefile
+++ b/Rakefile
@@ -1,3 +1,5 @@
+require 'bundler/gem_tasks'
+
 $:.unshift File.dirname(__FILE__) unless $:.include? '.'
 
 ROOT = '.'
diff --git a/bench/bench.rb b/bench/bench.rb
index 1958c73..92f9d07 100644
--- a/bench/bench.rb
+++ b/bench/bench.rb
@@ -1,142 +1,46 @@
-# The most ugly test script I've ever written!
-# Shame on me!
-
-require 'pathname'
-require 'profile' if ARGV.include? '-p'
-
-MYDIR = File.dirname(__FILE__)
-LIBDIR = Pathname.new(MYDIR).join('..', 'lib').cleanpath.to_s
-$:.unshift MYDIR, LIBDIR
+require 'benchmark'
+$: << File.expand_path('../../lib', __FILE__)
 require 'coderay'
 
-@size = ARGV.fetch(2, 100).to_i * 1000
-
-lang = ARGV.fetch(0) do
-  puts <<-HELP
-Usage:
-  ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
-
-  SIZE defaults to 100 kB (= 100,000 bytes).
-  SIZE = 0 means the whole input.
-
--p generates a profile (slow! use with SIZE = 1)
--o shows the output
-stream enabled streaming mode
-
-Sorry for the strange interface. I will improve it in the next release.
-  HELP
+if ARGV.include? '-h'
+  puts DATA.read
   exit
 end
 
-format = ARGV.fetch(1, 'html').downcase
-
-$stream = ARGV.include? 'stream'
-$optimize = ARGV.include? 'opt'
-$style = ARGV.include? 'style'
-
-require 'benchmark'
-require 'fileutils'
+lang = ARGV.fetch(0, 'ruby')
+data = nil
+File.open(File.expand_path("../example.#{lang}", __FILE__), 'rb') { |f| data = f.read }
+raise 'Example file is empty.' if data.empty?
 
-if format == 'comp'
-  format = 'page'
-  begin
-    require 'syntax'
-    require 'syntax/convertors/html.rb'
-  rescue LoadError
-    puts 'Syntax no found!! (Try % gem install syntax)'
-  end
-end
+format = ARGV.fetch(1, 'html').downcase
+encoder = CodeRay.encoder(format)
 
-def here fn = nil
-  return MYDIR unless fn
-  File.join here, fn
+size = ARGV.fetch(2, 1000).to_i * 1000
+unless size.zero?
+  data += data until data.size >= size
+  data = data[0, size]
 end
+size = data.size
+puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round]
 
-n = ARGV.find { |a| a[/^N/] }
-N = if n then n[/\d+/].to_i else 1 end
-$filename = ARGV.include?('strange') ? 'strange' : 'example'
-
-Benchmark.bm(20) do |bm|
-N.times do
-
-  data = nil
-  File.open(here("#$filename." + lang), 'rb') { |f| data = f.read }
-  raise 'Example file is empty.' if data.empty?
-  unless @size.zero?
-    data += data until data.size >= @size
-    data = data[0, @size]
-  end
-  @size = data.size
-  
-  options = {
-    :tab_width => 2,
-    # :line_numbers => :inline,
-    :css => $style ? :style : :class,
-  }
-  $hl = CodeRay.encoder(format, options)
-  time = bm.report('CodeRay') do
-    if $stream || true
-      $o = $hl.encode(data, lang, options)
-    else
-      tokens = CodeRay.scan(data, lang)
-      tokens.optimize! if $optimize
-      $o = tokens.encode($hl)
-    end
-  end
-  $file_created = here('test.' + $hl.file_extension)
-  File.open($file_created, 'wb') do |f|
-    # f.write $o
-  end
-  
-  time_real = time.real
-  
-  puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000]
-  puts $o if ARGV.include? '-o'
-  
-end
+n = ARGV.fetch(3, 5).to_s[/\d+/].to_i
+require 'profile' if ARGV.include? '-p'
+n.times do |i|
+  time = Benchmark.realtime { encoder.encode(data, lang) }
+  puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0]
 end
-puts "Files created: #$file_created"
 
-STDIN.gets if ARGV.include? 'wait'
+STDIN.gets if ARGV.include? '-w'
 
 __END__
-.ruby .normal {}
-.ruby .comment { color: #005; font-style: italic; }
-.ruby .keyword { color: #A00; font-weight: bold; }
-.ruby .method { color: #077; }
-.ruby .class { color: #074; }
-.ruby .module { color: #050; }
-.ruby .punct { color: #447; font-weight: bold; }
-.ruby .symbol { color: #099; }
-.ruby .string { color: #944; background: #FFE; }
-.ruby .char { color: #F07; }
-.ruby .ident { color: #004; }
-.ruby .constant { color: #07F; }
-.ruby .regex { color: #B66; background: #FEF; }
-.ruby .number { color: #F99; }
-.ruby .attribute { color: #7BB; }
-.ruby .global { color: #7FB; }
-.ruby .expr { color: #227; }
-.ruby .escape { color: #277; }
+Usage:
+  ruby bench.rb [lang] [format] [size in kB] [number of runs]
 
-.xml .normal {}
-.xml .namespace { color: #B66; font-weight: bold; }
-.xml .tag { color: #F88; }
-.xml .comment { color: #005; font-style: italic; }
-.xml .punct { color: #447; font-weight: bold; }
-.xml .string { color: #944; }
-.xml .number { color: #F99; }
-.xml .attribute { color: #BB7; }
+  - lang defaults to ruby.
+  - format defaults to html.
+  - size defaults to 1000 kB (= 1,000,000 bytes). 0 uses the whole example input.
+  - number of runs defaults to 5.
 
-.yaml .normal {}
-.yaml .document { font-weight: bold; color: #07F; }
-.yaml .type { font-weight: bold; color: #05C; }
-.yaml .key { color: #F88; }
-.yaml .comment { color: #005; font-style: italic; }
-.yaml .punct { color: #447; font-weight: bold; }
-.yaml .string { color: #944; }
-.yaml .number { color: #F99; }
-.yaml .time { color: #F99; }
-.yaml .date { color: #F99; }
-.yaml .ref { color: #944; }
-.yaml .anchor { color: #944; }
+-h prints this help
+-p generates a profile (slow, use with SIZE = 1)
+-w waits after the benchmark (for debugging memory usw)
diff --git a/lib/coderay.rb b/lib/coderay.rb
index 24ae5a2..f759ed6 100644
--- a/lib/coderay.rb
+++ b/lib/coderay.rb
@@ -127,14 +127,14 @@ module CodeRay
   
   $CODERAY_DEBUG ||= false
   
-  CODERAY_PATH = File.join File.dirname(__FILE__), 'coderay'
+  CODERAY_PATH = File.expand_path('../coderay', __FILE__)
   
   # Assuming the path is a subpath of lib/coderay/
   def self.coderay_path *path
     File.join CODERAY_PATH, *path
   end
   
-  require coderay_path('version')
+  require 'coderay/version'
   
   # helpers
   autoload :FileType,    coderay_path('helpers', 'file_type')
diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb
index eeb2a92..2c14186 100644
--- a/lib/coderay/encoders/debug_lint.rb
+++ b/lib/coderay/encoders/debug_lint.rb
@@ -1,6 +1,8 @@
 module CodeRay
 module Encoders
   
+  load :lint
+  
   # = Debug Lint Encoder
   #
   # Debug encoder with additional checks for:
@@ -15,17 +17,8 @@ module Encoders
     
     register_for :debug_lint
     
-    InvalidTokenStream = Class.new StandardError
-    EmptyToken = Class.new InvalidTokenStream
-    IncorrectTokenGroupNesting = Class.new InvalidTokenStream
-    
-    def initialize options = {}
-      super
-      @opened = []
-    end
-    
     def text_token text, kind
-      raise EmptyToken, 'empty token' if text.empty?
+      raise Lint::EmptyToken, 'empty token' if text.empty?
       super
     end
     
@@ -35,7 +28,8 @@ module Encoders
     end
     
     def end_group kind
-      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
+      raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
       super
     end
     
@@ -45,7 +39,20 @@ module Encoders
     end
     
     def end_line kind
-      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
+      raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+      super
+    end
+    
+    protected
+    
+    def setup options
+      super
+      @opened = []
+    end
+    
+    def finish options
+      raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
       super
     end
     
diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index 5a10721..1040d20 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -193,7 +193,6 @@ module Encoders
     
     def finish options
       unless @opened.empty?
-        warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
         @out << '</span>' while @opened.pop
         @last_opened = nil
       end
diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb
new file mode 100644
index 0000000..4601e90
--- /dev/null
+++ b/lib/coderay/encoders/lint.rb
@@ -0,0 +1,57 @@
+module CodeRay
+module Encoders
+  
+  # = Lint Encoder
+  #
+  # Checks for:
+  # 
+  # - empty tokens
+  # - incorrect nesting
+  # 
+  # It will raise an InvalidTokenStream exception when any of the above occurs.
+  # 
+  # See also: Encoders::DebugLint
+  class Lint < Debug
+    
+    register_for :lint
+    
+    InvalidTokenStream         = Class.new StandardError
+    EmptyToken                 = Class.new InvalidTokenStream
+    IncorrectTokenGroupNesting = Class.new InvalidTokenStream
+    
+    def text_token text, kind
+      raise EmptyToken, 'empty token' if text.empty?
+    end
+    
+    def begin_group kind
+      @opened << kind
+    end
+    
+    def end_group kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+    end
+    
+    def begin_line kind
+      @opened << kind
+    end
+    
+    def end_line kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+    end
+    
+    protected
+    
+    def setup options
+      @opened = []
+    end
+    
+    def finish options
+      raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
+    end
+    
+  end
+  
+end
+end
diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb
index 19f27ac..5e3a1e7 100644
--- a/lib/coderay/helpers/file_type.rb
+++ b/lib/coderay/helpers/file_type.rb
@@ -86,6 +86,7 @@ module CodeRay
       'dpr'       => :delphi,
       'erb'       => :erb,
       'gemspec'   => :ruby,
+      'go'        => :go, 
       'groovy'    => :groovy,
       'gvy'       => :groovy,
       'h'         => :c,
@@ -96,6 +97,7 @@ module CodeRay
       'java'      => :java,
       'js'        => :java_script,
       'json'      => :json,
+      'lua'       => :lua,
       'mab'       => :ruby,
       'pas'       => :delphi,
       'patch'     => :diff,
@@ -115,7 +117,7 @@ module CodeRay
       'rhtml'     => :erb,
       'rjs'       => :ruby,
       'rpdf'      => :ruby,
-      'ru'        => :ruby,
+      'ru'        => :ruby,  # config.ru
       'rxml'      => :ruby,
       'sass'      => :sass,
       'sql'       => :sql,
@@ -139,6 +141,9 @@ module CodeRay
       'Rakefile' => :ruby,
       'Rantfile' => :ruby,
       'Gemfile'  => :ruby,
+      'Guardfile' => :ruby,
+      'Vagrantfile' => :ruby,
+      'Appraisals' => :ruby
     }
     
   end
diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 732f9c5..55d5239 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -25,7 +25,7 @@ module Scanners
       
       HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
       
-      Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)/
+      Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)n?/
       Name = /#{NMChar}+/
       Ident = /-?#{NMStart}#{NMChar}*/
       AtKeyword = /@#{Ident}/
@@ -53,7 +53,7 @@ module Scanners
     end
     
     def scan_tokens encoder, options
-      states = Array(options[:state] || @state)
+      states = Array(options[:state] || @state).dup
       value_expected = @value_expected
       
       until eos?
@@ -145,10 +145,10 @@ module Scanners
           start = match[/^\w+\(/]
           encoder.text_token start, :delimiter
           if match[-1] == ?)
-            encoder.text_token match[start.size..-2], :content
+            encoder.text_token match[start.size..-2], :content if match.size > start.size + 1
             encoder.text_token ')', :delimiter
           else
-            encoder.text_token match[start.size..-1], :content
+            encoder.text_token match[start.size..-1], :content if match.size > start.size
           end
           encoder.end_group :function
           
diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb
index af0f755..fd1aed6 100644
--- a/lib/coderay/scanners/diff.rb
+++ b/lib/coderay/scanners/diff.rb
@@ -69,7 +69,7 @@ module Scanners
             state = :added
           elsif match = scan(/\\ .*/)
             encoder.text_token match, :comment
-          elsif match = scan(/@@(?>[^@\n]*)@@/)
+          elsif match = scan(/@@(?>[^@\n]+)@@/)
             content_scanner.state = :initial unless match?(/\n\+/)
             content_scanner_entry_state = nil
             if check(/\n|$/)
diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb
new file mode 100644
index 0000000..59473f6
--- /dev/null
+++ b/lib/coderay/scanners/go.rb
@@ -0,0 +1,209 @@
+module CodeRay
+module Scanners
+  
+  # Scanner for Go, copy from c
+  class Go < Scanner
+    
+    register_for :go
+    file_extension 'go'
+    
+    # http://golang.org/ref/spec#Keywords
+    KEYWORDS = [
+      'break', 'default', 'func', 'interface', 'select',
+      'case', 'defer', 'go', 'map', 'struct',
+      'chan', 'else', 'goto', 'package', 'switch',
+      'const', 'fallthrough', 'if', 'range', 'type',
+      'continue', 'for', 'import', 'return', 'var',
+    ]  # :nodoc:
+    
+    # http://golang.org/ref/spec#Types
+    PREDEFINED_TYPES = [
+      'bool',
+      'uint8', 'uint16', 'uint32', 'uint64',
+      'int8', 'int16', 'int32', 'int64',
+      'float32', 'float64',
+      'complex64', 'complex128',
+      'byte', 'rune', 'string', 'error',
+      'uint', 'int', 'uintptr',
+    ]  # :nodoc:
+    
+    PREDEFINED_CONSTANTS = [
+      'nil', 'iota',
+      'true', 'false',
+    ]  # :nodoc:
+    
+    PREDEFINED_FUNCTIONS = %w[
+      append cap close complex copy delete imag len
+      make new panic print println real recover
+    ] # :nodoc:
+    
+    IDENT_KIND = WordList.new(:ident).
+      add(KEYWORDS, :keyword).
+      add(PREDEFINED_TYPES, :predefined_type).
+      add(PREDEFINED_CONSTANTS, :predefined_constant).
+      add(PREDEFINED_FUNCTIONS, :predefined)  # :nodoc:
+    
+    ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
+    UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
+    
+    protected
+    
+    def scan_tokens encoder, options
+      
+      state = :initial
+      label_expected = true
+      case_expected = false
+      label_expected_before_preproc_line = nil
+      in_preproc_line = false
+      
+      until eos?
+        
+        case state
+        
+        when :initial
+          
+          if match = scan(/ \s+ | \\\n /x)
+            if in_preproc_line && match != "\\\n" && match.index(?\n)
+              in_preproc_line = false
+              case_expected = false
+              label_expected = label_expected_before_preproc_line
+            end
+            encoder.text_token match, :space
+          
+          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            encoder.text_token match, :comment
+          
+          elsif match = scan(/ <?- (?![\d.]) | [+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
+            if case_expected
+              label_expected = true if match == ':'
+              case_expected = false
+            end
+            encoder.text_token match, :operator
+          
+          elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+            kind = IDENT_KIND[match]
+            if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
+              kind = :label
+              label_expected = false
+              match << matched
+            else
+              label_expected = false
+              if kind == :keyword
+                case match
+                when 'case', 'default'
+                  case_expected = true
+                end
+              end
+            end
+            encoder.text_token match, kind
+          
+          elsif match = scan(/L?"/)
+            encoder.begin_group :string
+            if match[0] == ?L
+              encoder.text_token 'L', :modifier
+              match = '"'
+            end
+            encoder.text_token match, :delimiter
+            state = :string
+          
+          elsif match = scan(/ ` ([^`]+)? (`)? /x)
+            encoder.begin_group :shell
+            encoder.text_token '`', :delimiter
+            encoder.text_token self[1], :content if self[1]
+            encoder.text_token self[2], :delimiter if self[2]
+            encoder.end_group :shell
+          
+          elsif match = scan(/ \# \s* if \s* 0 /x)
+            match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
+            encoder.text_token match, :comment
+          
+          elsif match = scan(/#[ \t]*(\w*)/)
+            encoder.text_token match, :preprocessor
+            in_preproc_line = true
+            label_expected_before_preproc_line = label_expected
+            state = :include_expected if self[1] == 'include'
+          
+          elsif match = scan(/ L?' (?: [^\'\n\\] | \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) )? '? /ox)
+            label_expected = false
+            encoder.text_token match, :char
+          
+          elsif match = scan(/\$/)
+            encoder.text_token match, :ident
+          
+          elsif match = scan(/-?\d*(\.\d*)?([eE][+-]?\d+)?i/)
+            label_expected = false
+            encoder.text_token match, :imaginary
+          
+          elsif match = scan(/-?0[xX][0-9A-Fa-f]+/)
+            label_expected = false
+            encoder.text_token match, :hex
+          
+          elsif match = scan(/-?(?:0[0-7]+)(?![89.eEfF])/)
+            label_expected = false
+            encoder.text_token match, :octal
+          
+          elsif match = scan(/-?(?:\d*\.\d+|\d+\.)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
+            label_expected = false
+            encoder.text_token match, :float
+          
+          elsif match = scan(/-?(?:\d+)(?![.eEfF])L?L?/)
+            label_expected = false
+            encoder.text_token match, :integer
+          
+          else
+            encoder.text_token getch, :error
+          
+          end
+        
+        when :string
+          if match = scan(/[^\\\n"]+/)
+            encoder.text_token match, :content
+          elsif match = scan(/"/)
+            encoder.text_token match, :delimiter
+            encoder.end_group :string
+            state = :initial
+            label_expected = false
+          elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            encoder.text_token match, :char
+          elsif match = scan(/ \\ /x)
+            encoder.text_token match, :error
+          elsif match = scan(/$/)
+            encoder.end_group :string
+            state = :initial
+            label_expected = false
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
+          end
+        
+        when :include_expected
+          if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
+            encoder.text_token match, :include
+            state = :initial
+          
+          elsif match = scan(/\s+/)
+            encoder.text_token match, :space
+            state = :initial if match.index ?\n
+          
+          else
+            state = :initial
+          
+          end
+        
+        else
+          raise_inspect 'Unknown state', encoder
+        
+        end
+        
+      end
+      
+      if state == :string
+        encoder.end_group :string
+      end
+      
+      encoder
+    end
+    
+  end
+  
+end
+end
diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb
index cf55daf..c64454f 100644
--- a/lib/coderay/scanners/groovy.rb
+++ b/lib/coderay/scanners/groovy.rb
@@ -36,9 +36,12 @@ module Scanners
     
   protected
     
+    def setup
+      @state = :initial
+    end
+    
     def scan_tokens encoder, options
-      
-      state = :initial
+      state = options[:state] || @state
       inline_block_stack = []
       inline_block_paren_depth = nil
       string_delimiter = nil
@@ -223,7 +226,7 @@ module Scanners
             encoder.text_token match, :content  # TODO: Shouldn't this be :error?
             
           elsif match = scan(/ \\ | \n /x)
-            encoder.end_group state
+            encoder.end_group state == :regexp ? :regexp : :string
             encoder.text_token match, :error
             after_def = value_expected = false
             state = :initial
@@ -243,7 +246,17 @@ module Scanners
       end
       
       if [:multiline_string, :string, :regexp].include? state
-        encoder.end_group state
+        encoder.end_group state == :regexp ? :regexp : :string
+      end
+      
+      if options[:keep_state]
+        @state = state
+      end
+      
+      until inline_block_stack.empty?
+        state, = *inline_block_stack.pop
+        encoder.end_group :inline
+        encoder.end_group state == :regexp ? :regexp : :string
       end
       
       encoder
diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index 3ba3b79..ebe7b01 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -1,13 +1,13 @@
 module CodeRay
 module Scanners
-
+  
   # HTML Scanner
   # 
   # Alias: +xhtml+
   # 
   # See also: Scanners::XML
   class HTML < Scanner
-
+    
     register_for :html
     
     KINDS_NOT_LOC = [
@@ -33,7 +33,8 @@ module Scanners
     )
     
     IN_ATTRIBUTE = WordList::CaseIgnoring.new(nil).
-      add(EVENT_ATTRIBUTES, :script)
+      add(EVENT_ATTRIBUTES, :script).
+      add(['style'], :style)
     
     ATTR_NAME = /[\w.:-]+/  # :nodoc:
     TAG_END = /\/?>/  # :nodoc:
@@ -75,9 +76,14 @@ module Scanners
     def scan_java_script encoder, code
       if code && !code.empty?
         @java_script_scanner ||= Scanners::JavaScript.new '', :keep_tokens => true
-        # encoder.begin_group :inline
         @java_script_scanner.tokenize code, :tokens => encoder
-        # encoder.end_group :inline
+      end
+    end
+    
+    def scan_css encoder, code, state = [:initial]
+      if code && !code.empty?
+        @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true
+        @css_scanner.tokenize code, :tokens => encoder, :state => state
       end
     end
     
@@ -99,7 +105,15 @@ module Scanners
           case state
           
           when :initial
-            if match = scan(/<!--(?:.*?-->|.*)/m)
+            if match = scan(/<!\[CDATA\[/)
+              encoder.text_token match, :inline_delimiter
+              if match = scan(/.*?\]\]>/m)
+                encoder.text_token match[0..-4], :plain
+                encoder.text_token ']]>', :inline_delimiter
+              elsif match = scan(/.+/)
+                encoder.text_token match, :error
+              end
+            elsif match = scan(/<!--(?:.*?-->|.*)/m)
               encoder.text_token match, :comment
             elsif match = scan(/<!(\w+)(?:.*?>|.*)|\]>/m)
               encoder.text_token match, :doctype
@@ -110,7 +124,7 @@ module Scanners
             elsif match = scan(/<\/[-\w.:]*>?/m)
               in_tag = nil
               encoder.text_token match, :tag
-            elsif match = scan(/<(?:(script)|[-\w.:]+)(>)?/m)
+            elsif match = scan(/<(?:(script|style)|[-\w.:]+)(>)?/m)
               encoder.text_token match, :tag
               in_tag = self[1]
               if self[2]
@@ -161,17 +175,21 @@ module Scanners
               encoder.text_token match, :attribute_value
               state = :attribute
             elsif match = scan(/["']/)
-              if in_attribute == :script
-                encoder.begin_group :inline
-                encoder.text_token match, :inline_delimiter
+              if in_attribute == :script || in_attribute == :style
+                encoder.begin_group :string
+                encoder.text_token match, :delimiter
                 if scan(/javascript:[ \t]*/)
                   encoder.text_token matched, :comment
                 end
                 code = scan_until(match == '"' ? /(?="|\z)/ : /(?='|\z)/)
-                scan_java_script encoder, code
+                if in_attribute == :script
+                  scan_java_script encoder, code
+                else
+                  scan_css encoder, code, [:block]
+                end
                 match = scan(/["']/)
-                encoder.text_token match, :inline_delimiter if match
-                encoder.end_group :inline
+                encoder.text_token match, :delimiter if match
+                encoder.end_group :string
                 state = :attribute
                 in_attribute = nil
               else
@@ -206,19 +224,23 @@ module Scanners
             
           when :in_special_tag
             case in_tag
-            when 'script'
+            when 'script', 'style'
               encoder.text_token match, :space if match = scan(/[ \t]*\n/)
               if scan(/(\s*<!--)(?:(.*?)(-->)|(.*))/m)
                 code = self[2] || self[4]
                 closing = self[3]
                 encoder.text_token self[1], :comment
               else
-                code = scan_until(/(?=(?:\n\s*)?<\/script>)|\z/)
+                code = scan_until(/(?=(?:\n\s*)?<\/#{in_tag}>)|\z/)
                 closing = false
               end
               unless code.empty?
                 encoder.begin_group :inline
-                scan_java_script encoder, code
+                if in_tag == 'script'
+                  scan_java_script encoder, code
+                else
+                  scan_css encoder, code
+                end
                 encoder.end_group :inline
               end
               encoder.text_token closing, :comment if closing
diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb
index 3754a9b..b09970c 100644
--- a/lib/coderay/scanners/json.rb
+++ b/lib/coderay/scanners/json.rb
@@ -26,6 +26,10 @@ module Scanners
     def scan_tokens encoder, options
       state = options[:state] || @state
       
+      if [:string, :key].include? state
+        encoder.begin_group state
+      end
+      
       until eos?
         
         case state
diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
new file mode 100644
index 0000000..fb1e45a
--- /dev/null
+++ b/lib/coderay/scanners/lua.rb
@@ -0,0 +1,280 @@
+# encoding: utf-8
+
+module CodeRay
+module Scanners
+
+  # Scanner for the Lua[http://lua.org] programming lanuage.
+  #
+  # The language’s complete syntax is defined in
+  # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html],
+  # which is what this scanner tries to conform to.
+  class Lua < Scanner
+    
+    register_for :lua
+    file_extension 'lua'
+    title 'Lua'
+    
+    # Keywords used in Lua.
+    KEYWORDS = %w[and break do else elseif end
+      for function goto if in
+      local not or repeat return
+      then until while
+    ]
+    
+    # Constants set by the Lua core.
+    PREDEFINED_CONSTANTS = %w[false true nil]
+    
+    # The expressions contained in this array are parts of Lua’s `basic'
+    # library. Although it’s not entirely necessary to load that library,
+    # it is highly recommended and one would have to provide own implementations
+    # of some of these expressions if one does not do so. They however aren’t
+    # keywords, neither are they constants, but nearly predefined, so they
+    # get tagged as `predefined' rather than anything else.
+    #
+    # This list excludes values of form `_UPPERCASE' because the Lua manual
+    # requires such identifiers to be reserved by Lua anyway and they are
+    # highlighted directly accordingly, without the need for specific
+    # identifiers to be listed here.
+    PREDEFINED_EXPRESSIONS = %w[
+      assert collectgarbage dofile error getmetatable
+      ipairs load loadfile next pairs pcall print
+      rawequal rawget rawlen rawset select setmetatable
+      tonumber tostring type xpcall
+    ]
+    
+    # Automatic token kind selection for normal words.
+    IDENT_KIND = CodeRay::WordList.new(:ident).
+      add(KEYWORDS, :keyword).
+      add(PREDEFINED_CONSTANTS, :predefined_constant).
+      add(PREDEFINED_EXPRESSIONS, :predefined)
+    
+    protected
+    
+    # Scanner initialization.
+    def setup
+      @state = :initial
+      @brace_depth = 0
+    end
+    
+    # CodeRay entry hook. Starts parsing.
+    def scan_tokens(encoder, options)
+      state = options[:state] || @state
+      brace_depth = @brace_depth
+      num_equals = nil
+      
+      until eos?
+        case state
+        
+        when :initial
+          if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
+            num_equals = match.count("=") # Number must match for comment end
+            encoder.begin_group(:comment)
+            encoder.text_token(match, :delimiter)
+            state = :long_comment
+          
+          elsif match = scan(/--.*$/) # --Lua comment
+            encoder.text_token(match, :comment)
+          
+          elsif match = scan(/\[=*\[/)     # [[ long (possibly multiline) string ]]
+            num_equals = match.count("=") # Number must match for comment end
+            encoder.begin_group(:string)
+            encoder.text_token(match, :delimiter)
+            state = :long_string
+          
+          elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label::
+            encoder.text_token(match, :label)
+          
+          elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua
+            encoder.text_token(match, :predefined)
+          
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits)
+            kind = IDENT_KIND[match]
+            
+            # Extra highlighting for entities following certain keywords
+            if kind == :keyword and match == "function"
+              state = :function_expected
+            elsif kind == :keyword and match == "goto"
+              state = :goto_label_expected
+            elsif kind == :keyword and match == "local"
+              state = :local_var_expected
+            end
+            
+            encoder.text_token(match, kind)
+          
+          elsif match = scan(/\{/) # Opening table brace {
+            encoder.begin_group(:map)
+            encoder.text_token(match, brace_depth >= 1 ? :inline_delimiter : :delimiter)
+            brace_depth += 1
+            state        = :map
+          
+          elsif match = scan(/\}/) # Closing table brace }
+            if brace_depth == 1
+              brace_depth = 0
+              encoder.text_token(match, :delimiter)
+              encoder.end_group(:map)
+            elsif brace_depth == 0 # Mismatched brace
+              encoder.text_token(match, :error)
+            else
+              brace_depth -= 1
+              encoder.text_token(match, :inline_delimiter)
+              encoder.end_group(:map)
+              state = :map
+            end
+          
+          elsif match = scan(/["']/) # String delimiters " and '
+            encoder.begin_group(:string)
+            encoder.text_token(match, :delimiter)
+            start_delim = match
+            state       = :string
+          
+                            # ↓Prefix                hex number ←|→ decimal number
+          elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
+            encoder.text_token(match, :float)
+          
+                            # ↓Prefix         hex number ←|→ decimal number
+          elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power
+            encoder.text_token(match, :integer)
+          
+          elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators
+            encoder.text_token(match, :operator)
+          
+          elsif match = scan(/\s+/) # Space
+            encoder.text_token(match, :space)
+          
+          else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors.
+            encoder.text_token(getch, :error)
+          end
+          
+          # It may be that we’re scanning a full-blown subexpression of a table
+          # (tables can contain full expressions in parts).
+          # If this is the case, return to :map scanning state.
+          state = :map if state == :initial && brace_depth >= 1
+        
+        when :function_expected
+          if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
+            encoder.text_token(match, :operator)
+            state = :initial
+          elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator
+            encoder.text_token(match, :ident)
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo()
+            encoder.text_token(match, :function)
+            state = :initial
+          elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace
+            encoder.text_token(match, :space)
+          else
+            encoder.text_token(getch, :error)
+            state = :initial
+          end
+        
+        when :goto_label_expected
+          if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
+            encoder.text_token(match, :label)
+            state = :initial
+          elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace
+            encoder.text_token(match, :space)
+          else
+            encoder.text_token(getch, :error)
+          end
+        
+        when :local_var_expected
+          if match = scan(/function/) # local function ...
+            encoder.text_token(match, :keyword)
+            state = :function_expected
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/)
+            encoder.text_token(match, :local_variable)
+          elsif match = scan(/,/)
+            encoder.text_token(match, :operator)
+          elsif match = scan(/\=/)
+            encoder.text_token(match, :operator)
+            # After encountering the equal sign, arbitrary expressions are
+            # allowed again, so just return to the main state for further
+            # parsing.
+            state = :initial
+          elsif match = scan(/\n/)
+            encoder.text_token(match, :space)
+            state = :initial
+          elsif match = scan(/\s+/)
+            encoder.text_token(match, :space)
+          else
+            encoder.text_token(getch, :error)
+          end
+        
+        when :long_comment
+          if match = scan(/.*?(?=\]={#{num_equals}}\])/m)
+            encoder.text_token(match, :content)
+            
+            delim = scan(/\]={#{num_equals}}\]/)
+            encoder.text_token(delim, :delimiter)
+          else # No terminator found till EOF
+            encoder.text_token(rest, :error)
+            terminate
+          end
+          encoder.end_group(:comment)
+          state = :initial
+        
+        when :long_string
+          if match = scan(/.*?(?=\]={#{num_equals}}\])/m) # Long strings do not interpret any escape sequences
+            encoder.text_token(match, :content)
+            
+            delim = scan(/\]={#{num_equals}}\]/)
+            encoder.text_token(delim, :delimiter)
+          else # No terminator found till EOF
+            encoder.text_token(rest, :error)
+            terminate
+          end
+          encoder.end_group(:string)
+          state = :initial
+        
+        when :string
+          if match = scan(/[^\\#{start_delim}\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
+            encoder.text_token(match, :content)
+          elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
+            encoder.text_token(match, :char)
+          elsif match = scan(Regexp.compile(start_delim))
+            encoder.text_token(match, :delimiter)
+            encoder.end_group(:string)
+            state = :initial
+          elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings
+            encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings
+            encoder.end_group(:string)
+            state = :initial
+          else
+            encoder.text_token(getch, :error)
+          end
+        
+        when :map
+          if match = scan(/[,;]/)
+            encoder.text_token(match, :operator)
+          elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x)
+            encoder.text_token(match, :key)
+            encoder.text_token(scan(/\s+/), :space) if check(/\s+/)
+            encoder.text_token(scan(/\=/), :operator)
+            state = :initial
+          elsif match = scan(/\s+/m)
+            encoder.text_token(match, :space)
+          else
+            # Note this clause doesn’t advance the scan pointer, it’s a kind of
+            # "retry with other options" (the :initial state then of course
+            # advances the pointer).
+            state = :initial
+          end
+        else
+          raise
+        end
+        
+      end
+      
+      if options[:keep_state]
+        @state = state
+      end
+      
+      encoder.end_group :string if [:string].include? state
+      brace_depth.times { encoder.end_group :map }
+      
+      encoder
+    end
+    
+  end
+  
+end
+end
diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb
index 6c68834..7a8d75d 100644
--- a/lib/coderay/scanners/php.rb
+++ b/lib/coderay/scanners/php.rb
@@ -265,7 +265,7 @@ module Scanners
             @html_scanner.tokenize match unless match.empty?
           end
         
-        when :php
+        when :php, :php_inline
           if match = scan(/\s+/)
             encoder.text_token match, :space
           
@@ -332,7 +332,7 @@ module Scanners
             if states.size == 1
               encoder.text_token match, :error
             else
-              states.pop
+              state = states.pop
               if states.last.is_a?(::Array)
                 delimiter = states.last[1]
                 states[-1] = states.last[0]
@@ -340,6 +340,7 @@ module Scanners
                 encoder.end_group :inline
               else
                 encoder.text_token match, :operator
+                encoder.end_group :inline if state == :php_inline
                 label_expected = true
               end
             end
@@ -350,7 +351,14 @@ module Scanners
           
           elsif match = scan(RE::PHP_END)
             encoder.text_token match, :inline_delimiter
-            states = [:initial]
+            while state = states.pop
+              encoder.end_group :string if [:sqstring, :dqstring].include? state
+              if state.is_a? Array
+                encoder.end_group :inline
+                encoder.end_group :string if [:sqstring, :dqstring].include? state.first
+              end
+            end
+            states << :initial
           
           elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o)
             encoder.begin_group :string
@@ -400,6 +408,7 @@ module Scanners
           elsif match = scan(/\\/)
             encoder.text_token match, :error
           else
+            encoder.end_group :string
             states.pop
           end
         
@@ -459,7 +468,7 @@ module Scanners
               encoder.begin_group :inline
               states[-1] = [states.last, delimiter]
               delimiter = nil
-              states.push :php
+              states.push :php_inline
               encoder.text_token match, :delimiter
             else
               encoder.text_token match, :content
@@ -469,6 +478,7 @@ module Scanners
           elsif match = scan(/\$/)
             encoder.text_token match, :content
           else
+            encoder.end_group :string
             states.pop
           end
         
@@ -500,6 +510,14 @@ module Scanners
         
       end
       
+      while state = states.pop
+        encoder.end_group :string if [:sqstring, :dqstring].include? state
+        if state.is_a? Array
+          encoder.end_group :inline
+          encoder.end_group :string if [:sqstring, :dqstring].include? state.first
+        end
+      end
+      
       encoder
     end
     
diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb
index 7a21354..d39d962 100644
--- a/lib/coderay/scanners/raydebug.rb
+++ b/lib/coderay/scanners/raydebug.rb
@@ -1,11 +1,11 @@
 module CodeRay
 module Scanners
-
+  
   # = Debug Scanner
   # 
   # Parses the output of the Encoders::Debug encoder.
   class Raydebug < Scanner
-
+    
     register_for :raydebug
     file_extension 'raydebug'
     title 'CodeRay Token Dump'
@@ -13,11 +13,11 @@ module Scanners
   protected
     
     def scan_tokens encoder, options
-
+      
       opened_tokens = []
-
+      
       until eos?
-
+        
         if match = scan(/\s+/)
           encoder.text_token match, :space
           
@@ -26,7 +26,7 @@ module Scanners
           encoder.text_token kind, :class
           encoder.text_token '(', :operator
           match = self[2]
-          encoder.text_token match, kind.to_sym
+          encoder.text_token match, kind.to_sym unless match.empty?
           encoder.text_token match, :operator if match = scan(/\)/)
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
@@ -59,8 +59,8 @@ module Scanners
       
       encoder
     end
-
+    
   end
-
+  
 end
 end
diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index c282f31..80165ca 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -269,7 +269,7 @@ module Scanners
             end
             
             if last_state
-              state = last_state
+              state = last_state unless state.is_a?(StringState)  # otherwise, a simple 'def"' results in unclosed tokens
               last_state = nil
             end
             
diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
index 167051d..e3296b9 100644
--- a/lib/coderay/scanners/sass.rb
+++ b/lib/coderay/scanners/sass.rb
@@ -7,11 +7,6 @@ module Scanners
     register_for :sass
     file_extension 'sass'
     
-    STRING_CONTENT_PATTERN = {
-      "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
-      '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
-    }
-    
   protected
     
     def setup
@@ -19,8 +14,9 @@ module Scanners
     end
     
     def scan_tokens encoder, options
-      states = Array(options[:state] || @state)
-      string_delimiter = nil
+      states = Array(options[:state] || @state).dup
+      
+      encoder.begin_group :string if states.last == :sqstring || states.last == :dqstring
       
       until eos?
         
@@ -48,7 +44,7 @@ module Scanners
         elsif case states.last
           when :initial, :media, :sass_inline
             if match = scan(/(?>#{RE::Ident})(?!\()/ox)
-              encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag)
+              encoder.text_token match, value_expected ? :value : (check(/.*:(?![a-z])/) ? :key : :tag)
               next
             elsif !value_expected && (match = scan(/\*/))
               encoder.text_token match, :tag
@@ -91,24 +87,23 @@ module Scanners
               next
             end
             
-          when :string
-            if match = scan(STRING_CONTENT_PATTERN[string_delimiter])
+          when :sqstring, :dqstring
+            if match = scan(states.last == :sqstring ? /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o : /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o)
               encoder.text_token match, :content
             elsif match = scan(/['"]/)
               encoder.text_token match, :delimiter
               encoder.end_group :string
-              string_delimiter = nil
               states.pop
             elsif match = scan(/#\{/)
               encoder.begin_group :inline
               encoder.text_token match, :inline_delimiter
               states.push :sass_inline
             elsif match = scan(/ \\ | $ /x)
-              encoder.end_group :string
+              encoder.end_group states.last
               encoder.text_token match, :error unless match.empty?
               states.pop
             else
-              raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder
+              raise_inspect "else case #{states.last} reached; %p not handled." % peek(1), encoder
             end
           
           when :include
@@ -119,7 +114,7 @@ module Scanners
           
           else
             #:nocov:
-            raise_inspect 'Unknown state', encoder
+            raise_inspect 'Unknown state: %p' % [states.last], encoder
             #:nocov:
             
           end
@@ -157,15 +152,15 @@ module Scanners
           
         elsif match = scan(/['"]/)
           encoder.begin_group :string
-          string_delimiter = match
           encoder.text_token match, :delimiter
           if states.include? :sass_inline
-            content = scan_until(/(?=#{string_delimiter}|\}|\z)/)
+            # no nesting, just scan the string until delimiter
+            content = scan_until(/(?=#{match}|\}|\z)/)
             encoder.text_token content, :content unless content.empty?
-            encoder.text_token string_delimiter, :delimiter if scan(/#{string_delimiter}/)
+            encoder.text_token match, :delimiter if scan(/#{match}/)
             encoder.end_group :string
           else
-            states.push :string
+            states.push match == "'" ? :sqstring : :dqstring
           end
           
         elsif match = scan(/#{RE::Function}/o)
@@ -176,7 +171,7 @@ module Scanners
             encoder.text_token match[start.size..-2], :content
             encoder.text_token ')', :delimiter
           else
-            encoder.text_token match[start.size..-1], :content
+            encoder.text_token match[start.size..-1], :content if start.size < match.size
           end
           encoder.end_group :function
           
@@ -195,7 +190,7 @@ module Scanners
         elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
           encoder.text_token match, :color
           
-        elsif match = scan(/@else if\b|#{RE::AtKeyword}/)
+        elsif match = scan(/@else if\b|#{RE::AtKeyword}/o)
           encoder.text_token match, :directive
           value_expected = true
           
@@ -214,8 +209,18 @@ module Scanners
         
       end
       
+      states.pop if states.last == :include
+      
       if options[:keep_state]
-        @state = states
+        @state = states.dup
+      end
+      
+      while state = states.pop
+        if state == :sass_inline
+          encoder.end_group :inline
+        elsif state == :sqstring || state == :dqstring
+          encoder.end_group :string
+        end
       end
       
       encoder
diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb
index b757278..93aeaf3 100644
--- a/lib/coderay/scanners/sql.rb
+++ b/lib/coderay/scanners/sql.rb
@@ -1,8 +1,9 @@
-module CodeRay module Scanners
+module CodeRay
+module Scanners
   
   # by Josh Goebel
   class SQL < Scanner
-
+    
     register_for :sql
     
     KEYWORDS = %w(
@@ -149,6 +150,7 @@ module CodeRay module Scanners
               string_content = ''
             end
             encoder.text_token match, :error unless match.empty?
+            encoder.end_group :string
             state = :initial
           else
             raise "else case \" reached; %p not handled." % peek(1), encoder
@@ -171,4 +173,5 @@ module CodeRay module Scanners
     
   end
   
-end end
\ No newline at end of file
+end
+end
diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb
index 96f4e93..32c8e2c 100644
--- a/lib/coderay/scanners/yaml.rb
+++ b/lib/coderay/scanners/yaml.rb
@@ -47,7 +47,7 @@ module Scanners
           when !check(/(?:"[^"]*")(?=: |:$)/) && match = scan(/"/)
             encoder.begin_group :string
             encoder.text_token match, :delimiter
-            encoder.text_token match, :content if match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)
+            encoder.text_token match, :content if (match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)) && !match.empty?
             encoder.text_token match, :delimiter if match = scan(/"/)
             encoder.end_group :string
             next
@@ -84,7 +84,7 @@ module Scanners
           when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/)
             encoder.begin_group :key
             encoder.text_token match[0,1], :delimiter
-            encoder.text_token match[1..-2], :content
+            encoder.text_token match[1..-2], :content if match.size > 2
             encoder.text_token match[-1,1], :delimiter
             encoder.end_group :key
             key_indent = column(pos - match.size) - 1
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index f4d07f1..f4e9d7d 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -3,14 +3,14 @@ module Styles
   
   # A colorful theme using CSS 3 colors (with alpha channel).
   class Alpha < Style
-
+    
     register_for :alpha
-
+    
     code_background = 'hsl(0,0%,95%)'
     numbers_background = 'hsl(180,65%,90%)'
     border_color = 'silver'
     normal_color = 'black'
-
+    
     CSS_MAIN_STYLES = <<-MAIN  # :nodoc:
 .CodeRay {
   background-color: #{code_background};
@@ -98,13 +98,16 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .key .delimiter { color: #404 }
 .keyword { color:#080; font-weight:bold }
 .label { color:#970; font-weight:bold }
-.local-variable { color:#963 }
+.local-variable { color:#950 }
+.map .content { color:#808 }
+.map .delimiter { color:#40A}
+.map { background-color:hsla(200,100%,50%,0.06); }
 .namespace { color:#707; font-weight:bold }
 .octal { color:#40E }
 .operator { }
 .predefined { color:#369; font-weight:bold }
 .predefined-constant { color:#069 }
-.predefined-type { color:#0a5; font-weight:bold }
+.predefined-type { color:#0a8; font-weight:bold }
 .preprocessor { color:#579 }
 .pseudo-class { color:#00C; font-weight:bold }
 .regexp { background-color:hsla(300,100%,50%,0.06); }
@@ -142,8 +145,8 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .change .change { color: #88f }
 .head .head { color: #f4f }
     TOKENS
-
+    
   end
-
+  
 end
 end
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index 9154658..9137a49 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -51,6 +51,7 @@ module CodeRay
     :keyword             => 'keyword',            # reserved word that's actually implemented; most scanners
     :label               => 'label',              # C, PHP
     :local_variable      => 'local-variable',     # local and magic variables; some scanners
+    :map                 => 'map',                # Lua tables
     :modifier            => 'modifier',           # used inside on strings; lots of scanners
     :namespace           => 'namespace',          # Clojure, Java, Taskpaper
     :octal               => 'octal',              # lots of scanners
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index 54358d4..e7bffce 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -3,17 +3,16 @@ module CodeRay
   # The Tokens class represents a list of tokens returned from
   # a Scanner. It's actually just an Array with a few helper methods.
   #
-  # A token itself is not a special object, just a two-element Array
-  # consisting of
+  # A token itself is not a special object, just two elements in an Array:
   # * the _token_ _text_ (the original source of the token in a String) or
   #   a _token_ _action_ (begin_group, end_group, begin_line, end_line)
   # * the _token_ _kind_ (a Symbol representing the type of the token)
   #
   # It looks like this:
   #
-  #   ['# It looks like this', :comment]
-  #   ['3.1415926', :float]
-  #   ['$^', :error]
+  #   ..., '# It looks like this', :comment, ...
+  #   ..., '3.1415926', :float, ...
+  #   ..., '$^', :error, ...
   #
   # Some scanners also yield sub-tokens, represented by special
   # token actions, for example :begin_group and :end_group.
@@ -21,11 +20,11 @@ module CodeRay
   # The Ruby scanner, for example, splits "a string" into:
   #
   #  [
-  #   [:begin_group, :string],
-  #   ['"', :delimiter],
-  #   ['a string', :content],
-  #   ['"', :delimiter],
-  #   [:end_group, :string]
+  #   :begin_group, :string,
+  #   '"',          :delimiter,
+  #   'a string',   :content,
+  #   '"',          :delimiter,
+  #   :end_group,   :string
   #  ]
   #
   # Tokens can be used to save the output of a Scanners in a simple
diff --git a/rake_tasks/benchmark.rake b/rake_tasks/benchmark.rake
index 040951b..2e38b57 100644
--- a/rake_tasks/benchmark.rake
+++ b/rake_tasks/benchmark.rake
@@ -1,7 +1,6 @@
 desc 'Do a benchmark'
 task :benchmark do
-  ruby "-v"
-  ruby "-wIlib bench/bench.rb ruby div 3000 N5"
+  ruby 'bench/bench.rb ruby html 3000'
 end
 
 task :bench => :benchmark