rake test now runs in debug mode.

All .rb files converted to UNIX format (where did the \r come from?)

23 files changed, 1661 insertions, 1661 deletions

diff --git a/lib/coderay/duo.rb b/lib/coderay/duo.rb
index 3125568..0e5956e 100644
--- a/lib/coderay/duo.rb
+++ b/lib/coderay/duo.rb
@@ -1,29 +1,29 @@
-module CodeRay

-  

-  # = Duo

-  #

-  # $Id: scanner.rb 123 2006-03-21 14:46:34Z murphy $

-  #

-  # TODO: Doc.

-  class Duo

-

-    attr_accessor :scanner, :encoder

-

-    def initialize lang, format, options = {}

-      @scanner = CodeRay.scanner lang, CodeRay.get_scanner_options(options)

-      @encoder = CodeRay.encoder format, options

-    end

-

-    class << self

-      alias [] new

-    end

-

-    def encode code

-      @scanner.string = code

-      @encoder.encode_tokens(scanner.tokenize)

-    end

-    alias highlight encode

-

-  end

-

-end

+module CodeRay
+  
+  # = Duo
+  #
+  # $Id: scanner.rb 123 2006-03-21 14:46:34Z murphy $
+  #
+  # TODO: Doc.
+  class Duo
+
+    attr_accessor :scanner, :encoder
+
+    def initialize lang, format, options = {}
+      @scanner = CodeRay.scanner lang, CodeRay.get_scanner_options(options)
+      @encoder = CodeRay.encoder format, options
+    end
+
+    class << self
+      alias [] new
+    end
+
+    def encode code
+      @scanner.string = code
+      @encoder.encode_tokens(scanner.tokenize)
+    end
+    alias highlight encode
+
+  end
+
+end
diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb
index 1065a9c..221cd58 100644
--- a/lib/coderay/encoder.rb
+++ b/lib/coderay/encoder.rb
@@ -1,173 +1,173 @@
-module CodeRay

-

-  # This module holds the Encoder class and its subclasses.

-  # For example, the HTML encoder is named CodeRay::Encoders::HTML

-  # can be found in coderay/encoders/html.

-  #

-  # Encoders also provides methods and constants for the register

-  # mechanism and the [] method that returns the Encoder class

-  # belonging to the given format.

-  module Encoders

-    extend PluginHost

-    plugin_path File.dirname(__FILE__), 'encoders'

-

-    # = Encoder

-    #

-    # The Encoder base class. Together with Scanner and

-    # Tokens, it forms the highlighting triad.

-    #

-    # Encoder instances take a Tokens object and do something with it.

-    #

-    # The most common Encoder is surely the HTML encoder

-    # (CodeRay::Encoders::HTML). It highlights the code in a colorful

-    # html page.

-    # If you want the highlighted code in a div or a span instead,

-    # use its subclasses Div and Span.

-    class Encoder

-      extend Plugin

-      plugin_host Encoders

-

-      attr_reader :token_stream

-

-      class << self

-

-        # Returns if the Encoder can be used in streaming mode.

-        def streamable?

-          is_a? Streamable

-        end

-

-        # If FILE_EXTENSION isn't defined, this method returns the

-        # downcase class name instead.

-        def const_missing sym

-          if sym == :FILE_EXTENSION

-            sym.to_s.downcase

-          else

-            super

-          end

-        end

-

-      end

-

-      # Subclasses are to store their default options in this constant.

-      DEFAULT_OPTIONS = { :stream => false }

-

-      # The options you gave the Encoder at creating.

-      attr_accessor :options

-

-      # Creates a new Encoder.

-      # +options+ is saved and used for all encode operations, as long

-      # as you don't overwrite it there by passing additional options.

-      #

-      # Encoder objects provide three encode methods:

-      # - encode simply takes a +code+ string and a +lang+

-      # - encode_tokens expects a +tokens+ object instead

-      # - encode_stream is like encode, but uses streaming mode.

-      #

-      # Each method has an optional +options+ parameter. These are

-      # added to the options you passed at creation.

-      def initialize options = {}

-        @options = self.class::DEFAULT_OPTIONS.merge options

-        raise "I am only the basic Encoder class. I can't encode "\

-          "anything. :( Use my subclasses." if self.class == Encoder

-      end

-

-      # Encode a Tokens object.

-      def encode_tokens tokens, options = {}

-        options = @options.merge options

-        setup options

-        compile tokens, options

-        finish options

-      end

-

-      # Encode the given +code+ after tokenizing it using the Scanner

-      # for +lang+.

-      def encode code, lang, options = {}

-        options = @options.merge options

-        scanner_options = CodeRay.get_scanner_options(options)

-        tokens = CodeRay.scan code, lang, scanner_options

-        encode_tokens tokens, options

-      end

-

-      # You can use highlight instead of encode, if that seems

-      # more clear to you.

-      alias highlight encode

-

-      # Encode the given +code+ using the Scanner for +lang+ in

-      # streaming mode.

-      def encode_stream code, lang, options = {}

-        raise NotStreamableError, self unless kind_of? Streamable

-        options = @options.merge options

-        setup options

-        scanner_options = CodeRay.get_scanner_options options

-        @token_stream =

-          CodeRay.scan_stream code, lang, scanner_options, &self

-        finish options

-      end

-

-      # Behave like a proc. The token method is converted to a proc.

-      def to_proc

-        method(:token).to_proc

-      end

-

-      # Return the default file extension for outputs of this encoder.

-      def file_extension

-        self.class::FILE_EXTENSION

-      end

-

-    protected

-

-      # Called with merged options before encoding starts.

-      # Sets @out to an empty string.

-      #

-      # See the HTML Encoder for an example of option caching.

-      def setup options

-        @out = ''

-      end

-

-      # Called with +text+ and +kind+ of the currently scanned token.

-      # For simple scanners, it's enougth to implement this method.

-      #

-      # By default, it calls text_token or block_token, depending on

-      # whether +text+ is a String.

-      def token text, kind

-        if text.is_a? ::String

-          text_token text, kind

-        elsif text.is_a? ::Symbol

-          block_token text, kind

-        else

-          raise 'Unknown token text type: %p' % text

-        end

-      end

-

-      def text_token text, kind

-      end

-

-      def block_token action, kind

-        case action

-        when :open

-          open_token kind

-        when :close

-          close_token kind

-        else

-          raise 'unknown block action: %p' % action

-        end

-      end

-

-      # Called with merged options after encoding starts.

-      # The return value is the result of encoding, typically @out.

-      def finish options

-        @out

-      end

-

-      # Do the encoding.

-      #

-      # The already created +tokens+ object must be used; it can be a

-      # TokenStream or a Tokens object.

-      def compile tokens, options

-        tokens.each(&self)

-      end

-

-    end

-

-  end

-end

+module CodeRay
+
+  # This module holds the Encoder class and its subclasses.
+  # For example, the HTML encoder is named CodeRay::Encoders::HTML
+  # can be found in coderay/encoders/html.
+  #
+  # Encoders also provides methods and constants for the register
+  # mechanism and the [] method that returns the Encoder class
+  # belonging to the given format.
+  module Encoders
+    extend PluginHost
+    plugin_path File.dirname(__FILE__), 'encoders'
+
+    # = Encoder
+    #
+    # The Encoder base class. Together with Scanner and
+    # Tokens, it forms the highlighting triad.
+    #
+    # Encoder instances take a Tokens object and do something with it.
+    #
+    # The most common Encoder is surely the HTML encoder
+    # (CodeRay::Encoders::HTML). It highlights the code in a colorful
+    # html page.
+    # If you want the highlighted code in a div or a span instead,
+    # use its subclasses Div and Span.
+    class Encoder
+      extend Plugin
+      plugin_host Encoders
+
+      attr_reader :token_stream
+
+      class << self
+
+        # Returns if the Encoder can be used in streaming mode.
+        def streamable?
+          is_a? Streamable
+        end
+
+        # If FILE_EXTENSION isn't defined, this method returns the
+        # downcase class name instead.
+        def const_missing sym
+          if sym == :FILE_EXTENSION
+            sym.to_s.downcase
+          else
+            super
+          end
+        end
+
+      end
+
+      # Subclasses are to store their default options in this constant.
+      DEFAULT_OPTIONS = { :stream => false }
+
+      # The options you gave the Encoder at creating.
+      attr_accessor :options
+
+      # Creates a new Encoder.
+      # +options+ is saved and used for all encode operations, as long
+      # as you don't overwrite it there by passing additional options.
+      #
+      # Encoder objects provide three encode methods:
+      # - encode simply takes a +code+ string and a +lang+
+      # - encode_tokens expects a +tokens+ object instead
+      # - encode_stream is like encode, but uses streaming mode.
+      #
+      # Each method has an optional +options+ parameter. These are
+      # added to the options you passed at creation.
+      def initialize options = {}
+        @options = self.class::DEFAULT_OPTIONS.merge options
+        raise "I am only the basic Encoder class. I can't encode "\
+          "anything. :( Use my subclasses." if self.class == Encoder
+      end
+
+      # Encode a Tokens object.
+      def encode_tokens tokens, options = {}
+        options = @options.merge options
+        setup options
+        compile tokens, options
+        finish options
+      end
+
+      # Encode the given +code+ after tokenizing it using the Scanner
+      # for +lang+.
+      def encode code, lang, options = {}
+        options = @options.merge options
+        scanner_options = CodeRay.get_scanner_options(options)
+        tokens = CodeRay.scan code, lang, scanner_options
+        encode_tokens tokens, options
+      end
+
+      # You can use highlight instead of encode, if that seems
+      # more clear to you.
+      alias highlight encode
+
+      # Encode the given +code+ using the Scanner for +lang+ in
+      # streaming mode.
+      def encode_stream code, lang, options = {}
+        raise NotStreamableError, self unless kind_of? Streamable
+        options = @options.merge options
+        setup options
+        scanner_options = CodeRay.get_scanner_options options
+        @token_stream =
+          CodeRay.scan_stream code, lang, scanner_options, &self
+        finish options
+      end
+
+      # Behave like a proc. The token method is converted to a proc.
+      def to_proc
+        method(:token).to_proc
+      end
+
+      # Return the default file extension for outputs of this encoder.
+      def file_extension
+        self.class::FILE_EXTENSION
+      end
+
+    protected
+
+      # Called with merged options before encoding starts.
+      # Sets @out to an empty string.
+      #
+      # See the HTML Encoder for an example of option caching.
+      def setup options
+        @out = ''
+      end
+
+      # Called with +text+ and +kind+ of the currently scanned token.
+      # For simple scanners, it's enougth to implement this method.
+      #
+      # By default, it calls text_token or block_token, depending on
+      # whether +text+ is a String.
+      def token text, kind
+        if text.is_a? ::String
+          text_token text, kind
+        elsif text.is_a? ::Symbol
+          block_token text, kind
+        else
+          raise 'Unknown token text type: %p' % text
+        end
+      end
+
+      def text_token text, kind
+      end
+
+      def block_token action, kind
+        case action
+        when :open
+          open_token kind
+        when :close
+          close_token kind
+        else
+          raise 'unknown block action: %p' % action
+        end
+      end
+
+      # Called with merged options after encoding starts.
+      # The return value is the result of encoding, typically @out.
+      def finish options
+        @out
+      end
+
+      # Do the encoding.
+      #
+      # The already created +tokens+ object must be used; it can be a
+      # TokenStream or a Tokens object.
+      def compile tokens, options
+        tokens.each(&self)
+      end
+
+    end
+
+  end
+end
diff --git a/lib/coderay/encoders/_map.rb b/lib/coderay/encoders/_map.rb
index a22a951..fdd8ae4 100644
--- a/lib/coderay/encoders/_map.rb
+++ b/lib/coderay/encoders/_map.rb
@@ -1,8 +1,8 @@
-module CodeRay

-module Encoders

-

-  map :stats => :statistic,

-    :plain => :text

-

-end

-end

+module CodeRay
+module Encoders
+
+  map :stats => :statistic,
+    :plain => :text
+
+end
+end
diff --git a/lib/coderay/encoders/count.rb b/lib/coderay/encoders/count.rb
index 6885541..c9a6dfd 100644
--- a/lib/coderay/encoders/count.rb
+++ b/lib/coderay/encoders/count.rb
@@ -1,21 +1,21 @@
-module CodeRay

-module Encoders

-

-  class Count < Encoder

-

-    include Streamable

-    register_for :count

-

-    protected

-

-    def setup options

-      @out = 0

-    end

-

-    def token text, kind

-      @out += 1

-    end

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  class Count < Encoder
+
+    include Streamable
+    register_for :count
+
+    protected
+
+    def setup options
+      @out = 0
+    end
+
+    def token text, kind
+      @out += 1
+    end
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
index 2639e1f..eb9eaa4 100644
--- a/lib/coderay/encoders/debug.rb
+++ b/lib/coderay/encoders/debug.rb
@@ -1,46 +1,46 @@
-module CodeRay

-module Encoders

-

-  # = Debug Encoder

-  #

-  # Fast encoder producing simple debug output.

-  #

-  # It is readable and diff-able and is used for testing.

-  #

-  # You cannot fully restore the tokens information from the

-  # output, because consecutive :space tokens are merged.

-  # Use Tokens#dump for caching purposes.

-  class Debug < Encoder

-

-    include Streamable

-    register_for :debug

-

-    FILE_EXTENSION = 'raydebug'

-

-  protected

-    def text_token text, kind

-      @out <<

-        if kind == :space

-          text

-        else

-          text = text.gsub(/[)\\]/, '\\\\\0')

-          "#{kind}(#{text})"

-        end

-    end

-

-    def block_token action, kind

-      @out << super

-    end

-

-    def open_token kind

-      "#{kind}<"

-    end

-

-    def close_token kind

-      ">"

-    end

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  # = Debug Encoder
+  #
+  # Fast encoder producing simple debug output.
+  #
+  # It is readable and diff-able and is used for testing.
+  #
+  # You cannot fully restore the tokens information from the
+  # output, because consecutive :space tokens are merged.
+  # Use Tokens#dump for caching purposes.
+  class Debug < Encoder
+
+    include Streamable
+    register_for :debug
+
+    FILE_EXTENSION = 'raydebug'
+
+  protected
+    def text_token text, kind
+      @out <<
+        if kind == :space
+          text
+        else
+          text = text.gsub(/[)\\]/, '\\\\\0')
+          "#{kind}(#{text})"
+        end
+    end
+
+    def block_token action, kind
+      @out << super
+    end
+
+    def open_token kind
+      "#{kind}<"
+    end
+
+    def close_token kind
+      ">"
+    end
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/div.rb b/lib/coderay/encoders/div.rb
index ce558f2..3d55415 100644
--- a/lib/coderay/encoders/div.rb
+++ b/lib/coderay/encoders/div.rb
@@ -1,20 +1,20 @@
-module CodeRay

-module Encoders

-

-  load :html

-

-  class Div < HTML

-

-    FILE_EXTENSION = 'div.html'

-

-    register_for :div

-

-    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({

-      :css => :style,

-      :wrap => :div,

-    })

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  load :html
+
+  class Div < HTML
+
+    FILE_EXTENSION = 'div.html'
+
+    register_for :div
+
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+      :css => :style,
+      :wrap => :div,
+    })
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/html/classes.rb b/lib/coderay/encoders/html/classes.rb
index 8493fa0..ea15ca0 100644
--- a/lib/coderay/encoders/html/classes.rb
+++ b/lib/coderay/encoders/html/classes.rb
@@ -1,73 +1,73 @@
-module CodeRay

-module Encoders

-

-  class HTML

-

-    ClassOfKind = {

-      :attribute_name => 'an',

-      :attribute_name_fat => 'af',

-      :attribute_value => 'av',

-      :attribute_value_fat => 'aw',

-      :bin => 'bi',

-      :char => 'ch',

-      :class => 'cl',

-      :class_variable => 'cv',

-      :color => 'cr',

-      :comment => 'c',

-      :constant => 'co',

-      :content => 'k',

-      :definition => 'df',

-      :delimiter => 'dl',

-      :directive => 'di',

-      :doc => 'do',

-      :doc_string => 'ds',

-      :entity => 'en',

-      :error => 'er',

-      :escape => 'e',

-      :exception => 'ex',

-      :float => 'fl',

-      :function => 'fu',

-      :global_variable => 'gv',

-      :hex => 'hx',

-      :include => 'ic',

-      :inline => 'il',

-      :instance_variable => 'iv',

-      :integer => 'i',

-      :interpreted => 'in',

-      :label => 'la',

-      :local_variable => 'lv',

-      :modifier => 'mod',

-      :oct => 'oc',

-      :operator_name => 'on',

-      :pre_constant => 'pc',

-      :pre_type => 'pt',

-      :predefined => 'pd',

-      :preprocessor => 'pp',

-      :regexp => 'rx',

-      :reserved => 'r',

-      :shell => 'sh',

-      :string => 's',

-      :symbol => 'sy',

-      :tag => 'ta',

-      :tag_fat => 'tf',

-      :tag_special => 'ts',

-      :type => 'ty',

-      :variable => 'v',

-      :xml_text => 'xt',

-

-      :ident => :NO_HIGHLIGHT, # 'id'

-      #:operator => 'op',

-      :operator => :NO_HIGHLIGHT,  # 'op'

-      :space => :NO_HIGHLIGHT,  # 'sp'

-      :plain => :NO_HIGHLIGHT,

-    }

-    ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]

-    ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]

-    ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]

-    ClassOfKind[:escape] = ClassOfKind[:delimiter]

-    ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  class HTML
+
+    ClassOfKind = {
+      :attribute_name => 'an',
+      :attribute_name_fat => 'af',
+      :attribute_value => 'av',
+      :attribute_value_fat => 'aw',
+      :bin => 'bi',
+      :char => 'ch',
+      :class => 'cl',
+      :class_variable => 'cv',
+      :color => 'cr',
+      :comment => 'c',
+      :constant => 'co',
+      :content => 'k',
+      :definition => 'df',
+      :delimiter => 'dl',
+      :directive => 'di',
+      :doc => 'do',
+      :doc_string => 'ds',
+      :entity => 'en',
+      :error => 'er',
+      :escape => 'e',
+      :exception => 'ex',
+      :float => 'fl',
+      :function => 'fu',
+      :global_variable => 'gv',
+      :hex => 'hx',
+      :include => 'ic',
+      :inline => 'il',
+      :instance_variable => 'iv',
+      :integer => 'i',
+      :interpreted => 'in',
+      :label => 'la',
+      :local_variable => 'lv',
+      :modifier => 'mod',
+      :oct => 'oc',
+      :operator_name => 'on',
+      :pre_constant => 'pc',
+      :pre_type => 'pt',
+      :predefined => 'pd',
+      :preprocessor => 'pp',
+      :regexp => 'rx',
+      :reserved => 'r',
+      :shell => 'sh',
+      :string => 's',
+      :symbol => 'sy',
+      :tag => 'ta',
+      :tag_fat => 'tf',
+      :tag_special => 'ts',
+      :type => 'ty',
+      :variable => 'v',
+      :xml_text => 'xt',
+
+      :ident => :NO_HIGHLIGHT, # 'id'
+      #:operator => 'op',
+      :operator => :NO_HIGHLIGHT,  # 'op'
+      :space => :NO_HIGHLIGHT,  # 'sp'
+      :plain => :NO_HIGHLIGHT,
+    }
+    ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
+    ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
+    ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
+    ClassOfKind[:escape] = ClassOfKind[:delimiter]
+    ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb
index b76d682..d577602 100644
--- a/lib/coderay/encoders/html/css.rb
+++ b/lib/coderay/encoders/html/css.rb
@@ -1,65 +1,65 @@
-module CodeRay

-module Encoders

-

-  class HTML

-    class CSS

-

-      attr :stylesheet

-

-      def CSS.load_stylesheet style = nil

-        CodeRay::Styles[style]

-      end

-

-      def initialize style = :default

-        @classes = Hash.new

-        style = CSS.load_stylesheet style

-        @stylesheet = [

-          style::CSS_MAIN_STYLES,

-          style::TOKEN_COLORS.gsub(/^(?!$)/, '.CodeRay ')

-        ].join("\n")

-        parse style::TOKEN_COLORS

-      end

-

-      def [] *styles

-        cl = @classes[styles.first]

-        return '' unless cl

-        style = ''

-        1.upto(styles.size) do |offset|

-          break if style = cl[styles[offset .. -1]]

-        end

-        raise 'Style not found: %p' % [styles] if $DEBUG and style.empty?

-        return style

-      end

-

-    private

-

-      CSS_CLASS_PATTERN = /

-        ( (?:                # $1 = classes

-          \s* \. [-\w]+

-        )+ )

-        \s* \{ \s*

-        ( [^\}]+ )?          # $2 = style

-        \s* \} \s*

-      |

-        ( . )                # $3 = error

-      /mx

-      def parse stylesheet

-        stylesheet.scan CSS_CLASS_PATTERN do |classes, style, error|

-          raise "CSS parse error: '#{error.inspect}' not recognized" if error

-          styles = classes.scan(/[-\w]+/)

-          cl = styles.pop

-          @classes[cl] ||= Hash.new

-          @classes[cl][styles] = style.to_s.strip

-        end

-      end

-

-    end

-  end

-

-end

-end

-

-if $0 == __FILE__

-  require 'pp'

-  pp CodeRay::Encoders::HTML::CSS.new

-end

+module CodeRay
+module Encoders
+
+  class HTML
+    class CSS
+
+      attr :stylesheet
+
+      def CSS.load_stylesheet style = nil
+        CodeRay::Styles[style]
+      end
+
+      def initialize style = :default
+        @classes = Hash.new
+        style = CSS.load_stylesheet style
+        @stylesheet = [
+          style::CSS_MAIN_STYLES,
+          style::TOKEN_COLORS.gsub(/^(?!$)/, '.CodeRay ')
+        ].join("\n")
+        parse style::TOKEN_COLORS
+      end
+
+      def [] *styles
+        cl = @classes[styles.first]
+        return '' unless cl
+        style = ''
+        1.upto(styles.size) do |offset|
+          break if style = cl[styles[offset .. -1]]
+        end
+        raise 'Style not found: %p' % [styles] if $DEBUG and style.empty?
+        return style
+      end
+
+    private
+
+      CSS_CLASS_PATTERN = /
+        ( (?:                # $1 = classes
+          \s* \. [-\w]+
+        )+ )
+        \s* \{ \s*
+        ( [^\}]+ )?          # $2 = style
+        \s* \} \s*
+      |
+        ( . )                # $3 = error
+      /mx
+      def parse stylesheet
+        stylesheet.scan CSS_CLASS_PATTERN do |classes, style, error|
+          raise "CSS parse error: '#{error.inspect}' not recognized" if error
+          styles = classes.scan(/[-\w]+/)
+          cl = styles.pop
+          @classes[cl] ||= Hash.new
+          @classes[cl][styles] = style.to_s.strip
+        end
+      end
+
+    end
+  end
+
+end
+end
+
+if $0 == __FILE__
+  require 'pp'
+  pp CodeRay::Encoders::HTML::CSS.new
+end
diff --git a/lib/coderay/encoders/html/numerization.rb b/lib/coderay/encoders/html/numerization.rb
index 2960f87..1e4a4ed 100644
--- a/lib/coderay/encoders/html/numerization.rb
+++ b/lib/coderay/encoders/html/numerization.rb
@@ -1,122 +1,122 @@
-module CodeRay

-module Encoders

-

-  class HTML

-

-    module Output

-

-      def numerize *args

-        clone.numerize!(*args)

-      end

-

-=begin      NUMERIZABLE_WRAPPINGS = {

-        :table => [:div, :page, nil],

-        :inline => :all,

-        :list => [:div, :page, nil]

-      }

-      NUMERIZABLE_WRAPPINGS.default = :all

-=end

-      def numerize! mode = :table, options = {}

-        return self unless mode

-

-        options = DEFAULT_OPTIONS.merge options

-

-        start = options[:line_number_start]

-        unless start.is_a? Integer

-          raise ArgumentError, "Invalid value %p for :line_number_start; Integer expected." % start

-        end

-

-        #allowed_wrappings = NUMERIZABLE_WRAPPINGS[mode]

-        #unless allowed_wrappings == :all or allowed_wrappings.include? options[:wrap]

-        #  raise ArgumentError, "Can't numerize, :wrap must be in %p, but is %p" % [NUMERIZABLE_WRAPPINGS, options[:wrap]]

-        #end

-

-        bold_every = options[:bold_every]

-        bolding =

-          if bold_every == false

-            proc { |line| line.to_s }

-          elsif bold_every.is_a? Integer

-            raise ArgumentError, ":bolding can't be 0." if bold_every == 0

-            proc do |line|

-              if line % bold_every == 0

-                "<strong>#{line}</strong>"  # every bold_every-th number in bold

-              else

-                line.to_s

-              end

-            end

-          else

-            raise ArgumentError, 'Invalid value %p for :bolding; false or Integer expected.' % bold_every

-          end

-

-        case mode

-        when :inline

-          max_width = (start + line_count).to_s.size

-          line = start

-          gsub!(/^/) do

-            line_number = bolding.call line

-            indent = ' ' * (max_width - line.to_s.size)

-            res = "<span class=\"no\">#{indent}#{line_number}</span> "

-            line += 1

-            res

-          end

-

-        when :table

-          # This is really ugly.

-          # Because even monospace fonts seem to have different heights when bold,

-          # I make the newline bold, both in the code and the line numbers.

-          # FIXME Still not working perfect for Mr. Internet Exploder

-          # FIXME Firefox struggles with very long codes (> 200 lines)

-          line_numbers = (start ... start + line_count).to_a.map(&bolding).join("\n")

-          line_numbers << "\n"  # also for Mr. MS Internet Exploder :-/

-          line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }

-

-          line_numbers_table_tpl = TABLE.apply('LINE_NUMBERS', line_numbers)

-          gsub!(/\n/) { "<tt>\n</tt>" }

-          wrap_in! line_numbers_table_tpl

-          @wrapped_in = :div

-

-        when :list

-          opened_tags = []

-          gsub!(/^.*$\n?/) do |line|

-            line.chomp!

-

-            open = opened_tags.join

-            line.scan(%r!<(/)?span[^>]*>?!) do |close,|

-              if close

-                opened_tags.pop

-              else

-                opened_tags << $&

-              end

-            end

-            close = '</span>' * opened_tags.size

-

-            "<li>#{open}#{line}#{close}</li>"

-          end

-          wrap_in! LIST

-          @wrapped_in = :div

-

-        else

-          raise ArgumentError, 'Unknown value %p for mode: expected one of %p' %

-            [mode, [:table, :list, :inline]]

-        end

-

-        self

-      end

-

-      def line_count

-        line_count = count("\n")

-        position_of_last_newline = rindex(?\n)

-        if position_of_last_newline

-          after_last_newline = self[position_of_last_newline + 1 .. -1]

-          ends_with_newline = after_last_newline[/\A(?:<\/span>)*\z/]

-          line_count += 1 if not ends_with_newline

-        end

-        line_count

-      end

-

-    end

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  class HTML
+
+    module Output
+
+      def numerize *args
+        clone.numerize!(*args)
+      end
+
+=begin      NUMERIZABLE_WRAPPINGS = {
+        :table => [:div, :page, nil],
+        :inline => :all,
+        :list => [:div, :page, nil]
+      }
+      NUMERIZABLE_WRAPPINGS.default = :all
+=end
+      def numerize! mode = :table, options = {}
+        return self unless mode
+
+        options = DEFAULT_OPTIONS.merge options
+
+        start = options[:line_number_start]
+        unless start.is_a? Integer
+          raise ArgumentError, "Invalid value %p for :line_number_start; Integer expected." % start
+        end
+
+        #allowed_wrappings = NUMERIZABLE_WRAPPINGS[mode]
+        #unless allowed_wrappings == :all or allowed_wrappings.include? options[:wrap]
+        #  raise ArgumentError, "Can't numerize, :wrap must be in %p, but is %p" % [NUMERIZABLE_WRAPPINGS, options[:wrap]]
+        #end
+
+        bold_every = options[:bold_every]
+        bolding =
+          if bold_every == false
+            proc { |line| line.to_s }
+          elsif bold_every.is_a? Integer
+            raise ArgumentError, ":bolding can't be 0." if bold_every == 0
+            proc do |line|
+              if line % bold_every == 0
+                "<strong>#{line}</strong>"  # every bold_every-th number in bold
+              else
+                line.to_s
+              end
+            end
+          else
+            raise ArgumentError, 'Invalid value %p for :bolding; false or Integer expected.' % bold_every
+          end
+
+        case mode
+        when :inline
+          max_width = (start + line_count).to_s.size
+          line = start
+          gsub!(/^/) do
+            line_number = bolding.call line
+            indent = ' ' * (max_width - line.to_s.size)
+            res = "<span class=\"no\">#{indent}#{line_number}</span> "
+            line += 1
+            res
+          end
+
+        when :table
+          # This is really ugly.
+          # Because even monospace fonts seem to have different heights when bold,
+          # I make the newline bold, both in the code and the line numbers.
+          # FIXME Still not working perfect for Mr. Internet Exploder
+          # FIXME Firefox struggles with very long codes (> 200 lines)
+          line_numbers = (start ... start + line_count).to_a.map(&bolding).join("\n")
+          line_numbers << "\n"  # also for Mr. MS Internet Exploder :-/
+          line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }
+
+          line_numbers_table_tpl = TABLE.apply('LINE_NUMBERS', line_numbers)
+          gsub!(/\n/) { "<tt>\n</tt>" }
+          wrap_in! line_numbers_table_tpl
+          @wrapped_in = :div
+
+        when :list
+          opened_tags = []
+          gsub!(/^.*$\n?/) do |line|
+            line.chomp!
+
+            open = opened_tags.join
+            line.scan(%r!<(/)?span[^>]*>?!) do |close,|
+              if close
+                opened_tags.pop
+              else
+                opened_tags << $&
+              end
+            end
+            close = '</span>' * opened_tags.size
+
+            "<li>#{open}#{line}#{close}</li>"
+          end
+          wrap_in! LIST
+          @wrapped_in = :div
+
+        else
+          raise ArgumentError, 'Unknown value %p for mode: expected one of %p' %
+            [mode, [:table, :list, :inline]]
+        end
+
+        self
+      end
+
+      def line_count
+        line_count = count("\n")
+        position_of_last_newline = rindex(?\n)
+        if position_of_last_newline
+          after_last_newline = self[position_of_last_newline + 1 .. -1]
+          ends_with_newline = after_last_newline[/\A(?:<\/span>)*\z/]
+          line_count += 1 if not ends_with_newline
+        end
+        line_count
+      end
+
+    end
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/html/output.rb b/lib/coderay/encoders/html/output.rb
index 61258ee..e74e55e 100644
--- a/lib/coderay/encoders/html/output.rb
+++ b/lib/coderay/encoders/html/output.rb
@@ -1,195 +1,195 @@
-module CodeRay

-module Encoders

-

-  class HTML

-

-    # This module is included in the output String from thew HTML Encoder.

-    #

-    # It provides methods like wrap, div, page etc.

-    #

-    # Remember to use #clone instead of #dup to keep the modules the object was

-    # extended with.

-    #

-    # TODO: more doc.

-    module Output

-

-      require 'coderay/encoders/html/numerization.rb'

-

-      attr_accessor :css

-

-      class << self

-

-        # This makes Output look like a class.

-        #

-        # Example:

-        #

-        #  a = Output.new '<span class="co">Code</span>'

-        #  a.wrap! :page

-        def new string, css = CSS.new, element = nil

-          output = string.clone.extend self

-          output.wrapped_in = element

-          output.css = css

-          output

-        end

-

-        # Raises an exception if an object that doesn't respond to to_str is extended by Output,

-        # to prevent users from misuse. Use Module#remove_method to disable.

-        def extended o

-          warn "The Output module is intended to extend instances of String, not #{o.class}." unless o.respond_to? :to_str

-        end

-

-        def make_stylesheet css, in_tag = false

-          sheet = css.stylesheet

-          sheet = <<-CSS if in_tag

-<style type="text/css">

-#{sheet}

-</style>

-          CSS

-          sheet

-        end

-

-        def page_template_for_css css

-          sheet = make_stylesheet css

-          PAGE.apply 'CSS', sheet

-        end

-

-        # Define a new wrapper. This is meta programming.

-        def wrapper *wrappers

-          wrappers.each do |wrapper|

-            define_method wrapper do |*args|

-              wrap wrapper, *args

-            end

-            define_method "#{wrapper}!".to_sym do |*args|

-              wrap! wrapper, *args

-            end

-          end

-        end

-

-      end

-

-      wrapper :div, :span, :page

-

-      def wrapped_in? element

-        wrapped_in == element

-      end

-

-      def wrapped_in

-        @wrapped_in ||= nil

-      end

-      attr_writer :wrapped_in

-

-      def wrap_in template

-        clone.wrap_in! template

-      end

-

-      def wrap_in! template

-        Template.wrap! self, template, 'CONTENT'

-        self

-      end

-

-      def wrap! element, *args

-        return self if not element or element == wrapped_in

-        case element

-        when :div

-          raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil

-          wrap_in! DIV

-        when :span

-          raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil

-          wrap_in! SPAN

-        when :page

-          wrap! :div if wrapped_in? nil

-          raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? :div

-          wrap_in! Output.page_template_for_css(@css)

-        when nil

-          return self

-        else

-          raise "Unknown value %p for :wrap" % element

-        end

-        @wrapped_in = element

-        self

-      end

-

-      def wrap *args

-        clone.wrap!(*args)

-      end

-

-      def stylesheet in_tag = false

-        Output.make_stylesheet @css, in_tag

-      end

-

-      class Template < String

-

-        def self.wrap! str, template, target

-          target = Regexp.new(Regexp.escape("<%#{target}%>"))

-          if template =~ target

-            str[0,0] = $`

-            str << $'

-          else

-            raise "Template target <%%%p%%> not found" % target

-          end

-        end

-

-        def apply target, replacement

-          target = Regexp.new(Regexp.escape("<%#{target}%>"))

-          if self =~ target

-            Template.new($` + replacement + $')

-          else

-            raise "Template target <%%%p%%> not found" % target

-          end

-        end

-

-        module Simple

-          def ` str  #` <-- for stupid editors

-            Template.new str

-          end

-        end

-      end

-

-      extend Template::Simple

-

-#-- don't include the templates in docu

-

-      SPAN = `<span class="CodeRay"><%CONTENT%></span>`

-

-      DIV = <<-`DIV`

-<div class="CodeRay">

-  <div class="code"><pre><%CONTENT%></pre></div>

-</div>

-      DIV

-

-      TABLE = <<-`TABLE`

-<table class="CodeRay"><tr>

-  <td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><%LINE_NUMBERS%></pre></td>

-  <td class="code"><pre ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><%CONTENT%></pre></td>

-</tr></table>

-      TABLE

-      # title="double click to expand"

-

-      LIST = <<-`LIST`

-<ol class="CodeRay"><%CONTENT%></ol>

-      LIST

-

-      PAGE = <<-`PAGE`

-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"

-  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">

-<head>

-  <meta http-equiv="content-type" content="text/html; charset=utf-8" />

-  <title>CodeRay HTML Encoder Example</title>

-  <style type="text/css">

-<%CSS%>

-  </style>

-</head>

-<body style="background-color: white;">

-

-<%CONTENT%>

-</body>

-</html>

-      PAGE

-

-    end

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  class HTML
+
+    # This module is included in the output String from thew HTML Encoder.
+    #
+    # It provides methods like wrap, div, page etc.
+    #
+    # Remember to use #clone instead of #dup to keep the modules the object was
+    # extended with.
+    #
+    # TODO: more doc.
+    module Output
+
+      require 'coderay/encoders/html/numerization.rb'
+
+      attr_accessor :css
+
+      class << self
+
+        # This makes Output look like a class.
+        #
+        # Example:
+        #
+        #  a = Output.new '<span class="co">Code</span>'
+        #  a.wrap! :page
+        def new string, css = CSS.new, element = nil
+          output = string.clone.extend self
+          output.wrapped_in = element
+          output.css = css
+          output
+        end
+
+        # Raises an exception if an object that doesn't respond to to_str is extended by Output,
+        # to prevent users from misuse. Use Module#remove_method to disable.
+        def extended o
+          warn "The Output module is intended to extend instances of String, not #{o.class}." unless o.respond_to? :to_str
+        end
+
+        def make_stylesheet css, in_tag = false
+          sheet = css.stylesheet
+          sheet = <<-CSS if in_tag
+<style type="text/css">
+#{sheet}
+</style>
+          CSS
+          sheet
+        end
+
+        def page_template_for_css css
+          sheet = make_stylesheet css
+          PAGE.apply 'CSS', sheet
+        end
+
+        # Define a new wrapper. This is meta programming.
+        def wrapper *wrappers
+          wrappers.each do |wrapper|
+            define_method wrapper do |*args|
+              wrap wrapper, *args
+            end
+            define_method "#{wrapper}!".to_sym do |*args|
+              wrap! wrapper, *args
+            end
+          end
+        end
+
+      end
+
+      wrapper :div, :span, :page
+
+      def wrapped_in? element
+        wrapped_in == element
+      end
+
+      def wrapped_in
+        @wrapped_in ||= nil
+      end
+      attr_writer :wrapped_in
+
+      def wrap_in template
+        clone.wrap_in! template
+      end
+
+      def wrap_in! template
+        Template.wrap! self, template, 'CONTENT'
+        self
+      end
+
+      def wrap! element, *args
+        return self if not element or element == wrapped_in
+        case element
+        when :div
+          raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil
+          wrap_in! DIV
+        when :span
+          raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil
+          wrap_in! SPAN
+        when :page
+          wrap! :div if wrapped_in? nil
+          raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? :div
+          wrap_in! Output.page_template_for_css(@css)
+        when nil
+          return self
+        else
+          raise "Unknown value %p for :wrap" % element
+        end
+        @wrapped_in = element
+        self
+      end
+
+      def wrap *args
+        clone.wrap!(*args)
+      end
+
+      def stylesheet in_tag = false
+        Output.make_stylesheet @css, in_tag
+      end
+
+      class Template < String
+
+        def self.wrap! str, template, target
+          target = Regexp.new(Regexp.escape("<%#{target}%>"))
+          if template =~ target
+            str[0,0] = $`
+            str << $'
+          else
+            raise "Template target <%%%p%%> not found" % target
+          end
+        end
+
+        def apply target, replacement
+          target = Regexp.new(Regexp.escape("<%#{target}%>"))
+          if self =~ target
+            Template.new($` + replacement + $')
+          else
+            raise "Template target <%%%p%%> not found" % target
+          end
+        end
+
+        module Simple
+          def ` str  #` <-- for stupid editors
+            Template.new str
+          end
+        end
+      end
+
+      extend Template::Simple
+
+#-- don't include the templates in docu
+
+      SPAN = `<span class="CodeRay"><%CONTENT%></span>`
+
+      DIV = <<-`DIV`
+<div class="CodeRay">
+  <div class="code"><pre><%CONTENT%></pre></div>
+</div>
+      DIV
+
+      TABLE = <<-`TABLE`
+<table class="CodeRay"><tr>
+  <td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><%LINE_NUMBERS%></pre></td>
+  <td class="code"><pre ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><%CONTENT%></pre></td>
+</tr></table>
+      TABLE
+      # title="double click to expand"
+
+      LIST = <<-`LIST`
+<ol class="CodeRay"><%CONTENT%></ol>
+      LIST
+
+      PAGE = <<-`PAGE`
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">
+<head>
+  <meta http-equiv="content-type" content="text/html; charset=utf-8" />
+  <title>CodeRay HTML Encoder Example</title>
+  <style type="text/css">
+<%CSS%>
+  </style>
+</head>
+<body style="background-color: white;">
+
+<%CONTENT%>
+</body>
+</html>
+      PAGE
+
+    end
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/null.rb b/lib/coderay/encoders/null.rb
index 96d81fe..add3862 100644
--- a/lib/coderay/encoders/null.rb
+++ b/lib/coderay/encoders/null.rb
@@ -1,26 +1,26 @@
-module CodeRay

-module Encoders

-

-  # = Null Encoder

-  #

-  # Does nothing and returns an empty string.

-  class Null < Encoder

-

-    include Streamable

-    register_for :null

-

-    # Defined for faster processing

-    def to_proc

-      proc {}

-    end

-

-  protected

-

-    def token(*)

-      # do nothing

-    end

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  # = Null Encoder
+  #
+  # Does nothing and returns an empty string.
+  class Null < Encoder
+
+    include Streamable
+    register_for :null
+
+    # Defined for faster processing
+    def to_proc
+      proc {}
+    end
+
+  protected
+
+    def token(*)
+      # do nothing
+    end
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/page.rb b/lib/coderay/encoders/page.rb
index 1ed7985..c08f094 100644
--- a/lib/coderay/encoders/page.rb
+++ b/lib/coderay/encoders/page.rb
@@ -1,21 +1,21 @@
-module CodeRay

-module Encoders

-

-  load :html

-

-  class Page < HTML

-

-    FILE_EXTENSION = 'html'

-

-    register_for :page

-

-    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({

-      :css => :class,

-      :wrap => :page,

-      :line_numbers => :table

-    })

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  load :html
+
+  class Page < HTML
+
+    FILE_EXTENSION = 'html'
+
+    register_for :page
+
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+      :css => :class,
+      :wrap => :page,
+      :line_numbers => :table
+    })
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/span.rb b/lib/coderay/encoders/span.rb
index e892cb2..988afec 100644
--- a/lib/coderay/encoders/span.rb
+++ b/lib/coderay/encoders/span.rb
@@ -1,20 +1,20 @@
-module CodeRay

-module Encoders

-

-  load :html

-

-  class Span < HTML

-

-    FILE_EXTENSION = 'span.html'

-

-    register_for :span

-

-    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({

-      :css => :style,

-      :wrap => :span,

-    })

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  load :html
+
+  class Span < HTML
+
+    FILE_EXTENSION = 'span.html'
+
+    register_for :span
+
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+      :css => :style,
+      :wrap => :span,
+    })
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb
index f80d5c8..e2a0460 100644
--- a/lib/coderay/encoders/statistic.rb
+++ b/lib/coderay/encoders/statistic.rb
@@ -1,81 +1,81 @@
-module CodeRay

-module Encoders

-

-  # Makes a statistic for the given tokens.

-  class Statistic < Encoder

-

-    include Streamable

-    register_for :stats, :statistic

-

-    attr_reader :type_stats, :real_token_count

-

-  protected

-

-    TypeStats = Struct.new :count, :size

-

-    def setup options

-      @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }

-      @real_token_count = 0

-    end

-

-    def generate tokens, options

-      @tokens = tokens

-      super

-    end

-

-    def text_token text, kind

-      @real_token_count += 1 unless kind == :space

-      @type_stats[kind].count += 1

-      @type_stats[kind].size += text.size

-      @type_stats['TOTAL'].size += text.size

-    end

-

-    # TODO Hierarchy handling

-    def block_token action, kind

-      #@content_type = kind

-      @type_stats['open/close'].count += 1

-    end

-

-    def token text, kind

-      super

-      @type_stats['TOTAL'].count += 1

-    end

-

-    STATS = <<-STATS

-

-Code Statistics

-

-Tokens            %8d

-  Non-Whitespace  %8d

-Bytes Total       %8d

-

-Token Types (%d):

-  type                     count     ratio    size (average)

--------------------------------------------------------------

-%s

-      STATS

-# space                    12007   33.81 %     1.7

-    TOKEN_TYPES_ROW = <<-TKR

-  %-20s  %8d  %6.2f %%   %5.1f

-      TKR

-

-    def finish options

-      all = @type_stats['TOTAL']

-      all_count, all_size = all.count, all.size

-      @type_stats.each do |type, stat|

-        stat.size /= stat.count.to_f

-      end

-      types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|

-        TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]

-      end.join

-      STATS % [

-        all_count, @real_token_count, all_size,

-        @type_stats.delete_if { |k, v| k.is_a? String }.size,

-        types_stats

-      ]

-    end

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  # Makes a statistic for the given tokens.
+  class Statistic < Encoder
+
+    include Streamable
+    register_for :stats, :statistic
+
+    attr_reader :type_stats, :real_token_count
+
+  protected
+
+    TypeStats = Struct.new :count, :size
+
+    def setup options
+      @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
+      @real_token_count = 0
+    end
+
+    def generate tokens, options
+      @tokens = tokens
+      super
+    end
+
+    def text_token text, kind
+      @real_token_count += 1 unless kind == :space
+      @type_stats[kind].count += 1
+      @type_stats[kind].size += text.size
+      @type_stats['TOTAL'].size += text.size
+    end
+
+    # TODO Hierarchy handling
+    def block_token action, kind
+      #@content_type = kind
+      @type_stats['open/close'].count += 1
+    end
+
+    def token text, kind
+      super
+      @type_stats['TOTAL'].count += 1
+    end
+
+    STATS = <<-STATS
+
+Code Statistics
+
+Tokens            %8d
+  Non-Whitespace  %8d
+Bytes Total       %8d
+
+Token Types (%d):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+%s
+      STATS
+# space                    12007   33.81 %     1.7
+    TOKEN_TYPES_ROW = <<-TKR
+  %-20s  %8d  %6.2f %%   %5.1f
+      TKR
+
+    def finish options
+      all = @type_stats['TOTAL']
+      all_count, all_size = all.count, all.size
+      @type_stats.each do |type, stat|
+        stat.size /= stat.count.to_f
+      end
+      types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
+        TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
+      end.join
+      STATS % [
+        all_count, @real_token_count, all_size,
+        @type_stats.delete_if { |k, v| k.is_a? String }.size,
+        types_stats
+      ]
+    end
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb
index 31661ef..17256c6 100644
--- a/lib/coderay/encoders/text.rb
+++ b/lib/coderay/encoders/text.rb
@@ -1,33 +1,33 @@
-module CodeRay

-module Encoders

-

-  class Text < Encoder

-

-    include Streamable

-    register_for :text

-

-    FILE_EXTENSION = 'txt'

-

-    DEFAULT_OPTIONS = {

-      :separator => ''

-    }

-

-  protected

-    def setup options

-      super

-      @sep = options[:separator]

-    end

-

-    def token text, kind

-      return unless text.respond_to? :to_str

-      @out << text + @sep

-    end

-

-    def finish options

-      @out.chomp @sep

-    end

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  class Text < Encoder
+
+    include Streamable
+    register_for :text
+
+    FILE_EXTENSION = 'txt'
+
+    DEFAULT_OPTIONS = {
+      :separator => ''
+    }
+
+  protected
+    def setup options
+      super
+      @sep = options[:separator]
+    end
+
+    def token text, kind
+      return unless text.respond_to? :to_str
+      @out << text + @sep
+    end
+
+    def finish options
+      @out.chomp @sep
+    end
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/tokens.rb b/lib/coderay/encoders/tokens.rb
index 743cc0e..2428589 100644
--- a/lib/coderay/encoders/tokens.rb
+++ b/lib/coderay/encoders/tokens.rb
@@ -1,44 +1,44 @@
-module CodeRay

-module Encoders

-

-  # The Tokens encoder converts the tokens to a simple

-  # readable format. It doesn't use colors and is mainly

-  # intended for console output.

-  #

-  # The tokens are converted with Tokens.write_token.

-  #

-  # The format is:

-  #

-  #   <token-kind> \t <escaped token-text> \n

-  #

-  # Example:

-  #

-  #   require 'coderay'

-  #   puts CodeRay.scan("puts 3 + 4", :ruby).tokens

-  #

-  # prints:

-  #

-  #   ident   puts

-  #   space

-  #   integer 3

-  #   space

-  #   operator        +

-  #   space

-  #   integer 4

-  #

-  class Tokens < Encoder

-

-    include Streamable

-    register_for :tokens

-

-    FILE_EXTENSION = 'tok'

-

-    protected

-    def token *args

-      @out << CodeRay::Tokens.write_token(*args)

-    end

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  # The Tokens encoder converts the tokens to a simple
+  # readable format. It doesn't use colors and is mainly
+  # intended for console output.
+  #
+  # The tokens are converted with Tokens.write_token.
+  #
+  # The format is:
+  #
+  #   <token-kind> \t <escaped token-text> \n
+  #
+  # Example:
+  #
+  #   require 'coderay'
+  #   puts CodeRay.scan("puts 3 + 4", :ruby).tokens
+  #
+  # prints:
+  #
+  #   ident   puts
+  #   space
+  #   integer 3
+  #   space
+  #   operator        +
+  #   space
+  #   integer 4
+  #
+  class Tokens < Encoder
+
+    include Streamable
+    register_for :tokens
+
+    FILE_EXTENSION = 'tok'
+
+    protected
+    def token *args
+      @out << CodeRay::Tokens.write_token(*args)
+    end
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/xml.rb b/lib/coderay/encoders/xml.rb
index 21ef0cf..09e4549 100644
--- a/lib/coderay/encoders/xml.rb
+++ b/lib/coderay/encoders/xml.rb
@@ -1,71 +1,71 @@
-module CodeRay

-module Encoders

-

-  # = XML Encoder

-  #

-  # Uses REXML. Very slow.

-  class XML < Encoder

-

-    include Streamable

-    register_for :xml

-

-    FILE_EXTENSION = 'xml'

-

-    require 'rexml/document'

-

-    DEFAULT_OPTIONS = {

-      :tab_width => 8,

-      :pretty => -1,

-      :transitive => false,

-    }

-

-  protected

-

-    def setup options

-      @out = ''

-      @doc = REXML::Document.new

-      @doc << REXML::XMLDecl.new

-      @tab_width = options[:tab_width]

-      @root = @node = @doc.add_element('coderay-tokens')

-    end

-

-    def finish options

-      @doc.write @out, options[:pretty], options[:transitive], true

-      @out

-    end

-

-    def text_token text, kind

-      if kind == :space

-        token = @node

-      else

-        token = @node.add_element kind.to_s

-      end

-      text.scan(/(\x20+)|(\t+)|(\n)|[^\x20\t\n]+/) do |space, tab, nl|

-        case

-        when space

-          token << REXML::Text.new(space, true)

-        when tab

-          token << REXML::Text.new(tab, true)

-        when nl

-          token << REXML::Text.new(nl, true)

-        else

-          token << REXML::Text.new($&)

-        end

-      end

-    end

-

-    def open_token kind

-      @node = @node.add_element kind.to_s

-    end

-

-    def close_token kind

-      if @node == @root

-        raise 'no token to close!'

-      end

-      @node = @node.parent

-    end

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  # = XML Encoder
+  #
+  # Uses REXML. Very slow.
+  class XML < Encoder
+
+    include Streamable
+    register_for :xml
+
+    FILE_EXTENSION = 'xml'
+
+    require 'rexml/document'
+
+    DEFAULT_OPTIONS = {
+      :tab_width => 8,
+      :pretty => -1,
+      :transitive => false,
+    }
+
+  protected
+
+    def setup options
+      @out = ''
+      @doc = REXML::Document.new
+      @doc << REXML::XMLDecl.new
+      @tab_width = options[:tab_width]
+      @root = @node = @doc.add_element('coderay-tokens')
+    end
+
+    def finish options
+      @doc.write @out, options[:pretty], options[:transitive], true
+      @out
+    end
+
+    def text_token text, kind
+      if kind == :space
+        token = @node
+      else
+        token = @node.add_element kind.to_s
+      end
+      text.scan(/(\x20+)|(\t+)|(\n)|[^\x20\t\n]+/) do |space, tab, nl|
+        case
+        when space
+          token << REXML::Text.new(space, true)
+        when tab
+          token << REXML::Text.new(tab, true)
+        when nl
+          token << REXML::Text.new(nl, true)
+        else
+          token << REXML::Text.new($&)
+        end
+      end
+    end
+
+    def open_token kind
+      @node = @node.add_element kind.to_s
+    end
+
+    def close_token kind
+      if @node == @root
+        raise 'no token to close!'
+      end
+      @node = @node.parent
+    end
+
+  end
+
+end
+end
diff --git a/lib/coderay/encoders/yaml.rb b/lib/coderay/encoders/yaml.rb
index 47f64a4..5564e58 100644
--- a/lib/coderay/encoders/yaml.rb
+++ b/lib/coderay/encoders/yaml.rb
@@ -1,22 +1,22 @@
-module CodeRay

-module Encoders

-

-  # = YAML Encoder

-  #

-  # Slow.

-  class YAML < Encoder

-

-    register_for :yaml

-

-    FILE_EXTENSION = 'yaml'

-

-  protected

-    def compile tokens, options

-      require 'yaml'

-      @out = tokens.to_a.to_yaml

-    end

-

-  end

-

-end

-end

+module CodeRay
+module Encoders
+
+  # = YAML Encoder
+  #
+  # Slow.
+  class YAML < Encoder
+
+    register_for :yaml
+
+    FILE_EXTENSION = 'yaml'
+
+  protected
+    def compile tokens, options
+      require 'yaml'
+      @out = tokens.to_a.to_yaml
+    end
+
+  end
+
+end
+end
diff --git a/lib/coderay/scanners/plaintext.rb b/lib/coderay/scanners/plaintext.rb
index 9007646..432745f 100644
--- a/lib/coderay/scanners/plaintext.rb
+++ b/lib/coderay/scanners/plaintext.rb
@@ -1,15 +1,15 @@
-module CodeRay

-module Scanners

-

-  class Plaintext < Scanner

-

-    register_for :plaintext, :plain

-

-    def scan_tokens tokens, options

-      tokens << [scan_until(/\z/), :plain]

-    end

-

-  end

-

-end

-end

+module CodeRay
+module Scanners
+
+  class Plaintext < Scanner
+
+    register_for :plaintext, :plain
+
+    def scan_tokens tokens, options
+      tokens << [scan_until(/\z/), :plain]
+    end
+
+  end
+
+end
+end
diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb
index b1e0d1b..c601011 100644
--- a/lib/coderay/scanners/ruby/patterns.rb
+++ b/lib/coderay/scanners/ruby/patterns.rb
@@ -1,216 +1,216 @@
-module CodeRay

-module Scanners

-

-  module Ruby::Patterns  # :nodoc:

-

-    RESERVED_WORDS = %w[

-      and def end in or unless begin

-      defined? ensure module redo super until

-      BEGIN break do next rescue then

-      when END case else for retry

-      while alias class elsif if not return

-      undef yield

-    ]

-

-    DEF_KEYWORDS = %w[ def ]

-    UNDEF_KEYWORDS = %w[ undef ]

-    MODULE_KEYWORDS = %w[class module]

-    DEF_NEW_STATE = WordList.new(:initial).

-      add(DEF_KEYWORDS, :def_expected).

-      add(UNDEF_KEYWORDS, :undef_expected).

-      add(MODULE_KEYWORDS, :module_expected)

-

-    IDENTS_ALLOWING_REGEXP = %w[

-      and or not while until unless if then elsif when sub sub! gsub gsub! scan slice slice! split

-    ]

-    REGEXP_ALLOWED = WordList.new(false).

-      add(IDENTS_ALLOWING_REGEXP, :set)

-

-    PREDEFINED_CONSTANTS = %w[

-      nil true false self

-      DATA ARGV ARGF __FILE__ __LINE__

-    ]

-

-    IDENT_KIND = WordList.new(:ident).

-      add(RESERVED_WORDS, :reserved).

-      add(PREDEFINED_CONSTANTS, :pre_constant)

-

-    IDENT = /[a-z_][\w_]*/i

-

-    METHOD_NAME = / #{IDENT} [?!]? /ox

-    METHOD_NAME_OPERATOR = /

-      \*\*?           # multiplication and power

-      | [-+]@?        # plus, minus

-      | [\/%&|^`~]    # division, modulo or format strings, &and, |or, ^xor, `system`, tilde

-      | \[\]=?        # array getter and setter

-      | << | >>       # append or shift left, shift right

-      | <=?>? | >=?   # comparison, rocket operator

-      | ===?          # simple equality and case equality

-    /ox

-    METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox

-    INSTANCE_VARIABLE = / @ #{IDENT} /ox

-    CLASS_VARIABLE = / @@ #{IDENT} /ox

-    OBJECT_VARIABLE = / @@? #{IDENT} /ox

-    GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox

-    PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} |#{OBJECT_VARIABLE} /ox

-    VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox

-

-    QUOTE_TO_TYPE = {

-      '`' => :shell,

-      '/'=> :regexp,

-    }

-    QUOTE_TO_TYPE.default = :string

-

-    REGEXP_MODIFIERS = /[mixounse]*/

-    REGEXP_SYMBOLS = /[|?*+?(){}\[\].^$]/

-

-    DECIMAL = /\d+(?:_\d+)*/

-    OCTAL = /0_?[0-7]+(?:_[0-7]+)*/

-    HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/

-    BINARY = /0b[01]+(?:_[01]+)*/

-

-    EXPONENT = / [eE] [+-]? #{DECIMAL} /ox

-    FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox

-    FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox

-    NUMERIC = / [-+]? (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox

-

-    SYMBOL = /

-      :

-      (?:

-        #{METHOD_NAME_EX}

-      | #{PREFIX_VARIABLE}

-      | ['"]

-      )

-    /ox

-

-    # TODO investigste \M, \c and \C escape sequences

-    # (?: M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-)? (?: \\ (?: [0-7]{3} | x[0-9A-Fa-f]{2} | . ) )

-    # assert_equal(225, ?\M-a)

-    # assert_equal(129, ?\M-\C-a)

-    ESCAPE = /

-        [abefnrstv]

-      | M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-

-      |  [0-7]{1,3}

-      | x[0-9A-Fa-f]{1,2}

-      | .

-    /mx

-

-    CHARACTER = /

-      \?

-      (?:

-        [^\s\\]

-      | \\ #{ESCAPE}

-      )

-    /mx

-

-    # NOTE: This is not completely correct, but

-    # nobody needs heredoc delimiters ending with \n.

-    HEREDOC_OPEN = /

-      << (-)?              # $1 = float

-      (?:

-        ( [A-Za-z_0-9]+ )  # $2 = delim

-      |

-        ( ["'`] )          # $3 = quote, type

-        ( [^\n]*? ) \3     # $4 = delim

-      )

-    /mx

-

-    RUBYDOC = /

-      =begin (?!\S)

-      .*?

-      (?: \Z | ^=end (?!\S) [^\n]* )

-    /mx

-

-    DATA = /

-      __END__$

-      .*?

-      (?: \Z | (?=^\#CODE) )

-    /mx

-

-    RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo

-

-    RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x

-

-    # FIXME: \s and = are only a workaround, they are still allowed

-    # as delimiters.

-    FANCY_START_SAVE = / % ( [qQwWxsr] | (?![a-zA-Z0-9\s=]) ) ([^a-zA-Z0-9]) /mx

-    FANCY_START_CORRECT = / % ( [qQwWxsr] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /mx

-

-    FancyStringType = {

-      'q' => [:string, false],

-      'Q' => [:string, true],

-      'r' => [:regexp, true],

-      's' => [:symbol, false],

-      'x' => [:shell, true]

-    }

-    FancyStringType['w'] = FancyStringType['q']

-    FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q']

-

-    class StringState < Struct.new :type, :interpreted, :delim, :heredoc,

-      :paren, :paren_depth, :pattern, :next_state

-

-      CLOSING_PAREN = Hash[ *%w[

-        ( )

-        [ ]

-        < >

-        { }

-      ] ]

-

-      CLOSING_PAREN.values.each { |o| o.freeze }  # debug, if I try to change it with <<

-      OPENING_PAREN = CLOSING_PAREN.invert

-

-      STRING_PATTERN = Hash.new { |h, k|

-        delim, interpreted = *k

-        delim_pattern = Regexp.escape(delim.dup)

-        if closing_paren = CLOSING_PAREN[delim]

-          delim_pattern << Regexp.escape(closing_paren)

-        end

-

-

-        special_escapes =

-          case interpreted

-          when :regexp_symbols

-            '| ' + REGEXP_SYMBOLS.source

-          when :words

-            '| \s'

-          end

-

-        h[k] =

-          if interpreted and not delim == '#'

-            / (?= [#{delim_pattern}\\] | \# [{$@] #{special_escapes} ) /mx

-          else

-            / (?= [#{delim_pattern}\\] #{special_escapes} ) /mx

-          end

-      }

-

-      HEREDOC_PATTERN = Hash.new { |h, k|

-        delim, interpreted, indented = *k

-        delim_pattern = Regexp.escape(delim.dup)

-        delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x

-        h[k] =

-          if interpreted

-            / (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx  # $1 set == end of heredoc

-          else

-            / (?= #{delim_pattern}() | \\ ) /mx

-          end

-      }

-

-      def initialize kind, interpreted, delim, heredoc = false

-        if heredoc

-          pattern = HEREDOC_PATTERN[ [delim, interpreted, heredoc == :indented] ]

-          delim  = nil

-        else

-          pattern = STRING_PATTERN[ [delim, interpreted] ]

-          if paren = CLOSING_PAREN[delim]

-            delim, paren = paren, delim

-            paren_depth = 1

-          end

-        end

-        super kind, interpreted, delim, heredoc, paren, paren_depth, pattern, :initial

-      end

-    end unless defined? StringState

-

-  end

-

-end

-end

+module CodeRay
+module Scanners
+
+  module Ruby::Patterns  # :nodoc:
+
+    RESERVED_WORDS = %w[
+      and def end in or unless begin
+      defined? ensure module redo super until
+      BEGIN break do next rescue then
+      when END case else for retry
+      while alias class elsif if not return
+      undef yield
+    ]
+
+    DEF_KEYWORDS = %w[ def ]
+    UNDEF_KEYWORDS = %w[ undef ]
+    MODULE_KEYWORDS = %w[class module]
+    DEF_NEW_STATE = WordList.new(:initial).
+      add(DEF_KEYWORDS, :def_expected).
+      add(UNDEF_KEYWORDS, :undef_expected).
+      add(MODULE_KEYWORDS, :module_expected)
+
+    IDENTS_ALLOWING_REGEXP = %w[
+      and or not while until unless if then elsif when sub sub! gsub gsub! scan slice slice! split
+    ]
+    REGEXP_ALLOWED = WordList.new(false).
+      add(IDENTS_ALLOWING_REGEXP, :set)
+
+    PREDEFINED_CONSTANTS = %w[
+      nil true false self
+      DATA ARGV ARGF __FILE__ __LINE__
+    ]
+
+    IDENT_KIND = WordList.new(:ident).
+      add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_CONSTANTS, :pre_constant)
+
+    IDENT = /[a-z_][\w_]*/i
+
+    METHOD_NAME = / #{IDENT} [?!]? /ox
+    METHOD_NAME_OPERATOR = /
+      \*\*?           # multiplication and power
+      | [-+]@?        # plus, minus
+      | [\/%&|^`~]    # division, modulo or format strings, &and, |or, ^xor, `system`, tilde
+      | \[\]=?        # array getter and setter
+      | << | >>       # append or shift left, shift right
+      | <=?>? | >=?   # comparison, rocket operator
+      | ===?          # simple equality and case equality
+    /ox
+    METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox
+    INSTANCE_VARIABLE = / @ #{IDENT} /ox
+    CLASS_VARIABLE = / @@ #{IDENT} /ox
+    OBJECT_VARIABLE = / @@? #{IDENT} /ox
+    GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
+    PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} |#{OBJECT_VARIABLE} /ox
+    VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
+
+    QUOTE_TO_TYPE = {
+      '`' => :shell,
+      '/'=> :regexp,
+    }
+    QUOTE_TO_TYPE.default = :string
+
+    REGEXP_MODIFIERS = /[mixounse]*/
+    REGEXP_SYMBOLS = /[|?*+?(){}\[\].^$]/
+
+    DECIMAL = /\d+(?:_\d+)*/
+    OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
+    HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
+    BINARY = /0b[01]+(?:_[01]+)*/
+
+    EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
+    FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox
+    FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox
+    NUMERIC = / [-+]? (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
+
+    SYMBOL = /
+      :
+      (?:
+        #{METHOD_NAME_EX}
+      | #{PREFIX_VARIABLE}
+      | ['"]
+      )
+    /ox
+
+    # TODO investigste \M, \c and \C escape sequences
+    # (?: M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-)? (?: \\ (?: [0-7]{3} | x[0-9A-Fa-f]{2} | . ) )
+    # assert_equal(225, ?\M-a)
+    # assert_equal(129, ?\M-\C-a)
+    ESCAPE = /
+        [abefnrstv]
+      | M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-
+      |  [0-7]{1,3}
+      | x[0-9A-Fa-f]{1,2}
+      | .
+    /mx
+
+    CHARACTER = /
+      \?
+      (?:
+        [^\s\\]
+      | \\ #{ESCAPE}
+      )
+    /mx
+
+    # NOTE: This is not completely correct, but
+    # nobody needs heredoc delimiters ending with \n.
+    HEREDOC_OPEN = /
+      << (-)?              # $1 = float
+      (?:
+        ( [A-Za-z_0-9]+ )  # $2 = delim
+      |
+        ( ["'`] )          # $3 = quote, type
+        ( [^\n]*? ) \3     # $4 = delim
+      )
+    /mx
+
+    RUBYDOC = /
+      =begin (?!\S)
+      .*?
+      (?: \Z | ^=end (?!\S) [^\n]* )
+    /mx
+
+    DATA = /
+      __END__$
+      .*?
+      (?: \Z | (?=^\#CODE) )
+    /mx
+
+    RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo
+
+    RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x
+
+    # FIXME: \s and = are only a workaround, they are still allowed
+    # as delimiters.
+    FANCY_START_SAVE = / % ( [qQwWxsr] | (?![a-zA-Z0-9\s=]) ) ([^a-zA-Z0-9]) /mx
+    FANCY_START_CORRECT = / % ( [qQwWxsr] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /mx
+
+    FancyStringType = {
+      'q' => [:string, false],
+      'Q' => [:string, true],
+      'r' => [:regexp, true],
+      's' => [:symbol, false],
+      'x' => [:shell, true]
+    }
+    FancyStringType['w'] = FancyStringType['q']
+    FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q']
+
+    class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
+      :paren, :paren_depth, :pattern, :next_state
+
+      CLOSING_PAREN = Hash[ *%w[
+        ( )
+        [ ]
+        < >
+        { }
+      ] ]
+
+      CLOSING_PAREN.values.each { |o| o.freeze }  # debug, if I try to change it with <<
+      OPENING_PAREN = CLOSING_PAREN.invert
+
+      STRING_PATTERN = Hash.new { |h, k|
+        delim, interpreted = *k
+        delim_pattern = Regexp.escape(delim.dup)
+        if closing_paren = CLOSING_PAREN[delim]
+          delim_pattern << Regexp.escape(closing_paren)
+        end
+
+
+        special_escapes =
+          case interpreted
+          when :regexp_symbols
+            '| ' + REGEXP_SYMBOLS.source
+          when :words
+            '| \s'
+          end
+
+        h[k] =
+          if interpreted and not delim == '#'
+            / (?= [#{delim_pattern}\\] | \# [{$@] #{special_escapes} ) /mx
+          else
+            / (?= [#{delim_pattern}\\] #{special_escapes} ) /mx
+          end
+      }
+
+      HEREDOC_PATTERN = Hash.new { |h, k|
+        delim, interpreted, indented = *k
+        delim_pattern = Regexp.escape(delim.dup)
+        delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
+        h[k] =
+          if interpreted
+            / (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx  # $1 set == end of heredoc
+          else
+            / (?= #{delim_pattern}() | \\ ) /mx
+          end
+      }
+
+      def initialize kind, interpreted, delim, heredoc = false
+        if heredoc
+          pattern = HEREDOC_PATTERN[ [delim, interpreted, heredoc == :indented] ]
+          delim  = nil
+        else
+          pattern = STRING_PATTERN[ [delim, interpreted] ]
+          if paren = CLOSING_PAREN[delim]
+            delim, paren = paren, delim
+            paren_depth = 1
+          end
+        end
+        super kind, interpreted, delim, heredoc, paren, paren_depth, pattern, :initial
+      end
+    end unless defined? StringState
+
+  end
+
+end
+end
diff --git a/lib/coderay/scanners/xml.rb b/lib/coderay/scanners/xml.rb
index 5ce8ce9..ff923fb 100644
--- a/lib/coderay/scanners/xml.rb
+++ b/lib/coderay/scanners/xml.rb
@@ -1,18 +1,18 @@
-module CodeRay

-module Scanners

-

-  load :html

-

-  # XML Scanner

-  #

-  # $Id$

-  #

-  # Currently this is the same scanner as Scanners::HTML.

-  class XML < HTML

-

-    register_for :xml

-

-  end

-

-end

-end

+module CodeRay
+module Scanners
+
+  load :html
+
+  # XML Scanner
+  #
+  # $Id$
+  #
+  # Currently this is the same scanner as Scanners::HTML.
+  class XML < HTML
+
+    register_for :xml
+
+  end
+
+end
+end
diff --git a/lib/coderay/style.rb b/lib/coderay/style.rb
index 057f8d4..c2977c5 100644
--- a/lib/coderay/style.rb
+++ b/lib/coderay/style.rb
@@ -1,20 +1,20 @@
-module CodeRay

-

-  # This module holds the Style class and its subclasses.

-  #

-  # See Plugin.

-  module Styles

-    extend PluginHost

-    plugin_path File.dirname(__FILE__), 'styles'

-

-    class Style

-      extend Plugin

-      plugin_host Styles

-

-      DEFAULT_OPTIONS = { }

-

-    end

-

-  end

-

-end

+module CodeRay
+
+  # This module holds the Style class and its subclasses.
+  #
+  # See Plugin.
+  module Styles
+    extend PluginHost
+    plugin_path File.dirname(__FILE__), 'styles'
+
+    class Style
+      extend Plugin
+      plugin_host Styles
+
+      DEFAULT_OPTIONS = { }
+
+    end
+
+  end
+
+end
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index 8b8c692..c8c62e0 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -1,322 +1,322 @@
-module CodeRay

-

-  # = Tokens

-  #

-  # The Tokens class represents a list of tokens returnd from

-  # a Scanner.

-  #

-  # A token is not a special object, just a two-element Array

-  # consisting of

-  # * the _token_ _kind_ (a Symbol representing the type of the token)

-  # * the _token_ _text_ (the original source of the token in a String)

-  #

-  # A token looks like this:

-  #

-  #   [:comment, '# It looks like this']

-  #   [:float, '3.1415926']

-  #   [:error, 'äöü']

-  #

-  # Some scanners also yield some kind of sub-tokens, represented by special

-  # token texts, namely :open and :close .

-  #

-  # The Ruby scanner, for example, splits "a string" into:

-  #

-  #  [

-  #   [:open, :string],

-  #   [:delimiter, '"'],

-  #   [:content, 'a string'],

-  #   [:delimiter, '"'],

-  #   [:close, :string]

-  #  ]

-  #

-  # Tokens is also the interface between Scanners and Encoders:

-  # The input is split and saved into a Tokens object. The Encoder

-  # then builds the output from this object.

-  #

-  # Thus, the syntax below becomes clear:

-  #

-  #   CodeRay.scan('price = 2.59', :ruby).html

-  #   # the Tokens object is here -------^

-  #

-  # See how small it is? ;)

-  #

-  # Tokens gives you the power to handle pre-scanned code very easily:

-  # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string

-  # that you put in your DB.

-  #

-  # Tokens' subclass TokenStream allows streaming to save memory.

-  class Tokens < Array

-

-    class << self

-

-      # Convert the token to a string.

-      #

-      # This format is used by Encoders.Tokens.

-      # It can be reverted using read_token.

-      def write_token text, type

-        if text.is_a? String

-          "#{type}\t#{escape(text)}\n"

-        else

-          ":#{text}\t#{type}\t\n"

-        end

-      end

-

-      # Read a token from the string.

-      #

-      # Inversion of write_token.

-      #

-      # TODO Test this!

-      def read_token token

-        type, text = token.split("\t", 2)

-        if type[0] == ?:

-          [text.to_sym, type[1..-1].to_sym]

-        else

-          [type.to_sym, unescape(text)]

-        end

-      end

-

-      # Escapes a string for use in write_token.

-      def escape text

-        text.gsub(/[\n\\]/, '\\\\\&')

-      end

-

-      # Unescapes a string created by escape.

-      def unescape text

-        text.gsub(/\\[\n\\]/) { |m| m[1,1] }

-      end

-

-    end

-

-    # Whether the object is a TokenStream.

-    #

-    # Returns false.

-    def stream?

-      false

-    end

-

-    # Iterates over all tokens.

-    #

-    # If a filter is given, only tokens of that kind are yielded.

-    def each kind_filter = nil, &block

-      unless kind_filter

-        super(&block)

-      else

-        super() do |text, kind|

-          next unless kind == kind_filter

-          yield text, kind

-        end

-      end

-    end

-

-    # Iterates over all text tokens.

-    # Range tokens like [:open, :string] are left out.

-    #

-    # Example:

-    #   tokens.each_text_token { |text, kind| text.replace html_escape(text) }

-    def each_text_token

-      each do |text, kind|

-        next unless text.respond_to? :to_str

-        yield text, kind

-      end

-    end

-

-    # Encode the tokens using encoder.

-    #

-    # encoder can be

-    # * a symbol like :html oder :statistic

-    # * an Encoder class

-    # * an Encoder object

-    #

-    # options are passed to the encoder.

-    def encode encoder, options = {}

-      unless encoder.is_a? Encoders::Encoder

-        unless encoder.is_a? Class

-          encoder_class = Encoders[encoder]

-        end

-        encoder = encoder_class.new options

-      end

-      encoder.encode_tokens self, options

-    end

-

-

-    # Turn into a string using Encoders::Text.

-    #

-    # +options+ are passed to the encoder if given.

-    def to_s options = {}

-      encode :text, options

-    end

-

-

-    # Redirects unknown methods to encoder calls.

-    #

-    # For example, if you call +tokens.html+, the HTML encoder

-    # is used to highlight the tokens.

-    def method_missing meth, options = {}

-      Encoders[meth].new(options).encode_tokens self

-    end

-

-    # Returns the tokens compressed by joining consecutive

-    # tokens of the same kind.

-    #

-    # This can not be undone, but should yield the same output

-    # in most Encoders.  It basically makes the output smaller.

-    #

-    # Combined with dump, it saves space for the cost of time.

-    #

-    # If the scanner is written carefully, this is not required -

-    # for example, consecutive //-comment lines could already be

-    # joined in one comment token by the Scanner.

-    def optimize

-      print ' Tokens#optimize: before: %d - ' % size if $DEBUG

-      last_kind = last_text = nil

-      new = self.class.new

-      each do |text, kind|

-        if text.is_a? String

-          if kind == last_kind

-            last_text << text

-          else

-            new << [last_text, last_kind] if last_kind

-            last_text = text

-            last_kind = kind

-          end

-        else

-          new << [last_text, last_kind] if last_kind

-          last_kind = last_text = nil

-          new << [text, kind]

-        end

-      end

-      new << [last_text, last_kind] if last_kind

-      print 'after: %d (%d saved = %2.0f%%)' %

-        [new.size, size - new.size, 1.0 - (new.size.to_f / size)] if $DEBUG

-      new

-    end

-

-    # Compact the object itself; see optimize.

-    def optimize!

-      replace optimize

-    end

-

-    # Dumps the object into a String that can be saved

-    # in files or databases.

-    #

-    # The dump is created with Marshal.dump;

-    # In addition, it is gzipped using GZip.gzip.

-    #

-    # The returned String object includes Undumping

-    # so it has an #undump method. See Tokens.load.

-    #

-    # You can configure the level of compression,

-    # but the default value 7 should be what you want

-    # in most cases as it is a good comprimise between

-    # speed and compression rate.

-    #

-    # See GZip module.

-    def dump gzip_level = 7

-      require 'coderay/helpers/gzip_simple'

-      dump = Marshal.dump self

-      dump = dump.gzip gzip_level

-      dump.extend Undumping

-    end

-

-    # The total size of the tokens.

-    # Should be equal to the input size before

-    # scanning.

-    def text_size

-      map { |t, k| t }.join.size

-    end

-

-    # Include this module to give an object an #undump

-    # method.

-    #

-    # The string returned by Tokens.dump includes Undumping.

-    module Undumping

-      # Calls Tokens.load with itself.

-      def undump

-        Tokens.load self

-      end

-    end

-

-    # Undump the object using Marshal.load, then

-    # unzip it using GZip.gunzip.

-    #

-    # The result is commonly a Tokens object, but

-    # this is not guaranteed.

-    def Tokens.load dump

-      require 'coderay/helpers/gzip_simple'

-      dump = dump.gunzip

-      @dump = Marshal.load dump

-    end

-

-  end

-

-

-  # = TokenStream

-  #

-  # The TokenStream class is a fake Array without elements.

-  #

-  # It redirects the method << to a block given at creation.

-  #

-  # This allows scanners and Encoders to use streaming (no

-  # tokens are saved, the input is highlighted the same time it

-  # is scanned) with the same code.

-  #

-  # See CodeRay.encode_stream and CodeRay.scan_stream

-  class TokenStream < Tokens

-

-    # Whether the object is a TokenStream.

-    #

-    # Returns true.

-    def stream?

-      true

-    end

-

-    # The Array is empty, but size counts the tokens given by <<.

-    attr_reader :size

-

-    # Creates a new TokenStream that calls +block+ whenever

-    # its << method is called.

-    #

-    # Example:

-    #

-    #   require 'coderay'

-    #   

-    #   token_stream = CodeRay::TokenStream.new do |kind, text|

-    #     puts 'kind: %s, text size: %d.' % [kind, text.size]

-    #   end

-    #   

-    #   token_stream << [:regexp, '/\d+/']

-    #   #-> kind: rexpexp, text size: 5.

-    #

-    def initialize &block

-      raise ArgumentError, 'Block expected for streaming.' unless block

-      @callback = block

-      @size = 0

-    end

-

-    # Calls +block+ with +token+ and increments size.

-    #

-    # Returns self.

-    def << token

-      @callback.call token

-      @size += 1

-      self

-    end

-

-    # This method is not implemented due to speed reasons. Use Tokens.

-    def text_size

-      raise NotImplementedError, 'This method is not implemented due to speed reasons.'

-    end

-

-    # A TokenStream cannot be dumped. Use Tokens.

-    def dump

-      raise NotImplementedError, 'A TokenStream cannot be dumped.'

-    end

-

-    # A TokenStream cannot be optimized. Use Tokens.

-    def optimize

-      raise NotImplementedError, 'A TokenStream cannot be optimized.'

-    end

-

-  end

-

-end

+module CodeRay
+
+  # = Tokens
+  #
+  # The Tokens class represents a list of tokens returnd from
+  # a Scanner.
+  #
+  # A token is not a special object, just a two-element Array
+  # consisting of
+  # * the _token_ _kind_ (a Symbol representing the type of the token)
+  # * the _token_ _text_ (the original source of the token in a String)
+  #
+  # A token looks like this:
+  #
+  #   [:comment, '# It looks like this']
+  #   [:float, '3.1415926']
+  #   [:error, 'äöü']
+  #
+  # Some scanners also yield some kind of sub-tokens, represented by special
+  # token texts, namely :open and :close .
+  #
+  # The Ruby scanner, for example, splits "a string" into:
+  #
+  #  [
+  #   [:open, :string],
+  #   [:delimiter, '"'],
+  #   [:content, 'a string'],
+  #   [:delimiter, '"'],
+  #   [:close, :string]
+  #  ]
+  #
+  # Tokens is also the interface between Scanners and Encoders:
+  # The input is split and saved into a Tokens object. The Encoder
+  # then builds the output from this object.
+  #
+  # Thus, the syntax below becomes clear:
+  #
+  #   CodeRay.scan('price = 2.59', :ruby).html
+  #   # the Tokens object is here -------^
+  #
+  # See how small it is? ;)
+  #
+  # Tokens gives you the power to handle pre-scanned code very easily:
+  # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
+  # that you put in your DB.
+  #
+  # Tokens' subclass TokenStream allows streaming to save memory.
+  class Tokens < Array
+
+    class << self
+
+      # Convert the token to a string.
+      #
+      # This format is used by Encoders.Tokens.
+      # It can be reverted using read_token.
+      def write_token text, type
+        if text.is_a? String
+          "#{type}\t#{escape(text)}\n"
+        else
+          ":#{text}\t#{type}\t\n"
+        end
+      end
+
+      # Read a token from the string.
+      #
+      # Inversion of write_token.
+      #
+      # TODO Test this!
+      def read_token token
+        type, text = token.split("\t", 2)
+        if type[0] == ?:
+          [text.to_sym, type[1..-1].to_sym]
+        else
+          [type.to_sym, unescape(text)]
+        end
+      end
+
+      # Escapes a string for use in write_token.
+      def escape text
+        text.gsub(/[\n\\]/, '\\\\\&')
+      end
+
+      # Unescapes a string created by escape.
+      def unescape text
+        text.gsub(/\\[\n\\]/) { |m| m[1,1] }
+      end
+
+    end
+
+    # Whether the object is a TokenStream.
+    #
+    # Returns false.
+    def stream?
+      false
+    end
+
+    # Iterates over all tokens.
+    #
+    # If a filter is given, only tokens of that kind are yielded.
+    def each kind_filter = nil, &block
+      unless kind_filter
+        super(&block)
+      else
+        super() do |text, kind|
+          next unless kind == kind_filter
+          yield text, kind
+        end
+      end
+    end
+
+    # Iterates over all text tokens.
+    # Range tokens like [:open, :string] are left out.
+    #
+    # Example:
+    #   tokens.each_text_token { |text, kind| text.replace html_escape(text) }
+    def each_text_token
+      each do |text, kind|
+        next unless text.respond_to? :to_str
+        yield text, kind
+      end
+    end
+
+    # Encode the tokens using encoder.
+    #
+    # encoder can be
+    # * a symbol like :html oder :statistic
+    # * an Encoder class
+    # * an Encoder object
+    #
+    # options are passed to the encoder.
+    def encode encoder, options = {}
+      unless encoder.is_a? Encoders::Encoder
+        unless encoder.is_a? Class
+          encoder_class = Encoders[encoder]
+        end
+        encoder = encoder_class.new options
+      end
+      encoder.encode_tokens self, options
+    end
+
+
+    # Turn into a string using Encoders::Text.
+    #
+    # +options+ are passed to the encoder if given.
+    def to_s options = {}
+      encode :text, options
+    end
+
+
+    # Redirects unknown methods to encoder calls.
+    #
+    # For example, if you call +tokens.html+, the HTML encoder
+    # is used to highlight the tokens.
+    def method_missing meth, options = {}
+      Encoders[meth].new(options).encode_tokens self
+    end
+
+    # Returns the tokens compressed by joining consecutive
+    # tokens of the same kind.
+    #
+    # This can not be undone, but should yield the same output
+    # in most Encoders.  It basically makes the output smaller.
+    #
+    # Combined with dump, it saves space for the cost of time.
+    #
+    # If the scanner is written carefully, this is not required -
+    # for example, consecutive //-comment lines could already be
+    # joined in one comment token by the Scanner.
+    def optimize
+      print ' Tokens#optimize: before: %d - ' % size if $DEBUG
+      last_kind = last_text = nil
+      new = self.class.new
+      each do |text, kind|
+        if text.is_a? String
+          if kind == last_kind
+            last_text << text
+          else
+            new << [last_text, last_kind] if last_kind
+            last_text = text
+            last_kind = kind
+          end
+        else
+          new << [last_text, last_kind] if last_kind
+          last_kind = last_text = nil
+          new << [text, kind]
+        end
+      end
+      new << [last_text, last_kind] if last_kind
+      print 'after: %d (%d saved = %2.0f%%)' %
+        [new.size, size - new.size, 1.0 - (new.size.to_f / size)] if $DEBUG
+      new
+    end
+
+    # Compact the object itself; see optimize.
+    def optimize!
+      replace optimize
+    end
+
+    # Dumps the object into a String that can be saved
+    # in files or databases.
+    #
+    # The dump is created with Marshal.dump;
+    # In addition, it is gzipped using GZip.gzip.
+    #
+    # The returned String object includes Undumping
+    # so it has an #undump method. See Tokens.load.
+    #
+    # You can configure the level of compression,
+    # but the default value 7 should be what you want
+    # in most cases as it is a good comprimise between
+    # speed and compression rate.
+    #
+    # See GZip module.
+    def dump gzip_level = 7
+      require 'coderay/helpers/gzip_simple'
+      dump = Marshal.dump self
+      dump = dump.gzip gzip_level
+      dump.extend Undumping
+    end
+
+    # The total size of the tokens.
+    # Should be equal to the input size before
+    # scanning.
+    def text_size
+      map { |t, k| t }.join.size
+    end
+
+    # Include this module to give an object an #undump
+    # method.
+    #
+    # The string returned by Tokens.dump includes Undumping.
+    module Undumping
+      # Calls Tokens.load with itself.
+      def undump
+        Tokens.load self
+      end
+    end
+
+    # Undump the object using Marshal.load, then
+    # unzip it using GZip.gunzip.
+    #
+    # The result is commonly a Tokens object, but
+    # this is not guaranteed.
+    def Tokens.load dump
+      require 'coderay/helpers/gzip_simple'
+      dump = dump.gunzip
+      @dump = Marshal.load dump
+    end
+
+  end
+
+
+  # = TokenStream
+  #
+  # The TokenStream class is a fake Array without elements.
+  #
+  # It redirects the method << to a block given at creation.
+  #
+  # This allows scanners and Encoders to use streaming (no
+  # tokens are saved, the input is highlighted the same time it
+  # is scanned) with the same code.
+  #
+  # See CodeRay.encode_stream and CodeRay.scan_stream
+  class TokenStream < Tokens
+
+    # Whether the object is a TokenStream.
+    #
+    # Returns true.
+    def stream?
+      true
+    end
+
+    # The Array is empty, but size counts the tokens given by <<.
+    attr_reader :size
+
+    # Creates a new TokenStream that calls +block+ whenever
+    # its << method is called.
+    #
+    # Example:
+    #
+    #   require 'coderay'
+    #   
+    #   token_stream = CodeRay::TokenStream.new do |kind, text|
+    #     puts 'kind: %s, text size: %d.' % [kind, text.size]
+    #   end
+    #   
+    #   token_stream << [:regexp, '/\d+/']
+    #   #-> kind: rexpexp, text size: 5.
+    #
+    def initialize &block
+      raise ArgumentError, 'Block expected for streaming.' unless block
+      @callback = block
+      @size = 0
+    end
+
+    # Calls +block+ with +token+ and increments size.
+    #
+    # Returns self.
+    def << token
+      @callback.call token
+      @size += 1
+      self
+    end
+
+    # This method is not implemented due to speed reasons. Use Tokens.
+    def text_size
+      raise NotImplementedError, 'This method is not implemented due to speed reasons.'
+    end
+
+    # A TokenStream cannot be dumped. Use Tokens.
+    def dump
+      raise NotImplementedError, 'A TokenStream cannot be dumped.'
+    end
+
+    # A TokenStream cannot be optimized. Use Tokens.
+    def optimize
+      raise NotImplementedError, 'A TokenStream cannot be optimized.'
+    end
+
+  end
+
+end