Merge branch 'master' into go-scanner

7 files changed, 100 insertions, 140 deletions

diff --git a/.gitignore b/.gitignore
index dd001c8..deed1a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,5 +11,4 @@ Gemfile.lock
 test/executable/source.rb.html
 test/executable/source.rb.json
 test/scanners
-bench/test.div.html
 old-stuff
diff --git a/bench/bench.rb b/bench/bench.rb
index 1958c73..92f9d07 100644
--- a/bench/bench.rb
+++ b/bench/bench.rb
@@ -1,142 +1,46 @@
-# The most ugly test script I've ever written!
-# Shame on me!
-
-require 'pathname'
-require 'profile' if ARGV.include? '-p'
-
-MYDIR = File.dirname(__FILE__)
-LIBDIR = Pathname.new(MYDIR).join('..', 'lib').cleanpath.to_s
-$:.unshift MYDIR, LIBDIR
+require 'benchmark'
+$: << File.expand_path('../../lib', __FILE__)
 require 'coderay'
 
-@size = ARGV.fetch(2, 100).to_i * 1000
-
-lang = ARGV.fetch(0) do
-  puts <<-HELP
-Usage:
-  ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
-
-  SIZE defaults to 100 kB (= 100,000 bytes).
-  SIZE = 0 means the whole input.
-
--p generates a profile (slow! use with SIZE = 1)
--o shows the output
-stream enabled streaming mode
-
-Sorry for the strange interface. I will improve it in the next release.
-  HELP
+if ARGV.include? '-h'
+  puts DATA.read
   exit
 end
 
-format = ARGV.fetch(1, 'html').downcase
-
-$stream = ARGV.include? 'stream'
-$optimize = ARGV.include? 'opt'
-$style = ARGV.include? 'style'
-
-require 'benchmark'
-require 'fileutils'
+lang = ARGV.fetch(0, 'ruby')
+data = nil
+File.open(File.expand_path("../example.#{lang}", __FILE__), 'rb') { |f| data = f.read }
+raise 'Example file is empty.' if data.empty?
 
-if format == 'comp'
-  format = 'page'
-  begin
-    require 'syntax'
-    require 'syntax/convertors/html.rb'
-  rescue LoadError
-    puts 'Syntax no found!! (Try % gem install syntax)'
-  end
-end
+format = ARGV.fetch(1, 'html').downcase
+encoder = CodeRay.encoder(format)
 
-def here fn = nil
-  return MYDIR unless fn
-  File.join here, fn
+size = ARGV.fetch(2, 1000).to_i * 1000
+unless size.zero?
+  data += data until data.size >= size
+  data = data[0, size]
 end
+size = data.size
+puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round]
 
-n = ARGV.find { |a| a[/^N/] }
-N = if n then n[/\d+/].to_i else 1 end
-$filename = ARGV.include?('strange') ? 'strange' : 'example'
-
-Benchmark.bm(20) do |bm|
-N.times do
-
-  data = nil
-  File.open(here("#$filename." + lang), 'rb') { |f| data = f.read }
-  raise 'Example file is empty.' if data.empty?
-  unless @size.zero?
-    data += data until data.size >= @size
-    data = data[0, @size]
-  end
-  @size = data.size
-  
-  options = {
-    :tab_width => 2,
-    # :line_numbers => :inline,
-    :css => $style ? :style : :class,
-  }
-  $hl = CodeRay.encoder(format, options)
-  time = bm.report('CodeRay') do
-    if $stream || true
-      $o = $hl.encode(data, lang, options)
-    else
-      tokens = CodeRay.scan(data, lang)
-      tokens.optimize! if $optimize
-      $o = tokens.encode($hl)
-    end
-  end
-  $file_created = here('test.' + $hl.file_extension)
-  File.open($file_created, 'wb') do |f|
-    # f.write $o
-  end
-  
-  time_real = time.real
-  
-  puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000]
-  puts $o if ARGV.include? '-o'
-  
-end
+n = ARGV.fetch(3, 5).to_s[/\d+/].to_i
+require 'profile' if ARGV.include? '-p'
+n.times do |i|
+  time = Benchmark.realtime { encoder.encode(data, lang) }
+  puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0]
 end
-puts "Files created: #$file_created"
 
-STDIN.gets if ARGV.include? 'wait'
+STDIN.gets if ARGV.include? '-w'
 
 __END__
-.ruby .normal {}
-.ruby .comment { color: #005; font-style: italic; }
-.ruby .keyword { color: #A00; font-weight: bold; }
-.ruby .method { color: #077; }
-.ruby .class { color: #074; }
-.ruby .module { color: #050; }
-.ruby .punct { color: #447; font-weight: bold; }
-.ruby .symbol { color: #099; }
-.ruby .string { color: #944; background: #FFE; }
-.ruby .char { color: #F07; }
-.ruby .ident { color: #004; }
-.ruby .constant { color: #07F; }
-.ruby .regex { color: #B66; background: #FEF; }
-.ruby .number { color: #F99; }
-.ruby .attribute { color: #7BB; }
-.ruby .global { color: #7FB; }
-.ruby .expr { color: #227; }
-.ruby .escape { color: #277; }
+Usage:
+  ruby bench.rb [lang] [format] [size in kB] [number of runs]
 
-.xml .normal {}
-.xml .namespace { color: #B66; font-weight: bold; }
-.xml .tag { color: #F88; }
-.xml .comment { color: #005; font-style: italic; }
-.xml .punct { color: #447; font-weight: bold; }
-.xml .string { color: #944; }
-.xml .number { color: #F99; }
-.xml .attribute { color: #BB7; }
+  - lang defaults to ruby.
+  - format defaults to html.
+  - size defaults to 1000 kB (= 1,000,000 bytes). 0 uses the whole example input.
+  - number of runs defaults to 5.
 
-.yaml .normal {}
-.yaml .document { font-weight: bold; color: #07F; }
-.yaml .type { font-weight: bold; color: #05C; }
-.yaml .key { color: #F88; }
-.yaml .comment { color: #005; font-style: italic; }
-.yaml .punct { color: #447; font-weight: bold; }
-.yaml .string { color: #944; }
-.yaml .number { color: #F99; }
-.yaml .time { color: #F99; }
-.yaml .date { color: #F99; }
-.yaml .ref { color: #944; }
-.yaml .anchor { color: #944; }
+-h prints this help
+-p generates a profile (slow, use with SIZE = 1)
+-w waits after the benchmark (for debugging memory usw)
diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb
index 17a0795..2c14186 100644
--- a/lib/coderay/encoders/debug_lint.rb
+++ b/lib/coderay/encoders/debug_lint.rb
@@ -1,6 +1,8 @@
 module CodeRay
 module Encoders
   
+  load :lint
+  
   # = Debug Lint Encoder
   #
   # Debug encoder with additional checks for:
@@ -15,12 +17,8 @@ module Encoders
     
     register_for :debug_lint
     
-    InvalidTokenStream = Class.new StandardError
-    EmptyToken = Class.new InvalidTokenStream
-    IncorrectTokenGroupNesting = Class.new InvalidTokenStream
-    
     def text_token text, kind
-      raise EmptyToken, 'empty token' if text.empty?
+      raise Lint::EmptyToken, 'empty token' if text.empty?
       super
     end
     
@@ -30,7 +28,7 @@ module Encoders
     end
     
     def end_group kind
-      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
       @opened.pop
       super
     end
@@ -41,7 +39,7 @@ module Encoders
     end
     
     def end_line kind
-      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
       @opened.pop
       super
     end
diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb
new file mode 100644
index 0000000..4601e90
--- /dev/null
+++ b/lib/coderay/encoders/lint.rb
@@ -0,0 +1,57 @@
+module CodeRay
+module Encoders
+  
+  # = Lint Encoder
+  #
+  # Checks for:
+  # 
+  # - empty tokens
+  # - incorrect nesting
+  # 
+  # It will raise an InvalidTokenStream exception when any of the above occurs.
+  # 
+  # See also: Encoders::DebugLint
+  class Lint < Debug
+    
+    register_for :lint
+    
+    InvalidTokenStream         = Class.new StandardError
+    EmptyToken                 = Class.new InvalidTokenStream
+    IncorrectTokenGroupNesting = Class.new InvalidTokenStream
+    
+    def text_token text, kind
+      raise EmptyToken, 'empty token' if text.empty?
+    end
+    
+    def begin_group kind
+      @opened << kind
+    end
+    
+    def end_group kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+    end
+    
+    def begin_line kind
+      @opened << kind
+    end
+    
+    def end_line kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+    end
+    
+    protected
+    
+    def setup options
+      @opened = []
+    end
+    
+    def finish options
+      raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
+    end
+    
+  end
+  
+end
+end
diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb
index 9c36b62..5e3a1e7 100644
--- a/lib/coderay/helpers/file_type.rb
+++ b/lib/coderay/helpers/file_type.rb
@@ -117,7 +117,7 @@ module CodeRay
       'rhtml'     => :erb,
       'rjs'       => :ruby,
       'rpdf'      => :ruby,
-      'ru'        => :ruby,
+      'ru'        => :ruby,  # config.ru
       'rxml'      => :ruby,
       'sass'      => :sass,
       'sql'       => :sql,
@@ -141,6 +141,9 @@ module CodeRay
       'Rakefile' => :ruby,
       'Rantfile' => :ruby,
       'Gemfile'  => :ruby,
+      'Guardfile' => :ruby,
+      'Vagrantfile' => :ruby,
+      'Appraisals' => :ruby
     }
     
   end
diff --git a/rake_tasks/benchmark.rake b/rake_tasks/benchmark.rake
index 040951b..2e38b57 100644
--- a/rake_tasks/benchmark.rake
+++ b/rake_tasks/benchmark.rake
@@ -1,7 +1,6 @@
 desc 'Do a benchmark'
 task :benchmark do
-  ruby "-v"
-  ruby "-wIlib bench/bench.rb ruby div 3000 N5"
+  ruby 'bench/bench.rb ruby html 3000'
 end
 
 task :bench => :benchmark
diff --git a/test/executable/suite.rb b/test/executable/suite.rb
index ac0ff1d..997405c 100644
--- a/test/executable/suite.rb
+++ b/test/executable/suite.rb
@@ -18,9 +18,9 @@ class TestCodeRayExecutable < Test::Unit::TestCase
   EXE_COMMAND =
     if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success?
       # use Nailgun
-      "#{RUBY_COMMAND}--ng -w -I%s %s"
+      "#{RUBY_COMMAND}--ng -I%s %s"
     else
-      "#{RUBY_COMMAND} -w -I%s %s"
+      "#{RUBY_COMMAND} -I%s %s"
     end % [ROOT_DIR + 'lib', EXECUTABLE]
   
   def coderay args, options = {}