diff options
28 files changed, 17 insertions, 120 deletions
diff --git a/Changes.textile b/Changes.textile index 3145fee..12fc6ae 100644 --- a/Changes.textile +++ b/Changes.textile @@ -29,10 +29,6 @@ h3. @Tokens@ * *REMOVED* method @stream?@. * *NEW* methods @encode_with@, @count@, @begin_group@, @end_group@, @begin_line@, and @end_line@. -h3. @TokenStream@ - -Removed. - h3. *RENAMED*: @Tokens::AbbreviationForKind@ Renamed from @ClassOfKind@; the term "token class" is no longer used in CodeRay. Instead, tokens have _kinds_. diff --git a/bin/coderay b/bin/coderay index 7f8271d..ef819f7 100644 --- a/bin/coderay +++ b/bin/coderay @@ -73,7 +73,7 @@ Examples: end if tokens == :scan - output = CodeRay::Duo[lang => format].highlight input, :stream => true + output = CodeRay::Duo[lang => format].highlight input else output = tokens.encode format end diff --git a/lib/coderay.rb b/lib/coderay.rb index ef2574a..ad0bb5e 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -170,27 +170,6 @@ module CodeRay scan file, lang, options = {}, &block end - # Scan the +code+ (a string) with the scanner for +lang+. - # - # Calls scan. - # - # See CodeRay.scan. - def scan_stream code, lang, options = {}, &block - options[:stream] = true - scan code, lang, options, &block - end - - # Encode a string in Streaming mode. - # - # This starts scanning +code+ with the the Scanner for +lang+ - # while encodes the output with the Encoder for +format+. - # +options+ will be passed to the Encoder. - # - # See CodeRay::Encoder.encode_stream - def encode_stream code, lang, format, options = {} - encoder(format, options).encode_stream code, lang, options - end - # Encode a string. # # This scans +code+ with the the Scanner for +lang+ and then @@ -286,12 +265,6 @@ module CodeRay end - # A dummy module that is included by subclasses of - # CodeRay::Scanners::Scanner and CodeRay::Encoders::Encoder - # to show that they are able to handle streams. - module Streamable - end - end # Run a test script. diff --git a/lib/coderay/duo.rb b/lib/coderay/duo.rb index 5468dda..c11bd3f 100644 --- a/lib/coderay/duo.rb +++ b/lib/coderay/duo.rb @@ -23,9 +23,6 @@ module CodeRay # simple: # CodeRay::Duo[:ruby, :page].highlight 'bla 42' # - # streaming: - # CodeRay::Duo[:ruby, :page].highlight 'bar 23', :stream => true - # # with options: # CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??' # @@ -64,18 +61,9 @@ module CodeRay end # Tokenize and highlight the code using +scanner+ and +encoder+. - # - # If the :stream option is set, the Duo will go into streaming mode, - # saving memory for the cost of time. - def encode code, options = { :stream => false } - stream = options.delete :stream + def encode code, options = {} options = @options.merge options - if stream - encoder.encode_stream(code, @lang, options) - else - scanner.code = code - encoder.encode_tokens(scanner.tokenize, options) - end + encoder.encode(code, @lang, options) end alias highlight encode diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb index 82545c4..716f516 100644 --- a/lib/coderay/encoder.rb +++ b/lib/coderay/encoder.rb @@ -27,8 +27,6 @@ module CodeRay extend Plugin plugin_host Encoders - attr_reader :token_stream - class << self # If FILE_EXTENSION isn't defined, this method returns the @@ -44,7 +42,7 @@ module CodeRay end # Subclasses are to store their default options in this constant. - DEFAULT_OPTIONS = { :stream => false } + DEFAULT_OPTIONS = { } # The options you gave the Encoder at creating. attr_accessor :options @@ -56,7 +54,6 @@ module CodeRay # Encoder objects provide three encode methods: # - encode simply takes a +code+ string and a +lang+ # - encode_tokens expects a +tokens+ object instead - # - encode_stream is like encode, but uses streaming mode. # # Each method has an optional +options+ parameter. These are # added to the options you passed at creation. @@ -75,31 +72,20 @@ module CodeRay finish options end - # Encode the given +code+ after tokenizing it using the Scanner - # for +lang+. + # Encode the given +code+ using the Scanner for +lang+. def encode code, lang, options = {} options = @options.merge options - scanner_options = CodeRay.get_scanner_options(options) - tokens = CodeRay.scan code, lang, scanner_options - encode_tokens tokens, options - end - - # You can use highlight instead of encode, if that seems - # more clear to you. - alias highlight encode - - # Encode the given +code+ using the Scanner for +lang+ in - # streaming mode. - def encode_stream code, lang, options = {} - options = @options.merge options setup options scanner_options = CodeRay.get_scanner_options options scanner_options[:tokens] = self - @token_stream = - CodeRay.scan_stream code, lang, scanner_options + CodeRay.scan code, lang, scanner_options finish options end + # You can use highlight instead of encode, if that seems + # more clear to you. + alias highlight encode + # Return the default file extension for outputs of this encoder. def file_extension self.class::FILE_EXTENSION diff --git a/lib/coderay/encoders/count.rb b/lib/coderay/encoders/count.rb index 451a7f8..f4dd0c8 100644 --- a/lib/coderay/encoders/count.rb +++ b/lib/coderay/encoders/count.rb @@ -7,7 +7,6 @@ module Encoders # Text and block tokens are counted. class Count < Encoder - include Streamable register_for :count protected diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index 89e430f..c97631b 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -15,7 +15,6 @@ module Encoders # See also: Scanners::Debug class Debug < Encoder - include Streamable register_for :debug FILE_EXTENSION = 'raydebug' diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 807fb42..0a55a02 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -89,7 +89,6 @@ module Encoders # Default: false class HTML < Encoder - include Streamable register_for :html FILE_EXTENSION = 'html' diff --git a/lib/coderay/encoders/null.rb b/lib/coderay/encoders/null.rb index 7a8899b..b815c2f 100644 --- a/lib/coderay/encoders/null.rb +++ b/lib/coderay/encoders/null.rb @@ -6,7 +6,6 @@ module Encoders # Does nothing and returns an empty string. class Null < Encoder - include Streamable register_for :null # Defined for faster processing. diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb index d267b21..455da46 100644 --- a/lib/coderay/encoders/statistic.rb +++ b/lib/coderay/encoders/statistic.rb @@ -7,7 +7,6 @@ module Encoders # Alias: +stats+ class Statistic < Encoder - include Streamable register_for :stats, :statistic attr_reader :type_stats, :real_token_count # :nodoc: diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb index ecbf624..2dfb224 100644 --- a/lib/coderay/encoders/text.rb +++ b/lib/coderay/encoders/text.rb @@ -14,7 +14,6 @@ module Encoders # Default: empty String class Text < Encoder - include Streamable register_for :text FILE_EXTENSION = 'txt' diff --git a/lib/coderay/encoders/token_kind_filter.rb b/lib/coderay/encoders/token_kind_filter.rb index fd3df44..431794d 100644 --- a/lib/coderay/encoders/token_kind_filter.rb +++ b/lib/coderay/encoders/token_kind_filter.rb @@ -25,7 +25,6 @@ module Encoders # See also: CommentFilter class TokenKindFilter < Filter - include Streamable register_for :token_kind_filter DEFAULT_OPTIONS = { diff --git a/lib/coderay/encoders/xml.rb b/lib/coderay/encoders/xml.rb index 0006d75..d3e8832 100644 --- a/lib/coderay/encoders/xml.rb +++ b/lib/coderay/encoders/xml.rb @@ -6,7 +6,6 @@ module Encoders # Uses REXML. Very slow. class XML < Encoder - include Streamable register_for :xml FILE_EXTENSION = 'xml' diff --git a/lib/coderay/for_redcloth.rb b/lib/coderay/for_redcloth.rb index e439929..2af9082 100644 --- a/lib/coderay/for_redcloth.rb +++ b/lib/coderay/for_redcloth.rb @@ -57,7 +57,7 @@ module CodeRay @in_bc ||= nil format = @in_bc ? :div : :span opts[:text] = unescape(opts[:text]) unless @in_bc - highlighted_code = CodeRay.encode opts[:text], opts[:lang], format, :stream => true + highlighted_code = CodeRay.encode opts[:text], opts[:lang], format highlighted_code.sub!(/\A<(span|div)/) { |m| m + pba(@in_bc || opts) } highlighted_code else diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 286561d..22f1c67 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -55,7 +55,7 @@ module CodeRay # The default options for all scanner classes. # # Define @default_options for subclasses. - DEFAULT_OPTIONS = { :stream => false } + DEFAULT_OPTIONS = { } KINDS_NOT_LOC = [:comment, :doctype] @@ -108,10 +108,9 @@ module CodeRay # * +options+ is a Hash with Symbols as keys. # It is merged with the default options of the class (you can # overwrite default options here.) - # * +block+ is the callback for streamed highlighting. # # Else, a Tokens object is used. - def initialize code='', options = {}, &block + def initialize code='', options = {} raise "I am only the basic Scanner class. I can't scan "\ "anything. :( Use my subclasses." if self.class == Scanner @@ -119,14 +118,7 @@ module CodeRay super Scanner.normify(code) - @tokens = options[:tokens] - if @options[:stream] - raise NotImplementedError unless @tokens.is_a? Encoders::Encoder - else - warn "warning in CodeRay::Scanner.new: Block given, "\ - "but :stream is #{@options[:stream]}" if block_given? - @tokens ||= Tokens.new - end + @tokens = options[:tokens] || Tokens.new @tokens.scanner = self if @tokens.respond_to? :scanner= setup @@ -158,14 +150,9 @@ module CodeRay def tokenize new_string=nil, options = {} options = @options.merge(options) self.string = new_string if new_string - @cached_tokens = - if @options[:stream] # :stream must have been set already - reset unless new_string - scan_tokens @tokens, options - @tokens - else - scan_tokens @tokens, options - end + reset unless new_string + scan_tokens @tokens, options + @cached_tokens = @tokens end # Caches the result of tokenize. @@ -173,11 +160,6 @@ module CodeRay @cached_tokens ||= tokenize end - # Whether the scanner is in streaming mode. - def streaming? - !!@options[:stream] - end - # Traverses the tokens. def each &block tokens.each(&block) diff --git a/lib/coderay/scanners/c.rb b/lib/coderay/scanners/c.rb index 45ca42e..bd182e8 100644 --- a/lib/coderay/scanners/c.rb +++ b/lib/coderay/scanners/c.rb @@ -4,8 +4,6 @@ module Scanners # Scanner for C. class C < Scanner - include Streamable - register_for :c file_extension 'c' diff --git a/lib/coderay/scanners/cpp.rb b/lib/coderay/scanners/cpp.rb index 7531892..cb79768 100644 --- a/lib/coderay/scanners/cpp.rb +++ b/lib/coderay/scanners/cpp.rb @@ -6,8 +6,6 @@ module Scanners # Aliases: +cplusplus+, c++ class CPlusPlus < Scanner - include Streamable - register_for :cpp file_extension 'cpp' title 'C++' diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb index 0f2b89f..deaa30b 100644 --- a/lib/coderay/scanners/debug.rb +++ b/lib/coderay/scanners/debug.rb @@ -7,7 +7,6 @@ module Scanners # Parses the output of the Encoders::Debug encoder. class Debug < Scanner - include Streamable register_for :debug file_extension 'raydebug' title 'CodeRay Token Dump' diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index fdbbbc7..2334bc6 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -6,7 +6,6 @@ module Scanners # Scanner for Groovy. class Groovy < Java - include Streamable register_for :groovy # TODO: check list of keywords diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 8f71e0e..7fab3cc 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -8,7 +8,6 @@ module Scanners # See also: Scanners::XML class HTML < Scanner - include Streamable register_for :html KINDS_NOT_LOC = [ diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index e7becda..427e754 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -4,7 +4,6 @@ module Scanners # Scanner for Java. class Java < Scanner - include Streamable register_for :java helper :builtin_types diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb index 3ae8d80..f97e6da 100644 --- a/lib/coderay/scanners/java_script.rb +++ b/lib/coderay/scanners/java_script.rb @@ -6,8 +6,6 @@ module Scanners # Aliases: +ecmascript+, +ecma_script+, +javascript+ class JavaScript < Scanner - include Streamable - register_for :java_script file_extension 'js' diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb index 668fd82..b36811f 100644 --- a/lib/coderay/scanners/json.rb +++ b/lib/coderay/scanners/json.rb @@ -4,8 +4,6 @@ module Scanners # Scanner for JSON (JavaScript Object Notation). class JSON < Scanner - include Streamable - register_for :json file_extension 'json' diff --git a/lib/coderay/scanners/nitro_xhtml.rb b/lib/coderay/scanners/nitro_xhtml.rb index ba8ee71..cf8c5aa 100644 --- a/lib/coderay/scanners/nitro_xhtml.rb +++ b/lib/coderay/scanners/nitro_xhtml.rb @@ -9,7 +9,6 @@ module Scanners # Alias: +nitro+ class NitroXHTML < Scanner - include Streamable register_for :nitro_xhtml file_extension :xhtml title 'Nitro XHTML' diff --git a/lib/coderay/scanners/plaintext.rb b/lib/coderay/scanners/plaintext.rb index e176403..b8ea5f5 100644 --- a/lib/coderay/scanners/plaintext.rb +++ b/lib/coderay/scanners/plaintext.rb @@ -11,8 +11,6 @@ module Scanners register_for :plaintext, :plain title 'Plain text' - include Streamable - KINDS_NOT_LOC = [:plain] # :nodoc: protected diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index 568ed57..b01248a 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -7,8 +7,6 @@ module Scanners # http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py. class Python < Scanner - include Streamable - register_for :python file_extension 'py' diff --git a/lib/coderay/scanners/rhtml.rb b/lib/coderay/scanners/rhtml.rb index 064a92c..1922948 100644 --- a/lib/coderay/scanners/rhtml.rb +++ b/lib/coderay/scanners/rhtml.rb @@ -7,7 +7,6 @@ module Scanners # Scanner for HTML ERB templates. class RHTML < Scanner - include Streamable register_for :rhtml title 'HTML ERB Template' diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index dcbfce0..a2e5fed 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -17,8 +17,6 @@ module Scanners # Alias: +irb+ class Ruby < Scanner - include Streamable - register_for :ruby file_extension 'rb' |