diff options
| -rw-r--r-- | README | 55 | ||||
| -rw-r--r-- | Rakefile | 82 | ||||
| -rw-r--r-- | demo/demo_css.rb | 2 | ||||
| -rw-r--r-- | lib/coderay.rb | 120 | ||||
| -rw-r--r-- | lib/coderay/encoder.rb | 2 | ||||
| -rw-r--r-- | lib/coderay/tokens.rb | 22 | ||||
| -rw-r--r-- | rake_helpers/rdoctask2.rb | 125 | ||||
| -rw-r--r-- | rdoctask2.rb | 129 |
8 files changed, 348 insertions, 189 deletions
@@ -1,13 +1,13 @@ = CodeRay -You might want to see the original version of this documentation on -http://rd.cYcnus.de/coderay/doc (use Ctrl+Click to open it in its own frame.) +[- Tired of blue'n'gray? Try the original version of this documentation on +http://rd.cYcnus.de/coderay/doc (use Ctrl+Click to open it in its own frame.) -] == About CodeRay is a Ruby library for syntax highlighting. Syntax highlighting means: You put your code in, and you get it back colored; -Keywords, Strings, Floats, Comments - all in different colors. +Keywords, strings, floats, comments - all in different colors. And with line numbers. *Syntax* *Highlighting*... @@ -23,8 +23,8 @@ Author:: murphy Idea:: licenser Website:: rd.cYcnus.de/coderay[http://rd.cYcnus.de/coderay] Copyright:: (c) 2005 by cYcnus -License:: Not yet decided -Subversion: $Id$ +License:: GNU LGPL; see LICENSE file in the main directory. +Subversion:: $Id$ ----- @@ -32,7 +32,7 @@ Subversion: $Id$ You need RubyGems[http://rubyforge.org/frs/?group_id=126]. - % gem install coderay + % gem install coderay === Dependencies @@ -43,28 +43,61 @@ CodeRay needs Ruby 1.8 and the strscan[http://www.ruby-doc.org/stdlib/libdoc/str == Example Usage (Forgive me, but this is not highlighted.) - require 'coderay' - - hl = CodeRay.html :line_numbers => :column - puts hl.highlight_page "puts 'Hello, world!'", :ruby + require 'coderay' + + hl = CodeRay.html :line_numbers => :column + puts hl.highlight_page "puts 'Hello, world!'", :ruby == Documentation See CodeRay. +Please report errors in this documentation to <coderay cycnus de>. + ----- == Credits === Special Thanks to + * licenser (Heinz N. Gies) for ending my QBasic career, inventing the Coder project and the input/output plugin system. CodeRay would not exist without him. === Thanks to -* Caleb Clausen for writing RubyLexer (see http://rubyforge.org/projects/rubylexer) and lots of mails + +* Caleb Clausen for writing RubyLexer (see http://rubyforge.org/projects/rubylexer) and lots + of very interesting mails traffic * Jamis Buck for writing Syntax (see http://rubyforge.org/projects/syntax) + I got some useful ideas from it. +* Doug Kearns and everyone else who worked on ruby.vim - it not only helped me coding CodeRay, + but also gave me a wonderful target to reach for the Ruby scanner. * everyone who used CodeRay on http://www.rubyforen.de and http://www.infhu.de/mx * iGEL, magichisoka, manveru and everyone I forgot from rubyforen.de * Dookie (who is no longer with us...) and Leonidas from http://www.python-forum.de +* matz, nobu, why, dave, dhh, etc... you know, those Ruby gods and gurus +* The inventors of: the computer, the internet, the true color display, HTML & CSS, VIM, RUBY, + pizza, microwaves, guitars, scouting, programming, anime, manga, coke and green ice tea. + +Where would we be without all those people? + +=== Created using + +* Ruby[http://ruby-lang.org/en/] +* Chihiro (my Sony VAIO laptop) and Seras (my Athlon 2200+ tower) +* VIM[http://vim.org] +* RDE[http://homepage2.nifty.com/sakazuki/rde_e.html] +* Microsoft Windows (yes, I confess!) +* Firefox[http://www.mozilla.org/products/firefox/] and Thunderbird[http://www.mozilla.org/products/thunderbird/] +* Rake[http://rake.rubyforge.org/] +* RubyGems[http://docs.rubygems.org/] +* {Subversion/TortoiseSVN}[http://tortoisesvn.tigris.org/] using Apache via XAMPP[http://www.apachefriends.org/en/xampp.html] +* RDoc (though I'm quite unsatisfied with it) +* GNUWin32, MinGW and some other tools to make the shell under windows a bit more usefull + +--- + +* As you can see, CodeRay was created under heavy use of *free* software. +* So CodeRay is also *free*. +* If you use CodeRay to create software, think about making this software *free*, too. @@ -1,6 +1,6 @@ # excuse me, this is my first Rakefile :( [m]
require 'rake'
-require 'rdoctask2'
+require 'rake_helpers/rdoctask2'
require 'rake/gempackagetask.rb'
ROOT = ''
@@ -8,14 +8,14 @@ LIB_ROOT = ROOT + 'lib/' task :default => :make
-task :doc => [:deldoc, :rdoc]
-task :deldoc do
- rm_r 'doc' if File.directory? 'doc'
-end
+#task :doc => [:deldoc, :make_doc]
+#task :minidoc => [:deldoc, :make_minidoc]
+#task :deldoc do
+# rm_r 'doc' if File.directory? 'doc'
+#end
-desc 'Generate documentation for CodeRay'
-Rake::RDocTask.new :rdoc do |rd|
- rd.rdoc_dir = 'doc'
+def set_rdoc_info rd, small = false
+# rd.rdoc_dir = 'doc'
rd.main = ROOT + 'README'
rd.title = "CodeRay Documentation"
rd.options << '--line-numbers' << '--inline-source' << '--tab-width' << '2'
@@ -23,12 +23,18 @@ Rake::RDocTask.new :rdoc do |rd| rd.options << '--all'
rd.template = 'rake_helpers/coderay_rdoc_template.rb'
rd.rdoc_files.add ROOT + 'README'
- rd.rdoc_files.add *Dir[LIB_ROOT + '*.rb']
-# rd.rdoc_files.include ROOT + 'coderay/scanners/*.rb'
-# rd.rdoc_files.include ROOT + 'coderay/scanners/helpers/*.rb'
-# rd.rdoc_files.include ROOT + 'coderay/encoders/*.rb'
-# rd.rdoc_files.include ROOT + 'coderay/encoders/helpers/*.rb'
-# rd.rdoc_files.include ROOT + 'coderay/helpers/*.rb'
+ rd.rdoc_files.add ROOT + 'LICENSE'
+ rd.rdoc_files.add *Dir[LIB_ROOT + "#{'**/' unless small}*.rb"]
+end
+
+desc 'Generate documentation for CodeRay'
+Rake::RDocTask.new :rdoc do |rd|
+ set_rdoc_info rd
+end
+
+desc 'Generate test documentation for CodeRay'
+Rake::RDocTask.new :rdoc_small do |rd|
+ set_rdoc_info rd, true
end
desc 'Report code statistics (LOC) from the application'
@@ -54,8 +60,8 @@ def gemspec s.requirements = ['strscan']
s.date = Time.now.strftime '%Y-%m-%d'
s.has_rdoc = true
- s.rdoc_options = '-SNw2', '-mREADME', '-a'
- s.extra_rdoc_files = %w(./README)
+ s.rdoc_options = '-SNw2', '-mREADME', '-a', '-t CodeRay Documentation'
+ s.extra_rdoc_files = %w(./README ./LICENSE)
# Description
s.summary = <<-EOF
@@ -153,18 +159,22 @@ def cYcnus_ftp end
end
+def uploader_for ftp
+ proc do |l, r|
+ raise 'File %s not found!' % l unless File.exist? l
+ g 'Uploading %s to %s...' % [l, r]
+ ftp.putbinaryfile f, f
+ gd
+ end
+end
+
task :upload_gem do
gn 'Uploading gem:'
Dir.chdir 'gem_server' do
cYcnus_ftp do |ftp|
- uploader = proc do |f|
- raise 'File %s not found!' % f unless File.exist? f
- g 'Uploading %s...' % f
- ftp.putbinaryfile f, f
- gd
- end
+ uploader = uploader_for ftp
ftp.chdir 'public_html/raindark/coderay'
- %w(yaml yaml.Z).each &uploader
+ %w(yaml yaml.Z).each { |f| uploader.call f, f }
Dir.chdir 'gems' do
ftp.chdir 'gems'
uploader.call $gemfile
@@ -175,20 +185,18 @@ task :upload_gem do end
task :example do
- Dir.chdir 'demo' do
- g 'Highlighting self...'
- system 'ruby -w highlight_self.rb -o -L'
- gd
- gn 'Uploading example:'
- cYcnus_ftp do |ftp|
- ftp.chdir 'public_html/raindark/coderay'
- uploader = proc do |l, r|
- g 'Uploading %s to %s...' % [l, r]
- ftp.putbinaryfile l, r
- gd
- end
- uploader.call 'highlight_self/all_in_one.html', 'example.html'
+ g 'Highlighting self...'
+ system 'ruby -wIlib ../hidden/highlight.rb -r -1 lib demo bin rake_helpers'
+ gd
+ gn 'Uploading example:'
+ cYcnus_ftp do |ftp|
+ ftp.chdir 'public_html/raindark/coderay'
+ uploader = proc do |l, r|
+ g 'Uploading %s to %s...' % [l, r]
+ ftp.putbinaryfile l, r
+ gd
end
- gn 'Example uploaded.'
+ uploader.call 'highlighted/all_in_one.html', 'example.html'
end
+ gn 'Example uploaded.'
end
diff --git a/demo/demo_css.rb b/demo/demo_css.rb index 972bbfa..e4cd455 100644 --- a/demo/demo_css.rb +++ b/demo/demo_css.rb @@ -1,6 +1,6 @@ require 'coderay'
-data = File.read 'L:\bench\strange.ruby'
+data = File.read $0
page = CodeRay.scan(data, :ruby).optimize.html(:css => :style, :debug => $DEBUG).page
puts page
diff --git a/lib/coderay.rb b/lib/coderay.rb index 17c315d..012668b 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -56,6 +56,76 @@ # # and look at the file it created. # + +# = CodeRay +# +# The CodeRay module provides convenience methods for the engine. +# +# * The +lang+ and +format+ arguments select Scanner and Encoder to use. These are +# simply lower-case symbols, like <tt>:python</tt> or <tt>:html</tt>. +# * All methods take an optional hash as last parameter, +options+, that is send to +# the Encoder / Scanner. +# * Input and language are always sorted in this order: +code+, +lang+. +# (This is in alphabetical order, if you need a mnemonic ;) +# +# You should be able to highlight everything you want just using this methods; +# so there is no need to dive into CodeRay's deep class hierarchy. +# +# The exmaples in the demo/ directory demonstrate common cases using this interface. +# +# = Basic Access Ways +# +# Read this to get a general view what CodeRay provides. +# +# == Scanning +# +# Scanning means analysing an input string, splitting it up into Tokens. +# Each Token knows about what type it is: string, comment, class name, etc. +# +# Each +lang+ (language) has its own Scanner; for example, <tt>:ruby</tt> code is +# handled by CodeRay::Scanners::RubyScanner. +# +# CodeRay.scan:: Scan a string in a given language into Tokens. +# This is the most common method to use. +# CodeRay.scan_file:: Scan a file and guess the language using FileType. +# +# The Tokens object you get from these methods can encode itself; see Tokens. +# +# == Encoding +# +# Encoding means compiling Tokens into an output. This can be colored HTML or +# LaTeX, a textual statistic or just the number of non-whitespace tokens. +# +# Each Encoder provides output in a specific +format+, so you select Encoders via +# formats like <tt>:html</tt> or <tt>:statistic</tt>. +# +# CodeRay.encode:: Scan and encode a string in a given language. +# CodeRay.encode_tokens:: Encode the given tokens. +# CodeRay.encode_file:: Scan a file, guess the language using FileType and encode it. +# +# == Streaming +# +# Streaming saves RAM by running Scanner and Encoder in some sort of +# pipe mode; see TokenStream. +# +# CodeRay.scan_stream:: Scan in stream mode. +# +# == All-in-One Encoding +# +# CodeRay.encode:: Highlight a string with a given input and output format. +# +# == Instanciating +# +# You can use an Encoder instance to highlight multiple inputs. This way, the setup +# for this Encoder must only be done once. +# +# CodeRay.encoder:: Create an Encoder instance with format and options. +# +# There is no CodeRay.scanner method because Scanners are bound to an input string +# on creation; you can't re-use them with another string. +# +# The scanning methods provide more flexibility; we recommend to use these. +# module CodeRay Version = '0.4.2' @@ -108,7 +178,10 @@ module CodeRay scan code, lang, options, &block end - # Encode +code+ with the Encoder for +format+ and the Scanner for +lang+. + # Encode a string in Streaming mode. + # + # This starts scanning +code+ with the the Scanner for +lang+ + # while encodes the output with the Encoder for +format+. # +options+ will be passed to the Encoder. # # See CodeRay::Encoder.encode_stream @@ -116,17 +189,48 @@ module CodeRay encoder(format, options).encode_stream code, lang, options end + # Encode a string. + # + # This scans +code+ with the the Scanner for +lang+ and then + # encodes it with the Encoder for +format+. + # +options+ will be passed to the Encoder. + # + # See CodeRay::Encoder.encode def encode code, lang, format, options = {} encoder(format, options).encode code, lang, options end + + # Encode pre-scanned Tokens. + # Use this together with CodeRay.scan: + # + # require 'coderay' + # + # # Highlight a short Ruby code example in a HTML span + # tokens = CodeRay.scan '1 + 2', :ruby + # puts CodeRay.encode_tokens(tokens, :span) + # + def encode_tokens tokens, format, options = {} + encoder(format, options).encode_tokens tokens, options + end + + # Encodes +filename+ (a path to a code file) with the Scanner for +lang+. + # + # See CodeRay.scan_file. + # Notice that the second argument is the output +format+, not the input language. + # + # Example: + # require 'coderay' + # page = CodeRay.encode_file 'some_c_code.c', :html + def encode_file filename, format, options = {} + tokens = scan_file filename, auto, get_scanner_options(options) + encode_tokens tokens, format, options + end # Finds the Encoder class for +format+ and creates an instance, passing # +options+ to it. # # Example: # require 'coderay' - # token_count = CodeRay.encoder(:count).encodea("puts 17 + 4\n", :ruby).to_i #-> 8 - # require 'coderay' # # stats = CodeRay.encoder(:statistic) # stats.encode("puts 17 + 4\n", :ruby) @@ -140,6 +244,16 @@ module CodeRay Encoders[format].new options end + # Extract the options for the scanner from the +options+ hash. + # + # Returns an empty Hash if <tt>:scanner_options</tt> is not set. + # + # This is used if a method like CodeRay.encode has to provide options + # for Encoder _and_ scanner. + def get_scanner_options options + options.fetch :scanner_options, {} + end + end # This Exception is raised when you try to stream with something that is not diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb index b6a22f0..53febc2 100644 --- a/lib/coderay/encoder.rb +++ b/lib/coderay/encoder.rb @@ -165,7 +165,7 @@ module CodeRay raise NotStreamableError, self unless kind_of? Streamable
options = @options.merge options
setup options
- scanner_options = options.fetch :scanner_options, {}
+ scanner_options = CodeRay.get_scanner_options options
@token_stream = CodeRay.scan_stream code, lang, scanner_options, &self
finish options
end
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 71ad33a..988008e 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -151,9 +151,15 @@ module CodeRay # This can not be undone, but should yield the same output
# in most Encoders. It basically makes the output smaller.
#
- # Combined with dump, it saves database space.
+ # Combined with dump, it saves space for the cost
+ # calculating time.
+ #
+ # If the scanner is written carefully, this is not required -
+ # for example, consecutive //-comment lines can already be
+ # joined in one token by the Scanner.
def optimize
- last_kind, last_text = nil, nil
+ print ' Tokens#optimize: before: %d - ' % size if $DEBUG
+ last_kind = last_text = nil
new = self.class.new
each do |text, kind|
if text.is_a? String
@@ -166,15 +172,17 @@ module CodeRay end
else
new << [last_text, last_kind] if last_kind
- last_kind, last_text = nil, nil
+ last_kind = last_text = nil
new << [text, kind]
end
end
new << [last_text, last_kind] if last_kind
+ print 'after: %d (%d saved = %2.0f%%)' %
+ [new.size, size - new.size, 1.0 - (new.size.to_f / size)] if $DEBUG
new
end
- # Compact the object itself; see compact.
+ # Compact the object itself; see optimize.
def optimize!
replace optimize
end
@@ -290,9 +298,9 @@ module CodeRay raise NotImplementedError, 'A TokenStream cannot be dumped.'
end
- # A TokenStream cannot be compacted. Use Tokens.
- def compact
- raise NotImplementedError, 'A TokenStream cannot be compacted.'
+ # A TokenStream cannot be optimized. Use Tokens.
+ def optimize
+ raise NotImplementedError, 'A TokenStream cannot be optimized.'
end
end
diff --git a/rake_helpers/rdoctask2.rb b/rake_helpers/rdoctask2.rb new file mode 100644 index 0000000..26a2673 --- /dev/null +++ b/rake_helpers/rdoctask2.rb @@ -0,0 +1,125 @@ +#!/usr/bin/env ruby + +require 'rake' +require 'rake/tasklib' +require 'rdoc/rdoc' + +module Rake + + # Create a documentation task that will generate the RDoc files for + # a project. + # + # The RDocTask will create the following targets: + # + # [<b><em>rdoc</em></b>] + # Main task for this RDOC task. + # + # [<b>:clobber_<em>rdoc</em></b>] + # Delete all the rdoc files. This target is automatically + # added to the main clobber target. + # + # [<b>:re<em>rdoc</em></b>] + # Rebuild the rdoc files from scratch, even if they are not out + # of date. + # + # Simple Example: + # + # Rake::RDocTask.new do |rd| + # rd.main = "README.rdoc" + # rd.rdoc_files.include("README.rdoc", "lib/**/*.rb") + # end + # + # You may wish to give the task a different name, such as if you are + # generating two sets of documentation. For instance, if you want to have a + # development set of documentation including private methods: + # + # Rake::RDocTask.new(:rdoc_dev) do |rd| + # rd.main = "README.doc" + # rd.rdoc_files.include("README.rdoc", "lib/**/*.rb") + # rd.options << "--all" + # end + # + # The tasks would then be named :<em>rdoc_dev</em>, :clobber_<em>rdoc_dev</em>, and + # :re<em>rdoc_dev</em>. + # + class RDocTask < TaskLib + # Name of the main, top level task. (default is :rdoc) + attr_accessor :name + + # Name of directory to receive the html output files. (default is "html") + attr_accessor :rdoc_dir + + # Title of RDoc documentation. (default is none) + attr_accessor :title + + # Name of file to be used as the main, top level file of the + # RDoc. (default is none) + attr_accessor :main + + # Name of template to be used by rdoc. (default is 'html') + attr_accessor :template + + # List of files to be included in the rdoc generation. (default is []) + attr_accessor :rdoc_files + + # List of options to be passed rdoc. (default is []) + attr_accessor :options + + # Create an RDoc task named <em>rdoc</em>. Default task name is +rdoc+. + def initialize(name=:rdoc) # :yield: self + @name = name + @rdoc_files = Rake::FileList.new + @rdoc_dir = name.to_s + @main = nil + @title = nil + @template = 'html' + @options = [] + yield self if block_given? + define + end + + # Create the tasks defined by this task lib. + def define + desc "Build the #{name} HTML Files" + task name + + desc "Force a rebuild of the RDOC files" + task paste("re", name) => [paste("clobber_", name), name] + + desc "Remove rdoc products" + task paste("clobber_", name) do + rm_r rdoc_dir rescue nil + end + + #task :clobber => [paste("clobber_", name)] + + directory @rdoc_dir + task name => [rdoc_target] + file rdoc_target => @rdoc_files + [$rakefile] do + rm_r @rdoc_dir rescue nil + r = RDoc::RDoc.new + r.document(['-o', @rdoc_dir] + option_list + @rdoc_files) + end + self + end + + def option_list + result = @options.dup + result << "--main" << main if main + result << "--title" << "'#{title}'" if title + result << "-T" << template if template + result + end + + def option_string + option_list.join(' ') + end + + private + + def rdoc_target + "#{rdoc_dir}/index.html" + end + + end +end diff --git a/rdoctask2.rb b/rdoctask2.rb deleted file mode 100644 index 311dd36..0000000 --- a/rdoctask2.rb +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env ruby - -require 'rake' -require 'rake/tasklib' -require 'rdoc/rdoc' - -module Rake - - # Create a documentation task that will generate the RDoc files for - # a project. - # - # The RDocTask will create the following targets: - # - # [<b><em>rdoc</em></b>] - # Main task for this RDOC task. - # - # [<b>:clobber_<em>rdoc</em></b>] - # Delete all the rdoc files. This target is automatically - # added to the main clobber target. - # - # [<b>:re<em>rdoc</em></b>] - # Rebuild the rdoc files from scratch, even if they are not out - # of date. - # - # Simple Example: - # - # Rake::RDocTask.new do |rd| - # rd.main = "README.rdoc" - # rd.rdoc_files.include("README.rdoc", "lib/**/*.rb") - # end - # - # You may wish to give the task a different name, such as if you are - # generating two sets of documentation. For instance, if you want to have a - # development set of documentation including private methods: - # - # Rake::RDocTask.new(:rdoc_dev) do |rd| - # rd.main = "README.doc" - # rd.rdoc_files.include("README.rdoc", "lib/**/*.rb") - # rd.options << "--all" - # end - # - # The tasks would then be named :<em>rdoc_dev</em>, :clobber_<em>rdoc_dev</em>, and - # :re<em>rdoc_dev</em>. - # - class RDocTask < TaskLib - # Name of the main, top level task. (default is :rdoc) - attr_accessor :name - - # Name of directory to receive the html output files. (default is "html") - attr_accessor :rdoc_dir - - # Title of RDoc documentation. (default is none) - attr_accessor :title - - # Name of file to be used as the main, top level file of the - # RDoc. (default is none) - attr_accessor :main - - # Name of template to be used by rdoc. (default is 'html') - attr_accessor :template - - # List of files to be included in the rdoc generation. (default is []) - attr_accessor :rdoc_files - - # List of options to be passed rdoc. (default is []) - attr_accessor :options - - # Create an RDoc task named <em>rdoc</em>. Default task name is +rdoc+. - def initialize(name=:rdoc) # :yield: self - @name = name - @rdoc_files = Rake::FileList.new - @rdoc_dir = 'html' - @main = nil - @title = nil - @template = 'html' - @options = [] - yield self if block_given? - define - end - - # Create the tasks defined by this task lib. - def define - if name.to_s != "rdoc" - desc "Build the RDOC HTML Files" - end - - desc "Build the #{name} HTML Files" - task name - - desc "Force a rebuild of the RDOC files" - task paste("re", name) => [paste("clobber_", name), name] - - desc "Remove rdoc products" - task paste("clobber_", name) do - rm_r rdoc_dir rescue nil - end - - task :clobber => [paste("clobber_", name)] - - directory @rdoc_dir - task name => [rdoc_target] - file rdoc_target => @rdoc_files + [$rakefile] do - rm_r @rdoc_dir rescue nil - r = RDoc::RDoc.new - r.document(['-o', @rdoc_dir] + option_list + @rdoc_files) - end - self - end - - def option_list - result = @options.dup - result << "--main" << main if main - result << "--title" << "'#{title}'" if title - result << "-T" << template if template - result - end - - def option_string - option_list.join(' ') - end - - private - - def rdoc_target - "#{rdoc_dir}/index.html" - end - - end -end |