From 501df7613c67bcd743eaa00071f0dbb179607394 Mon Sep 17 00:00:00 2001 From: Quintus Date: Sun, 22 Apr 2012 22:10:42 +0200 Subject: Lua scanner for CodeRay. Meta-commit. This commit is a super-commit containing all the subcommits for implementing the Lua scanner. --- lib/coderay/scanners/lua.rb | 267 ++++++++++++++++++++++++++++++++++++++++++++ lib/coderay/styles/alpha.rb | 3 + lib/coderay/token_kinds.rb | 1 + 3 files changed, 271 insertions(+) create mode 100644 lib/coderay/scanners/lua.rb diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb new file mode 100644 index 0000000..e7706fc --- /dev/null +++ b/lib/coderay/scanners/lua.rb @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- + +# Scanner for the Lua[http://lua.org] programming lanuage. +# +# The language’s complete syntax is defined in +# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], +# which is what this scanner tries to conform to. +class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + + register_for :lua + file_extension "lua" + title "Lua" + + # Keywords used in Lua. + KEYWORDS = %w[and break do else elseif end + for function goto if in + local not or repeat return + then until while + ] + + # Constants set by the Lua core. + PREDEFINED_CONSTANTS = %w[false true nil] + + # The expressions contained in this array are parts of Lua’s `basic' + # library. Although it’s not entirely necessary to load that library, + # it is highly recommended and one would have to provide own implementations + # of some of these expressions if one does not do so. They however aren’t + # keywords, neither are they constants, but nearly predefined, so they + # get tagged as `predefined' rather than anything else. + # + # This list excludes values of form `_UPPERCASE' because the Lua manual + # requires such identifiers to be reserved by Lua anyway and they are + # highlighted directly accordingly, without the need for specific + # identifiers to be listed here. + PREDEFINED_EXPRESSIONS = %w[ + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall + ] + + # Automatic token kind selection for normal words. + IDENT_KIND = CodeRay::WordList.new(:ident) + .add(KEYWORDS, :keyword) + .add(PREDEFINED_CONSTANTS, :predefined_constant) + .add(PREDEFINED_EXPRESSIONS, :predefined) + + protected + + # Scanner initialization. + def setup + @state = :initial + @brace_depth = 0 + end + + # CodeRay entry hook. Starts parsing. + def scan_tokens(encoder, options) + @encoder = encoder + @options = options + + send(:"handle_state_#@state") until eos? + + @encoder + end + + def handle_state_initial + if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] + @num_equals = match.count("=") # Number must match for comment end + @encoder.begin_group(:comment) + @encoder.text_token(match, :delimiter) + @state = :long_comment + + elsif match = scan(/--.*?$/) # --Lua comment + @encoder.text_token(match, :comment) + + elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] + @num_equals = match.count("=") # Number must match for comment end + @encoder.begin_group(:string) + @encoder.text_token(match, :delimiter) + @state = :long_string + + elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: + @encoder.text_token(match, :label) + + elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua + @encoder.text_token(match, :predefined) + + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) + kind = IDENT_KIND[match] + + # Extra highlighting for entities following certain keywords + if kind == :keyword and match == "function" + @state = :function_expected + elsif kind == :keyword and match == "goto" + @state = :goto_label_expected + elsif kind == :keyword and match == "local" + @state = :local_var_expected + end + + @encoder.text_token(match, kind) + + elsif match = scan(/{/) # Opening table brace { + @encoder.begin_group(:table) + @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) + @brace_depth += 1 + @state = :table + + elsif match = scan(/}/) # Closing table brace } + if @brace_depth == 1 + @brace_depth = 0 + @encoder.text_token(match, :delimiter) + elsif @brace_depth == 0 # Mismatched brace + @encoder.text_token(match, :error) + else + @brace_depth -= 1 + @encoder.text_token(match, :inline_delimiter) + @state = :table + end + @encoder.end_group(:table) + + elsif match = scan(/["']/) # String delimiters " and ' + @encoder.begin_group(:string) + @encoder.text_token(match, :delimiter) + @start_delim = match + @state = :string + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + @encoder.text_token(match, :float) + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + @encoder.text_token(match, :integer) + + elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators + @encoder.text_token(match, :operator) + + elsif match = scan(/\s+/) # Space + @encoder.text_token(match, :space) + + else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. + @encoder.text_token(getch, :error) + end + + # It may be that we’re scanning a full-blown subexpression of a table + # (tables can contain full expressions in parts). + # If this is the case, return to :table scanning state. + @state = :table if @state == :initial && @brace_depth >= 1 + end + + def handle_state_function_expected + if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name + @encoder.text_token(match, :operator) + @state = :initial + elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator + @encoder.text_token(match, :ident) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() + @encoder.text_token(match, :function) + @state = :initial + elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + @state = :initial + end + end + + def handle_state_goto_label_expected + if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + @encoder.text_token(match, :label) + @state = :initial + elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_local_var_expected + if match = scan(/function/) # local function ... + @encoder.text_token(match, :keyword) + @state = :function_expected + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + @encoder.text_token(match, :local_variable) + elsif match = scan(/,/) + @encoder.text_token(match, :operator) + elsif match = scan(/=/) + @encoder.text_token(match, :operator) + # After encountering the equal sign, arbitrary expressions are + # allowed again, so just return to the main state for further + # parsing. + @state = :initial + elsif match = scan(/\n/) + @encoder.text_token(match, :space) + @state = :initial + elsif match = scan(/\s+/) + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_long_comment + if match = scan(/.*?(?=\]={#@num_equals}\])/m) + @encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + @encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + @encoder.text_token(rest, :error) + terminate + end + @encoder.end_group(:comment) + @state = :initial + end + + def handle_state_long_string + if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + @encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + @encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + @encoder.text_token(rest, :error) + terminate + end + @encoder.end_group(:string) + @state = :initial + end + + def handle_state_string + if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + @encoder.text_token(match, :content) + elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) + @encoder.text_token(match, :char) + elsif match = scan(Regexp.compile(@start_delim)) + @encoder.text_token(match, :delimiter) + @encoder.end_group(:string) + @state = :initial + elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings + @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings + @encoder.end_group(:string) + @state = :initial + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_table + if match = scan(/[,;]/) + @encoder.text_token(match, :operator) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) + @encoder.text_token(match, :key) + @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) + @encoder.text_token(scan(/=/), :operator) + @state = :initial + elsif match = scan(/\s+/m) + @encoder.text_token(match, :space) + else + # Note this clause doesn’t advance the scan pointer, it’s a kind of + # "retry with other options" (the :initial state then of course + # advances the pointer). + @state = :initial + end + end + +end diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 8506d10..257083e 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -116,6 +116,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } .symbol { color:#A60 } +.table .content { color:#808 } +.table .delimiter { color:#40A} +.table { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070 } .type { color:#339; font-weight:bold } .value { color: #088; } diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 3b8d07e..e245623 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -63,6 +63,7 @@ module CodeRay :shell => 'shell', :string => 'string', :symbol => 'symbol', + :table => 'table', :tag => 'tag', :type => 'type', :value => 'value', -- cgit v1.2.1 From 8979cc621431248fded86c341e2102a67c5344bb Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 19 Jun 2012 17:45:49 +0200 Subject: use case+when instead of send and methods --- lib/coderay/scanners/lua.rb | 64 ++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e7706fc..e712061 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -40,10 +40,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ] # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident) - .add(KEYWORDS, :keyword) - .add(PREDEFINED_CONSTANTS, :predefined_constant) - .add(PREDEFINED_EXPRESSIONS, :predefined) + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) protected @@ -57,13 +57,11 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner def scan_tokens(encoder, options) @encoder = encoder @options = options - - send(:"handle_state_#@state") until eos? - - @encoder - end - - def handle_state_initial + + until eos? + case state + + when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] @num_equals = match.count("=") # Number must match for comment end @encoder.begin_group(:comment) @@ -99,13 +97,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, kind) - elsif match = scan(/{/) # Opening table brace { + elsif match = scan(/\{/) # Opening table brace { @encoder.begin_group(:table) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 @state = :table - elsif match = scan(/}/) # Closing table brace } + elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @brace_depth = 0 @encoder.text_token(match, :delimiter) @@ -146,9 +144,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # (tables can contain full expressions in parts). # If this is the case, return to :table scanning state. @state = :table if @state == :initial && @brace_depth >= 1 - end - - def handle_state_function_expected + + when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @encoder.text_token(match, :operator) @state = :initial @@ -163,9 +160,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(getch, :error) @state = :initial end - end - def handle_state_goto_label_expected + when :goto_label_expected if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) @encoder.text_token(match, :label) @state = :initial @@ -174,9 +170,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - end - - def handle_state_local_var_expected + + when :local_var_expected if match = scan(/function/) # local function ... @encoder.text_token(match, :keyword) @state = :function_expected @@ -198,9 +193,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - end - - def handle_state_long_comment + + when :long_comment if match = scan(/.*?(?=\]={#@num_equals}\])/m) @encoder.text_token(match, :content) @@ -212,9 +206,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:comment) @state = :initial - end - - def handle_state_long_string + + when :long_string if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences @encoder.text_token(match, :content) @@ -226,9 +219,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:string) @state = :initial - end - - def handle_state_string + + when :string if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) @encoder.text_token(match, :content) elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) @@ -244,9 +236,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - end - - def handle_state_table + + when :table if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) @@ -262,6 +253,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # advances the pointer). @state = :initial end + else + raise + end + + end + + @encoder end end -- cgit v1.2.1 From 2b16d115f7d6caf21864934df763556e126d6357 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 19 Jun 2012 18:05:04 +0200 Subject: fix for Ruby 1.8, escape { and } in regexps --- lib/coderay/scanners/lua.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e7706fc..8464ef7 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -40,10 +40,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ] # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident) - .add(KEYWORDS, :keyword) - .add(PREDEFINED_CONSTANTS, :predefined_constant) - .add(PREDEFINED_EXPRESSIONS, :predefined) + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) protected @@ -99,13 +99,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, kind) - elsif match = scan(/{/) # Opening table brace { + elsif match = scan(/\{/) # Opening table brace { @encoder.begin_group(:table) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 @state = :table - elsif match = scan(/}/) # Closing table brace } + elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @brace_depth = 0 @encoder.text_token(match, :delimiter) -- cgit v1.2.1 From b081c1f3ab670301199005c04ecbc11614593285 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Tue, 11 Sep 2012 15:04:45 -0600 Subject: escape = in regex (my editor gets confused:-) --- lib/coderay/scanners/lua.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e712061..e640397 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -57,10 +57,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner def scan_tokens(encoder, options) @encoder = encoder @options = options - + until eos? case state - + when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] @num_equals = match.count("=") # Number must match for comment end @@ -144,7 +144,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # (tables can contain full expressions in parts). # If this is the case, return to :table scanning state. @state = :table if @state == :initial && @brace_depth >= 1 - + when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @encoder.text_token(match, :operator) @@ -170,7 +170,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - + when :local_var_expected if match = scan(/function/) # local function ... @encoder.text_token(match, :keyword) @@ -179,7 +179,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, :local_variable) elsif match = scan(/,/) @encoder.text_token(match, :operator) - elsif match = scan(/=/) + elsif match = scan(/\=/) @encoder.text_token(match, :operator) # After encountering the equal sign, arbitrary expressions are # allowed again, so just return to the main state for further @@ -193,7 +193,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - + when :long_comment if match = scan(/.*?(?=\]={#@num_equals}\])/m) @encoder.text_token(match, :content) @@ -206,7 +206,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:comment) @state = :initial - + when :long_string if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences @encoder.text_token(match, :content) @@ -219,7 +219,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner end @encoder.end_group(:string) @state = :initial - + when :string if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) @encoder.text_token(match, :content) @@ -236,14 +236,14 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @encoder.text_token(getch, :error) end - + when :table if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) @encoder.text_token(match, :key) @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) - @encoder.text_token(scan(/=/), :operator) + @encoder.text_token(scan(/\=/), :operator) @state = :initial elsif match = scan(/\s+/m) @encoder.text_token(match, :space) -- cgit v1.2.1 From 279348d3c2159df4ce6ac5949ada0177aa7c7159 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sat, 27 Oct 2012 23:41:52 -0600 Subject: :map token kind Use :map instead of :table. It's more generic, and won't be confused with the :table rendering style. --- lib/coderay/scanners/lua.rb | 14 +++++++------- lib/coderay/styles/alpha.rb | 8 ++++---- lib/coderay/token_kinds.rb | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e640397..64763dc 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -98,10 +98,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(match, kind) elsif match = scan(/\{/) # Opening table brace { - @encoder.begin_group(:table) + @encoder.begin_group(:map) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 - @state = :table + @state = :map elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @@ -112,9 +112,9 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner else @brace_depth -= 1 @encoder.text_token(match, :inline_delimiter) - @state = :table + @state = :map end - @encoder.end_group(:table) + @encoder.end_group(:map) elsif match = scan(/["']/) # String delimiters " and ' @encoder.begin_group(:string) @@ -142,8 +142,8 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # It may be that we’re scanning a full-blown subexpression of a table # (tables can contain full expressions in parts). - # If this is the case, return to :table scanning state. - @state = :table if @state == :initial && @brace_depth >= 1 + # If this is the case, return to :map scanning state. + @state = :map if @state == :initial && @brace_depth >= 1 when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @@ -237,7 +237,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner @encoder.text_token(getch, :error) end - when :table + when :map if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 257083e..a60725b 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -1,6 +1,6 @@ module CodeRay module Styles - + # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style @@ -116,9 +116,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } .symbol { color:#A60 } -.table .content { color:#808 } -.table .delimiter { color:#40A} -.table { background-color:hsla(200,100%,50%,0.06); } +.map .content { color:#808 } +.map .delimiter { color:#40A} +.map { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070 } .type { color:#339; font-weight:bold } .value { color: #088; } diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index e245623..a0ad5e4 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,14 +1,14 @@ module CodeRay - + # A Hash of all known token kinds and their associated CSS classes. TokenKinds = Hash.new do |h, k| warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG false end - + # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity - + TokenKinds.update( # :nodoc: :annotation => 'annotation', :attribute_name => 'attribute-name', @@ -50,6 +50,7 @@ module CodeRay :keyword => 'keyword', :label => 'label', :local_variable => 'local-variable', + :map => 'map', :modifier => 'modifier', :namespace => 'namespace', :octal => 'octal', @@ -63,29 +64,28 @@ module CodeRay :shell => 'shell', :string => 'string', :symbol => 'symbol', - :table => 'table', :tag => 'tag', :type => 'type', :value => 'value', :variable => 'variable', - + :change => 'change', :delete => 'delete', :head => 'head', :insert => 'insert', - + :eyecatcher => 'eyecatcher', - + :ident => false, :operator => false, - + :space => false, :plain => false ) - + TokenKinds[:method] = TokenKinds[:function] TokenKinds[:escape] = TokenKinds[:delimiter] TokenKinds[:docstring] = TokenKinds[:comment] - + TokenKinds.freeze end -- cgit v1.2.1 From d9d6dd5f4a73363ea5d353ecda142f77ed4eba5a Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Thu, 6 Jun 2013 10:43:19 -0600 Subject: Support CDATA blocks in HTML/XML --- lib/coderay/scanners/html.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 3ba3b79..fcf249a 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -99,7 +99,17 @@ module Scanners case state when :initial - if match = scan(/|.*)/m) + if match = scan(//m) + encoder.text_token match[0..-4], :content + encoder.text_token ']]>', :delimiter + else + encoder.text_token scan(/.*/m), :error + end + encoder.end_group :string + elsif match = scan(/|.*)/m) encoder.text_token match, :comment elsif match = scan(/|.*)|\]>/m) encoder.text_token match, :doctype -- cgit v1.2.1 From 65eeaca6c18bea379b966559084e47d6fb45c75c Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Wed, 12 Jun 2013 16:19:41 +0200 Subject: add python binary string --- lib/coderay/scanners/python.rb | 4 ++-- lib/coderay/styles/alpha.rb | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index a9492ab..09c8b6e 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -157,12 +157,12 @@ module Scanners encoder.text_token match, :operator elsif match = scan(/(u?r?|b)?("""|"|'''|')/i) + modifiers = self[1] string_delimiter = self[2] - string_type = docstring_coming ? :docstring : :string + string_type = docstring_coming ? :docstring : (modifiers == 'b' ? :binary : :string) docstring_coming = false if docstring_coming encoder.begin_group string_type string_raw = false - modifiers = self[1] unless modifiers.empty? string_raw = !!modifiers.index(?r) encoder.text_token modifiers, :modifier diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index f57e4a1..2ad088a 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -56,7 +56,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .annotation { color:#007 } .attribute-name { color:#b48 } .attribute-value { color:#700 } -.binary { color:#509 } +.binary { color:#549 } +.binary .delimiter { color:#325 } +.binary .char { color:#325 } .char .content { color:#D20 } .char .delimiter { color:#710 } .char { color:#D20 } -- cgit v1.2.1 From 3cc51316b3836c030b7da079f5177a79405700a7 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:05:06 +0200 Subject: add taskpaper file extension --- lib/coderay/helpers/file_type.rb | 97 ++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index a5d83ff..6144e8c 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -77,54 +77,55 @@ module CodeRay end TypeFromExt = { - 'c' => :c, - 'cfc' => :xml, - 'cfm' => :xml, - 'clj' => :clojure, - 'css' => :css, - 'diff' => :diff, - 'dpr' => :delphi, - 'erb' => :erb, - 'gemspec' => :ruby, - 'groovy' => :groovy, - 'gvy' => :groovy, - 'h' => :c, - 'haml' => :haml, - 'htm' => :html, - 'html' => :html, - 'html.erb' => :erb, - 'java' => :java, - 'js' => :java_script, - 'json' => :json, - 'mab' => :ruby, - 'pas' => :delphi, - 'patch' => :diff, - 'phtml' => :php, - 'php' => :php, - 'php3' => :php, - 'php4' => :php, - 'php5' => :php, - 'prawn' => :ruby, - 'py' => :python, - 'py3' => :python, - 'pyw' => :python, - 'rake' => :ruby, - 'raydebug' => :raydebug, - 'rb' => :ruby, - 'rbw' => :ruby, - 'rhtml' => :erb, - 'rjs' => :ruby, - 'rpdf' => :ruby, - 'ru' => :ruby, - 'rxml' => :ruby, - 'sass' => :sass, - 'sql' => :sql, - 'tmproj' => :xml, - 'xaml' => :xml, - 'xhtml' => :html, - 'xml' => :xml, - 'yaml' => :yaml, - 'yml' => :yaml, + 'c' => :c, + 'cfc' => :xml, + 'cfm' => :xml, + 'clj' => :clojure, + 'css' => :css, + 'diff' => :diff, + 'dpr' => :delphi, + 'erb' => :erb, + 'gemspec' => :ruby, + 'groovy' => :groovy, + 'gvy' => :groovy, + 'h' => :c, + 'haml' => :haml, + 'htm' => :html, + 'html' => :html, + 'html.erb' => :erb, + 'java' => :java, + 'js' => :java_script, + 'json' => :json, + 'mab' => :ruby, + 'pas' => :delphi, + 'patch' => :diff, + 'phtml' => :php, + 'php' => :php, + 'php3' => :php, + 'php4' => :php, + 'php5' => :php, + 'prawn' => :ruby, + 'py' => :python, + 'py3' => :python, + 'pyw' => :python, + 'rake' => :ruby, + 'raydebug' => :raydebug, + 'rb' => :ruby, + 'rbw' => :ruby, + 'rhtml' => :erb, + 'rjs' => :ruby, + 'rpdf' => :ruby, + 'ru' => :ruby, + 'rxml' => :ruby, + 'sass' => :sass, + 'sql' => :sql, + 'taskpaper' => :taskpaper, + 'tmproj' => :xml, + 'xaml' => :xml, + 'xhtml' => :html, + 'xml' => :xml, + 'yaml' => :yaml, + 'yml' => :yaml, } for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp -- cgit v1.2.1 From 4a85580f76421bb0afb92501119c4dfcfa6540ef Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:12:14 +0200 Subject: tweak Terminal encoder, cleanup token kinds --- lib/coderay/encoders/terminal.rb | 179 ++++++++++++++++++++++----------------- lib/coderay/styles/alpha.rb | 29 +++---- lib/coderay/token_kinds.rb | 136 +++++++++++++++-------------- 3 files changed, 181 insertions(+), 163 deletions(-) diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb index 9894b91..c7ae014 100644 --- a/lib/coderay/encoders/terminal.rb +++ b/lib/coderay/encoders/terminal.rb @@ -19,105 +19,135 @@ module CodeRay register_for :terminal TOKEN_COLORS = { - :annotation => "\e[35m", - :attribute_name => "\e[33m", + :debug => "\e[1;37;44m", + + :annotation => "\e[34m", + :attribute_name => "\e[35m", :attribute_value => "\e[31m", - :binary => "\e[1;35m", + :binary => { + :self => "\e[31m", + :char => "\e[1;31m", + :delimiter => "\e[1;31m", + }, :char => { - :self => "\e[36m", :delimiter => "\e[1;34m" + :self => "\e[35m", + :delimiter => "\e[1;35m" }, - :class => "\e[1;35m", + :class => "\e[1;35;4m", :class_variable => "\e[36m", :color => "\e[32m", - :comment => "\e[37m", - :complex => "\e[1;34m", - :constant => "\e[1;34m\e[4m", - :decoration => "\e[35m", - :definition => "\e[1;32m", - :directive => "\e[32m\e[4m", - :doc => "\e[46m", - :doctype => "\e[1;30m", - :docstring => "\e[31m\e[4m", - :entity => "\e[33m", - :error => "\e[1;33m\e[41m", + :comment => { + :self => "\e[1;30m", + :char => "\e[37m", + :delimiter => "\e[37m", + }, + :constant => "\e[1;34;4m", + :decorator => "\e[35m", + :definition => "\e[1;33m", + :directive => "\e[33m", + :docstring => "\e[31m", + :doctype => "\e[1;34m", + :done => "\e[1;30;2m", + :entity => "\e[31m", + :error => "\e[1;37;41m", :exception => "\e[1;31m", :float => "\e[1;35m", :function => "\e[1;34m", - :global_variable => "\e[42m", + :global_variable => "\e[1;32m", :hex => "\e[1;36m", - :include => "\e[33m", + :id => "\e[1;34m", + :include => "\e[31m", :integer => "\e[1;34m", - :key => "\e[35m", - :label => "\e[1;15m", + :imaginary => "\e[1;34m", + :important => "\e[1;31m", + :key => { + :self => "\e[35m", + :char => "\e[1;35m", + :delimiter => "\e[1;35m", + }, + :keyword => "\e[32m", + :label => "\e[1;33m", :local_variable => "\e[33m", - :octal => "\e[1;35m", - :operator_name => "\e[1;29m", + :namespace => "\e[1;35m", + :octal => "\e[1;34m", + :predefined => "\e[36m", :predefined_constant => "\e[1;36m", - :predefined_type => "\e[1;30m", - :predefined => "\e[4m\e[1;34m", - :preprocessor => "\e[36m", + :predefined_type => "\e[1;32m", + :preprocessor => "\e[1;36m", :pseudo_class => "\e[1;34m", :regexp => { - :self => "\e[31m", - :content => "\e[31m", - :delimiter => "\e[1;29m", + :self => "\e[35m", + :delimiter => "\e[1;35m", :modifier => "\e[35m", + :char => "\e[1;35m", }, - :reserved => "\e[1;31m", + :reserved => "\e[32m", :shell => { - :self => "\e[42m", - :content => "\e[1;29m", - :delimiter => "\e[37m", + :self => "\e[33m", + :char => "\e[1;33m", + :delimiter => "\e[1;33m", + :escape => "\e[1;33m", }, :string => { - :self => "\e[32m", - :modifier => "\e[1;32m", - :escape => "\e[1;36m", - :delimiter => "\e[1;32m", - :char => "\e[1;36m", + :self => "\e[31m", + :modifier => "\e[1;31m", + :char => "\e[1;35m", + :delimiter => "\e[1;31m", + :escape => "\e[1;31m", + }, + :symbol => { + :self => "\e[33m", + :delimiter => "\e[1;33m", }, - :symbol => "\e[1;32m", - :tag => "\e[1;34m", + :tag => "\e[32m", :type => "\e[1;34m", :value => "\e[36m", - :variable => "\e[1;34m", + :variable => "\e[34m", - :insert => "\e[42m", - :delete => "\e[41m", - :change => "\e[44m", - :head => "\e[45m" + :insert => { + :self => "\e[42m", + :insert => "\e[1;32;42m", + :eyecatcher => "\e[102m", + }, + :delete => { + :self => "\e[41m", + :delete => "\e[1;31;41m", + :eyecatcher => "\e[101m", + }, + :change => { + :self => "\e[44m", + :change => "\e[37;44m", + }, + :head => { + :self => "\e[45m", + :filename => "\e[37;45m" + }, } + TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved] TOKEN_COLORS[:method] = TOKEN_COLORS[:function] - TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex] - TOKEN_COLORS[:begin_group] = TOKEN_COLORS[:end_group] = - TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter] + TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter] protected def setup(options) super @opened = [] - @subcolors = nil + @color_scopes = [TOKEN_COLORS] end public def text_token text, kind - if color = (@subcolors || TOKEN_COLORS)[kind] - if Hash === color - if color[:self] - color = color[:self] - else - @out << text - return - end - end + if color = @color_scopes.last[kind] + color = color[:self] if color.is_a? Hash @out << color - @out << text.gsub("\n", "\e[0m\n" + color) + @out << (text.index("\n") ? text.gsub("\n", "\e[0m\n" + color) : text) @out << "\e[0m" - @out << @subcolors[:self] if @subcolors + if outer_color = @color_scopes.last[:self] + @out << outer_color + end else @out << text end @@ -130,40 +160,33 @@ module CodeRay alias begin_line begin_group def end_group kind - if @opened.empty? - # nothing to close - else - @opened.pop + if @opened.pop + @color_scopes.pop @out << "\e[0m" - @out << open_token(@opened.last) + if outer_color = @color_scopes.last[:self] + @out << outer_color + end end end def end_line kind - if @opened.empty? - # nothing to close - else - @opened.pop - # whole lines to be highlighted, - # eg. added/modified/deleted lines in a diff - @out << (@line_filler ||= "\t" * 100 + "\e[0m") - @out << open_token(@opened.last) - end + @out << (@line_filler ||= "\t" * 100) + end_group kind end private def open_token kind - if color = TOKEN_COLORS[kind] - if Hash === color - @subcolors = color + if color = @color_scopes.last[kind] + if color.is_a? Hash + @color_scopes << color color[:self] else - @subcolors = {} + @color_scopes << @color_scopes.last color end else - @subcolors = nil + @color_scopes << @color_scopes.last '' end end diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 2ad088a..9a94091 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -57,26 +57,25 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .attribute-name { color:#b48 } .attribute-value { color:#700 } .binary { color:#549 } -.binary .delimiter { color:#325 } .binary .char { color:#325 } +.binary .delimiter { color:#325 } +.char { color:#D20 } .char .content { color:#D20 } .char .delimiter { color:#710 } -.char { color:#D20 } .class { color:#B06; font-weight:bold } .class-variable { color:#369 } .color { color:#0A0 } .comment { color:#777 } .comment .char { color:#444 } .comment .delimiter { color:#444 } -.complex { color:#A08 } .constant { color:#036; font-weight:bold } .decorator { color:#B0B } .definition { color:#099; font-weight:bold } .delimiter { color:black } .directive { color:#088; font-weight:bold } -.doc { color:#970 } -.doc-string { color:#D42; font-weight:bold } +.docstring { color:#D42; } .doctype { color:#34b } +.done { text-decoration: line-through; color: gray } .entity { color:#800; font-weight:bold } .error { color:#F00; background-color:#FAA } .escape { color:#666 } @@ -87,16 +86,16 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .global-variable { color:#d70 } .hex { color:#02b } .id { color:#33D; font-weight:bold } -.imaginary { color:#f00 } .include { color:#B44; font-weight:bold } .inline { background-color: hsla(0,0%,0%,0.07); color: black } .inline-delimiter { font-weight: bold; color: #666 } .instance-variable { color:#33B } .integer { color:#00D } +.imaginary { color:#f00 } .important { color:#D00 } +.key { color: #606 } .key .char { color: #60f } .key .delimiter { color: #404 } -.key { color: #606 } .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } .local-variable { color:#963 } @@ -108,30 +107,30 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .predefined-type { color:#0a5; font-weight:bold } .preprocessor { color:#579 } .pseudo-class { color:#00C; font-weight:bold } +.regexp { background-color:hsla(300,100%,50%,0.06); } .regexp .content { color:#808 } .regexp .delimiter { color:#404 } .regexp .modifier { color:#C2C } -.regexp { background-color:hsla(300,100%,50%,0.06); } .reserved { color:#080; font-weight:bold } +.shell { background-color:hsla(120,100%,50%,0.06); } .shell .content { color:#2B2 } .shell .delimiter { color:#161 } -.shell { background-color:hsla(120,100%,50%,0.06); } +.string { background-color:hsla(0,100%,50%,0.05); } .string .char { color: #b0b } .string .content { color: #D20 } .string .delimiter { color: #710 } .string .modifier { color: #E40 } -.string { background-color:hsla(0,100%,50%,0.05); } +.symbol { color:#A60 } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } -.symbol { color:#A60 } .tag { color:#070 } .type { color:#339; font-weight:bold } -.value { color: #088; } -.variable { color:#037 } +.value { color: #088 } +.variable { color:#037 } .insert { background: hsla(120,100%,50%,0.12) } .delete { background: hsla(0,100%,50%,0.12) } -.change { color: #bbf; background: #007; } +.change { color: #bbf; background: #007 } .head { color: #f8f; background: #505 } .head .filename { color: white; } @@ -142,8 +141,6 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .delete .delete { color: #c00; background:transparent; font-weight:bold } .change .change { color: #88f } .head .head { color: #f4f } - -.done { text-decoration: line-through; color: gray } TOKENS end diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index de3a0d0..42ea427 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -10,79 +10,77 @@ module CodeRay TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity TokenKinds.update( # :nodoc: - :annotation => 'annotation', - :attribute_name => 'attribute-name', - :attribute_value => 'attribute-value', - :binary => 'bin', - :char => 'char', - :class => 'class', - :class_variable => 'class-variable', - :color => 'color', - :comment => 'comment', - :complex => 'complex', - :constant => 'constant', - :content => 'content', - :debug => 'debug', - :decorator => 'decorator', - :definition => 'definition', - :delimiter => 'delimiter', - :directive => 'directive', - :doc => 'doc', - :doctype => 'doctype', - :docstring => 'doc-string', - :done => 'done', - :entity => 'entity', - :error => 'error', - :escape => 'escape', - :exception => 'exception', - :filename => 'filename', - :float => 'float', - :function => 'function', - :global_variable => 'global-variable', - :hex => 'hex', - :id => 'id', - :imaginary => 'imaginary', - :important => 'important', - :include => 'include', - :inline => 'inline', - :inline_delimiter => 'inline-delimiter', - :instance_variable => 'instance-variable', - :integer => 'integer', - :key => 'key', - :keyword => 'keyword', - :label => 'label', - :local_variable => 'local-variable', - :modifier => 'modifier', - :namespace => 'namespace', - :octal => 'octal', - :predefined => 'predefined', - :predefined_constant => 'predefined-constant', - :predefined_type => 'predefined-type', - :preprocessor => 'preprocessor', - :pseudo_class => 'pseudo-class', - :regexp => 'regexp', - :reserved => 'reserved', - :shell => 'shell', - :string => 'string', - :symbol => 'symbol', - :tag => 'tag', - :type => 'type', - :value => 'value', - :variable => 'variable', + :debug => 'debug', # highlight for debugging (white on blue background) - :change => 'change', - :delete => 'delete', - :head => 'head', - :insert => 'insert', + :annotation => 'annotation', # Groovy, Java + :attribute_name => 'attribute-name', # HTML, CSS + :attribute_value => 'attribute-value', # HTML + :binary => 'binary', # Python, Ruby + :char => 'char', # most scanners, also inside of strings + :class => 'class', # lots of scanners, for different purposes also in CSS + :class_variable => 'class-variable', # Ruby, YAML + :color => 'color', # CSS + :comment => 'comment', # most scanners + :constant => 'constant', # PHP, Ruby + :content => 'content', # inside of strings, most scanners + :decorator => 'decorator', # Python + :definition => 'definition', # CSS + :delimiter => 'delimiter', # inside strings, comments and other types + :directive => 'directive', # lots of scanners + :doctype => 'doctype', # Goorvy, HTML, Ruby, YAML + :docstring => 'docstring', # Python + :done => 'done', # Taskpaper + :entity => 'entity', # HTML + :error => 'error', # invalid token, most scanners + :escape => 'escape', # Ruby (string inline variables like #$foo, #@bar) + :exception => 'exception', # Java, PHP, Python + :filename => 'filename', # Diff + :float => 'float', # most scanners + :function => 'function', # CSS, JavaScript, PHP + :global_variable => 'global-variable', # Ruby, YAML + :hex => 'hex', # hexadecimal number; lots of scanners + :id => 'id', # CSS + :imaginary => 'imaginary', # Python + :important => 'important', # CSS, Taskpaper + :include => 'include', # C, Groovy, Java, Python, Sass + :inline => 'inline', # nested code, eg. inline string evaluation; lots of scanners + :inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why? + :instance_variable => 'instance-variable', # Ruby + :integer => 'integer', # most scanners + :key => 'key', # lots of scanners, used together with :value + :keyword => 'keyword', # reserved word that's actually implemented; most scanners + :label => 'label', # C, PHP + :local_variable => 'local-variable', # local and magic variables; some scanners + :modifier => 'modifier', # used inside on strings; lots of scanners + :namespace => 'namespace', # Clojure, Java, Taskpaper + :octal => 'octal', # lots of scanners + :predefined => 'predefined', # predefined function: lots of scanners + :predefined_constant => 'predefined-constant',# lots of scanners + :predefined_type => 'predefined-type', # C, Java, PHP + :preprocessor => 'preprocessor', # C, Delphi, HTML + :pseudo_class => 'pseudo-class', # CSS + :regexp => 'regexp', # Groovy, JavaScript, Ruby + :reserved => 'reserved', # most scanners + :shell => 'shell', # Ruby + :string => 'string', # most scanners + :symbol => 'symbol', # Clojure, Ruby, YAML + :tag => 'tag', # CSS, HTML + :type => 'type', # CSS, Java, SQL, YAML + :value => 'value', # used together with :key; CSS, JSON, YAML + :variable => 'variable', # Sass, SQL, YAML - :eyecatcher => 'eyecatcher', + :change => 'change', # Diff + :delete => 'delete', # Diff + :head => 'head', # Diff, YAML + :insert => 'insert', # Diff + :eyecatcher => 'eyecatcher', # Diff - :ident => false, - :operator => false, + :ident => false, # almost all scanners + :operator => false, # almost all scanners - :space => false, - :plain => false + :space => false, # almost all scanners + :plain => false # almost all scanners ) - TokenKinds[:method] = TokenKinds[:function] + TokenKinds[:method] = TokenKinds[:function] end -- cgit v1.2.1 From 108a5c93d3c91969a928f9896e145e0b5d3eeb25 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:35:32 +0200 Subject: fix examples test --- test/functional/examples.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/examples.rb b/test/functional/examples.rb index 15f9ca3..985ef87 100755 --- a/test/functional/examples.rb +++ b/test/functional/examples.rb @@ -114,7 +114,7 @@ Token Types (7): # format the tokens term = terminal_encoder.encode_tokens(tokens) - assert_equal "\e[1;31mimport\e[0m \e[33mthis\e[0m; \e[37m# The Zen of Python\e[0m", term + assert_equal "\e[32mimport\e[0m \e[31mthis\e[0m; \e[1;30m# The Zen of Python\e[0m", term # re-using scanner and encoder ruby_highlighter = CodeRay::Duo[:ruby, :div] -- cgit v1.2.1 From df21150ed089f1e3d1d35badde23b2d905a421cd Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:35:39 +0200 Subject: speedup/tweak debug encoder --- lib/coderay/encoders/debug.rb | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index c03d3fb..94d52af 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -25,34 +25,37 @@ module Encoders def text_token text, kind raise 'empty token' if $CODERAY_DEBUG && text.empty? + if kind == :space @out << text else - # TODO: Escape ( - text = text.gsub(/[)\\]/, '\\\\\0') if text.index(/[)\\]/) - @out << kind.to_s << '(' << text << ')' + text = text.gsub('\\', '\\\\\\\\') if text.index('\\') + text = text.gsub(')', '\\\\)') if text.index(')') + @out << "#{kind}(#{text})" end end def begin_group kind - @opened << kind - @out << kind.to_s << '<' + @opened << kind if $CODERAY_DEBUG + + @out << "#{kind}<" end def end_group kind - if @opened.last != kind - puts @out - raise "we are inside #{@opened.inspect}, not #{kind}" - end - @opened.pop + raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind + @out << '>' end def begin_line kind - @out << kind.to_s << '[' + @opened << kind if $CODERAY_DEBUG + + @out << "#{kind}[" end def end_line kind + raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind + @out << ']' end -- cgit v1.2.1 From 88b812274a808f69675445a9265dab1d4e41cc44 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 04:46:11 +0200 Subject: refactor check functionality to DebugLint encoder --- Changes.textile | 1 + lib/coderay/encoders/debug.rb | 15 ----------- lib/coderay/encoders/debug_lint.rb | 55 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 lib/coderay/encoders/debug_lint.rb diff --git a/Changes.textile b/Changes.textile index ff9e6a9..15b0cc6 100644 --- a/Changes.textile +++ b/Changes.textile @@ -20,6 +20,7 @@ h2. Changes in 1.1 * Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05] * Fixed @:docstring@ token type style * @Plugin@ does not warn about fallback when default is defined +* @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now h2. Changes in 1.0.9 diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index 94d52af..61520a1 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -18,14 +18,7 @@ module Encoders FILE_EXTENSION = 'raydebug' - def initialize options = {} - super - @opened = [] - end - def text_token text, kind - raise 'empty token' if $CODERAY_DEBUG && text.empty? - if kind == :space @out << text else @@ -36,26 +29,18 @@ module Encoders end def begin_group kind - @opened << kind if $CODERAY_DEBUG - @out << "#{kind}<" end def end_group kind - raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind - @out << '>' end def begin_line kind - @opened << kind if $CODERAY_DEBUG - @out << "#{kind}[" end def end_line kind - raise "We are inside #{@opened.inspect}, not #{kind}" if $CODERAY_DEBUG && @opened.pop != kind - @out << ']' end diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb new file mode 100644 index 0000000..eeb2a92 --- /dev/null +++ b/lib/coderay/encoders/debug_lint.rb @@ -0,0 +1,55 @@ +module CodeRay +module Encoders + + # = Debug Lint Encoder + # + # Debug encoder with additional checks for: + # + # - empty tokens + # - incorrect nesting + # + # It will raise an InvalidTokenStream exception when any of the above occurs. + # + # See also: Encoders::Debug + class DebugLint < Debug + + register_for :debug_lint + + InvalidTokenStream = Class.new StandardError + EmptyToken = Class.new InvalidTokenStream + IncorrectTokenGroupNesting = Class.new InvalidTokenStream + + def initialize options = {} + super + @opened = [] + end + + def text_token text, kind + raise EmptyToken, 'empty token' if text.empty? + super + end + + def begin_group kind + @opened << kind + super + end + + def end_group kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + super + end + + def begin_line kind + @opened << kind + super + end + + def end_line kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + super + end + + end + +end +end -- cgit v1.2.1 From 6936def023edcdd3c10be99ecb955778f537f42b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 05:27:29 +0200 Subject: cleanup rake_helpers, samples --- rake_helpers/ca.rb | 40 - rake_helpers/code_statistics.rb | 171 ---- rake_helpers/coderay_rdoc_template.rb | 636 ------------- rake_helpers/html_coderay_generator.rb | 1517 -------------------------------- rake_tasks/code_statistics.rb | 171 ++++ rake_tasks/statistic.rake | 2 +- rake_tasks/test.rake | 2 +- sample/README | 1 - sample/cache.expected | 2 - sample/cache.rb | 12 - sample/count.expected | 1 - sample/count.rb | 10 - sample/css.expected | 130 --- sample/css.rb | 4 - sample/div.expected | 17 - sample/div.rb | 19 - sample/dump.expected | 21 - sample/dump.rb | 15 - sample/encoder.expected | 65 -- sample/encoder.rb | 39 - sample/global_vars.expected | 3 - sample/global_vars.rb | 13 - sample/global_vars2.expected | 10 - sample/global_vars2.rb | 28 - sample/highlight.expected | 175 ---- sample/highlight.rb | 14 - sample/html.expected | 919 ------------------- sample/html.rb | 394 --------- sample/html2.expected | 185 ---- sample/html2.rb | 11 - sample/html_list.expected | 160 ---- sample/html_list.rb | 12 - sample/load_encoder.expected | 8 - sample/load_encoder.rb | 25 - sample/load_scanner.expected | 8 - sample/load_scanner.rb | 25 - sample/more.expected | 2 - sample/more.rb | 205 ----- sample/scanner.expected | 16 - sample/scanner.rb | 36 - sample/server.rb | 110 --- sample/simple.expected | 1 - sample/simple.rb | 10 - sample/stream.rb | 25 - sample/stream2.expected | 2 - sample/stream2.rb | 8 - sample/suite.rb | 86 -- sample/tokens.expected | 1 - sample/tokens.rb | 3 - test/samples/README | 1 + test/samples/cache.actual | 2 + test/samples/cache.expected | 2 + test/samples/cache.rb | 12 + test/samples/count.expected | 1 + test/samples/count.rb | 10 + test/samples/css.actual | 127 +++ test/samples/css.expected | 127 +++ test/samples/css.rb | 4 + test/samples/div.actual | 17 + test/samples/div.expected | 17 + test/samples/div.rb | 19 + test/samples/encoder.actual | 65 ++ test/samples/encoder.expected | 65 ++ test/samples/encoder.rb | 40 + test/samples/global_vars.actual | 0 test/samples/global_vars.diff | 5 + test/samples/global_vars.expected | 3 + test/samples/global_vars.rb | 13 + test/samples/global_vars2.expected | 10 + test/samples/global_vars2.rb | 28 + test/samples/highlight.expected | 175 ++++ test/samples/highlight.rb | 14 + test/samples/html.expected | 919 +++++++++++++++++++ test/samples/html.rb | 394 +++++++++ test/samples/html2.expected | 185 ++++ test/samples/html2.rb | 11 + test/samples/html_list.expected | 160 ++++ test/samples/html_list.rb | 12 + test/samples/load_encoder.expected | 8 + test/samples/load_encoder.rb | 25 + test/samples/load_scanner.expected | 8 + test/samples/load_scanner.rb | 25 + test/samples/more.expected | 2 + test/samples/more.rb | 205 +++++ test/samples/scanner.expected | 16 + test/samples/scanner.rb | 36 + test/samples/server.rb | 110 +++ test/samples/simple.expected | 1 + test/samples/simple.rb | 10 + test/samples/stream.rb | 25 + test/samples/stream2.expected | 2 + test/samples/stream2.rb | 8 + test/samples/suite.rb | 86 ++ test/samples/tokens.expected | 1 + test/samples/tokens.rb | 3 + 95 files changed, 3182 insertions(+), 5197 deletions(-) delete mode 100644 rake_helpers/ca.rb delete mode 100644 rake_helpers/code_statistics.rb delete mode 100644 rake_helpers/coderay_rdoc_template.rb delete mode 100644 rake_helpers/html_coderay_generator.rb create mode 100644 rake_tasks/code_statistics.rb delete mode 100644 sample/README delete mode 100644 sample/cache.expected delete mode 100644 sample/cache.rb delete mode 100644 sample/count.expected delete mode 100644 sample/count.rb delete mode 100644 sample/css.expected delete mode 100644 sample/css.rb delete mode 100644 sample/div.expected delete mode 100644 sample/div.rb delete mode 100644 sample/dump.expected delete mode 100644 sample/dump.rb delete mode 100644 sample/encoder.expected delete mode 100644 sample/encoder.rb delete mode 100644 sample/global_vars.expected delete mode 100644 sample/global_vars.rb delete mode 100644 sample/global_vars2.expected delete mode 100644 sample/global_vars2.rb delete mode 100644 sample/highlight.expected delete mode 100644 sample/highlight.rb delete mode 100644 sample/html.expected delete mode 100644 sample/html.rb delete mode 100644 sample/html2.expected delete mode 100644 sample/html2.rb delete mode 100644 sample/html_list.expected delete mode 100644 sample/html_list.rb delete mode 100644 sample/load_encoder.expected delete mode 100644 sample/load_encoder.rb delete mode 100644 sample/load_scanner.expected delete mode 100644 sample/load_scanner.rb delete mode 100644 sample/more.expected delete mode 100644 sample/more.rb delete mode 100644 sample/scanner.expected delete mode 100644 sample/scanner.rb delete mode 100644 sample/server.rb delete mode 100644 sample/simple.expected delete mode 100644 sample/simple.rb delete mode 100644 sample/stream.rb delete mode 100644 sample/stream2.expected delete mode 100644 sample/stream2.rb delete mode 100644 sample/suite.rb delete mode 100644 sample/tokens.expected delete mode 100644 sample/tokens.rb create mode 100644 test/samples/README create mode 100644 test/samples/cache.actual create mode 100644 test/samples/cache.expected create mode 100644 test/samples/cache.rb create mode 100644 test/samples/count.expected create mode 100644 test/samples/count.rb create mode 100644 test/samples/css.actual create mode 100644 test/samples/css.expected create mode 100644 test/samples/css.rb create mode 100644 test/samples/div.actual create mode 100644 test/samples/div.expected create mode 100644 test/samples/div.rb create mode 100644 test/samples/encoder.actual create mode 100644 test/samples/encoder.expected create mode 100644 test/samples/encoder.rb create mode 100644 test/samples/global_vars.actual create mode 100644 test/samples/global_vars.diff create mode 100644 test/samples/global_vars.expected create mode 100644 test/samples/global_vars.rb create mode 100644 test/samples/global_vars2.expected create mode 100644 test/samples/global_vars2.rb create mode 100644 test/samples/highlight.expected create mode 100644 test/samples/highlight.rb create mode 100644 test/samples/html.expected create mode 100644 test/samples/html.rb create mode 100644 test/samples/html2.expected create mode 100644 test/samples/html2.rb create mode 100644 test/samples/html_list.expected create mode 100644 test/samples/html_list.rb create mode 100644 test/samples/load_encoder.expected create mode 100644 test/samples/load_encoder.rb create mode 100644 test/samples/load_scanner.expected create mode 100644 test/samples/load_scanner.rb create mode 100644 test/samples/more.expected create mode 100644 test/samples/more.rb create mode 100644 test/samples/scanner.expected create mode 100644 test/samples/scanner.rb create mode 100644 test/samples/server.rb create mode 100644 test/samples/simple.expected create mode 100644 test/samples/simple.rb create mode 100644 test/samples/stream.rb create mode 100644 test/samples/stream2.expected create mode 100644 test/samples/stream2.rb create mode 100644 test/samples/suite.rb create mode 100644 test/samples/tokens.expected create mode 100644 test/samples/tokens.rb diff --git a/rake_helpers/ca.rb b/rake_helpers/ca.rb deleted file mode 100644 index ba6fe1e..0000000 --- a/rake_helpers/ca.rb +++ /dev/null @@ -1,40 +0,0 @@ -#!c:/ruby/bin/rubyw -# Hy-Ca 0.2 by murphy - -module Hy - def self.ca str - str.gsub! %r-^(\s*)(?://)(.*)?-, '\1/*\2*/' - str.gsub! %r-\s*/\*.*?\*/\n?-m, '' - str.gsub!(/<<(.*?)>>/m) do - begin - eval $1 - '' - rescue Exception => boom - "<<\n#{boom}\n>>" - end - end - - str.gsub!(/\$([\w_]+)/m) do - begin - eval $1 - rescue - '' - end - end - - str - end -end - -begin - if file = ENV['PATH_TRANSLATED'] - puts "Content-Type: text/css" - puts - ca = File.read file - else - ca = ARGF.read - end - print Hy.ca(ca) -rescue => boom - p boom -end if __FILE__ == $0 diff --git a/rake_helpers/code_statistics.rb b/rake_helpers/code_statistics.rb deleted file mode 100644 index 0a2016b..0000000 --- a/rake_helpers/code_statistics.rb +++ /dev/null @@ -1,171 +0,0 @@ -# From rails (http://rubyonrails.com) -# -# Improved by murphy -class CodeStatistics - - TEST_TYPES = /\btest/i - - # Create a new Code Statistic. - # - # Rakefile Example: - # - # desc 'Report code statistics (LOC) from the application' - # task :stats => :copy_files do - # require 'rake_helpers/code_statistics' - # CodeStatistics.new( - # ["Main", "lib"], - # ["Tests", "test"], - # ["Demos", "demo"] - # ).to_s - # end - def initialize(*pairs) - @pairs = pairs - @statistics = calculate_statistics - @total = if pairs.empty? then nil else calculate_total end - end - - # Print a textual table viewing the stats - # - # Intended for console output. - def print - print_header - @pairs.each { |name, path| print_line name, @statistics[name] } - print_splitter - - if @total - print_line 'Total', @total - print_splitter - end - - print_code_test_stats - end - -private - - DEFAULT_FILE_PATTERN = /\.rb$/ - - def calculate_statistics - @pairs.inject({}) do |stats, (name, path, pattern, is_ruby_code)| - pattern ||= DEFAULT_FILE_PATTERN - path = File.join path, '*.rb' - stats[name] = calculate_directory_statistics path, pattern, is_ruby_code - stats - end - end - - def calculate_directory_statistics directory, pattern = DEFAULT_FILE_PATTERN, is_ruby_code = true - is_ruby_code = true if is_ruby_code.nil? - stats = Hash.new 0 - - Dir[directory].each do |file_name| - p "Scanning #{file_name}..." if $VERBOSE - next unless file_name =~ pattern - - lines = codelines = classes = modules = methods = 0 - empty_lines = comment_lines = 0 - in_comment_block = false - - File.readlines(file_name).each do |line| - lines += 1 - if line[/^\s*$/] - empty_lines += 1 - elsif is_ruby_code - case line - when /^=end\b/ - comment_lines += 1 - in_comment_block = false - when in_comment_block - comment_lines += 1 - when /^\s*class\b/ - classes += 1 - when /^\s*module\b/ - modules += 1 - when /^\s*def\b/ - methods += 1 - when /^\s*#/ - comment_lines += 1 - when /^=begin\b/ - in_comment_block = false - comment_lines += 1 - when /^__END__$/ - in_comment_block = true - end - end - end - - codelines = lines - comment_lines - empty_lines - - stats[:lines] += lines - stats[:comments] += comment_lines - stats[:codelines] += codelines - stats[:classes] += classes - stats[:modules] += modules - stats[:methods] += methods - stats[:files] += 1 - end - - stats - end - - def calculate_total - total = Hash.new 0 - @statistics.each_value { |pair| pair.each { |k, v| total[k] += v } } - total - end - - def calculate_code - code_loc = 0 - @statistics.each { |k, v| code_loc += v[:codelines] unless k[TEST_TYPES] } - code_loc - end - - def calculate_tests - test_loc = 0 - @statistics.each { |k, v| test_loc += v[:codelines] if k[TEST_TYPES] } - test_loc - end - - def print_header - print_splitter - puts "| T=Test Name | Files | Lines | LOC | Comments | Classes | Modules | Methods | M/C | LOC/M |" - print_splitter - end - - def print_splitter - puts "+---------------------------+-------+-------+-------+----------+---------+---------+---------+-----+-------+" - end - - def print_line name, statistics - m_over_c = (statistics[:methods] / (statistics[:classes] + statistics[:modules])) rescue m_over_c = 0 - loc_over_m = (statistics[:codelines] / statistics[:methods]) - 2 rescue loc_over_m = 0 - - if name[TEST_TYPES] - name = "T #{name}" - else - name = " #{name}" - end - - line = "| %-25s | %5d | %5d | %5d | %8d | %7d | %7d | %7d | %3d | %5d |" % ( - [name, *statistics.values_at(:files, :lines, :codelines, :comments, :classes, :modules, :methods)] + - [m_over_c, loc_over_m] ) - - puts line - end - - def print_code_test_stats - code = calculate_code - tests = calculate_tests - - puts " Code LOC = #{code} Test LOC = #{tests} Code:Test Ratio = [1 : #{sprintf("%.2f", tests.to_f/code)}]" - puts "" - end - -end - -# Run a test script. -if $0 == __FILE__ - $VERBOSE = true - CodeStatistics.new( - ['This dir', File.dirname(__FILE__)] - ).print -end diff --git a/rake_helpers/coderay_rdoc_template.rb b/rake_helpers/coderay_rdoc_template.rb deleted file mode 100644 index 21a3231..0000000 --- a/rake_helpers/coderay_rdoc_template.rb +++ /dev/null @@ -1,636 +0,0 @@ -# This goes to /lib/ruby/1.8/rdoc/generators/template/html/ -module RDoc::Page - -FONTS = "Tahoma, Verdana, sans-serif" - -require 'rake_helpers/ca.rb' - -Hy.ca <> -CA - -require 'coderay' - -STYLE = Hy.ca < pre { - border: 1px solid silver; - background: #112; - color: white; - padding: 0.5em; -} -CSS - -XHTML_FRAMESET_PREAMBLE = # -%{ -} - -XHTML_PREAMBLE = -%{ -} - -HEADER = XHTML_PREAMBLE + < - - %title% - - - - - - - -ENDHEADER - -FILE_PAGE = < - - - - - - - -
File %short_name%
- - - - - - - - - -
Path:%full_path% -IF:cvsurl -  (CVS) -ENDIF:cvsurl -
Modified:%dtm_modified%
-
- - - Ruby-Chan - - -
-HTML - -################################################################### - -CLASS_PAGE = < - - %classmod% %full_name% - - Ruby-Chan - - - - - - - - - -IF:parent - - - - -ENDIF:parent -
In: -START:infiles -HREF:full_path_url:full_path: -IF:cvsurl - (CVS) -ENDIF:cvsurl -END:infiles -
Parent: -IF:par_url - -ENDIF:par_url -%parent% -IF:par_url - -ENDIF:par_url -
- - - -HTML - -################################################################### - -METHOD_LIST = < -IF:diagram -
- %diagram% -
-ENDIF:diagram - -IF:description -
%description%
-ENDIF:description - -IF:requires -
Required Files
-
    -START:requires -
  • HREF:aref:name:
    • -END:requires -
-ENDIF:requires - -IF:toc -
Contents
- -ENDIF:toc - -IF:methods -
Methods
-
    -START:methods -
  • HREF:aref:name:
    • -END:methods -
-ENDIF:methods - -IF:includes -
Included Modules
-
    -START:includes -
  • HREF:aref:name:
    • -END:includes -
-ENDIF:includes - -START:sections -IF:sectitle -
%sectitle%
-IF:seccomment -
-%seccomment% -
-ENDIF:seccomment -ENDIF:sectitle - -IF:classlist -
Classes and Modules
- %classlist% -ENDIF:classlist - -IF:constants -
Constants
- -START:constants - - - - - -IF:desc - - - - -ENDIF:desc -END:constants -
%name%=%value%
 %desc%
-ENDIF:constants - -IF:attributes -
Attributes
- -START:attributes - - - - - -END:attributes -
-IF:rw -[%rw%] -ENDIF:rw - %name%%a_desc%
-ENDIF:attributes - -IF:method_list -START:method_list -IF:methods -
%type% %category% methods
-START:methods -
-
-IF:callseq - %callseq% -ENDIF:callseq -IFNOT:callseq - %name%%params% -ENDIF:callseq -IF:codeurl -[ source ] -ENDIF:codeurl -
-IF:m_desc -
- %m_desc% -
-ENDIF:m_desc -IF:aka -
- --- This method is also aliased as -START:aka - %name% -END:aka - --- -
-ENDIF:aka -IF:sourcecode -
-

→ show source

-
-%sourcecode% -
-
-ENDIF:sourcecode -
-END:methods -ENDIF:methods -END:method_list -ENDIF:method_list -END:sections - -HTML - -FOOTER = < - -ENDFOOTER - -BODY = HEADER + < - -
- #{METHOD_LIST} -
- - #{FOOTER} -ENDBODY - -########################## Source code ########################## - -SRC_PAGE = XHTML_PREAMBLE + < -%title% - - - - -
%code%
- - -HTML - -########################## Index ################################ - -FR_INDEX_BODY = < -List - - - - - - -
-START:entries -%name%
-END:entries -
- -HTML - -CLASS_INDEX = FILE_INDEX -METHOD_INDEX = FILE_INDEX - -INDEX = XHTML_FRAMESET_PREAMBLE + < - - %title% - - - - - - - - - -IF:inline_source - -ENDIF:inline_source -IFNOT:inline_source - - - - -ENDIF:inline_source - - <body bgcolor="white"> - Click <a href="html/index.html">here</a> for a non-frames - version of this page. - </body> - - - - -HTML - -end diff --git a/rake_helpers/html_coderay_generator.rb b/rake_helpers/html_coderay_generator.rb deleted file mode 100644 index 3c77705..0000000 --- a/rake_helpers/html_coderay_generator.rb +++ /dev/null @@ -1,1517 +0,0 @@ -# We're responsible for generating all the HTML files -# from the object tree defined in code_objects.rb. We -# generate: -# -# [files] an html file for each input file given. These -# input files appear as objects of class -# TopLevel -# -# [classes] an html file for each class or module encountered. -# These classes are not grouped by file: if a file -# contains four classes, we'll generate an html -# file for the file itself, and four html files -# for the individual classes. -# -# [indices] we generate three indices for files, classes, -# and methods. These are displayed in a browser -# like window with three index panes across the -# top and the selected description below -# -# Method descriptions appear in whatever entity (file, class, -# or module) that contains them. -# -# We generate files in a structure below a specified subdirectory, -# normally +doc+. -# -# opdir -# | -# |___ files -# | |__ per file summaries -# | -# |___ classes -# |__ per class/module descriptions -# -# HTML is generated using the Template class. -# - -require 'ftools' - -require 'rdoc/options' -require 'rdoc/template' -require 'rdoc/markup/simple_markup' -require 'rdoc/markup/simple_markup/to_html' -require 'cgi' - -module Generators - - # Name of sub-direcories that hold file and class/module descriptions - - FILE_DIR = "files" - CLASS_DIR = "classes" - CSS_NAME = "rdoc-style.css" - - - ## - # Build a hash of all items that can be cross-referenced. - # This is used when we output required and included names: - # if the names appear in this hash, we can generate - # an html cross reference to the appropriate description. - # We also use this when parsing comment blocks: any decorated - # words matching an entry in this list are hyperlinked. - - class AllReferences - @@refs = {} - - def AllReferences::reset - @@refs = {} - end - - def AllReferences.add(name, html_class) - @@refs[name] = html_class - end - - def AllReferences.[](name) - @@refs[name] - end - - def AllReferences.keys - @@refs.keys - end - end - - - ## - # Subclass of the SM::ToHtml class that supports looking - # up words in the AllReferences list. Those that are - # found (like AllReferences in this comment) will - # be hyperlinked - - class HyperlinkHtml < SM::ToHtml - # We need to record the html path of our caller so we can generate - # correct relative paths for any hyperlinks that we find - def initialize(from_path, context) - super() - @from_path = from_path - - @parent_name = context.parent_name - @parent_name += "::" if @parent_name - @context = context - end - - # We're invoked when any text matches the CROSSREF pattern - # (defined in MarkUp). If we fine the corresponding reference, - # generate a hyperlink. If the name we're looking for contains - # no punctuation, we look for it up the module/class chain. For - # example, HyperlinkHtml is found, even without the Generators:: - # prefix, because we look for it in module Generators first. - - def handle_special_CROSSREF(special) - name = special.text - if name[0,1] == '#' - lookup = name[1..-1] - name = lookup unless Options.instance.show_hash - else - lookup = name - end - - if /([A-Z].*)[.\#](.*)/ =~ lookup - container = $1 - method = $2 - ref = @context.find_symbol(container, method) - else - ref = @context.find_symbol(lookup) - end - - if ref and ref.document_self - "#{name}" - else - name - end - end - - - # Generate a hyperlink for url, labeled with text. Handle the - # special cases for img: and link: described under handle_special_HYPEDLINK - def gen_url(url, text) - if url =~ /([A-Za-z]+):(.*)/ - type = $1 - path = $2 - else - type = "http" - path = url - url = "http://#{url}" - end - - if type == "link" - if path[0,1] == '#' # is this meaningful? - url = path - else - url = HTMLGenerator.gen_url(@from_path, path) - end - end - - if (type == "http" || type == "link") && - url =~ /\.(gif|png|jpg|jpeg|bmp)$/ - - "" - else - "#{text.sub(%r{^#{type}:/*}, '')}" - end - end - - # And we're invoked with a potential external hyperlink mailto: - # just gets inserted. http: links are checked to see if they - # reference an image. If so, that image gets inserted using an - # tag. Otherwise a conventional is used. We also - # support a special type of hyperlink, link:, which is a reference - # to a local file whose path is relative to the --op directory. - - def handle_special_HYPERLINK(special) - url = special.text - gen_url(url, url) - end - - # HEre's a hypedlink where the label is different to the URL - #

/, '') - res.sub!(/<\/p>$/, '') - end - res - end - - # Qualify a stylesheet URL; if if +css_name+ does not begin with '/' or - # 'http[s]://', prepend a prefix relative to +path+. Otherwise, return it - # unmodified. - - def style_url(path, css_name=nil) -# $stderr.puts "style_url( #{path.inspect}, #{css_name.inspect} )" - css_name ||= CSS_NAME - if %r{^(https?:/)?/} =~ css_name - return css_name - else - return HTMLGenerator.gen_url(path, css_name) - end - end - - # Build a webcvs URL with the given 'url' argument. URLs with a '%s' in them - # get the file's path sprintfed into them; otherwise they're just catenated - # together. - - def cvs_url(url, full_path) - if /%s/ =~ url - return sprintf( url, full_path ) - else - return url + full_path - end - end - end - - - ##################################################################### - # - # A Context is built by the parser to represent a container: contexts - # hold classes, modules, methods, require lists and include lists. - # ClassModule and TopLevel are the context objects we process here - # - class ContextUser - - include MarkUp - - attr_reader :context - - def initialize(context, options) - @context = context - @options = options - end - - # convenience method to build a hyperlink - def href(link, cls, name) - %{#{name}} #" - end - - # return a reference to outselves to be used as an href= - # the form depends on whether we're all in one file - # or in multiple files - - def as_href(from_path) - if @options.all_one_file - "#" + path - else - HTMLGenerator.gen_url(from_path, path) - end - end - - # Create a list of HtmlMethod objects for each method - # in the corresponding context object. If the @options.show_all - # variable is set (corresponding to the --all option, - # we include all methods, otherwise just the public ones. - - def collect_methods - list = @context.method_list - unless @options.show_all - list = list.find_all {|m| m.visibility == :public || m.force_documentation } - end - @methods = list.collect {|m| HtmlMethod.new(m, self, @options) } - end - - # Build a summary list of all the methods in this context - def build_method_summary_list(path_prefix="") - collect_methods unless @methods - meths = @methods.sort - res = [] - meths.each do |meth| - res << { - "name" => CGI.escapeHTML(meth.name), - "aref" => "#{path_prefix}\##{meth.aref}" - } - end - res - end - - - # Build a list of aliases for which we couldn't find a - # corresponding method - def build_alias_summary_list(section) - values = [] - @context.aliases.each do |al| - next unless al.section == section - res = { - 'old_name' => al.old_name, - 'new_name' => al.new_name, - } - if al.comment && !al.comment.empty? - res['desc'] = markup(al.comment, true) - end - values << res - end - values - end - - # Build a list of constants - def build_constants_summary_list(section) - values = [] - @context.constants.each do |co| - next unless co.section == section - res = { - 'name' => co.name, - 'value' => CGI.escapeHTML(co.value) - } - res['desc'] = markup(co.comment, true) if co.comment && !co.comment.empty? - values << res - end - values - end - - def build_requires_list(context) - potentially_referenced_list(context.requires) {|fn| [fn + ".rb"] } - end - - def build_include_list(context) - potentially_referenced_list(context.includes) - end - - # Build a list from an array of Htmlxxx items. Look up each - # in the AllReferences hash: if we find a corresponding entry, - # we generate a hyperlink to it, otherwise just output the name. - # However, some names potentially need massaging. For example, - # you may require a Ruby file without the .rb extension, - # but the file names we know about may have it. To deal with - # this, we pass in a block which performs the massaging, - # returning an array of alternative names to match - - def potentially_referenced_list(array) - res = [] - array.each do |i| - ref = AllReferences[i.name] -# if !ref -# container = @context.parent -# while !ref && container -# name = container.name + "::" + i.name -# ref = AllReferences[name] -# container = container.parent -# end -# end - - ref = @context.find_symbol(i.name) - ref = ref.viewer if ref - - if !ref && block_given? - possibles = yield(i.name) - while !ref and !possibles.empty? - ref = AllReferences[possibles.shift] - end - end - h_name = CGI.escapeHTML(i.name) - if ref and ref.document_self - path = url(ref.path) - res << { "name" => h_name, "aref" => path } - else - res << { "name" => h_name } - end - end - res - end - - # Build an array of arrays of method details. The outer array has up - # to six entries, public, private, and protected for both class - # methods, the other for instance methods. The inner arrays contain - # a hash for each method - - def build_method_detail_list(section) - outer = [] - - methods = @methods.sort - for singleton in [true, false] - for vis in [ :public, :protected, :private ] - res = [] - methods.each do |m| - if m.section == section and - m.document_self and - m.visibility == vis and - m.singleton == singleton - row = {} - if m.call_seq - row["callseq"] = m.call_seq.gsub(/->/, '→') - else - row["name"] = CGI.escapeHTML(m.name) - row["params"] = m.params - end - desc = m.description.strip - row["m_desc"] = desc unless desc.empty? - row["aref"] = m.aref - row["visibility"] = m.visibility.to_s - - alias_names = [] - m.aliases.each do |other| - if other.viewer # won't be if the alias is private - alias_names << { - 'name' => other.name, - 'aref' => other.viewer.as_href(path) - } - end - end - unless alias_names.empty? - row["aka"] = alias_names - end - - if @options.inline_source - code = m.source_code - row["sourcecode"] = code if code - else - code = m.src_url - if code - row["codeurl"] = code - row["imgurl"] = m.img_url - end - end - res << row - end - end - if res.size > 0 - outer << { - "type" => vis.to_s.capitalize, - "category" => singleton ? "Class" : "Instance", - "methods" => res - } - end - end - end - outer - end - - # Build the structured list of classes and modules contained - # in this context. - - def build_class_list(level, from, section, infile=nil) - res = "" - prefix = "  ::" * level; - - from.modules.sort.each do |mod| - next unless mod.section == section - next if infile && !mod.defined_in?(infile) - if mod.document_self - res << - prefix << - "Module " << - href(url(mod.viewer.path), "link", mod.full_name) << - "
\n" << - build_class_list(level + 1, mod, section, infile) - end - end - - from.classes.sort.each do |cls| - next unless cls.section == section - next if infile && !cls.defined_in?(infile) - if cls.document_self - res << - prefix << - "Class " << - href(url(cls.viewer.path), "link", cls.full_name) << - "
\n" << - build_class_list(level + 1, cls, section, infile) - end - end - - res - end - - def url(target) - HTMLGenerator.gen_url(path, target) - end - - def aref_to(target) - if @options.all_one_file - "#" + target - else - url(target) - end - end - - def document_self - @context.document_self - end - - def diagram_reference(diagram) - res = diagram.gsub(/((?:src|href)=")(.*?)"/) { - $1 + url($2) + '"' - } - res - end - - - # Find a symbol in ourselves or our parent - def find_symbol(symbol, method=nil) - res = @context.find_symbol(symbol, method) - if res - res = res.viewer - end - res - end - - # create table of contents if we contain sections - - def add_table_of_sections - toc = [] - @context.sections.each do |section| - if section.title - toc << { - 'secname' => section.title, - 'href' => section.sequence - } - end - end - - @values['toc'] = toc unless toc.empty? - end - - - end - - ##################################################################### - # - # Wrap a ClassModule context - - class HtmlClass < ContextUser - - attr_reader :path - - def initialize(context, html_file, prefix, options) - super(context, options) - - @html_file = html_file - @is_module = context.is_module? - @values = {} - - context.viewer = self - - if options.all_one_file - @path = context.full_name - else - @path = http_url(context.full_name, prefix) - end - - collect_methods - - AllReferences.add(name, self) - end - - # return the relative file name to store this class in, - # which is also its url - def http_url(full_name, prefix) - path = full_name.dup - if path['<<'] - path.gsub!(/<<\s*(\w*)/) { "from-#$1" } - end - File.join(prefix, path.split("::")) + ".html" - end - - - def name - @context.full_name - end - - def parent_name - @context.parent.full_name - end - - def index_name - name - end - - def write_on(f) - value_hash - template = TemplatePage.new(RDoc::Page::BODY, - RDoc::Page::CLASS_PAGE, - RDoc::Page::METHOD_LIST) - template.write_html_on(f, @values) - end - - def value_hash - class_attribute_values - add_table_of_sections - - @values["charset"] = @options.charset - @values["style_url"] = style_url(path, @options.css) - - d = markup(@context.comment) - @values["description"] = d unless d.empty? - - ml = build_method_summary_list - @values["methods"] = ml unless ml.empty? - - il = build_include_list(@context) - @values["includes"] = il unless il.empty? - - @values["sections"] = @context.sections.map do |section| - - secdata = { - "sectitle" => section.title, - "secsequence" => section.sequence, - "seccomment" => markup(section.comment) - } - - al = build_alias_summary_list(section) - secdata["aliases"] = al unless al.empty? - - co = build_constants_summary_list(section) - secdata["constants"] = co unless co.empty? - - al = build_attribute_list(section) - secdata["attributes"] = al unless al.empty? - - cl = build_class_list(0, @context, section) - secdata["classlist"] = cl unless cl.empty? - - mdl = build_method_detail_list(section) - secdata["method_list"] = mdl unless mdl.empty? - - secdata - end - - @values - end - - def build_attribute_list(section) - atts = @context.attributes.sort - res = [] - atts.each do |att| - next unless att.section == section - if att.visibility == :public || @options.show_all - entry = { - "name" => CGI.escapeHTML(att.name), - "rw" => att.rw, - "a_desc" => markup(att.comment, true) - } - unless att.visibility == :public - entry["rw"] << "-" - end - res << entry - end - end - res - end - - def class_attribute_values - h_name = CGI.escapeHTML(name) - - @values["classmod"] = @is_module ? "Module" : "Class" - @values["title"] = "#{@values['classmod']}: #{h_name}" - - c = @context - c = c.parent while c and !c.diagram - if c && c.diagram - @values["diagram"] = diagram_reference(c.diagram) - end - - @values["full_name"] = h_name - - parent_class = @context.superclass - - if parent_class - @values["parent"] = CGI.escapeHTML(parent_class) - - if parent_name - lookup = parent_name + "::" + parent_class - else - lookup = parent_class - end - - parent_url = AllReferences[lookup] || AllReferences[parent_class] - - if parent_url and parent_url.document_self - @values["par_url"] = aref_to(parent_url.path) - end - end - - files = [] - @context.in_files.each do |f| - res = {} - full_path = CGI.escapeHTML(f.file_absolute_name) - - res["full_path"] = full_path - res["full_path_url"] = aref_to(f.viewer.path) if f.document_self - - if @options.webcvs - res["cvsurl"] = cvs_url( @options.webcvs, full_path ) - end - - files << res - end - - @values['infiles'] = files - end - - def <=>(other) - self.name <=> other.name - end - - end - - ##################################################################### - # - # Handles the mapping of a file's information to HTML. In reality, - # a file corresponds to a +TopLevel+ object, containing modules, - # classes, and top-level methods. In theory it _could_ contain - # attributes and aliases, but we ignore these for now. - - class HtmlFile < ContextUser - - attr_reader :path - attr_reader :name - - def initialize(context, options, file_dir) - super(context, options) - - @values = {} - - if options.all_one_file - @path = filename_to_label - else - @path = http_url(file_dir) - end - - @name = @context.file_relative_name - - collect_methods - AllReferences.add(name, self) - context.viewer = self - end - - def http_url(file_dir) - File.join(file_dir, @context.file_relative_name.tr('.', '_')) + - ".html" - end - - def filename_to_label - @context.file_relative_name.gsub(/%|\/|\?|\#/) {|s| '%' + ("%x" % s[0]) } - end - - def index_name - name - end - - def parent_name - nil - end - - def value_hash - file_attribute_values - add_table_of_sections - - @values["charset"] = @options.charset - @values["href"] = path - @values["style_url"] = style_url(path, @options.css) - - if @context.comment - d = markup(@context.comment) - @values["description"] = d if d.size > 0 - end - - ml = build_method_summary_list - @values["methods"] = ml unless ml.empty? - - il = build_include_list(@context) - @values["includes"] = il unless il.empty? - - rl = build_requires_list(@context) - @values["requires"] = rl unless rl.empty? - - if @options.promiscuous - file_context = nil - else - file_context = @context - end - - - @values["sections"] = @context.sections.map do |section| - - secdata = { - "sectitle" => section.title, - "secsequence" => section.sequence, - "seccomment" => markup(section.comment) - } - - cl = build_class_list(0, @context, section, file_context) - @values["classlist"] = cl unless cl.empty? - - mdl = build_method_detail_list(section) - secdata["method_list"] = mdl unless mdl.empty? - - al = build_alias_summary_list(section) - secdata["aliases"] = al unless al.empty? - - co = build_constants_summary_list(section) - @values["constants"] = co unless co.empty? - - secdata - end - - @values - end - - def write_on(f) - value_hash - template = TemplatePage.new(RDoc::Page::BODY, - RDoc::Page::FILE_PAGE, - RDoc::Page::METHOD_LIST) - template.write_html_on(f, @values) - end - - def file_attribute_values - full_path = @context.file_absolute_name - short_name = File.basename(full_path) - - @values["title"] = CGI.escapeHTML("File: #{short_name}") - - if @context.diagram - @values["diagram"] = diagram_reference(@context.diagram) - end - - @values["short_name"] = CGI.escapeHTML(short_name) - @values["full_path"] = CGI.escapeHTML(full_path) - @values["dtm_modified"] = @context.file_stat.mtime.to_s - - if @options.webcvs - @values["cvsurl"] = cvs_url( @options.webcvs, @values["full_path"] ) - end - end - - def <=>(other) - self.name <=> other.name - end - end - - ##################################################################### - - class HtmlMethod - include MarkUp - - attr_reader :context - attr_reader :src_url - attr_reader :img_url - attr_reader :source_code - - @@seq = "M000000" - - @@all_methods = [] - - def HtmlMethod::reset - @@all_methods = [] - end - - def initialize(context, html_class, options) - @context = context - @html_class = html_class - @options = options - @@seq = @@seq.succ - @seq = @@seq - @@all_methods << self - - context.viewer = self - - if (ts = @context.token_stream) - @source_code = markup_code(ts) - unless @options.inline_source - @src_url = create_source_code_file(@source_code) - @img_url = HTMLGenerator.gen_url(path, 'source.png') - end - end - - AllReferences.add(name, self) - end - - # return a reference to outselves to be used as an href= - # the form depends on whether we're all in one file - # or in multiple files - - def as_href(from_path) - if @options.all_one_file - "#" + path - else - HTMLGenerator.gen_url(from_path, path) - end - end - - def name - @context.name - end - - def section - @context.section - end - - def index_name - "#{@context.name} (#{@html_class.name})" - end - - def parent_name - if @context.parent.parent - @context.parent.parent.full_name - else - nil - end - end - - def aref - @seq - end - - def path - if @options.all_one_file - aref - else - @html_class.path + "#" + aref - end - end - - def description - markup(@context.comment) - end - - def visibility - @context.visibility - end - - def singleton - @context.singleton - end - - def call_seq - cs = @context.call_seq - if cs - cs.gsub(/\n/, "
\n") - else - nil - end - end - - def params - # params coming from a call-seq in 'C' will start with the - # method name - p = @context.params - if p !~ /^\w/ - p = @context.params.gsub(/\s*\#.*/, '') - p = p.tr("\n", " ").squeeze(" ") - p = "(" + p + ")" unless p[0] == ?( - - if (block = @context.block_params) - # If this method has explicit block parameters, remove any - # explicit &block - - p.sub!(/,?\s*&\w+/, '') - - block.gsub!(/\s*\#.*/, '') - block = block.tr("\n", " ").squeeze(" ") - if block[0] == ?( - block.sub!(/^\(/, '').sub!(/\)/, '') - end - p << " {|#{block.strip}| ...}" - end - end - CGI.escapeHTML(p) - end - - def create_source_code_file(code_body) - meth_path = @html_class.path.sub(/\.html$/, '.src') - File.makedirs(meth_path) - file_path = File.join(meth_path, @seq) + ".html" - - template = TemplatePage.new(RDoc::Page::SRC_PAGE) - File.open(file_path, "w") do |f| - values = { - 'title' => CGI.escapeHTML(index_name), - 'code' => code_body, - 'style_url' => style_url(file_path, @options.css), - 'charset' => @options.charset - } - template.write_html_on(f, values) - end - HTMLGenerator.gen_url(path, file_path) - end - - def HtmlMethod.all_methods - @@all_methods - end - - def <=>(other) - @context <=> other.context - end - - ## - # Given a sequence of source tokens, mark up the source code - # to make it look purty. - def old_markup_code(tokens) - src = "" - tokens.each do |t| - next unless t - # p t.class -# style = STYLE_MAP[t.class] - style = case t - when RubyToken::TkCONSTANT then "ruby-constant" - when RubyToken::TkKW then "ruby-keyword kw" - when RubyToken::TkIVAR then "ruby-ivar" - when RubyToken::TkOp then "ruby-operator" - when RubyToken::TkId then "ruby-identifier" - when RubyToken::TkNode then "ruby-node" - when RubyToken::TkCOMMENT then "ruby-comment cmt" - when RubyToken::TkREGEXP then "ruby-regexp re" - when RubyToken::TkSTRING then "ruby-value str" - when RubyToken::TkVal then "ruby-value" - else - nil - end - - text = CGI.escapeHTML(t.text) - - if style - src << "#{text}" - else - src << text - end - end - - add_line_numbers(src) if Options.instance.include_line_numbers - src - end - - require 'coderay' - CodeRay::Scanners.load_all - CodeRay::Encoders.load_all - CodeRay::Styles.load_all - - def markup_code(tokens) - code = tokens.map { |t| t.text }.join - options = { - :css => :class, - :line_numbers_start => code[/\A.*?, line (\d+)/,1].to_i - 1, - :bold_every => :no_bolding, - } - options[:line_numbers] = nil unless Options.instance.include_line_numbers - CodeRay.scan(code, :ruby).div(options) - end - - # we rely on the fact that the first line of a source code - # listing has - # # File xxxxx, line dddd - - def add_line_numbers(src) - if src =~ /\A.*, line (\d+)/ - first = $1.to_i - 1 - last = first + src.count("\n") - size = last.to_s.length - real_fmt = "%#{size}d: " - fmt = " " * (size+2) - src.gsub!(/^/) do - res = sprintf(fmt, first) - first += 1 - fmt = real_fmt - res - end - end - end - - def document_self - @context.document_self - end - - def aliases - @context.aliases - end - - def find_symbol(symbol, method=nil) - res = @context.parent.find_symbol(symbol, method) - if res - res = res.viewer - end - res - end - end - - ##################################################################### - - class HTMLGenerator - - include MarkUp - - ## - # convert a target url to one that is relative to a given - # path - - def HTMLGenerator.gen_url(path, target) - from = File.dirname(path) - to, to_file = File.split(target) - - from = from.split("/") - to = to.split("/") - - while from.size > 0 and to.size > 0 and from[0] == to[0] - from.shift - to.shift - end - - from.fill("..") - from.concat(to) - from << to_file - File.join(*from) - end - - # Generators may need to return specific subclasses depending - # on the options they are passed. Because of this - # we create them using a factory - - def HTMLGenerator.for(options) - AllReferences::reset - HtmlMethod::reset - - if options.all_one_file - HTMLGeneratorInOne.new(options) - else - HTMLGenerator.new(options) - end - end - - class < RDoc::Page::FONTS } - template.write_html_on(f, values) - end - end - end - - ## - # See the comments at the top for a description of the - # directory structure - - def gen_sub_directories - File.makedirs(FILE_DIR, CLASS_DIR) - rescue - $stderr.puts $!.message - exit 1 - end - - ## - # Generate: - # - # * a list of HtmlFile objects for each TopLevel object. - # * a list of HtmlClass objects for each first level - # class or module in the TopLevel objects - # * a complete list of all hyperlinkable terms (file, - # class, module, and method names) - - def build_indices - - @toplevels.each do |toplevel| - @files << HtmlFile.new(toplevel, @options, FILE_DIR) - end - - RDoc::TopLevel.all_classes_and_modules.each do |cls| - build_class_list(cls, @files[0], CLASS_DIR) - end - end - - def build_class_list(from, html_file, class_dir) - @classes << HtmlClass.new(from, html_file, class_dir, @options) - from.each_classmodule do |mod| - build_class_list(mod, html_file, class_dir) - end - end - - ## - # Generate all the HTML - # - def generate_html - # the individual descriptions for files and classes - gen_into(@files) - gen_into(@classes) - # and the index files - gen_file_index - gen_class_index - gen_method_index - gen_main_index - - # this method is defined in the template file - write_extra_pages if defined? write_extra_pages - end - - def gen_into(list) - list.each do |item| - if item.document_self - op_file = item.path - File.makedirs(File.dirname(op_file)) - File.open(op_file, "w") { |file| item.write_on(file) } - end - end - - end - - def gen_file_index - gen_an_index(@files, 'Files', - RDoc::Page::FILE_INDEX, - "fr_file_index.html") - end - - def gen_class_index - gen_an_index(@classes, 'Classes', - RDoc::Page::CLASS_INDEX, - "fr_class_index.html") - end - - def gen_method_index - gen_an_index(HtmlMethod.all_methods, 'Methods', - RDoc::Page::METHOD_INDEX, - "fr_method_index.html") - end - - - def gen_an_index(collection, title, template, filename) - template = TemplatePage.new(RDoc::Page::FR_INDEX_BODY, template) - res = [] - collection.sort.each do |f| - if f.document_self - res << { "href" => f.path, "name" => f.index_name } - end - end - - values = { - "entries" => res, - 'list_title' => CGI.escapeHTML(title), - 'index_url' => main_url, - 'charset' => @options.charset, - 'style_url' => style_url('', @options.css), - } - - File.open(filename, "w") do |f| - template.write_html_on(f, values) - end - end - - # The main index page is mostly a template frameset, but includes - # the initial page. If the --main option was given, - # we use this as our main page, otherwise we use the - # first file specified on the command line. - - def gen_main_index - template = TemplatePage.new(RDoc::Page::INDEX) - File.open("index.html", "w") do |f| - values = { - "initial_page" => main_url, - 'title' => CGI.escapeHTML(@options.title), - 'charset' => @options.charset - } - if @options.inline_source - values['inline_source'] = true - end - template.write_html_on(f, values) - end - end - - # return the url of the main page - def main_url - main_page = @options.main_page - ref = nil - if main_page - ref = AllReferences[main_page] - if ref - ref = ref.path - else - $stderr.puts "Could not find main page #{main_page}" - end - end - - unless ref - for file in @files - if file.document_self - ref = file.path - break - end - end - end - - unless ref - $stderr.puts "Couldn't find anything to document" - $stderr.puts "Perhaps you've used :stopdoc: in all classes" - exit(1) - end - - ref - end - - - end - HTML_CODERAYGenerator = HTMLGenerator - - - ###################################################################### - - - class HTMLGeneratorInOne < HTMLGenerator - - def initialize(*args) - super - end - - ## - # Build the initial indices and output objects - # based on an array of TopLevel objects containing - # the extracted information. - - def generate(info) - @toplevels = info - @files = [] - @classes = [] - @hyperlinks = {} - - build_indices - generate_xml - end - - - ## - # Generate: - # - # * a list of HtmlFile objects for each TopLevel object. - # * a list of HtmlClass objects for each first level - # class or module in the TopLevel objects - # * a complete list of all hyperlinkable terms (file, - # class, module, and method names) - - def build_indices - - @toplevels.each do |toplevel| - @files << HtmlFile.new(toplevel, @options, FILE_DIR) - end - - RDoc::TopLevel.all_classes_and_modules.each do |cls| - build_class_list(cls, @files[0], CLASS_DIR) - end - end - - def build_class_list(from, html_file, class_dir) - @classes << HtmlClass.new(from, html_file, class_dir, @options) - from.each_classmodule do |mod| - build_class_list(mod, html_file, class_dir) - end - end - - ## - # Generate all the HTML. For the one-file case, we generate - # all the information in to one big hash - # - def generate_xml - values = { - 'charset' => @options.charset, - 'files' => gen_into(@files), - 'classes' => gen_into(@classes), - 'title' => CGI.escapeHTML(@options.title), - } - - # this method is defined in the template file - write_extra_pages if defined? write_extra_pages - - template = TemplatePage.new(RDoc::Page::ONE_PAGE) - - if @options.op_name - opfile = File.open(@options.op_name, "w") - else - opfile = $stdout - end - template.write_html_on(opfile, values) - end - - def gen_into(list) - res = [] - list.each do |item| - res << item.value_hash - end - res - end - - def gen_file_index - gen_an_index(@files, 'Files') - end - - def gen_class_index - gen_an_index(@classes, 'Classes') - end - - def gen_method_index - gen_an_index(HtmlMethod.all_methods, 'Methods') - end - - - def gen_an_index(collection, title) - res = [] - collection.sort.each do |f| - if f.document_self - res << { "href" => f.path, "name" => f.index_name } - end - end - - return { - "entries" => res, - 'list_title' => title, - 'index_url' => main_url, - } - end - - end -end diff --git a/rake_tasks/code_statistics.rb b/rake_tasks/code_statistics.rb new file mode 100644 index 0000000..0a2016b --- /dev/null +++ b/rake_tasks/code_statistics.rb @@ -0,0 +1,171 @@ +# From rails (http://rubyonrails.com) +# +# Improved by murphy +class CodeStatistics + + TEST_TYPES = /\btest/i + + # Create a new Code Statistic. + # + # Rakefile Example: + # + # desc 'Report code statistics (LOC) from the application' + # task :stats => :copy_files do + # require 'rake_helpers/code_statistics' + # CodeStatistics.new( + # ["Main", "lib"], + # ["Tests", "test"], + # ["Demos", "demo"] + # ).to_s + # end + def initialize(*pairs) + @pairs = pairs + @statistics = calculate_statistics + @total = if pairs.empty? then nil else calculate_total end + end + + # Print a textual table viewing the stats + # + # Intended for console output. + def print + print_header + @pairs.each { |name, path| print_line name, @statistics[name] } + print_splitter + + if @total + print_line 'Total', @total + print_splitter + end + + print_code_test_stats + end + +private + + DEFAULT_FILE_PATTERN = /\.rb$/ + + def calculate_statistics + @pairs.inject({}) do |stats, (name, path, pattern, is_ruby_code)| + pattern ||= DEFAULT_FILE_PATTERN + path = File.join path, '*.rb' + stats[name] = calculate_directory_statistics path, pattern, is_ruby_code + stats + end + end + + def calculate_directory_statistics directory, pattern = DEFAULT_FILE_PATTERN, is_ruby_code = true + is_ruby_code = true if is_ruby_code.nil? + stats = Hash.new 0 + + Dir[directory].each do |file_name| + p "Scanning #{file_name}..." if $VERBOSE + next unless file_name =~ pattern + + lines = codelines = classes = modules = methods = 0 + empty_lines = comment_lines = 0 + in_comment_block = false + + File.readlines(file_name).each do |line| + lines += 1 + if line[/^\s*$/] + empty_lines += 1 + elsif is_ruby_code + case line + when /^=end\b/ + comment_lines += 1 + in_comment_block = false + when in_comment_block + comment_lines += 1 + when /^\s*class\b/ + classes += 1 + when /^\s*module\b/ + modules += 1 + when /^\s*def\b/ + methods += 1 + when /^\s*#/ + comment_lines += 1 + when /^=begin\b/ + in_comment_block = false + comment_lines += 1 + when /^__END__$/ + in_comment_block = true + end + end + end + + codelines = lines - comment_lines - empty_lines + + stats[:lines] += lines + stats[:comments] += comment_lines + stats[:codelines] += codelines + stats[:classes] += classes + stats[:modules] += modules + stats[:methods] += methods + stats[:files] += 1 + end + + stats + end + + def calculate_total + total = Hash.new 0 + @statistics.each_value { |pair| pair.each { |k, v| total[k] += v } } + total + end + + def calculate_code + code_loc = 0 + @statistics.each { |k, v| code_loc += v[:codelines] unless k[TEST_TYPES] } + code_loc + end + + def calculate_tests + test_loc = 0 + @statistics.each { |k, v| test_loc += v[:codelines] if k[TEST_TYPES] } + test_loc + end + + def print_header + print_splitter + puts "| T=Test Name | Files | Lines | LOC | Comments | Classes | Modules | Methods | M/C | LOC/M |" + print_splitter + end + + def print_splitter + puts "+---------------------------+-------+-------+-------+----------+---------+---------+---------+-----+-------+" + end + + def print_line name, statistics + m_over_c = (statistics[:methods] / (statistics[:classes] + statistics[:modules])) rescue m_over_c = 0 + loc_over_m = (statistics[:codelines] / statistics[:methods]) - 2 rescue loc_over_m = 0 + + if name[TEST_TYPES] + name = "T #{name}" + else + name = " #{name}" + end + + line = "| %-25s | %5d | %5d | %5d | %8d | %7d | %7d | %7d | %3d | %5d |" % ( + [name, *statistics.values_at(:files, :lines, :codelines, :comments, :classes, :modules, :methods)] + + [m_over_c, loc_over_m] ) + + puts line + end + + def print_code_test_stats + code = calculate_code + tests = calculate_tests + + puts " Code LOC = #{code} Test LOC = #{tests} Code:Test Ratio = [1 : #{sprintf("%.2f", tests.to_f/code)}]" + puts "" + end + +end + +# Run a test script. +if $0 == __FILE__ + $VERBOSE = true + CodeStatistics.new( + ['This dir', File.dirname(__FILE__)] + ).print +end diff --git a/rake_tasks/statistic.rake b/rake_tasks/statistic.rake index 99de378..d30e9b1 100644 --- a/rake_tasks/statistic.rake +++ b/rake_tasks/statistic.rake @@ -1,6 +1,6 @@ desc 'Report code statistics (LOC) from the application' task :stats do - require 'rake_helpers/code_statistics' + require './rake_tasks/code_statistics' CodeStatistics.new( ['Main', 'lib', /coderay.rb$/], ['CodeRay', 'lib/coderay/'], diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index a60699d..371214a 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -1,7 +1,7 @@ namespace :test do desc 'run all sample tests' task :samples do - ruby './sample/suite.rb' + ruby './test/samples/suite.rb' end desc 'run functional tests' diff --git a/sample/README b/sample/README deleted file mode 100644 index b0ab6e4..0000000 --- a/sample/README +++ /dev/null @@ -1 +0,0 @@ -These demos rely on Ruby 1.8.5, so the tests might fail on other versions. \ No newline at end of file diff --git a/sample/cache.expected b/sample/cache.expected deleted file mode 100644 index f815e88..0000000 --- a/sample/cache.expected +++ /dev/null @@ -1,2 +0,0 @@ -test <test> -test <test> diff --git a/sample/cache.rb b/sample/cache.rb deleted file mode 100644 index 0c0b847..0000000 --- a/sample/cache.rb +++ /dev/null @@ -1,12 +0,0 @@ -require 'coderay' - -html_encoder = CodeRay.encoder :html - -scanner = Hash.new do |h, lang| - h[lang] = CodeRay.scanner lang -end - -for lang in [:ruby, :html] - tokens = scanner[lang].tokenize 'test ' - puts html_encoder.encode_tokens(tokens) -end diff --git a/sample/count.expected b/sample/count.expected deleted file mode 100644 index 7f493b6..0000000 --- a/sample/count.expected +++ /dev/null @@ -1 +0,0 @@ -2 out of 4 tokens have the kind :integer. diff --git a/sample/count.rb b/sample/count.rb deleted file mode 100644 index bcb7c2d..0000000 --- a/sample/count.rb +++ /dev/null @@ -1,10 +0,0 @@ -require 'coderay' - -stats = CodeRay.encoder(:statistic) -stats.encode("puts 17 + 4\n", :ruby) - -puts '%d out of %d tokens have the kind :integer.' % [ - stats.type_stats[:integer].count, - stats.real_token_count -] -#-> 2 out of 4 tokens have the kind :integer. diff --git a/sample/css.expected b/sample/css.expected deleted file mode 100644 index 09709ff..0000000 --- a/sample/css.expected +++ /dev/null @@ -1,130 +0,0 @@ -.CodeRay { - background-color: #f8f8f8; - border: 1px solid silver; - font-family: 'Courier New', 'Terminal', monospace; - color: #000; -} -.CodeRay pre { margin: 0px } - -div.CodeRay { } - -span.CodeRay { white-space: pre; border: 0px; padding: 2px } - -table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px } -table.CodeRay td { padding: 2px 4px; vertical-align: top } - -.CodeRay .line_numbers, .CodeRay .no { - background-color: #def; - color: gray; - text-align: right; -} -.CodeRay .line_numbers tt { font-weight: bold } -.CodeRay .line_numbers .highlighted { color: red } -.CodeRay .no { padding: 0px 4px } -.CodeRay .code { width: 100% } - -ol.CodeRay { font-size: 10pt } -ol.CodeRay li { white-space: pre } - -.CodeRay .code pre { overflow: auto } - -.CodeRay .debug { color:white ! important; background:blue ! important; } - -.CodeRay .af { color:#00C } -.CodeRay .an { color:#007 } -.CodeRay .at { color:#f08 } -.CodeRay .av { color:#700 } -.CodeRay .aw { color:#C00 } -.CodeRay .bi { color:#509; font-weight:bold } -.CodeRay .c { color:#888; } - -.CodeRay .ch { color:#04D } -.CodeRay .ch .k { color:#04D } -.CodeRay .ch .dl { color:#039 } - -.CodeRay .cl { color:#B06; font-weight:bold } -.CodeRay .cm { color:#A08; font-weight:bold } -.CodeRay .co { color:#036; font-weight:bold } -.CodeRay .cr { color:#0A0 } -.CodeRay .cv { color:#369 } -.CodeRay .de { color:#B0B; } -.CodeRay .df { color:#099; font-weight:bold } -.CodeRay .di { color:#088; font-weight:bold } -.CodeRay .dl { color:black } -.CodeRay .do { color:#970 } -.CodeRay .dt { color:#34b } -.CodeRay .ds { color:#D42; font-weight:bold } -.CodeRay .e { color:#666; font-weight:bold } -.CodeRay .en { color:#800; font-weight:bold } -.CodeRay .er { color:#F00; background-color:#FAA } -.CodeRay .ex { color:#C00; font-weight:bold } -.CodeRay .fl { color:#60E; font-weight:bold } -.CodeRay .fu { color:#06B; font-weight:bold } -.CodeRay .gv { color:#d70; font-weight:bold } -.CodeRay .hx { color:#058; font-weight:bold } -.CodeRay .i { color:#00D; font-weight:bold } -.CodeRay .ic { color:#B44; font-weight:bold } - -.CodeRay .il { background: #ddd; color: black } -.CodeRay .il .il { background: #ccc } -.CodeRay .il .il .il { background: #bbb } -.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 } -.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; } - -.CodeRay .im { color:#f00; } -.CodeRay .in { color:#B2B; font-weight:bold } -.CodeRay .iv { color:#33B } -.CodeRay .la { color:#970; font-weight:bold } -.CodeRay .lv { color:#963 } -.CodeRay .oc { color:#40E; font-weight:bold } -.CodeRay .of { color:#000; font-weight:bold } -.CodeRay .op { } -.CodeRay .pc { color:#038; font-weight:bold } -.CodeRay .pd { color:#369; font-weight:bold } -.CodeRay .pp { color:#579; } -.CodeRay .ps { color:#00C; font-weight:bold } -.CodeRay .pt { color:#074; font-weight:bold } -.CodeRay .r, .kw { color:#080; font-weight:bold } - -.CodeRay .ke { color: #808; } -.CodeRay .ke .dl { color: #606; } -.CodeRay .ke .ch { color: #80f; } -.CodeRay .vl { color: #088; } - -.CodeRay .rx { background-color:#fff0ff } -.CodeRay .rx .k { color:#808 } -.CodeRay .rx .dl { color:#404 } -.CodeRay .rx .mod { color:#C2C } -.CodeRay .rx .fu { color:#404; font-weight: bold } - -.CodeRay .s { background-color:#fff0f0; color: #D20; } -.CodeRay .s .s { background-color:#ffe0e0 } -.CodeRay .s .s .s { background-color:#ffd0d0 } -.CodeRay .s .k { } -.CodeRay .s .ch { color: #b0b; } -.CodeRay .s .dl { color: #710; } - -.CodeRay .sh { background-color:#f0fff0; color:#2B2 } -.CodeRay .sh .k { } -.CodeRay .sh .dl { color:#161 } - -.CodeRay .sy { color:#A60 } -.CodeRay .sy .k { color:#A60 } -.CodeRay .sy .dl { color:#630 } - -.CodeRay .ta { color:#070 } -.CodeRay .tf { color:#070; font-weight:bold } -.CodeRay .ts { color:#D70; font-weight:bold } -.CodeRay .ty { color:#339; font-weight:bold } -.CodeRay .v { color:#036 } -.CodeRay .xt { color:#444 } - -.CodeRay .ins { background: #afa; } -.CodeRay .del { background: #faa; } -.CodeRay .chg { color: #aaf; background: #007; } -.CodeRay .head { color: #f8f; background: #505 } - -.CodeRay .ins .ins { color: #080; font-weight:bold } -.CodeRay .del .del { color: #800; font-weight:bold } -.CodeRay .chg .chg { color: #66f; } -.CodeRay .head .head { color: #f4f; } diff --git a/sample/css.rb b/sample/css.rb deleted file mode 100644 index 52e4bcc..0000000 --- a/sample/css.rb +++ /dev/null @@ -1,4 +0,0 @@ -require 'coderay' - -# print the default stylesheet for HTML codes -puts CodeRay::Encoders[:html]::CSS.new.stylesheet diff --git a/sample/div.expected b/sample/div.expected deleted file mode 100644 index f28ede3..0000000 --- a/sample/div.expected +++ /dev/null @@ -1,17 +0,0 @@ -

- 
for a in 0..255
-        a = a.chr
-        begin
-                x = eval("?\\#{a}")
-                if x == a[0]
-                        next
-                else
-                        print "#{a}: #{x}"
-                end
-        rescue SyntaxError => boom
-                print "#{a}: error"
-        end
-        puts
-end
-
-
diff --git a/sample/div.rb b/sample/div.rb deleted file mode 100644 index 27b6f32..0000000 --- a/sample/div.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'coderay' - -puts CodeRay.scan(DATA.read, :ruby).div - -__END__ -for a in 0..255 - a = a.chr - begin - x = eval("?\\#{a}") - if x == a[0] - next - else - print "#{a}: #{x}" - end - rescue SyntaxError => boom - print "#{a}: error" - end - puts -end diff --git a/sample/dump.expected b/sample/dump.expected deleted file mode 100644 index a451686..0000000 --- a/sample/dump.expected +++ /dev/null @@ -1,21 +0,0 @@ -YAML: 2358 bytes -Dump: 1109 bytes -undumped: -
- 
require 'coderay'
-
-# scan some code
-tokens = CodeRay.scan(File.read($0), :ruby)
-
-# dump using YAML
-yaml = tokens.yaml
-puts 'YAML: %4d bytes' % yaml.size
-
-# dump using Marshal
-dump = tokens.dump(0)
-puts 'Dump: %4d bytes' % dump.size
-
-# undump and encode
-puts 'undumped:', dump.undump.div(:css => :class)
-
-
diff --git a/sample/dump.rb b/sample/dump.rb deleted file mode 100644 index cd68dc8..0000000 --- a/sample/dump.rb +++ /dev/null @@ -1,15 +0,0 @@ -require 'coderay' - -# scan some code -tokens = CodeRay.scan(File.read($0), :ruby) - -# dump using YAML -yaml = tokens.yaml -puts 'YAML: %4d bytes' % yaml.size - -# dump using Marshal -dump = tokens.dump(0) -puts 'Dump: %4d bytes' % dump.size - -# undump and encode -puts 'undumped:', dump.undump.div(:css => :class) diff --git a/sample/encoder.expected b/sample/encoder.expected deleted file mode 100644 index 438032a..0000000 --- a/sample/encoder.expected +++ /dev/null @@ -1,65 +0,0 @@ -Encoders Demo: puts 17 + 4 - -Statistic: - -Code Statistics - -Tokens 8 - Non-Whitespace 4 -Bytes Total 12 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 8 100.00 % 1.5 - space 4 50.00 % 1.0 - integer 2 25.00 % 1.5 - ident 1 12.50 % 4.0 - operator 1 12.50 % 1.0 - - -Original text: -[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}] - -YAML: ---- -- - puts - - :ident -- - " " - - :space -- - "17" - - :integer -- - " " - - :space -- - + - - :operator -- - " " - - :space -- - "4" - - :integer -- - | - - - - :space - -Dump: -"x\332\355\330\273\n\302@\020\005PQIL4\235\245E\260\265\022\004a\266\021\002B\332\250U\2525\031$\210\233\260\273)\202?o\036\370\370\006\271\325\354\314\345\334\017\330\351,\216h\031\2259'\262!:\227wV&\035\207\223\324]{Um\r\371E\316\312\266\253\023\222o*\231q\373v\267{Z\024\312\362\215u\037\t\267\e\e\n\312\212\265\264\345\357u'f\335\360\373\255/\025\3167n\253\206\374\335!.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL -ODER -TEXT = <--$BLA-->.TEST(...) \ No newline at end of file diff --git a/sample/global_vars.rb b/sample/global_vars.rb deleted file mode 100644 index 8066d67..0000000 --- a/sample/global_vars.rb +++ /dev/null @@ -1,13 +0,0 @@ -code = <<'CODE' -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) -CODE - -require 'coderay' - -tokens = CodeRay.scan code, :ruby -tokens.each_text_token { |text, kind| text.upcase! } -tokens.each(:global_variable) { |text, kind| text.replace '<--%s-->' % text } - -print tokens diff --git a/sample/global_vars2.expected b/sample/global_vars2.expected deleted file mode 100644 index 964cf50..0000000 --- a/sample/global_vars2.expected +++ /dev/null @@ -1,10 +0,0 @@ - - - - - -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) - - diff --git a/sample/global_vars2.rb b/sample/global_vars2.rb deleted file mode 100644 index 7646890..0000000 --- a/sample/global_vars2.rb +++ /dev/null @@ -1,28 +0,0 @@ -require 'coderay' -require 'erb' -include ERB::Util - -code = <<'CODE' -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) -CODE -puts < - - - - -HTML - -CodeRay.scan_stream code, :ruby do |text, kind| - next if text.is_a? Symbol - text = h(text) - text = '%s' % text if kind == :global_variable - print text -end - -puts < - -HTML diff --git a/sample/highlight.expected b/sample/highlight.expected deleted file mode 100644 index 6a9b278..0000000 --- a/sample/highlight.expected +++ /dev/null @@ -1,175 +0,0 @@ -
- 
puts "Hello, World!"
-
- - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-
require 'coderay'
-
-puts CodeRay.highlight('puts "Hello, World!"', :ruby)
-
-output = CodeRay.highlight_file($0, :line_numbers => :table)
-puts <<HTML
-<html>
-<head>
-#{output.stylesheet true}
-<body>
-#{output}
-</body>
-</html>
-HTML
-
- - - diff --git a/sample/highlight.rb b/sample/highlight.rb deleted file mode 100644 index 846efa4..0000000 --- a/sample/highlight.rb +++ /dev/null @@ -1,14 +0,0 @@ -require 'coderay' - -puts CodeRay.highlight('puts "Hello, World!"', :ruby) - -output = CodeRay.highlight_file($0, :line_numbers => :table) -puts < - -#{output.stylesheet true} - -#{output} - - -HTML diff --git a/sample/html.expected b/sample/html.expected deleted file mode 100644 index e98d589..0000000 --- a/sample/html.expected +++ /dev/null @@ -1,919 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-15
-16
-17
-18
-19
-20
-21
-22
-23
-24
-25
-26
-27
-28
-29
-30
-31
-32
-33
-34
-35
-36
-37
-38
-39
-40
-41
-42
-43
-44
-45
-46
-47
-48
-49
-50
-51
-52
-53
-54
-55
-56
-57
-58
-59
-60
-61
-62
-63
-64
-65
-66
-67
-68
-69
-70
-71
-72
-73
-74
-75
-76
-77
-78
-79
-80
-81
-82
-83
-84
-85
-86
-87
-88
-89
-90
-91
-92
-93
-94
-95
-96
-97
-98
-99
-100
-101
-102
-103
-104
-105
-106
-107
-108
-109
-110
-111
-112
-113
-114
-115
-116
-117
-118
-119
-120
-121
-122
-123
-124
-125
-126
-127
-128
-129
-130
-131
-132
-133
-134
-135
-136
-137
-138
-139
-140
-141
-142
-143
-144
-145
-146
-147
-148
-149
-150
-151
-152
-153
-154
-155
-156
-157
-158
-159
-160
-161
-162
-163
-164
-165
-166
-167
-168
-169
-170
-171
-172
-173
-174
-175
-176
-177
-178
-179
-180
-181
-182
-183
-184
-185
-186
-187
-188
-189
-190
-191
-192
-193
-194
-195
-196
-197
-198
-199
-200
-201
-202
-203
-204
-205
-206
-207
-208
-209
-210
-211
-212
-213
-214
-215
-216
-217
-218
-219
-220
-221
-222
-223
-224
-225
-226
-227
-228
-229
-230
-231
-232
-233
-234
-235
-236
-237
-238
-239
-240
-241
-242
-243
-244
-245
-246
-247
-248
-249
-250
-251
-252
-253
-254
-255
-256
-257
-258
-259
-260
-261
-262
-263
-264
-265
-266
-267
-268
-269
-270
-271
-272
-273
-274
-275
-276
-277
-278
-279
-280
-281
-282
-283
-284
-285
-286
-287
-288
-289
-290
-291
-292
-293
-294
-295
-296
-297
-298
-299
-300
-301
-302
-303
-304
-305
-306
-307
-308
-309
-310
-311
-312
-313
-314
-315
-316
-317
-318
-319
-320
-321
-322
-323
-324
-325
-326
-327
-328
-329
-330
-331
-332
-333
-334
-335
-336
-337
-338
-339
-340
-341
-342
-343
-344
-345
-346
-347
-348
-349
-350
-351
-352
-353
-354
-355
-356
-357
-358
-359
-360
-361
-362
-363
-364
-365
-366
-367
-368
-369
-370
-371
-372
-373
-374
-375
-376
-377
-378
-379
-380
-381
-382
-383
-384
-385
-
require 'scanner'
-
-module CodeRay
-  
-  class RubyScanner < Scanner
-    
-    RESERVED_WORDS = [
-      'and', 'def', 'end', 'in', 'or', 'unless', 'begin',
-      'defined?', 'ensure', 'module', 'redo', 'super', 'until',
-      'BEGIN', 'break', 'do', 'next', 'rescue', 'then',
-      'when', 'END', 'case', 'else', 'for', 'retry',
-      'while', 'alias', 'class', 'elsif', 'if', 'not', 'return',
-      'undef', 'yield',
-    ]
-
-    DEF_KEYWORDS = ['def']
-    MODULE_KEYWORDS = ['class', 'module']
-    DEF_NEW_STATE = WordList.new(:initial).
-      add(DEF_KEYWORDS, :def_expected).
-      add(MODULE_KEYWORDS, :module_expected)
-
-    WORDS_ALLOWING_REGEXP = [
-      'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when'
-    ]
-    REGEXP_ALLOWED = WordList.new(false).
-      add(WORDS_ALLOWING_REGEXP, :set)
-    
-    PREDEFINED_CONSTANTS = [
-      'nil', 'true', 'false', 'self',
-      'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__',
-    ]
-
-    IDENT_KIND = WordList.new(:ident).
-      add(RESERVED_WORDS, :reserved).
-      add(PREDEFINED_CONSTANTS, :pre_constant)
-
-    METHOD_NAME = / #{IDENT} [?!]? /xo
-    METHOD_NAME_EX = /
-     #{METHOD_NAME}  # common methods: split, foo=, empty?, gsub!
-     | \*\*?         # multiplication and power
-     | [-+~]@?       # plus, minus
-     | [\/%&|^`]     # division, modulo or format strings, &and, |or, ^xor, `system`
-     | \[\]=?        # array getter and setter
-     | <=?>? | >=?   # comparison, rocket operator
-     | << | >>       # append or shift left, shift right
-     | ===?          # simple equality and case equality
-    /ox
-    GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox
-
-    DOUBLEQ = / "  [^"\#\\]*  (?: (?: \#\{.*?\} | \#(?:$")?  | \\. ) [^"\#\\]*  )* "?  /ox
-    SINGLEQ = / '  [^'\\]*    (?:                              \\.   [^'\\]*    )* '?  /ox
-    STRING  = / #{SINGLEQ} | #{DOUBLEQ} /ox
-    SHELL   = / `  [^`\#\\]*  (?: (?: \#\{.*?\} | \#(?:$`)?  | \\. ) [^`\#\\]*  )* `?  /ox
-    REGEXP  = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox
-    
-    DECIMAL = /\d+(?:_\d+)*/  # doesn't recognize 09 as octal error
-    OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
-    HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
-    BINARY = /0b[01]+(?:_[01]+)*/
-
-    EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
-    FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /
-    INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/
-    
-    def reset
-      super
-      @regexp_allowed = false
-    end
-    
-    def next_token
-      return if @scanner.eos?
-
-      kind = :error
-      if @scanner.scan(/\s+/)  # in every state
-        kind = :space
-        @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n)  # delayed flag setting
-
-      elsif @state == :def_expected
-        if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox)
-          kind = :method
-          @state = :initial
-        else
-          @scanner.scan(/./)
-          kind = :error
-        end
-        @state = :initial
-        
-      elsif @state == :module_expected
-        if @scanner.scan(/<</)
-          kind = :operator
-        else
-          if @scanner.scan(/ (?: #{IDENT} (?:\.|::))* #{IDENT} /ox)
-            kind = :method
-          else
-            @scanner.scan(/./)
-            kind = :error
-          end
-          @state = :initial
-        end
-        
-      elsif # state == :initial
-        # IDENTIFIERS, KEYWORDS
-        if @scanner.scan(GLOBAL_VARIABLE)
-          kind = :global_variable
-        elsif @scanner.scan(/ @@ #{IDENT} /ox)
-          kind = :class_variable
-        elsif @scanner.scan(/ @ #{IDENT} /ox)
-          kind = :instance_variable
-        elsif @scanner.scan(/ __END__\n ( (?!\#CODE\#) .* )? | \#[^\n]* | =begin(?=\s).*? \n=end(?=\s|\z)(?:[^\n]*)? /x)
-          kind = :comment
-        elsif @scanner.scan(METHOD_NAME)
-          if @last_token_dot
-            kind = :ident
-          else
-            matched = @scanner.matched
-            kind = IDENT_KIND[matched]
-            if kind == :ident and matched =~ /^[A-Z]/
-              kind = :constant
-            elsif kind == :reserved
-              @state = DEF_NEW_STATE[matched]
-              @regexp_allowed = REGEXP_ALLOWED[matched]
-            end
-          end
-          
-        elsif @scanner.scan(STRING)
-          kind = :string
-        elsif @scanner.scan(SHELL)
-          kind = :shell
-        ## HEREDOCS
-        elsif @scanner.scan(/\//) and @regexp_allowed
-           @scanner.unscan
-           @scanner.scan(REGEXP)
-          kind = :regexp
-        ## %strings
-        elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
-          kind = :global_variable
-        elsif @scanner.scan(/
-          \? (?:
-            [^\s\\]
-          | 
-            \\ (?:M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-))? (?: \\ (?: . | [0-7]{3} | x[0-9A-Fa-f][0-9A-Fa-f] )
-          )
-        /ox)
-          kind = :integer
-          
-        elsif @scanner.scan(/ [-+*\/%=<>;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x)
-          kind = :operator
-          @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/
-        elsif @scanner.scan(FLOAT)
-          kind = :float
-        elsif @scanner.scan(INTEGER)
-          kind = :integer
-        elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
-          kind = :global_variable
-        else
-          @scanner.scan(/./m)
-        end
-      end
-      
-      token = Token.new @scanner.matched, kind
-
-      if kind == :regexp
-        token.text << @scanner.scan(/[eimnosux]*/)
-      end
-      
-      @regexp_allowed = (@regexp_allowed == :set)  # delayed flag setting
-
-      token
-    end
-  end
-  
-  ScannerList.register RubyScanner, 'ruby'
-
-end
-
-module CodeRay
-  require 'scanner'
-
-  class Highlighter
-
-    def initialize lang
-      @scanner = Scanner[lang].new
-    end
-
-    def highlight code
-      @scanner.feed code
-      @scanner.all_tokens.map { |t| t.inspect }.join "\n"
-    end
-
-  end
-
-  class HTMLHighlighter < Highlighter
-    
-    ClassOfKind = {
-      :attribute_name => 'an',
-      :attribute_name_fat => 'af',
-      :attribute_value => 'av',
-      :attribute_value_fat => 'aw',
-      :bin => 'bi',
-       :char => 'ch',
-      :class => 'cl',
-      :class_variable => 'cv',
-      :color => 'cr',
-      :comment => 'c',
-      :constant => 'co',
-      :definition => 'df',
-      :directive => 'di',
-      :doc => 'do',
-      :doc_string => 'ds',
-      :exception => 'ex',
-      :error => 'er',
-      :float => 'fl',
-      :function => 'fu',
-      :global_variable => 'gv',
-      :hex => 'hx',
-      :include => 'ic',
-      :instance_variable => 'iv',
-      :integer => 'i',
-      :interpreted => 'in',
-      :label => 'la',
-      :local_variable => 'lv',
-      :oct => 'oc',
-      :operator_name => 'on',
-      :pre_constant => 'pc',
-      :pre_type => 'pt',
-      :predefined => 'pd',
-      :preprocessor => 'pp',
-      :regexp => 'rx',
-      :reserved => 'r',
-      :shell => 'sh',
-      :string => 's',
-      :symbol => 'sy',
-      :tag => 'ta',
-      :tag_fat => 'tf',
-      :tag_special => 'ts',
-      :type => 'ty',
-      :variable => 'v',
-      :xml_text => 'xt',
-
-      :ident => :NO_HIGHLIGHT,
-      :operator => :NO_HIGHLIGHT,
-      :space => :NO_HIGHLIGHT,
-    }
-    ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
-    ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
-    
-    def initialize lang, options = {}
-      super lang
-      
-      @HTML_TAB = ' ' * options.fetch(:tabs2space, 8)
-      case level = options.fetch(:level, 'xhtml')
-        when 'html'
-          @HTML_BR = "<BR>\n"
-        when 'xhtml'
-          @HTML_BR = "<br />\n"
-      else
-        raise "Unknown HTML level: #{level}"
-      end
-    end
-
-    def highlight code
-      @scanner.feed code
-      
-      out = ''
-      while t = @scanner.next_token
-        warn t.inspect if t.text.nil?
-        out << to_html(t)
-      end
-      TEMPLATE =~ /<%CONTENT%>/
-      $` + out + $'
-    end
-    
-  private
-    def to_html token
-      css_class = ClassOfKind[token.kind]
-      if defined? ::DEBUG and not ClassOfKind.has_key? token.kind
-        warn "no token class found for :#{token.kind}"
-      end
-        
-      text = text_to_html token.text
-      if css_class == :NO_HIGHLIGHT
-        text
-      else
-        "<span class=\"#{css_class}\">#{text}</span>"
-      end
-    end
-    
-    def text_to_html text
-      return '' if text.empty?
-      text = text.dup  # important
-      if text.index(/["><&]/)
-        text.gsub!('&', '&amp;')
-        text.gsub!('"', '&quot;')
-        text.gsub!('>', '&gt;')
-        text.gsub!('<', '&lt;')
-      end
-      if text.index(/\s/)
-        text.gsub!("\n", @HTML_BR)
-        text.gsub!("\t", @HTML_TAB)
-        text.gsub!(/^ /, '&nbsp;')
-        text.gsub!('  ', ' &nbsp;')
-      end
-      text
-    end
-    
-    TEMPLATE = <<-'TEMPLATE'
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
-<html dir="ltr">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
-<meta http-equiv="Content-Style-Type" content="text/css">
-
-<title>RubyBB BBCode</title>
-<style type="text/css">
-.code {
-  width: 100%;
-  background-color: #FAFAFA;
-  border: 1px solid #D1D7DC;
-  font-family: 'Courier New', 'Terminal', monospace;
-  font-size: 10pt;
-  color: black;
-  vertical-align: top;
-  text-align: left;
-}
-.code .af { color:#00C; }
-.code .an { color:#007; }
-.code .av { color:#700; }
-.code .aw { color:#C00; }
-.code .bi { color:#509; font-weight:bold; }
-.code .c  { color:#888; }
-.code .ch { color:#C28; font-weight:bold; }
-.code .cl { color:#B06; font-weight:bold; }
-.code .co { color:#036; font-weight:bold; }
-.code .cr { color:#0A0; }
-.code .cv { color:#369; }
-.code .df { color:#099; font-weight:bold; }
-.code .di { color:#088; font-weight:bold; }
-.code .do { color:#970; }
-.code .ds { color:#D42; font-weight:bold; }
-.code .er { color:#F00; background-color:#FAA; }
-.code .ex { color:#F00; font-weight:bold; }
-.code .fl { color:#60E; font-weight:bold; }
-.code .fu { color:#06B; font-weight:bold; }
-.code .gv { color:#800; font-weight:bold; }
-.code .hx { color:#058; font-weight:bold; }
-.code .i  { color:#00D; font-weight:bold; }
-.code .ic { color:#B44; font-weight:bold; }
-.code .in { color:#B2B; font-weight:bold; }
-.code .iv { color:#33B; }
-.code .la { color:#970; font-weight:bold; }
-.code .lv { color:#963; }
-.code .oc { color:#40E; font-weight:bold; }
-.code .on { color:#000; font-weight:bold; }
-.code .pc { color:#038; font-weight:bold; }
-.code .pd { color:#369; font-weight:bold; }
-.code .pp { color:#579; }
-.code .pt { color:#339; font-weight:bold; }
-.code .r  { color:#080; font-weight:bold; }
-.code .rx { color:#927; font-weight:bold; }
-.code .s  { color:#D42; font-weight:bold; }
-.code .sh { color:#B2B; font-weight:bold; }
-.code .sy { color:#A60; }
-.code .ta { color:#070; }
-.code .tf { color:#070; font-weight:bold; }
-.code .ts { color:#D70; font-weight:bold; }
-.code .ty { color:#339; font-weight:bold; }
-.code .v  { color:#036; }
-.code .xt { color:#444; }
-</style>
-</head>
-<body>
-<div class="code">
-<%CONTENT%>
-</div>
-<div class="validators">
-<a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88" style="border:none;"></a>
-<img style="border:0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!" >
-</div>    
-</body>
-</html>
-    TEMPLATE
-
-  end
-
-end
-
- - - diff --git a/sample/html.rb b/sample/html.rb deleted file mode 100644 index c18284a..0000000 --- a/sample/html.rb +++ /dev/null @@ -1,394 +0,0 @@ -$: << '..' -require 'coderay' - -tokens = CodeRay.scan DATA.read, :ruby -html = tokens.page(:tab_width => 2, :line_numbers => :table, :title => 'CodeRay HTML Encoder Example') - -puts html - -__END__ -require 'scanner' - -module CodeRay - - class RubyScanner < Scanner - - RESERVED_WORDS = [ - 'and', 'def', 'end', 'in', 'or', 'unless', 'begin', - 'defined?', 'ensure', 'module', 'redo', 'super', 'until', - 'BEGIN', 'break', 'do', 'next', 'rescue', 'then', - 'when', 'END', 'case', 'else', 'for', 'retry', - 'while', 'alias', 'class', 'elsif', 'if', 'not', 'return', - 'undef', 'yield', - ] - - DEF_KEYWORDS = ['def'] - MODULE_KEYWORDS = ['class', 'module'] - DEF_NEW_STATE = WordList.new(:initial). - add(DEF_KEYWORDS, :def_expected). - add(MODULE_KEYWORDS, :module_expected) - - WORDS_ALLOWING_REGEXP = [ - 'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when' - ] - REGEXP_ALLOWED = WordList.new(false). - add(WORDS_ALLOWING_REGEXP, :set) - - PREDEFINED_CONSTANTS = [ - 'nil', 'true', 'false', 'self', - 'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__', - ] - - IDENT_KIND = WordList.new(:ident). - add(RESERVED_WORDS, :reserved). - add(PREDEFINED_CONSTANTS, :pre_constant) - - METHOD_NAME = / #{IDENT} [?!]? /xo - METHOD_NAME_EX = / - #{METHOD_NAME} # common methods: split, foo=, empty?, gsub! - | \*\*? # multiplication and power - | [-+~]@? # plus, minus - | [\/%&|^`] # division, modulo or format strings, &and, |or, ^xor, `system` - | \[\]=? # array getter and setter - | <=?>? | >=? # comparison, rocket operator - | << | >> # append or shift left, shift right - | ===? # simple equality and case equality - /ox - GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox - - DOUBLEQ = / " [^"\#\\]* (?: (?: \#\{.*?\} | \#(?:$")? | \\. ) [^"\#\\]* )* "? /ox - SINGLEQ = / ' [^'\\]* (?: \\. [^'\\]* )* '? /ox - STRING = / #{SINGLEQ} | #{DOUBLEQ} /ox - SHELL = / ` [^`\#\\]* (?: (?: \#\{.*?\} | \#(?:$`)? | \\. ) [^`\#\\]* )* `? /ox - REGEXP = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox - - DECIMAL = /\d+(?:_\d+)*/ # doesn't recognize 09 as octal error - OCTAL = /0_?[0-7]+(?:_[0-7]+)*/ - HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/ - BINARY = /0b[01]+(?:_[01]+)*/ - - EXPONENT = / [eE] [+-]? #{DECIMAL} /ox - FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) / - INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/ - - def reset - super - @regexp_allowed = false - end - - def next_token - return if @scanner.eos? - - kind = :error - if @scanner.scan(/\s+/) # in every state - kind = :space - @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n) # delayed flag setting - - elsif @state == :def_expected - if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox) - kind = :method - @state = :initial - else - @scanner.scan(/./) - kind = :error - end - @state = :initial - - elsif @state == :module_expected - if @scanner.scan(/<;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x) - kind = :operator - @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/ - elsif @scanner.scan(FLOAT) - kind = :float - elsif @scanner.scan(INTEGER) - kind = :integer - elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox) - kind = :global_variable - else - @scanner.scan(/./m) - end - end - - token = Token.new @scanner.matched, kind - - if kind == :regexp - token.text << @scanner.scan(/[eimnosux]*/) - end - - @regexp_allowed = (@regexp_allowed == :set) # delayed flag setting - - token - end - end - - ScannerList.register RubyScanner, 'ruby' - -end - -module CodeRay - require 'scanner' - - class Highlighter - - def initialize lang - @scanner = Scanner[lang].new - end - - def highlight code - @scanner.feed code - @scanner.all_tokens.map { |t| t.inspect }.join "\n" - end - - end - - class HTMLHighlighter < Highlighter - - ClassOfKind = { - :attribute_name => 'an', - :attribute_name_fat => 'af', - :attribute_value => 'av', - :attribute_value_fat => 'aw', - :bin => 'bi', - :char => 'ch', - :class => 'cl', - :class_variable => 'cv', - :color => 'cr', - :comment => 'c', - :constant => 'co', - :definition => 'df', - :directive => 'di', - :doc => 'do', - :doc_string => 'ds', - :exception => 'ex', - :error => 'er', - :float => 'fl', - :function => 'fu', - :global_variable => 'gv', - :hex => 'hx', - :include => 'ic', - :instance_variable => 'iv', - :integer => 'i', - :interpreted => 'in', - :label => 'la', - :local_variable => 'lv', - :oct => 'oc', - :operator_name => 'on', - :pre_constant => 'pc', - :pre_type => 'pt', - :predefined => 'pd', - :preprocessor => 'pp', - :regexp => 'rx', - :reserved => 'r', - :shell => 'sh', - :string => 's', - :symbol => 'sy', - :tag => 'ta', - :tag_fat => 'tf', - :tag_special => 'ts', - :type => 'ty', - :variable => 'v', - :xml_text => 'xt', - - :ident => :NO_HIGHLIGHT, - :operator => :NO_HIGHLIGHT, - :space => :NO_HIGHLIGHT, - } - ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function] - ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!' - - def initialize lang, options = {} - super lang - - @HTML_TAB = ' ' * options.fetch(:tabs2space, 8) - case level = options.fetch(:level, 'xhtml') - when 'html' - @HTML_BR = "
\n" - when 'xhtml' - @HTML_BR = "
\n" - else - raise "Unknown HTML level: #{level}" - end - end - - def highlight code - @scanner.feed code - - out = '' - while t = @scanner.next_token - warn t.inspect if t.text.nil? - out << to_html(t) - end - TEMPLATE =~ /<%CONTENT%>/ - $` + out + $' - end - - private - def to_html token - css_class = ClassOfKind[token.kind] - if defined? ::DEBUG and not ClassOfKind.has_key? token.kind - warn "no token class found for :#{token.kind}" - end - - text = text_to_html token.text - if css_class == :NO_HIGHLIGHT - text - else - "#{text}" - end - end - - def text_to_html text - return '' if text.empty? - text = text.dup # important - if text.index(/["><&]/) - text.gsub!('&', '&') - text.gsub!('"', '"') - text.gsub!('>', '>') - text.gsub!('<', '<') - end - if text.index(/\s/) - text.gsub!("\n", @HTML_BR) - text.gsub!("\t", @HTML_TAB) - text.gsub!(/^ /, ' ') - text.gsub!(' ', '  ') - end - text - end - - TEMPLATE = <<-'TEMPLATE' - - - - - - -RubyBB BBCode - - - -
-<%CONTENT%> -
-
-Valid HTML 4.01! -Valid CSS! -
- - - TEMPLATE - - end - -end diff --git a/sample/html2.expected b/sample/html2.expected deleted file mode 100644 index c8ae56a..0000000 --- a/sample/html2.expected +++ /dev/null @@ -1,185 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-
require 'coderay'
-
-# scan this file
-tokens = CodeRay.scan(File.read($0) * 1, :ruby)
-
-# output it with two styles of line numbers
-out = tokens.div(:line_numbers => :table)
-out << '<hr />'
-out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
-
-puts out.page(:title => 'CodeRay HTML Encoder Example')
-
-
- 
 8 require 'coderay'
- 9 
-10 # scan this file
-11 tokens = CodeRay.scan(File.read($0) * 1, :ruby)
-12 
-13 # output it with two styles of line numbers
-14 out = tokens.div(:line_numbers => :table)
-15 out << '<hr />'
-16 out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
-17 
-18 puts out.page(:title => 'CodeRay HTML Encoder Example')
-
-
- - - diff --git a/sample/html2.rb b/sample/html2.rb deleted file mode 100644 index 618d168..0000000 --- a/sample/html2.rb +++ /dev/null @@ -1,11 +0,0 @@ -require 'coderay' - -# scan this file -tokens = CodeRay.scan(File.read($0) * 1, :ruby) - -# output it with two styles of line numbers -out = tokens.div(:line_numbers => :table) -out << '
' -out << tokens.div(:line_numbers => :inline, :line_number_start => 8) - -puts out.page(:title => 'CodeRay HTML Encoder Example') diff --git a/sample/html_list.expected b/sample/html_list.expected deleted file mode 100644 index a4092c8..0000000 --- a/sample/html_list.expected +++ /dev/null @@ -1,160 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - -
- 
-1 $: << '..'
- 0 require 'coderay'
- 1 
- 2 tokens = CodeRay.scan File.read(__FILE__), :ruby
- 3 html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1)
- 4 
- 5 puts html.page(:title => 'CodeRay HTML Encoder Example')
- 6 
- 7 commment = <<_
- 8 This code must be > 10 lines
- 9 because I want to test the correct adjustment of the line numbers.
-10 _
-
-
- - - diff --git a/sample/html_list.rb b/sample/html_list.rb deleted file mode 100644 index fdfa512..0000000 --- a/sample/html_list.rb +++ /dev/null @@ -1,12 +0,0 @@ -$: << '..' -require 'coderay' - -tokens = CodeRay.scan File.read(__FILE__), :ruby -html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1) - -puts html.page(:title => 'CodeRay HTML Encoder Example') - -commment = <<_ -This code must be > 10 lines -because I want to test the correct adjustment of the line numbers. -_ diff --git a/sample/load_encoder.expected b/sample/load_encoder.expected deleted file mode 100644 index 1cff356..0000000 --- a/sample/load_encoder.expected +++ /dev/null @@ -1,8 +0,0 @@ -CodeRay::Encoders::YAML is not defined; you must load it first. -Now it is loaded: CodeRay::Encoders::YAML -See? -Require is also possible: CodeRay::Encoders::Tokens -See? -Now load some mapped encoders: stats and plain. -Require all Encoders: -[[:count, CodeRay::Encoders::Count], [:debug, CodeRay::Encoders::Debug], [:div, CodeRay::Encoders::Div], [:html, CodeRay::Encoders::HTML], [:null, CodeRay::Encoders::Null], [:page, CodeRay::Encoders::Page], [:plain, :text], [:span, CodeRay::Encoders::Span], [:statistic, CodeRay::Encoders::Statistic], [:stats, CodeRay::Encoders::Statistic], [:text, CodeRay::Encoders::Text], [:tokens, CodeRay::Encoders::Tokens], [:xml, CodeRay::Encoders::XML], [:yaml, CodeRay::Encoders::YAML]] diff --git a/sample/load_encoder.rb b/sample/load_encoder.rb deleted file mode 100644 index 9594bfa..0000000 --- a/sample/load_encoder.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -begin - CodeRay::Encoders::YAML -rescue - puts 'CodeRay::Encoders::YAML is not defined; you must load it first.' -end - -yaml_encoder = CodeRay::Encoders[:yaml] -print 'Now it is loaded: ' -p yaml_encoder -puts 'See?' - -tokens_encoder = CodeRay.require_plugin 'CodeRay::Encoders/tokens' -print 'Require is also possible: ' -p tokens_encoder -puts 'See?' - -puts 'Now load some mapped encoders: stats and plain.' -CodeRay.require_plugin 'CodeRay::Encoders/stats' -CodeRay.require_plugin 'CodeRay::Encoders/plain' - -puts 'Require all Encoders:' -CodeRay::Encoders.load_all -p CodeRay::Encoders.plugin_hash.sort_by { |k,v| k.to_s } diff --git a/sample/load_scanner.expected b/sample/load_scanner.expected deleted file mode 100644 index a2d200d..0000000 --- a/sample/load_scanner.expected +++ /dev/null @@ -1,8 +0,0 @@ -CodeRay::Encoders::Ruby is not defined; you must load it first. -Now it is loaded: CodeRay::Scanners::Ruby -See? -Require is also possible: CodeRay::Scanners::C -See? -Now load some mapped scanners: cpp and plain. -Require all Scanners: -[[nil, :plain], [:c, CodeRay::Scanners::C], [:cpp, :c], [:delphi, CodeRay::Scanners::Delphi], [:html, CodeRay::Scanners::HTML], [:irb, :ruby], [:nitro, :nitro_xhtml], [:nitro_xhtml, CodeRay::Scanners::NitroXHTML], [:pascal, :delphi], [:plain, CodeRay::Scanners::Plaintext], [:plaintext, CodeRay::Scanners::Plaintext], [:rhtml, CodeRay::Scanners::RHTML], [:ruby, CodeRay::Scanners::Ruby], [:xhtml, :nitro_xhtml], [:xml, :html]] diff --git a/sample/load_scanner.rb b/sample/load_scanner.rb deleted file mode 100644 index 23be8a2..0000000 --- a/sample/load_scanner.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -begin - CodeRay::Scanners::Ruby -rescue - puts 'CodeRay::Encoders::Ruby is not defined; you must load it first.' -end - -ruby_scanner = CodeRay::Scanners[:ruby] -print 'Now it is loaded: ' -p ruby_scanner -puts 'See?' - -c_scanner = CodeRay.require_plugin 'CodeRay::Scanners/c' -print 'Require is also possible: ' -p c_scanner -puts 'See?' - -puts 'Now load some mapped scanners: cpp and plain.' -CodeRay.require_plugin 'CodeRay::Scanners/cpp' -CodeRay.require_plugin 'CodeRay::Scanners/plain' - -puts 'Require all Scanners:' -CodeRay::Scanners.load_all -p CodeRay::Scanners.plugin_hash.sort_by { |k,v| k.to_s } diff --git a/sample/more.expected b/sample/more.expected deleted file mode 100644 index 196904d..0000000 --- a/sample/more.expected +++ /dev/null @@ -1,2 +0,0 @@ -Input: 4983B, Output: 23484B -Take a look with your browser. diff --git a/sample/more.rb b/sample/more.rb deleted file mode 100644 index 0db7ba4..0000000 --- a/sample/more.rb +++ /dev/null @@ -1,205 +0,0 @@ -require 'coderay' - -c, ruby = DATA.read.split(/^---$/) -DATA.rewind -me = DATA.read[/.*^__END__$/m] -$input = c + ruby + me - -require 'benchmark' -time = Benchmark.realtime do - - # here CodeRay comes to play - hl = CodeRay.encoder(:html, :tab_width => 2, :line_numbers => :table, :wrap => :div) - c = hl.highlight c, :c - ruby = hl.highlight ruby, :ruby - me = hl.highlight me, :ruby - - body = %w[C Ruby Genereated\ by].zip([c, ruby, me]).map do |title, code| - "

#{title}

\n#{code}" - end.join - body = hl.class::Output.new(body, hl.css, :div).page! - - # CodeRay also provides a simple page generator - $output = body #hl.class.wrap_in_page body -end - -File.open('test.html', 'w') do |f| - f.write $output -end -puts 'Input: %dB, Output: %dB' % [$input.size, $output.size] -#puts 'Created "test.html" in %0.3f seconds (%d KB/s).' % [time, $input.size / 1024.0 / time] -puts 'Take a look with your browser.' - -__END__ -/********************************************************************** - - version.c - - - $Author: nobu $ - $Date: 2004/03/25 12:01:40 $ - created at: Thu Sep 30 20:08:01 JST 1993 - - Copyright (C) 1993-2003 Yukihiro Matsumoto - -**********************************************************************/ - -#include "ruby.h" -#include "version.h" -#include - -const char ruby_version[] = RUBY_VERSION; -const char ruby_release_date[] = RUBY_RELEASE_DATE; -const char ruby_platform[] = RUBY_PLATFORM; - -void -Init_version() -{ - VALUE v = rb_obj_freeze(rb_str_new2(ruby_version)); - VALUE d = rb_obj_freeze(rb_str_new2(ruby_release_date)); - VALUE p = rb_obj_freeze(rb_str_new2(ruby_platform)); - - rb_define_global_const("RUBY_VERSION", v); - rb_define_global_const("RUBY_RELEASE_DATE", d); - rb_define_global_const("RUBY_PLATFORM", p); -} - -void -ruby_show_version() -{ - printf("ruby %s (%s) [%s]\n", RUBY_VERSION, RUBY_RELEASE_DATE, RUBY_PLATFORM); -} - -void -ruby_show_copyright() -{ - printf("ruby - Copyright (C) 1993-%d Yukihiro Matsumoto\n", RUBY_RELEASE_YEAR); - exit(0); -} ---- -# -# = ostruct.rb: OpenStruct implementation -# -# Author:: Yukihiro Matsumoto -# Documentation:: Gavin Sinclair -# -# OpenStruct allows the creation of data objects with arbitrary attributes. -# See OpenStruct for an example. -# - -# -# OpenStruct allows you to create data objects and set arbitrary attributes. -# For example: -# -# require 'ostruct' -# -# record = OpenStruct.new -# record.name = "John Smith" -# record.age = 70 -# record.pension = 300 -# -# puts record.name # -> "John Smith" -# puts record.address # -> nil -# -# It is like a hash with a different way to access the data. In fact, it is -# implemented with a hash, and you can initialize it with one. -# -# hash = { "country" => "Australia", :population => 20_000_000 } -# data = OpenStruct.new(hash) -# -# p data # -> -# -class OpenStruct - # - # Create a new OpenStruct object. The optional +hash+, if given, will - # generate attributes and values. For example. - # - # require 'ostruct' - # hash = { "country" => "Australia", :population => 20_000_000 } - # data = OpenStruct.new(hash) - # - # p data # -> - # - # By default, the resulting OpenStruct object will have no attributes. - # - def initialize(hash=nil) - @table = {} - if hash - for k,v in hash - @table[k.to_sym] = v - new_ostruct_member(k) - end - end - end - - # Duplicate an OpenStruct object members. - def initialize_copy(orig) - super - @table = @table.dup - end - - def marshal_dump - @table - end - def marshal_load(x) - @table = x - @table.each_key{|key| new_ostruct_member(key)} - end - - def new_ostruct_member(name) - unless self.respond_to?(name) - self.instance_eval %{ - def #{name}; @table[:#{name}]; end - def #{name}=(x); @table[:#{name}] = x; end - } - end - end - - def method_missing(mid, *args) # :nodoc: - mname = mid.id2name - len = args.length - if mname =~ /=$/ - if len != 1 - raise ArgumentError, "wrong number of arguments (#{len} for 1)", caller(1) - end - if self.frozen? - raise TypeError, "can't modify frozen #{self.class}", caller(1) - end - mname.chop! - @table[mname.intern] = args[0] - self.new_ostruct_member(mname) - elsif len == 0 - @table[mid] - else - raise NoMethodError, "undefined method `#{mname}' for #{self}", caller(1) - end - end - - # - # Remove the named field from the object. - # - def delete_field(name) - @table.delete name.to_sym - end - - # - # Returns a string containing a detailed summary of the keys and values. - # - def inspect - str = "<#{self.class}" - for k,v in @table - str << " #{k}=#{v.inspect}" - end - str << ">" - end - - attr_reader :table # :nodoc: - protected :table - - # - # Compare this object and +other+ for equality. - # - def ==(other) - return false unless(other.kind_of?(OpenStruct)) - return @table == other.table - end -end diff --git a/sample/scanner.expected b/sample/scanner.expected deleted file mode 100644 index 5015168..0000000 --- a/sample/scanner.expected +++ /dev/null @@ -1,16 +0,0 @@ -C Code: if (*p == '{') nest++; - -> print only operators: -(*==)++; ------------------------------- - -Ruby Code: ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee) - -> has a string? -false - -> number of regexps? -2 - -> has a string? -"ruby_code" (ident), "(" (operator), ":can" (symbol), "," (operator), " " (space), "BE" (constant), "," (operator), " " (space), "%r[" (delimiter), "q" (content), "[" (nesting_delimiter), "ui" (content), "]" (nesting_delimiter), "te " (content), "#{" (inline_delimiter), " " (space), "/" (delimiter), "comple" (content), "/" (delimiter), "x" (modifier), " " (space), "}" (inline_delimiter), "," (content), "]" (delimiter), " " (space), "=" (operator), ">" (operator), " " (space), "$-s" (global_variable), "," (operator), " " (space), "&" (operator), "?\xee" (integer), ")" (operator) diff --git a/sample/scanner.rb b/sample/scanner.rb deleted file mode 100644 index 6a0245e..0000000 --- a/sample/scanner.rb +++ /dev/null @@ -1,36 +0,0 @@ -require 'coderay' - -c_code = "if (*p == '{') nest++;" -puts 'C Code: ' + c_code -puts - -c_scanner = CodeRay::Scanners[:c].new c_code - -puts '> print only operators:' -for text, kind in c_scanner - print text if kind == :operator -end -puts -puts '-' * 30 -puts - -ruby_code = %q!ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee)! -puts 'Ruby Code: ' + ruby_code -puts - -ruby_scanner = CodeRay::Scanners[:ruby].new ruby_code - -puts '> has a string?' -puts ruby_scanner. - any? { |text, kind| kind == :string } -puts - -puts '> number of regexps?' -puts ruby_scanner. - select { |token| token == [:open, :regexp] }.size -puts - -puts '> has a string?' -puts ruby_scanner. - reject { |text, kind| not text.is_a? String }. - map { |text, kind| %("#{text}" (#{kind})) }.join(', ') diff --git a/sample/server.rb b/sample/server.rb deleted file mode 100644 index ccdff32..0000000 --- a/sample/server.rb +++ /dev/null @@ -1,110 +0,0 @@ -# CodeRay dynamic highlighter - -unless ARGV.grep(/-[hv]|--(help|version)/).empty? - puts <<-USAGE -CodeRay Server 0.5 -$Id: demo_server.rb 113 2006-03-15 23:24:37Z murphy $ - -Usage: - 1) Start this and your browser. - 2) Go to http://localhost:2468/? - and you should get the highlighted version. - -Parameters: - -d Debug mode; reload CodeRay engine for every file. - (prepare for MANY "already initialized" and "method redefined" - messages - ingore it.) - - ... More to come. - USAGE - exit -end - -require 'webrick' -require 'pathname' - -class << File - alias dir? directory? -end - -require 'erb' -include ERB::Util -def url_decode s - s.to_s.gsub(/%([0-9a-f]{2})/i) { [$1.hex].pack 'C' } -end - -class String - def to_link name = File.basename(self) - "#{name}" - end -end - -require 'coderay' -class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet - - STYLE = 'style="font-family: sans-serif; color: navy;"' - BANNER = '

Highlighted by CodeRay

' - - def do_GET req, res - q = req.query_string || '' - args = Hash[*q.scan(/(.*?)=(.*?)(?:&|$)/).flatten].each_value { |v| v.replace url_decode(v) } - path = args.fetch 'path', '.' - - backlinks = '

current path: %s
' % html_escape(path) + - (Pathname.new(path) + '..').cleanpath.to_s.to_link('up') + ' - ' + - '.'.to_link('current') + '

' - - res.body = - if File.dir? path - path = Pathname.new(path).cleanpath.to_s - dirs, files = Dir[File.join(path, '*')].sort.partition { |p| File.dir? p } - - page = "" - page << backlinks - - page << '
' - page << "
Directories
\n" + dirs.map do |p| - "
#{p.to_link}
\n" - end.join << "\n" - page << "
Files
\n" + files.map do |p| - "
#{p.to_link}
\n" - end.join << "\n" - page << "
\n" - page << "#{BANNER}" - - elsif File.exist? path - if $DEBUG - $".delete_if { |f| f =~ /coderay/ } - require 'coderay' - end - div = CodeRay.scan_file(path).html :tab_width => 8, :wrap => :div, :hint => :info - div.replace <<-DIV -
- #{backlinks} -#{div} -
- #{BANNER} - DIV - div.page - end - - res['Content-Type'] = 'text/html' - end -end - -# This port is taken by "qip_msgd" - I don't know that. Do you? -module CodeRay - PORT = 0xC0DE / 20 -end - -server = WEBrick::HTTPServer.new :Port => CodeRay::PORT - -server.mount '/', CodeRayServlet - -server.mount_proc '/version' do |req, res| - res.body = 'CodeRay::Version = ' + CodeRay::Version - res['Content-Type'] = "text/plain" -end - -trap("INT") { server.shutdown } -server.start diff --git a/sample/simple.expected b/sample/simple.expected deleted file mode 100644 index b3d7875..0000000 --- a/sample/simple.expected +++ /dev/null @@ -1 +0,0 @@ -puts 'Hello, world!' diff --git a/sample/simple.rb b/sample/simple.rb deleted file mode 100644 index a3129b0..0000000 --- a/sample/simple.rb +++ /dev/null @@ -1,10 +0,0 @@ - -# Load CodeRay -# If this doesn't work, try ruby -rubygems. -require 'coderay' - -# Generate HTML page for Ruby code. -page = CodeRay.scan("puts 'Hello, world!'", :ruby).span - -# Print it -puts page diff --git a/sample/stream.rb b/sample/stream.rb deleted file mode 100644 index 7ed8a22..0000000 --- a/sample/stream.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -code = File.read($0) * 500 -puts "Size of code: %d KB" % [code.size / 1024] - -puts "Use your system's memory tracker to see how much RAM this takes." -print 'Press some key to continue...'; gets - -require 'benchmark' -e = CodeRay.encoder(:div) -for do_stream in [true, false] - puts "Scanning and encoding in %s mode, please wait..." % - [do_stream ? 'streaming' : 'normal'] - output = '' - time = Benchmark.realtime do - if do_stream - output = e.encode_stream(code, :ruby) - else - output = e.encode_tokens(t = CodeRay.scan(code, :ruby)) - end - end - puts 'Finished after %4.2f seconds.' % time - puts "Size of output: %d KB" % [output.size / 1024] - print 'Press some key to continue...'; gets -end diff --git a/sample/stream2.expected b/sample/stream2.expected deleted file mode 100644 index 83aee98..0000000 --- a/sample/stream2.expected +++ /dev/null @@ -1,2 +0,0 @@ -kind: regexp, text size: 5. -kind: space, text size: 1. diff --git a/sample/stream2.rb b/sample/stream2.rb deleted file mode 100644 index d43cc9a..0000000 --- a/sample/stream2.rb +++ /dev/null @@ -1,8 +0,0 @@ -require 'coderay' - -token_stream = CodeRay::TokenStream.new do |kind, text| - puts 'kind: %s, text size: %d.' % [kind, text.size] -end - -token_stream << [:regexp, '/\d+/'] << [:space, "\n"] -#-> kind: rexpexp, text size: 5. diff --git a/sample/suite.rb b/sample/suite.rb deleted file mode 100644 index fa24114..0000000 --- a/sample/suite.rb +++ /dev/null @@ -1,86 +0,0 @@ -mydir = File.dirname(__FILE__) -$:.unshift mydir + '/../lib/' - -$VERBOSE = true - -require 'test/unit' -include Test::Unit - -class CodeRaySuite < TestCase - - def self.dir &block - @dir ||= File.dirname(__FILE__) - if block - Dir.chdir @dir, &block - end - @dir - end - - def dir &block - self.class.dir(&block) - end - - def test_ALL - dir do - for input in Dir["*.rb"] - %w(server.rb stream.rb suite.rb) - next if input[/^load_/] - puts "[ testing #{input}... ]" - name = File.basename(input, ".rb") - output = name + '.expected' - code = File.open(input, 'rb') { |f| break f.read } - - result = `ruby -wI../lib #{input}` - - diff = output.sub '.expected', '.diff' - File.delete diff if File.exist? diff - computed = output.sub '.expected', '.actual' - if File.exist? output - expected = File.read output - ok = expected == result - unless ok - File.open(computed, 'w') { |f| f.write result } - `diff #{output} #{computed} > #{diff}` if $DEBUG - puts "Test failed; output written to #{diff}." - end - assert(ok, "Output error: #{computed} != #{output}") unless $DEBUG - else - File.open(output, 'w') do |f| f.write result end - puts "New test: #{output}" - end - - end - end - end - -end - -require 'test/unit/testsuite' -$suite = TestSuite.new 'CodeRay Demos Test' -$suite << CodeRaySuite.suite - -def load_suite name - begin - require name + '/suite.rb' - rescue LoadError - $stderr.puts <<-ERR - -!! Folder #{File.split(__FILE__).first + '/' + name} not found - - ERR - false - end -end - -if subsuite = ARGV.find { |a| break $1 if a[/^([^-].*)/] } - load_suite(subsuite) or exit -else - Dir[mydir + '/*/'].each { |suite| load_suite suite } -end - -if ARGV.include? '-f' - require 'test/unit/ui/fox/testrunner' - UI::Fox::TestRunner.run $suite -else - require 'test/unit/ui/console/testrunner' - UI::Console::TestRunner.run $suite -end diff --git a/sample/tokens.expected b/sample/tokens.expected deleted file mode 100644 index 747904e..0000000 --- a/sample/tokens.expected +++ /dev/null @@ -1 +0,0 @@ -[["puts", :ident], [" ", :space], ["3", :integer], [" ", :space], ["+", :operator], [" ", :space], ["4", :integer], [",", :operator], [" ", :space], [:open, :string], ["'", :delimiter], ["3 + 4", :content], ["'", :delimiter], [:close, :string]] diff --git a/sample/tokens.rb b/sample/tokens.rb deleted file mode 100644 index 91b8abb..0000000 --- a/sample/tokens.rb +++ /dev/null @@ -1,3 +0,0 @@ -require 'coderay' - -p CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby) diff --git a/test/samples/README b/test/samples/README new file mode 100644 index 0000000..b0ab6e4 --- /dev/null +++ b/test/samples/README @@ -0,0 +1 @@ +These demos rely on Ruby 1.8.5, so the tests might fail on other versions. \ No newline at end of file diff --git a/test/samples/cache.actual b/test/samples/cache.actual new file mode 100644 index 0000000..c131857 --- /dev/null +++ b/test/samples/cache.actual @@ -0,0 +1,2 @@ +test <test> +test <test> diff --git a/test/samples/cache.expected b/test/samples/cache.expected new file mode 100644 index 0000000..c131857 --- /dev/null +++ b/test/samples/cache.expected @@ -0,0 +1,2 @@ +test <test> +test <test> diff --git a/test/samples/cache.rb b/test/samples/cache.rb new file mode 100644 index 0000000..0c0b847 --- /dev/null +++ b/test/samples/cache.rb @@ -0,0 +1,12 @@ +require 'coderay' + +html_encoder = CodeRay.encoder :html + +scanner = Hash.new do |h, lang| + h[lang] = CodeRay.scanner lang +end + +for lang in [:ruby, :html] + tokens = scanner[lang].tokenize 'test ' + puts html_encoder.encode_tokens(tokens) +end diff --git a/test/samples/count.expected b/test/samples/count.expected new file mode 100644 index 0000000..7f493b6 --- /dev/null +++ b/test/samples/count.expected @@ -0,0 +1 @@ +2 out of 4 tokens have the kind :integer. diff --git a/test/samples/count.rb b/test/samples/count.rb new file mode 100644 index 0000000..bcb7c2d --- /dev/null +++ b/test/samples/count.rb @@ -0,0 +1,10 @@ +require 'coderay' + +stats = CodeRay.encoder(:statistic) +stats.encode("puts 17 + 4\n", :ruby) + +puts '%d out of %d tokens have the kind :integer.' % [ + stats.type_stats[:integer].count, + stats.real_token_count +] +#-> 2 out of 4 tokens have the kind :integer. diff --git a/test/samples/css.actual b/test/samples/css.actual new file mode 100644 index 0000000..be73a7f --- /dev/null +++ b/test/samples/css.actual @@ -0,0 +1,127 @@ +.CodeRay { + background-color: hsl(0,0%,95%); + border: 1px solid silver; + color: black; +} +.CodeRay pre { + margin: 0px; +} + +span.CodeRay { white-space: pre; border: 0px; padding: 2px; } + +table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; } +table.CodeRay td { padding: 2px 4px; vertical-align: top; } + +.CodeRay .line-numbers { + background-color: hsl(180,65%,90%); + color: gray; + text-align: right; + -webkit-user-select: none; + -moz-user-select: none; + user-select: none; +} +.CodeRay .line-numbers a { + background-color: hsl(180,65%,90%) !important; + color: gray !important; + text-decoration: none !important; +} +.CodeRay .line-numbers pre { + word-break: normal; +} +.CodeRay .line-numbers a:target { color: blue !important; } +.CodeRay .line-numbers .highlighted { color: red !important; } +.CodeRay .line-numbers .highlighted a { color: red !important; } +.CodeRay span.line-numbers { padding: 0px 4px; } +.CodeRay .line { display: block; float: left; width: 100%; } +.CodeRay .code { width: 100%; } + +.CodeRay .debug { color: white !important; background: blue !important; } + +.CodeRay .annotation { color:#007 } +.CodeRay .attribute-name { color:#b48 } +.CodeRay .attribute-value { color:#700 } +.CodeRay .binary { color:#549 } +.CodeRay .binary .char { color:#325 } +.CodeRay .binary .delimiter { color:#325 } +.CodeRay .char { color:#D20 } +.CodeRay .char .content { color:#D20 } +.CodeRay .char .delimiter { color:#710 } +.CodeRay .class { color:#B06; font-weight:bold } +.CodeRay .class-variable { color:#369 } +.CodeRay .color { color:#0A0 } +.CodeRay .comment { color:#777 } +.CodeRay .comment .char { color:#444 } +.CodeRay .comment .delimiter { color:#444 } +.CodeRay .constant { color:#036; font-weight:bold } +.CodeRay .decorator { color:#B0B } +.CodeRay .definition { color:#099; font-weight:bold } +.CodeRay .delimiter { color:black } +.CodeRay .directive { color:#088; font-weight:bold } +.CodeRay .docstring { color:#D42; } +.CodeRay .doctype { color:#34b } +.CodeRay .done { text-decoration: line-through; color: gray } +.CodeRay .entity { color:#800; font-weight:bold } +.CodeRay .error { color:#F00; background-color:#FAA } +.CodeRay .escape { color:#666 } +.CodeRay .exception { color:#C00; font-weight:bold } +.CodeRay .float { color:#60E } +.CodeRay .function { color:#06B; font-weight:bold } +.CodeRay .function .delimiter { color:#024; font-weight:bold } +.CodeRay .global-variable { color:#d70 } +.CodeRay .hex { color:#02b } +.CodeRay .id { color:#33D; font-weight:bold } +.CodeRay .include { color:#B44; font-weight:bold } +.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black } +.CodeRay .inline-delimiter { font-weight: bold; color: #666 } +.CodeRay .instance-variable { color:#33B } +.CodeRay .integer { color:#00D } +.CodeRay .imaginary { color:#f00 } +.CodeRay .important { color:#D00 } +.CodeRay .key { color: #606 } +.CodeRay .key .char { color: #60f } +.CodeRay .key .delimiter { color: #404 } +.CodeRay .keyword { color:#080; font-weight:bold } +.CodeRay .label { color:#970; font-weight:bold } +.CodeRay .local-variable { color:#963 } +.CodeRay .namespace { color:#707; font-weight:bold } +.CodeRay .octal { color:#40E } +.CodeRay .operator { } +.CodeRay .predefined { color:#369; font-weight:bold } +.CodeRay .predefined-constant { color:#069 } +.CodeRay .predefined-type { color:#0a5; font-weight:bold } +.CodeRay .preprocessor { color:#579 } +.CodeRay .pseudo-class { color:#00C; font-weight:bold } +.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); } +.CodeRay .regexp .content { color:#808 } +.CodeRay .regexp .delimiter { color:#404 } +.CodeRay .regexp .modifier { color:#C2C } +.CodeRay .reserved { color:#080; font-weight:bold } +.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); } +.CodeRay .shell .content { color:#2B2 } +.CodeRay .shell .delimiter { color:#161 } +.CodeRay .string { background-color:hsla(0,100%,50%,0.05); } +.CodeRay .string .char { color: #b0b } +.CodeRay .string .content { color: #D20 } +.CodeRay .string .delimiter { color: #710 } +.CodeRay .string .modifier { color: #E40 } +.CodeRay .symbol { color:#A60 } +.CodeRay .symbol .content { color:#A60 } +.CodeRay .symbol .delimiter { color:#630 } +.CodeRay .tag { color:#070 } +.CodeRay .type { color:#339; font-weight:bold } +.CodeRay .value { color: #088 } +.CodeRay .variable { color:#037 } + +.CodeRay .insert { background: hsla(120,100%,50%,0.12) } +.CodeRay .delete { background: hsla(0,100%,50%,0.12) } +.CodeRay .change { color: #bbf; background: #007 } +.CodeRay .head { color: #f8f; background: #505 } +.CodeRay .head .filename { color: white; } + +.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; } +.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; } + +.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold } +.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold } +.CodeRay .change .change { color: #88f } +.CodeRay .head .head { color: #f4f } diff --git a/test/samples/css.expected b/test/samples/css.expected new file mode 100644 index 0000000..be73a7f --- /dev/null +++ b/test/samples/css.expected @@ -0,0 +1,127 @@ +.CodeRay { + background-color: hsl(0,0%,95%); + border: 1px solid silver; + color: black; +} +.CodeRay pre { + margin: 0px; +} + +span.CodeRay { white-space: pre; border: 0px; padding: 2px; } + +table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; } +table.CodeRay td { padding: 2px 4px; vertical-align: top; } + +.CodeRay .line-numbers { + background-color: hsl(180,65%,90%); + color: gray; + text-align: right; + -webkit-user-select: none; + -moz-user-select: none; + user-select: none; +} +.CodeRay .line-numbers a { + background-color: hsl(180,65%,90%) !important; + color: gray !important; + text-decoration: none !important; +} +.CodeRay .line-numbers pre { + word-break: normal; +} +.CodeRay .line-numbers a:target { color: blue !important; } +.CodeRay .line-numbers .highlighted { color: red !important; } +.CodeRay .line-numbers .highlighted a { color: red !important; } +.CodeRay span.line-numbers { padding: 0px 4px; } +.CodeRay .line { display: block; float: left; width: 100%; } +.CodeRay .code { width: 100%; } + +.CodeRay .debug { color: white !important; background: blue !important; } + +.CodeRay .annotation { color:#007 } +.CodeRay .attribute-name { color:#b48 } +.CodeRay .attribute-value { color:#700 } +.CodeRay .binary { color:#549 } +.CodeRay .binary .char { color:#325 } +.CodeRay .binary .delimiter { color:#325 } +.CodeRay .char { color:#D20 } +.CodeRay .char .content { color:#D20 } +.CodeRay .char .delimiter { color:#710 } +.CodeRay .class { color:#B06; font-weight:bold } +.CodeRay .class-variable { color:#369 } +.CodeRay .color { color:#0A0 } +.CodeRay .comment { color:#777 } +.CodeRay .comment .char { color:#444 } +.CodeRay .comment .delimiter { color:#444 } +.CodeRay .constant { color:#036; font-weight:bold } +.CodeRay .decorator { color:#B0B } +.CodeRay .definition { color:#099; font-weight:bold } +.CodeRay .delimiter { color:black } +.CodeRay .directive { color:#088; font-weight:bold } +.CodeRay .docstring { color:#D42; } +.CodeRay .doctype { color:#34b } +.CodeRay .done { text-decoration: line-through; color: gray } +.CodeRay .entity { color:#800; font-weight:bold } +.CodeRay .error { color:#F00; background-color:#FAA } +.CodeRay .escape { color:#666 } +.CodeRay .exception { color:#C00; font-weight:bold } +.CodeRay .float { color:#60E } +.CodeRay .function { color:#06B; font-weight:bold } +.CodeRay .function .delimiter { color:#024; font-weight:bold } +.CodeRay .global-variable { color:#d70 } +.CodeRay .hex { color:#02b } +.CodeRay .id { color:#33D; font-weight:bold } +.CodeRay .include { color:#B44; font-weight:bold } +.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black } +.CodeRay .inline-delimiter { font-weight: bold; color: #666 } +.CodeRay .instance-variable { color:#33B } +.CodeRay .integer { color:#00D } +.CodeRay .imaginary { color:#f00 } +.CodeRay .important { color:#D00 } +.CodeRay .key { color: #606 } +.CodeRay .key .char { color: #60f } +.CodeRay .key .delimiter { color: #404 } +.CodeRay .keyword { color:#080; font-weight:bold } +.CodeRay .label { color:#970; font-weight:bold } +.CodeRay .local-variable { color:#963 } +.CodeRay .namespace { color:#707; font-weight:bold } +.CodeRay .octal { color:#40E } +.CodeRay .operator { } +.CodeRay .predefined { color:#369; font-weight:bold } +.CodeRay .predefined-constant { color:#069 } +.CodeRay .predefined-type { color:#0a5; font-weight:bold } +.CodeRay .preprocessor { color:#579 } +.CodeRay .pseudo-class { color:#00C; font-weight:bold } +.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); } +.CodeRay .regexp .content { color:#808 } +.CodeRay .regexp .delimiter { color:#404 } +.CodeRay .regexp .modifier { color:#C2C } +.CodeRay .reserved { color:#080; font-weight:bold } +.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); } +.CodeRay .shell .content { color:#2B2 } +.CodeRay .shell .delimiter { color:#161 } +.CodeRay .string { background-color:hsla(0,100%,50%,0.05); } +.CodeRay .string .char { color: #b0b } +.CodeRay .string .content { color: #D20 } +.CodeRay .string .delimiter { color: #710 } +.CodeRay .string .modifier { color: #E40 } +.CodeRay .symbol { color:#A60 } +.CodeRay .symbol .content { color:#A60 } +.CodeRay .symbol .delimiter { color:#630 } +.CodeRay .tag { color:#070 } +.CodeRay .type { color:#339; font-weight:bold } +.CodeRay .value { color: #088 } +.CodeRay .variable { color:#037 } + +.CodeRay .insert { background: hsla(120,100%,50%,0.12) } +.CodeRay .delete { background: hsla(0,100%,50%,0.12) } +.CodeRay .change { color: #bbf; background: #007 } +.CodeRay .head { color: #f8f; background: #505 } +.CodeRay .head .filename { color: white; } + +.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; } +.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; } + +.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold } +.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold } +.CodeRay .change .change { color: #88f } +.CodeRay .head .head { color: #f4f } diff --git a/test/samples/css.rb b/test/samples/css.rb new file mode 100644 index 0000000..52e4bcc --- /dev/null +++ b/test/samples/css.rb @@ -0,0 +1,4 @@ +require 'coderay' + +# print the default stylesheet for HTML codes +puts CodeRay::Encoders[:html]::CSS.new.stylesheet diff --git a/test/samples/div.actual b/test/samples/div.actual new file mode 100644 index 0000000..d1e692a --- /dev/null +++ b/test/samples/div.actual @@ -0,0 +1,17 @@ +
+ 
for a in 0..255
+        a = a.chr
+        begin
+                x = eval("?\\#{a}")
+                if x == a[0]
+                        next
+                else
+                        print "#{a}: #{x}"
+                end
+        rescue SyntaxError => boom
+                print "#{a}: error"
+        end
+        puts
+end
+
+
diff --git a/test/samples/div.expected b/test/samples/div.expected new file mode 100644 index 0000000..d1e692a --- /dev/null +++ b/test/samples/div.expected @@ -0,0 +1,17 @@ +
+ 
for a in 0..255
+        a = a.chr
+        begin
+                x = eval("?\\#{a}")
+                if x == a[0]
+                        next
+                else
+                        print "#{a}: #{x}"
+                end
+        rescue SyntaxError => boom
+                print "#{a}: error"
+        end
+        puts
+end
+
+
diff --git a/test/samples/div.rb b/test/samples/div.rb new file mode 100644 index 0000000..27b6f32 --- /dev/null +++ b/test/samples/div.rb @@ -0,0 +1,19 @@ +require 'coderay' + +puts CodeRay.scan(DATA.read, :ruby).div + +__END__ +for a in 0..255 + a = a.chr + begin + x = eval("?\\#{a}") + if x == a[0] + next + else + print "#{a}: #{x}" + end + rescue SyntaxError => boom + print "#{a}: error" + end + puts +end diff --git a/test/samples/encoder.actual b/test/samples/encoder.actual new file mode 100644 index 0000000..8bd83a9 --- /dev/null +++ b/test/samples/encoder.actual @@ -0,0 +1,65 @@ +Encoders Demo: puts 17 + 4 + +Statistic: + +Code Statistics + +Tokens 8 + Non-Whitespace 4 +Bytes Total 12 + +Token Types (4): + type count ratio size (average) +------------------------------------------------------------- + TOTAL 8 100.00 % 1.5 + space 4 50.00 % 1.0 + integer 2 25.00 % 1.5 + ident 1 12.50 % 4.0 + operator 1 12.50 % 1.0 + + +Original text: +[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}] + +YAML: +--- +- - puts + - :ident +- - " " + - :space +- - "17" + - :integer +- - " " + - :space +- - + + - :operator +- - " " + - :space +- - "4" + - :integer +- - | + + + - :space + +Dump: +"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257" +compressed: 79 byte < 1200 byte + +Undump: + +Code Statistics + +Tokens 800 + Non-Whitespace 400 +Bytes Total 1200 + +Token Types (4): + type count ratio size (average) +------------------------------------------------------------- + TOTAL 800 100.00 % 1.5 + space 400 50.00 % 1.0 + integer 200 25.00 % 1.5 + ident 100 12.50 % 4.0 + operator 100 12.50 % 1.0 + diff --git a/test/samples/encoder.expected b/test/samples/encoder.expected new file mode 100644 index 0000000..8bd83a9 --- /dev/null +++ b/test/samples/encoder.expected @@ -0,0 +1,65 @@ +Encoders Demo: puts 17 + 4 + +Statistic: + +Code Statistics + +Tokens 8 + Non-Whitespace 4 +Bytes Total 12 + +Token Types (4): + type count ratio size (average) +------------------------------------------------------------- + TOTAL 8 100.00 % 1.5 + space 4 50.00 % 1.0 + integer 2 25.00 % 1.5 + ident 1 12.50 % 4.0 + operator 1 12.50 % 1.0 + + +Original text: +[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}] + +YAML: +--- +- - puts + - :ident +- - " " + - :space +- - "17" + - :integer +- - " " + - :space +- - + + - :operator +- - " " + - :space +- - "4" + - :integer +- - | + + + - :space + +Dump: +"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257" +compressed: 79 byte < 1200 byte + +Undump: + +Code Statistics + +Tokens 800 + Non-Whitespace 400 +Bytes Total 1200 + +Token Types (4): + type count ratio size (average) +------------------------------------------------------------- + TOTAL 800 100.00 % 1.5 + space 400 50.00 % 1.0 + integer 200 25.00 % 1.5 + ident 100 12.50 % 4.0 + operator 100 12.50 % 1.0 + diff --git a/test/samples/encoder.rb b/test/samples/encoder.rb new file mode 100644 index 0000000..bfcfbfa --- /dev/null +++ b/test/samples/encoder.rb @@ -0,0 +1,40 @@ +require 'coderay' + +SAMPLE = "puts 17 + 4\n" +puts 'Encoders Demo: ' + SAMPLE +scanner = CodeRay::Scanners[:ruby].new SAMPLE +encoder = CodeRay::Encoders[:statistic].new + +tokens = scanner.tokenize +stats = encoder.encode_tokens tokens + +puts +puts 'Statistic:' +puts stats + +# alternative 1 +tokens = CodeRay.scan SAMPLE, :ruby +encoder = CodeRay.encoder(:json) +textual = encoder.encode_tokens tokens +puts +puts 'Original text:' +puts textual + +# alternative 2 +yaml = CodeRay.encoder(:yaml).encode SAMPLE, :ruby +puts +puts 'YAML:' +puts yaml + +# alternative 3 +require 'zlib' +BIGSAMPLE = SAMPLE * 100 +dump = Zlib::Deflate.deflate(CodeRay.scan(BIGSAMPLE, :ruby).debug) +puts +puts 'Dump:' +p dump +puts 'compressed: %d byte < %d byte' % [dump.size, BIGSAMPLE.size] + +puts +puts 'Undump:' +puts CodeRay.scan(Zlib::Inflate.inflate(dump), :debug).statistic diff --git a/test/samples/global_vars.actual b/test/samples/global_vars.actual new file mode 100644 index 0000000..e69de29 diff --git a/test/samples/global_vars.diff b/test/samples/global_vars.diff new file mode 100644 index 0000000..2ed3382 --- /dev/null +++ b/test/samples/global_vars.diff @@ -0,0 +1,5 @@ +1,3d0 +< <--$IE-->.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL +< ODER +< TEXT = <--$BLA-->.TEST(...) +\ No newline at end of file diff --git a/test/samples/global_vars.expected b/test/samples/global_vars.expected new file mode 100644 index 0000000..0dc13c8 --- /dev/null +++ b/test/samples/global_vars.expected @@ -0,0 +1,3 @@ +<--$IE-->.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL +ODER +TEXT = <--$BLA-->.TEST(...) \ No newline at end of file diff --git a/test/samples/global_vars.rb b/test/samples/global_vars.rb new file mode 100644 index 0000000..8066d67 --- /dev/null +++ b/test/samples/global_vars.rb @@ -0,0 +1,13 @@ +code = <<'CODE' +$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel +oder +text = $bla.test(...) +CODE + +require 'coderay' + +tokens = CodeRay.scan code, :ruby +tokens.each_text_token { |text, kind| text.upcase! } +tokens.each(:global_variable) { |text, kind| text.replace '<--%s-->' % text } + +print tokens diff --git a/test/samples/global_vars2.expected b/test/samples/global_vars2.expected new file mode 100644 index 0000000..964cf50 --- /dev/null +++ b/test/samples/global_vars2.expected @@ -0,0 +1,10 @@ + + + + + +$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel +oder +text = $bla.test(...) + + diff --git a/test/samples/global_vars2.rb b/test/samples/global_vars2.rb new file mode 100644 index 0000000..7646890 --- /dev/null +++ b/test/samples/global_vars2.rb @@ -0,0 +1,28 @@ +require 'coderay' +require 'erb' +include ERB::Util + +code = <<'CODE' +$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel +oder +text = $bla.test(...) +CODE +puts < + + + + +HTML + +CodeRay.scan_stream code, :ruby do |text, kind| + next if text.is_a? Symbol + text = h(text) + text = '%s' % text if kind == :global_variable + print text +end + +puts < + +HTML diff --git a/test/samples/highlight.expected b/test/samples/highlight.expected new file mode 100644 index 0000000..6a9b278 --- /dev/null +++ b/test/samples/highlight.expected @@ -0,0 +1,175 @@ +
+ 
puts "Hello, World!"
+
+ + + + + + + + +
1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+
require 'coderay'
+
+puts CodeRay.highlight('puts "Hello, World!"', :ruby)
+
+output = CodeRay.highlight_file($0, :line_numbers => :table)
+puts <<HTML
+<html>
+<head>
+#{output.stylesheet true}
+<body>
+#{output}
+</body>
+</html>
+HTML
+
+ + + diff --git a/test/samples/highlight.rb b/test/samples/highlight.rb new file mode 100644 index 0000000..846efa4 --- /dev/null +++ b/test/samples/highlight.rb @@ -0,0 +1,14 @@ +require 'coderay' + +puts CodeRay.highlight('puts "Hello, World!"', :ruby) + +output = CodeRay.highlight_file($0, :line_numbers => :table) +puts < + +#{output.stylesheet true} + +#{output} + + +HTML diff --git a/test/samples/html.expected b/test/samples/html.expected new file mode 100644 index 0000000..e98d589 --- /dev/null +++ b/test/samples/html.expected @@ -0,0 +1,919 @@ + + + + + CodeRay HTML Encoder Example + + + + + + + +
1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
+343
+344
+345
+346
+347
+348
+349
+350
+351
+352
+353
+354
+355
+356
+357
+358
+359
+360
+361
+362
+363
+364
+365
+366
+367
+368
+369
+370
+371
+372
+373
+374
+375
+376
+377
+378
+379
+380
+381
+382
+383
+384
+385
+
require 'scanner'
+
+module CodeRay
+  
+  class RubyScanner < Scanner
+    
+    RESERVED_WORDS = [
+      'and', 'def', 'end', 'in', 'or', 'unless', 'begin',
+      'defined?', 'ensure', 'module', 'redo', 'super', 'until',
+      'BEGIN', 'break', 'do', 'next', 'rescue', 'then',
+      'when', 'END', 'case', 'else', 'for', 'retry',
+      'while', 'alias', 'class', 'elsif', 'if', 'not', 'return',
+      'undef', 'yield',
+    ]
+
+    DEF_KEYWORDS = ['def']
+    MODULE_KEYWORDS = ['class', 'module']
+    DEF_NEW_STATE = WordList.new(:initial).
+      add(DEF_KEYWORDS, :def_expected).
+      add(MODULE_KEYWORDS, :module_expected)
+
+    WORDS_ALLOWING_REGEXP = [
+      'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when'
+    ]
+    REGEXP_ALLOWED = WordList.new(false).
+      add(WORDS_ALLOWING_REGEXP, :set)
+    
+    PREDEFINED_CONSTANTS = [
+      'nil', 'true', 'false', 'self',
+      'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__',
+    ]
+
+    IDENT_KIND = WordList.new(:ident).
+      add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_CONSTANTS, :pre_constant)
+
+    METHOD_NAME = / #{IDENT} [?!]? /xo
+    METHOD_NAME_EX = /
+     #{METHOD_NAME}  # common methods: split, foo=, empty?, gsub!
+     | \*\*?         # multiplication and power
+     | [-+~]@?       # plus, minus
+     | [\/%&|^`]     # division, modulo or format strings, &and, |or, ^xor, `system`
+     | \[\]=?        # array getter and setter
+     | <=?>? | >=?   # comparison, rocket operator
+     | << | >>       # append or shift left, shift right
+     | ===?          # simple equality and case equality
+    /ox
+    GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox
+
+    DOUBLEQ = / "  [^"\#\\]*  (?: (?: \#\{.*?\} | \#(?:$")?  | \\. ) [^"\#\\]*  )* "?  /ox
+    SINGLEQ = / '  [^'\\]*    (?:                              \\.   [^'\\]*    )* '?  /ox
+    STRING  = / #{SINGLEQ} | #{DOUBLEQ} /ox
+    SHELL   = / `  [^`\#\\]*  (?: (?: \#\{.*?\} | \#(?:$`)?  | \\. ) [^`\#\\]*  )* `?  /ox
+    REGEXP  = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox
+    
+    DECIMAL = /\d+(?:_\d+)*/  # doesn't recognize 09 as octal error
+    OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
+    HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
+    BINARY = /0b[01]+(?:_[01]+)*/
+
+    EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
+    FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /
+    INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/
+    
+    def reset
+      super
+      @regexp_allowed = false
+    end
+    
+    def next_token
+      return if @scanner.eos?
+
+      kind = :error
+      if @scanner.scan(/\s+/)  # in every state
+        kind = :space
+        @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n)  # delayed flag setting
+
+      elsif @state == :def_expected
+        if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox)
+          kind = :method
+          @state = :initial
+        else
+          @scanner.scan(/./)
+          kind = :error
+        end
+        @state = :initial
+        
+      elsif @state == :module_expected
+        if @scanner.scan(/<</)
+          kind = :operator
+        else
+          if @scanner.scan(/ (?: #{IDENT} (?:\.|::))* #{IDENT} /ox)
+            kind = :method
+          else
+            @scanner.scan(/./)
+            kind = :error
+          end
+          @state = :initial
+        end
+        
+      elsif # state == :initial
+        # IDENTIFIERS, KEYWORDS
+        if @scanner.scan(GLOBAL_VARIABLE)
+          kind = :global_variable
+        elsif @scanner.scan(/ @@ #{IDENT} /ox)
+          kind = :class_variable
+        elsif @scanner.scan(/ @ #{IDENT} /ox)
+          kind = :instance_variable
+        elsif @scanner.scan(/ __END__\n ( (?!\#CODE\#) .* )? | \#[^\n]* | =begin(?=\s).*? \n=end(?=\s|\z)(?:[^\n]*)? /x)
+          kind = :comment
+        elsif @scanner.scan(METHOD_NAME)
+          if @last_token_dot
+            kind = :ident
+          else
+            matched = @scanner.matched
+            kind = IDENT_KIND[matched]
+            if kind == :ident and matched =~ /^[A-Z]/
+              kind = :constant
+            elsif kind == :reserved
+              @state = DEF_NEW_STATE[matched]
+              @regexp_allowed = REGEXP_ALLOWED[matched]
+            end
+          end
+          
+        elsif @scanner.scan(STRING)
+          kind = :string
+        elsif @scanner.scan(SHELL)
+          kind = :shell
+        ## HEREDOCS
+        elsif @scanner.scan(/\//) and @regexp_allowed
+           @scanner.unscan
+           @scanner.scan(REGEXP)
+          kind = :regexp
+        ## %strings
+        elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
+          kind = :global_variable
+        elsif @scanner.scan(/
+          \? (?:
+            [^\s\\]
+          | 
+            \\ (?:M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-))? (?: \\ (?: . | [0-7]{3} | x[0-9A-Fa-f][0-9A-Fa-f] )
+          )
+        /ox)
+          kind = :integer
+          
+        elsif @scanner.scan(/ [-+*\/%=<>;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x)
+          kind = :operator
+          @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/
+        elsif @scanner.scan(FLOAT)
+          kind = :float
+        elsif @scanner.scan(INTEGER)
+          kind = :integer
+        elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
+          kind = :global_variable
+        else
+          @scanner.scan(/./m)
+        end
+      end
+      
+      token = Token.new @scanner.matched, kind
+
+      if kind == :regexp
+        token.text << @scanner.scan(/[eimnosux]*/)
+      end
+      
+      @regexp_allowed = (@regexp_allowed == :set)  # delayed flag setting
+
+      token
+    end
+  end
+  
+  ScannerList.register RubyScanner, 'ruby'
+
+end
+
+module CodeRay
+  require 'scanner'
+
+  class Highlighter
+
+    def initialize lang
+      @scanner = Scanner[lang].new
+    end
+
+    def highlight code
+      @scanner.feed code
+      @scanner.all_tokens.map { |t| t.inspect }.join "\n"
+    end
+
+  end
+
+  class HTMLHighlighter < Highlighter
+    
+    ClassOfKind = {
+      :attribute_name => 'an',
+      :attribute_name_fat => 'af',
+      :attribute_value => 'av',
+      :attribute_value_fat => 'aw',
+      :bin => 'bi',
+       :char => 'ch',
+      :class => 'cl',
+      :class_variable => 'cv',
+      :color => 'cr',
+      :comment => 'c',
+      :constant => 'co',
+      :definition => 'df',
+      :directive => 'di',
+      :doc => 'do',
+      :doc_string => 'ds',
+      :exception => 'ex',
+      :error => 'er',
+      :float => 'fl',
+      :function => 'fu',
+      :global_variable => 'gv',
+      :hex => 'hx',
+      :include => 'ic',
+      :instance_variable => 'iv',
+      :integer => 'i',
+      :interpreted => 'in',
+      :label => 'la',
+      :local_variable => 'lv',
+      :oct => 'oc',
+      :operator_name => 'on',
+      :pre_constant => 'pc',
+      :pre_type => 'pt',
+      :predefined => 'pd',
+      :preprocessor => 'pp',
+      :regexp => 'rx',
+      :reserved => 'r',
+      :shell => 'sh',
+      :string => 's',
+      :symbol => 'sy',
+      :tag => 'ta',
+      :tag_fat => 'tf',
+      :tag_special => 'ts',
+      :type => 'ty',
+      :variable => 'v',
+      :xml_text => 'xt',
+
+      :ident => :NO_HIGHLIGHT,
+      :operator => :NO_HIGHLIGHT,
+      :space => :NO_HIGHLIGHT,
+    }
+    ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
+    ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
+    
+    def initialize lang, options = {}
+      super lang
+      
+      @HTML_TAB = ' ' * options.fetch(:tabs2space, 8)
+      case level = options.fetch(:level, 'xhtml')
+        when 'html'
+          @HTML_BR = "<BR>\n"
+        when 'xhtml'
+          @HTML_BR = "<br />\n"
+      else
+        raise "Unknown HTML level: #{level}"
+      end
+    end
+
+    def highlight code
+      @scanner.feed code
+      
+      out = ''
+      while t = @scanner.next_token
+        warn t.inspect if t.text.nil?
+        out << to_html(t)
+      end
+      TEMPLATE =~ /<%CONTENT%>/
+      $` + out + $'
+    end
+    
+  private
+    def to_html token
+      css_class = ClassOfKind[token.kind]
+      if defined? ::DEBUG and not ClassOfKind.has_key? token.kind
+        warn "no token class found for :#{token.kind}"
+      end
+        
+      text = text_to_html token.text
+      if css_class == :NO_HIGHLIGHT
+        text
+      else
+        "<span class=\"#{css_class}\">#{text}</span>"
+      end
+    end
+    
+    def text_to_html text
+      return '' if text.empty?
+      text = text.dup  # important
+      if text.index(/["><&]/)
+        text.gsub!('&', '&amp;')
+        text.gsub!('"', '&quot;')
+        text.gsub!('>', '&gt;')
+        text.gsub!('<', '&lt;')
+      end
+      if text.index(/\s/)
+        text.gsub!("\n", @HTML_BR)
+        text.gsub!("\t", @HTML_TAB)
+        text.gsub!(/^ /, '&nbsp;')
+        text.gsub!('  ', ' &nbsp;')
+      end
+      text
+    end
+    
+    TEMPLATE = <<-'TEMPLATE'
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html dir="ltr">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta http-equiv="Content-Style-Type" content="text/css">
+
+<title>RubyBB BBCode</title>
+<style type="text/css">
+.code {
+  width: 100%;
+  background-color: #FAFAFA;
+  border: 1px solid #D1D7DC;
+  font-family: 'Courier New', 'Terminal', monospace;
+  font-size: 10pt;
+  color: black;
+  vertical-align: top;
+  text-align: left;
+}
+.code .af { color:#00C; }
+.code .an { color:#007; }
+.code .av { color:#700; }
+.code .aw { color:#C00; }
+.code .bi { color:#509; font-weight:bold; }
+.code .c  { color:#888; }
+.code .ch { color:#C28; font-weight:bold; }
+.code .cl { color:#B06; font-weight:bold; }
+.code .co { color:#036; font-weight:bold; }
+.code .cr { color:#0A0; }
+.code .cv { color:#369; }
+.code .df { color:#099; font-weight:bold; }
+.code .di { color:#088; font-weight:bold; }
+.code .do { color:#970; }
+.code .ds { color:#D42; font-weight:bold; }
+.code .er { color:#F00; background-color:#FAA; }
+.code .ex { color:#F00; font-weight:bold; }
+.code .fl { color:#60E; font-weight:bold; }
+.code .fu { color:#06B; font-weight:bold; }
+.code .gv { color:#800; font-weight:bold; }
+.code .hx { color:#058; font-weight:bold; }
+.code .i  { color:#00D; font-weight:bold; }
+.code .ic { color:#B44; font-weight:bold; }
+.code .in { color:#B2B; font-weight:bold; }
+.code .iv { color:#33B; }
+.code .la { color:#970; font-weight:bold; }
+.code .lv { color:#963; }
+.code .oc { color:#40E; font-weight:bold; }
+.code .on { color:#000; font-weight:bold; }
+.code .pc { color:#038; font-weight:bold; }
+.code .pd { color:#369; font-weight:bold; }
+.code .pp { color:#579; }
+.code .pt { color:#339; font-weight:bold; }
+.code .r  { color:#080; font-weight:bold; }
+.code .rx { color:#927; font-weight:bold; }
+.code .s  { color:#D42; font-weight:bold; }
+.code .sh { color:#B2B; font-weight:bold; }
+.code .sy { color:#A60; }
+.code .ta { color:#070; }
+.code .tf { color:#070; font-weight:bold; }
+.code .ts { color:#D70; font-weight:bold; }
+.code .ty { color:#339; font-weight:bold; }
+.code .v  { color:#036; }
+.code .xt { color:#444; }
+</style>
+</head>
+<body>
+<div class="code">
+<%CONTENT%>
+</div>
+<div class="validators">
+<a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88" style="border:none;"></a>
+<img style="border:0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!" >
+</div>    
+</body>
+</html>
+    TEMPLATE
+
+  end
+
+end
+
+ + + diff --git a/test/samples/html.rb b/test/samples/html.rb new file mode 100644 index 0000000..c18284a --- /dev/null +++ b/test/samples/html.rb @@ -0,0 +1,394 @@ +$: << '..' +require 'coderay' + +tokens = CodeRay.scan DATA.read, :ruby +html = tokens.page(:tab_width => 2, :line_numbers => :table, :title => 'CodeRay HTML Encoder Example') + +puts html + +__END__ +require 'scanner' + +module CodeRay + + class RubyScanner < Scanner + + RESERVED_WORDS = [ + 'and', 'def', 'end', 'in', 'or', 'unless', 'begin', + 'defined?', 'ensure', 'module', 'redo', 'super', 'until', + 'BEGIN', 'break', 'do', 'next', 'rescue', 'then', + 'when', 'END', 'case', 'else', 'for', 'retry', + 'while', 'alias', 'class', 'elsif', 'if', 'not', 'return', + 'undef', 'yield', + ] + + DEF_KEYWORDS = ['def'] + MODULE_KEYWORDS = ['class', 'module'] + DEF_NEW_STATE = WordList.new(:initial). + add(DEF_KEYWORDS, :def_expected). + add(MODULE_KEYWORDS, :module_expected) + + WORDS_ALLOWING_REGEXP = [ + 'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when' + ] + REGEXP_ALLOWED = WordList.new(false). + add(WORDS_ALLOWING_REGEXP, :set) + + PREDEFINED_CONSTANTS = [ + 'nil', 'true', 'false', 'self', + 'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__', + ] + + IDENT_KIND = WordList.new(:ident). + add(RESERVED_WORDS, :reserved). + add(PREDEFINED_CONSTANTS, :pre_constant) + + METHOD_NAME = / #{IDENT} [?!]? /xo + METHOD_NAME_EX = / + #{METHOD_NAME} # common methods: split, foo=, empty?, gsub! + | \*\*? # multiplication and power + | [-+~]@? # plus, minus + | [\/%&|^`] # division, modulo or format strings, &and, |or, ^xor, `system` + | \[\]=? # array getter and setter + | <=?>? | >=? # comparison, rocket operator + | << | >> # append or shift left, shift right + | ===? # simple equality and case equality + /ox + GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox + + DOUBLEQ = / " [^"\#\\]* (?: (?: \#\{.*?\} | \#(?:$")? | \\. ) [^"\#\\]* )* "? /ox + SINGLEQ = / ' [^'\\]* (?: \\. [^'\\]* )* '? /ox + STRING = / #{SINGLEQ} | #{DOUBLEQ} /ox + SHELL = / ` [^`\#\\]* (?: (?: \#\{.*?\} | \#(?:$`)? | \\. ) [^`\#\\]* )* `? /ox + REGEXP = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox + + DECIMAL = /\d+(?:_\d+)*/ # doesn't recognize 09 as octal error + OCTAL = /0_?[0-7]+(?:_[0-7]+)*/ + HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/ + BINARY = /0b[01]+(?:_[01]+)*/ + + EXPONENT = / [eE] [+-]? #{DECIMAL} /ox + FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) / + INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/ + + def reset + super + @regexp_allowed = false + end + + def next_token + return if @scanner.eos? + + kind = :error + if @scanner.scan(/\s+/) # in every state + kind = :space + @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n) # delayed flag setting + + elsif @state == :def_expected + if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox) + kind = :method + @state = :initial + else + @scanner.scan(/./) + kind = :error + end + @state = :initial + + elsif @state == :module_expected + if @scanner.scan(/<;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x) + kind = :operator + @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/ + elsif @scanner.scan(FLOAT) + kind = :float + elsif @scanner.scan(INTEGER) + kind = :integer + elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox) + kind = :global_variable + else + @scanner.scan(/./m) + end + end + + token = Token.new @scanner.matched, kind + + if kind == :regexp + token.text << @scanner.scan(/[eimnosux]*/) + end + + @regexp_allowed = (@regexp_allowed == :set) # delayed flag setting + + token + end + end + + ScannerList.register RubyScanner, 'ruby' + +end + +module CodeRay + require 'scanner' + + class Highlighter + + def initialize lang + @scanner = Scanner[lang].new + end + + def highlight code + @scanner.feed code + @scanner.all_tokens.map { |t| t.inspect }.join "\n" + end + + end + + class HTMLHighlighter < Highlighter + + ClassOfKind = { + :attribute_name => 'an', + :attribute_name_fat => 'af', + :attribute_value => 'av', + :attribute_value_fat => 'aw', + :bin => 'bi', + :char => 'ch', + :class => 'cl', + :class_variable => 'cv', + :color => 'cr', + :comment => 'c', + :constant => 'co', + :definition => 'df', + :directive => 'di', + :doc => 'do', + :doc_string => 'ds', + :exception => 'ex', + :error => 'er', + :float => 'fl', + :function => 'fu', + :global_variable => 'gv', + :hex => 'hx', + :include => 'ic', + :instance_variable => 'iv', + :integer => 'i', + :interpreted => 'in', + :label => 'la', + :local_variable => 'lv', + :oct => 'oc', + :operator_name => 'on', + :pre_constant => 'pc', + :pre_type => 'pt', + :predefined => 'pd', + :preprocessor => 'pp', + :regexp => 'rx', + :reserved => 'r', + :shell => 'sh', + :string => 's', + :symbol => 'sy', + :tag => 'ta', + :tag_fat => 'tf', + :tag_special => 'ts', + :type => 'ty', + :variable => 'v', + :xml_text => 'xt', + + :ident => :NO_HIGHLIGHT, + :operator => :NO_HIGHLIGHT, + :space => :NO_HIGHLIGHT, + } + ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function] + ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!' + + def initialize lang, options = {} + super lang + + @HTML_TAB = ' ' * options.fetch(:tabs2space, 8) + case level = options.fetch(:level, 'xhtml') + when 'html' + @HTML_BR = "
\n" + when 'xhtml' + @HTML_BR = "
\n" + else + raise "Unknown HTML level: #{level}" + end + end + + def highlight code + @scanner.feed code + + out = '' + while t = @scanner.next_token + warn t.inspect if t.text.nil? + out << to_html(t) + end + TEMPLATE =~ /<%CONTENT%>/ + $` + out + $' + end + + private + def to_html token + css_class = ClassOfKind[token.kind] + if defined? ::DEBUG and not ClassOfKind.has_key? token.kind + warn "no token class found for :#{token.kind}" + end + + text = text_to_html token.text + if css_class == :NO_HIGHLIGHT + text + else + "#{text}" + end + end + + def text_to_html text + return '' if text.empty? + text = text.dup # important + if text.index(/["><&]/) + text.gsub!('&', '&') + text.gsub!('"', '"') + text.gsub!('>', '>') + text.gsub!('<', '<') + end + if text.index(/\s/) + text.gsub!("\n", @HTML_BR) + text.gsub!("\t", @HTML_TAB) + text.gsub!(/^ /, ' ') + text.gsub!(' ', '  ') + end + text + end + + TEMPLATE = <<-'TEMPLATE' + + + + + + +RubyBB BBCode + + + +
+<%CONTENT%> +
+
+Valid HTML 4.01! +Valid CSS! +
+ + + TEMPLATE + + end + +end diff --git a/test/samples/html2.expected b/test/samples/html2.expected new file mode 100644 index 0000000..c8ae56a --- /dev/null +++ b/test/samples/html2.expected @@ -0,0 +1,185 @@ + + + + + CodeRay HTML Encoder Example + + + + + + + +
1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+
require 'coderay'
+
+# scan this file
+tokens = CodeRay.scan(File.read($0) * 1, :ruby)
+
+# output it with two styles of line numbers
+out = tokens.div(:line_numbers => :table)
+out << '<hr />'
+out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
+
+puts out.page(:title => 'CodeRay HTML Encoder Example')
+
+
+ 
 8 require 'coderay'
+ 9 
+10 # scan this file
+11 tokens = CodeRay.scan(File.read($0) * 1, :ruby)
+12 
+13 # output it with two styles of line numbers
+14 out = tokens.div(:line_numbers => :table)
+15 out << '<hr />'
+16 out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
+17 
+18 puts out.page(:title => 'CodeRay HTML Encoder Example')
+
+
+ + + diff --git a/test/samples/html2.rb b/test/samples/html2.rb new file mode 100644 index 0000000..618d168 --- /dev/null +++ b/test/samples/html2.rb @@ -0,0 +1,11 @@ +require 'coderay' + +# scan this file +tokens = CodeRay.scan(File.read($0) * 1, :ruby) + +# output it with two styles of line numbers +out = tokens.div(:line_numbers => :table) +out << '
' +out << tokens.div(:line_numbers => :inline, :line_number_start => 8) + +puts out.page(:title => 'CodeRay HTML Encoder Example') diff --git a/test/samples/html_list.expected b/test/samples/html_list.expected new file mode 100644 index 0000000..a4092c8 --- /dev/null +++ b/test/samples/html_list.expected @@ -0,0 +1,160 @@ + + + + + CodeRay HTML Encoder Example + + + + +
+ 
-1 $: << '..'
+ 0 require 'coderay'
+ 1 
+ 2 tokens = CodeRay.scan File.read(__FILE__), :ruby
+ 3 html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1)
+ 4 
+ 5 puts html.page(:title => 'CodeRay HTML Encoder Example')
+ 6 
+ 7 commment = <<_
+ 8 This code must be > 10 lines
+ 9 because I want to test the correct adjustment of the line numbers.
+10 _
+
+
+ + + diff --git a/test/samples/html_list.rb b/test/samples/html_list.rb new file mode 100644 index 0000000..fdfa512 --- /dev/null +++ b/test/samples/html_list.rb @@ -0,0 +1,12 @@ +$: << '..' +require 'coderay' + +tokens = CodeRay.scan File.read(__FILE__), :ruby +html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1) + +puts html.page(:title => 'CodeRay HTML Encoder Example') + +commment = <<_ +This code must be > 10 lines +because I want to test the correct adjustment of the line numbers. +_ diff --git a/test/samples/load_encoder.expected b/test/samples/load_encoder.expected new file mode 100644 index 0000000..1cff356 --- /dev/null +++ b/test/samples/load_encoder.expected @@ -0,0 +1,8 @@ +CodeRay::Encoders::YAML is not defined; you must load it first. +Now it is loaded: CodeRay::Encoders::YAML +See? +Require is also possible: CodeRay::Encoders::Tokens +See? +Now load some mapped encoders: stats and plain. +Require all Encoders: +[[:count, CodeRay::Encoders::Count], [:debug, CodeRay::Encoders::Debug], [:div, CodeRay::Encoders::Div], [:html, CodeRay::Encoders::HTML], [:null, CodeRay::Encoders::Null], [:page, CodeRay::Encoders::Page], [:plain, :text], [:span, CodeRay::Encoders::Span], [:statistic, CodeRay::Encoders::Statistic], [:stats, CodeRay::Encoders::Statistic], [:text, CodeRay::Encoders::Text], [:tokens, CodeRay::Encoders::Tokens], [:xml, CodeRay::Encoders::XML], [:yaml, CodeRay::Encoders::YAML]] diff --git a/test/samples/load_encoder.rb b/test/samples/load_encoder.rb new file mode 100644 index 0000000..9594bfa --- /dev/null +++ b/test/samples/load_encoder.rb @@ -0,0 +1,25 @@ +require 'coderay' + +begin + CodeRay::Encoders::YAML +rescue + puts 'CodeRay::Encoders::YAML is not defined; you must load it first.' +end + +yaml_encoder = CodeRay::Encoders[:yaml] +print 'Now it is loaded: ' +p yaml_encoder +puts 'See?' + +tokens_encoder = CodeRay.require_plugin 'CodeRay::Encoders/tokens' +print 'Require is also possible: ' +p tokens_encoder +puts 'See?' + +puts 'Now load some mapped encoders: stats and plain.' +CodeRay.require_plugin 'CodeRay::Encoders/stats' +CodeRay.require_plugin 'CodeRay::Encoders/plain' + +puts 'Require all Encoders:' +CodeRay::Encoders.load_all +p CodeRay::Encoders.plugin_hash.sort_by { |k,v| k.to_s } diff --git a/test/samples/load_scanner.expected b/test/samples/load_scanner.expected new file mode 100644 index 0000000..a2d200d --- /dev/null +++ b/test/samples/load_scanner.expected @@ -0,0 +1,8 @@ +CodeRay::Encoders::Ruby is not defined; you must load it first. +Now it is loaded: CodeRay::Scanners::Ruby +See? +Require is also possible: CodeRay::Scanners::C +See? +Now load some mapped scanners: cpp and plain. +Require all Scanners: +[[nil, :plain], [:c, CodeRay::Scanners::C], [:cpp, :c], [:delphi, CodeRay::Scanners::Delphi], [:html, CodeRay::Scanners::HTML], [:irb, :ruby], [:nitro, :nitro_xhtml], [:nitro_xhtml, CodeRay::Scanners::NitroXHTML], [:pascal, :delphi], [:plain, CodeRay::Scanners::Plaintext], [:plaintext, CodeRay::Scanners::Plaintext], [:rhtml, CodeRay::Scanners::RHTML], [:ruby, CodeRay::Scanners::Ruby], [:xhtml, :nitro_xhtml], [:xml, :html]] diff --git a/test/samples/load_scanner.rb b/test/samples/load_scanner.rb new file mode 100644 index 0000000..23be8a2 --- /dev/null +++ b/test/samples/load_scanner.rb @@ -0,0 +1,25 @@ +require 'coderay' + +begin + CodeRay::Scanners::Ruby +rescue + puts 'CodeRay::Encoders::Ruby is not defined; you must load it first.' +end + +ruby_scanner = CodeRay::Scanners[:ruby] +print 'Now it is loaded: ' +p ruby_scanner +puts 'See?' + +c_scanner = CodeRay.require_plugin 'CodeRay::Scanners/c' +print 'Require is also possible: ' +p c_scanner +puts 'See?' + +puts 'Now load some mapped scanners: cpp and plain.' +CodeRay.require_plugin 'CodeRay::Scanners/cpp' +CodeRay.require_plugin 'CodeRay::Scanners/plain' + +puts 'Require all Scanners:' +CodeRay::Scanners.load_all +p CodeRay::Scanners.plugin_hash.sort_by { |k,v| k.to_s } diff --git a/test/samples/more.expected b/test/samples/more.expected new file mode 100644 index 0000000..196904d --- /dev/null +++ b/test/samples/more.expected @@ -0,0 +1,2 @@ +Input: 4983B, Output: 23484B +Take a look with your browser. diff --git a/test/samples/more.rb b/test/samples/more.rb new file mode 100644 index 0000000..0db7ba4 --- /dev/null +++ b/test/samples/more.rb @@ -0,0 +1,205 @@ +require 'coderay' + +c, ruby = DATA.read.split(/^---$/) +DATA.rewind +me = DATA.read[/.*^__END__$/m] +$input = c + ruby + me + +require 'benchmark' +time = Benchmark.realtime do + + # here CodeRay comes to play + hl = CodeRay.encoder(:html, :tab_width => 2, :line_numbers => :table, :wrap => :div) + c = hl.highlight c, :c + ruby = hl.highlight ruby, :ruby + me = hl.highlight me, :ruby + + body = %w[C Ruby Genereated\ by].zip([c, ruby, me]).map do |title, code| + "

#{title}

\n#{code}" + end.join + body = hl.class::Output.new(body, hl.css, :div).page! + + # CodeRay also provides a simple page generator + $output = body #hl.class.wrap_in_page body +end + +File.open('test.html', 'w') do |f| + f.write $output +end +puts 'Input: %dB, Output: %dB' % [$input.size, $output.size] +#puts 'Created "test.html" in %0.3f seconds (%d KB/s).' % [time, $input.size / 1024.0 / time] +puts 'Take a look with your browser.' + +__END__ +/********************************************************************** + + version.c - + + $Author: nobu $ + $Date: 2004/03/25 12:01:40 $ + created at: Thu Sep 30 20:08:01 JST 1993 + + Copyright (C) 1993-2003 Yukihiro Matsumoto + +**********************************************************************/ + +#include "ruby.h" +#include "version.h" +#include + +const char ruby_version[] = RUBY_VERSION; +const char ruby_release_date[] = RUBY_RELEASE_DATE; +const char ruby_platform[] = RUBY_PLATFORM; + +void +Init_version() +{ + VALUE v = rb_obj_freeze(rb_str_new2(ruby_version)); + VALUE d = rb_obj_freeze(rb_str_new2(ruby_release_date)); + VALUE p = rb_obj_freeze(rb_str_new2(ruby_platform)); + + rb_define_global_const("RUBY_VERSION", v); + rb_define_global_const("RUBY_RELEASE_DATE", d); + rb_define_global_const("RUBY_PLATFORM", p); +} + +void +ruby_show_version() +{ + printf("ruby %s (%s) [%s]\n", RUBY_VERSION, RUBY_RELEASE_DATE, RUBY_PLATFORM); +} + +void +ruby_show_copyright() +{ + printf("ruby - Copyright (C) 1993-%d Yukihiro Matsumoto\n", RUBY_RELEASE_YEAR); + exit(0); +} +--- +# +# = ostruct.rb: OpenStruct implementation +# +# Author:: Yukihiro Matsumoto +# Documentation:: Gavin Sinclair +# +# OpenStruct allows the creation of data objects with arbitrary attributes. +# See OpenStruct for an example. +# + +# +# OpenStruct allows you to create data objects and set arbitrary attributes. +# For example: +# +# require 'ostruct' +# +# record = OpenStruct.new +# record.name = "John Smith" +# record.age = 70 +# record.pension = 300 +# +# puts record.name # -> "John Smith" +# puts record.address # -> nil +# +# It is like a hash with a different way to access the data. In fact, it is +# implemented with a hash, and you can initialize it with one. +# +# hash = { "country" => "Australia", :population => 20_000_000 } +# data = OpenStruct.new(hash) +# +# p data # -> +# +class OpenStruct + # + # Create a new OpenStruct object. The optional +hash+, if given, will + # generate attributes and values. For example. + # + # require 'ostruct' + # hash = { "country" => "Australia", :population => 20_000_000 } + # data = OpenStruct.new(hash) + # + # p data # -> + # + # By default, the resulting OpenStruct object will have no attributes. + # + def initialize(hash=nil) + @table = {} + if hash + for k,v in hash + @table[k.to_sym] = v + new_ostruct_member(k) + end + end + end + + # Duplicate an OpenStruct object members. + def initialize_copy(orig) + super + @table = @table.dup + end + + def marshal_dump + @table + end + def marshal_load(x) + @table = x + @table.each_key{|key| new_ostruct_member(key)} + end + + def new_ostruct_member(name) + unless self.respond_to?(name) + self.instance_eval %{ + def #{name}; @table[:#{name}]; end + def #{name}=(x); @table[:#{name}] = x; end + } + end + end + + def method_missing(mid, *args) # :nodoc: + mname = mid.id2name + len = args.length + if mname =~ /=$/ + if len != 1 + raise ArgumentError, "wrong number of arguments (#{len} for 1)", caller(1) + end + if self.frozen? + raise TypeError, "can't modify frozen #{self.class}", caller(1) + end + mname.chop! + @table[mname.intern] = args[0] + self.new_ostruct_member(mname) + elsif len == 0 + @table[mid] + else + raise NoMethodError, "undefined method `#{mname}' for #{self}", caller(1) + end + end + + # + # Remove the named field from the object. + # + def delete_field(name) + @table.delete name.to_sym + end + + # + # Returns a string containing a detailed summary of the keys and values. + # + def inspect + str = "<#{self.class}" + for k,v in @table + str << " #{k}=#{v.inspect}" + end + str << ">" + end + + attr_reader :table # :nodoc: + protected :table + + # + # Compare this object and +other+ for equality. + # + def ==(other) + return false unless(other.kind_of?(OpenStruct)) + return @table == other.table + end +end diff --git a/test/samples/scanner.expected b/test/samples/scanner.expected new file mode 100644 index 0000000..5015168 --- /dev/null +++ b/test/samples/scanner.expected @@ -0,0 +1,16 @@ +C Code: if (*p == '{') nest++; + +> print only operators: +(*==)++; +------------------------------ + +Ruby Code: ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee) + +> has a string? +false + +> number of regexps? +2 + +> has a string? +"ruby_code" (ident), "(" (operator), ":can" (symbol), "," (operator), " " (space), "BE" (constant), "," (operator), " " (space), "%r[" (delimiter), "q" (content), "[" (nesting_delimiter), "ui" (content), "]" (nesting_delimiter), "te " (content), "#{" (inline_delimiter), " " (space), "/" (delimiter), "comple" (content), "/" (delimiter), "x" (modifier), " " (space), "}" (inline_delimiter), "," (content), "]" (delimiter), " " (space), "=" (operator), ">" (operator), " " (space), "$-s" (global_variable), "," (operator), " " (space), "&" (operator), "?\xee" (integer), ")" (operator) diff --git a/test/samples/scanner.rb b/test/samples/scanner.rb new file mode 100644 index 0000000..6a0245e --- /dev/null +++ b/test/samples/scanner.rb @@ -0,0 +1,36 @@ +require 'coderay' + +c_code = "if (*p == '{') nest++;" +puts 'C Code: ' + c_code +puts + +c_scanner = CodeRay::Scanners[:c].new c_code + +puts '> print only operators:' +for text, kind in c_scanner + print text if kind == :operator +end +puts +puts '-' * 30 +puts + +ruby_code = %q!ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee)! +puts 'Ruby Code: ' + ruby_code +puts + +ruby_scanner = CodeRay::Scanners[:ruby].new ruby_code + +puts '> has a string?' +puts ruby_scanner. + any? { |text, kind| kind == :string } +puts + +puts '> number of regexps?' +puts ruby_scanner. + select { |token| token == [:open, :regexp] }.size +puts + +puts '> has a string?' +puts ruby_scanner. + reject { |text, kind| not text.is_a? String }. + map { |text, kind| %("#{text}" (#{kind})) }.join(', ') diff --git a/test/samples/server.rb b/test/samples/server.rb new file mode 100644 index 0000000..ccdff32 --- /dev/null +++ b/test/samples/server.rb @@ -0,0 +1,110 @@ +# CodeRay dynamic highlighter + +unless ARGV.grep(/-[hv]|--(help|version)/).empty? + puts <<-USAGE +CodeRay Server 0.5 +$Id: demo_server.rb 113 2006-03-15 23:24:37Z murphy $ + +Usage: + 1) Start this and your browser. + 2) Go to http://localhost:2468/? + and you should get the highlighted version. + +Parameters: + -d Debug mode; reload CodeRay engine for every file. + (prepare for MANY "already initialized" and "method redefined" + messages - ingore it.) + + ... More to come. + USAGE + exit +end + +require 'webrick' +require 'pathname' + +class << File + alias dir? directory? +end + +require 'erb' +include ERB::Util +def url_decode s + s.to_s.gsub(/%([0-9a-f]{2})/i) { [$1.hex].pack 'C' } +end + +class String + def to_link name = File.basename(self) + "#{name}" + end +end + +require 'coderay' +class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet + + STYLE = 'style="font-family: sans-serif; color: navy;"' + BANNER = '

Highlighted by CodeRay

' + + def do_GET req, res + q = req.query_string || '' + args = Hash[*q.scan(/(.*?)=(.*?)(?:&|$)/).flatten].each_value { |v| v.replace url_decode(v) } + path = args.fetch 'path', '.' + + backlinks = '

current path: %s
' % html_escape(path) + + (Pathname.new(path) + '..').cleanpath.to_s.to_link('up') + ' - ' + + '.'.to_link('current') + '

' + + res.body = + if File.dir? path + path = Pathname.new(path).cleanpath.to_s + dirs, files = Dir[File.join(path, '*')].sort.partition { |p| File.dir? p } + + page = "" + page << backlinks + + page << '
' + page << "
Directories
\n" + dirs.map do |p| + "
#{p.to_link}
\n" + end.join << "\n" + page << "
Files
\n" + files.map do |p| + "
#{p.to_link}
\n" + end.join << "\n" + page << "
\n" + page << "#{BANNER}" + + elsif File.exist? path + if $DEBUG + $".delete_if { |f| f =~ /coderay/ } + require 'coderay' + end + div = CodeRay.scan_file(path).html :tab_width => 8, :wrap => :div, :hint => :info + div.replace <<-DIV +
+ #{backlinks} +#{div} +
+ #{BANNER} + DIV + div.page + end + + res['Content-Type'] = 'text/html' + end +end + +# This port is taken by "qip_msgd" - I don't know that. Do you? +module CodeRay + PORT = 0xC0DE / 20 +end + +server = WEBrick::HTTPServer.new :Port => CodeRay::PORT + +server.mount '/', CodeRayServlet + +server.mount_proc '/version' do |req, res| + res.body = 'CodeRay::Version = ' + CodeRay::Version + res['Content-Type'] = "text/plain" +end + +trap("INT") { server.shutdown } +server.start diff --git a/test/samples/simple.expected b/test/samples/simple.expected new file mode 100644 index 0000000..b3d7875 --- /dev/null +++ b/test/samples/simple.expected @@ -0,0 +1 @@ +puts 'Hello, world!' diff --git a/test/samples/simple.rb b/test/samples/simple.rb new file mode 100644 index 0000000..a3129b0 --- /dev/null +++ b/test/samples/simple.rb @@ -0,0 +1,10 @@ + +# Load CodeRay +# If this doesn't work, try ruby -rubygems. +require 'coderay' + +# Generate HTML page for Ruby code. +page = CodeRay.scan("puts 'Hello, world!'", :ruby).span + +# Print it +puts page diff --git a/test/samples/stream.rb b/test/samples/stream.rb new file mode 100644 index 0000000..7ed8a22 --- /dev/null +++ b/test/samples/stream.rb @@ -0,0 +1,25 @@ +require 'coderay' + +code = File.read($0) * 500 +puts "Size of code: %d KB" % [code.size / 1024] + +puts "Use your system's memory tracker to see how much RAM this takes." +print 'Press some key to continue...'; gets + +require 'benchmark' +e = CodeRay.encoder(:div) +for do_stream in [true, false] + puts "Scanning and encoding in %s mode, please wait..." % + [do_stream ? 'streaming' : 'normal'] + output = '' + time = Benchmark.realtime do + if do_stream + output = e.encode_stream(code, :ruby) + else + output = e.encode_tokens(t = CodeRay.scan(code, :ruby)) + end + end + puts 'Finished after %4.2f seconds.' % time + puts "Size of output: %d KB" % [output.size / 1024] + print 'Press some key to continue...'; gets +end diff --git a/test/samples/stream2.expected b/test/samples/stream2.expected new file mode 100644 index 0000000..83aee98 --- /dev/null +++ b/test/samples/stream2.expected @@ -0,0 +1,2 @@ +kind: regexp, text size: 5. +kind: space, text size: 1. diff --git a/test/samples/stream2.rb b/test/samples/stream2.rb new file mode 100644 index 0000000..d43cc9a --- /dev/null +++ b/test/samples/stream2.rb @@ -0,0 +1,8 @@ +require 'coderay' + +token_stream = CodeRay::TokenStream.new do |kind, text| + puts 'kind: %s, text size: %d.' % [kind, text.size] +end + +token_stream << [:regexp, '/\d+/'] << [:space, "\n"] +#-> kind: rexpexp, text size: 5. diff --git a/test/samples/suite.rb b/test/samples/suite.rb new file mode 100644 index 0000000..cfe53c0 --- /dev/null +++ b/test/samples/suite.rb @@ -0,0 +1,86 @@ +mydir = File.dirname(__FILE__) +$:.unshift mydir + '/../../lib/' + +$VERBOSE = true + +require 'test/unit' +include Test::Unit + +class CodeRaySuite < TestCase + + def self.dir &block + @dir ||= File.dirname(__FILE__) + if block + Dir.chdir @dir, &block + end + @dir + end + + def dir &block + self.class.dir(&block) + end + + def test_ALL + dir do + for input in Dir["*.rb"] - %w(server.rb stream.rb suite.rb) + next if input[/^load_/] + puts "[ testing #{input}... ]" + name = File.basename(input, ".rb") + output = name + '.expected' + code = File.open(input, 'rb') { |f| break f.read } + + result = `ruby -wI../../lib #{input}` + + diff = output.sub '.expected', '.diff' + File.delete diff if File.exist? diff + computed = output.sub '.expected', '.actual' + if File.exist? output + expected = File.read output + ok = expected == result + unless ok + File.open(computed, 'w') { |f| f.write result } + `diff #{output} #{computed} > #{diff}` + puts "Test failed; output written to #{diff}." + end + assert(ok, "Output error: #{computed} != #{output}") + else + File.open(output, 'w') do |f| f.write result end + puts "New test: #{output}" + end + + end + end + end + +end + +require 'test/unit/testsuite' +$suite = TestSuite.new 'CodeRay Demos Test' +$suite << CodeRaySuite.suite + +def load_suite name + begin + require name + '/suite.rb' + rescue LoadError + $stderr.puts <<-ERR + +!! Folder #{File.split(__FILE__).first + '/' + name} not found + + ERR + false + end +end + +if subsuite = ARGV.find { |a| break $1 if a[/^([^-].*)/] } + load_suite(subsuite) or exit +else + Dir[mydir + '/*/'].each { |suite| load_suite suite } +end + +if ARGV.include? '-f' + require 'test/unit/ui/fox/testrunner' + UI::Fox::TestRunner.run $suite +else + require 'test/unit/ui/console/testrunner' + UI::Console::TestRunner.run $suite +end diff --git a/test/samples/tokens.expected b/test/samples/tokens.expected new file mode 100644 index 0000000..747904e --- /dev/null +++ b/test/samples/tokens.expected @@ -0,0 +1 @@ +[["puts", :ident], [" ", :space], ["3", :integer], [" ", :space], ["+", :operator], [" ", :space], ["4", :integer], [",", :operator], [" ", :space], [:open, :string], ["'", :delimiter], ["3 + 4", :content], ["'", :delimiter], [:close, :string]] diff --git a/test/samples/tokens.rb b/test/samples/tokens.rb new file mode 100644 index 0000000..91b8abb --- /dev/null +++ b/test/samples/tokens.rb @@ -0,0 +1,3 @@ +require 'coderay' + +p CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby) -- cgit v1.2.1 From 8e67efef5b412b16d755fb47538e24a2b6bafce0 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 05:31:14 +0200 Subject: cleanup rake tasks --- rake_tasks/bundler.rake | 6 --- rake_tasks/diff.rake | 93 ------------------------------------------- rake_tasks/documentation.rake | 9 ----- rake_tasks/ruby-versions.rake | 10 ----- 4 files changed, 118 deletions(-) delete mode 100644 rake_tasks/bundler.rake delete mode 100644 rake_tasks/diff.rake delete mode 100644 rake_tasks/ruby-versions.rake diff --git a/rake_tasks/bundler.rake b/rake_tasks/bundler.rake deleted file mode 100644 index 8de149d..0000000 --- a/rake_tasks/bundler.rake +++ /dev/null @@ -1,6 +0,0 @@ -begin - require 'bundler' - Bundler::GemHelper.install_tasks -rescue LoadError - puts 'Please gem install bundler.' -end diff --git a/rake_tasks/diff.rake b/rake_tasks/diff.rake deleted file mode 100644 index f0af55a..0000000 --- a/rake_tasks/diff.rake +++ /dev/null @@ -1,93 +0,0 @@ -# A simple differ using svn. Handles externals. -class Differ < Hash - - include Rake::DSL if defined? Rake::DSL - - def initialize path - @path = path - super 0 - end - - def count key, value - self[key] += value - value - end - - FORMAT = ' %-30s %8d lines, %3d changes in %2d files' - - def scan(path = @path) - Dir.chdir path do - diff_file_name = 'diff' - if File.directory? 'diff' - diff_file_name = 'diff.diff' - end - system "svn diff > #{diff_file_name}" - if File.size? diff_file_name - puts FORMAT % - [ - path, - count(:LOC, `wc -l #{diff_file_name}`.to_i), - count(:changes, `grep ^@@ #{diff_file_name} | wc -l`.to_i), - count(:files, `grep ^Index #{diff_file_name} | wc -l`.to_i), - ] - else - rm diff_file_name - end - end - end - - def scan_with_externals(path = @path) - scan path - `svn status`.scan(/^X\s*(.*)/) do |external,| - scan external - end - end - - def clean(path = @path) - Dir.chdir path do - rm 'diff' if File.file? 'diff' - rm 'diff.diff' if File.file? 'diff.diff' - end - end - - def clean_with_externals(path = @path) - clean path - `svn status`.scan(/^X\s*(.*)/) do |external,| - clean external - end - end - - def differences? - self[:LOC] > 0 - end - - def inspect - FORMAT % - [ 'Total', self[:LOC], self[:changes], self[:files] ] - end - -end - -namespace :diff do - - desc 'Make a diff and print a summary' - task :summary do - differ = Differ.new '.' - differ.scan_with_externals - if differ.empty? - puts 'No differences found.' - else - p differ - end - end - - desc 'Remove all diffs' - task :clean do - differ = Differ.new '.' - differ.clean_with_externals - end - -end - -desc 'Make a diff and print a summary' -task :diff => 'diff:summary' diff --git a/rake_tasks/documentation.rake b/rake_tasks/documentation.rake index d555022..4f7cef7 100644 --- a/rake_tasks/documentation.rake +++ b/rake_tasks/documentation.rake @@ -15,18 +15,9 @@ Rake::RDocTask.new :doc do |rd| rd.title = 'CodeRay Documentation' rd.options << '--line-numbers' << '--tab-width' << '2' - # rd.options << '--fmt' << ENV.fetch('format', 'html_coderay') - # require 'pathname' - # template = File.join ROOT, 'rake_helpers', 'coderay_rdoc_template.rb' - # rd.template = Pathname.new(template).expand_path.to_s rd.main = 'README_INDEX.rdoc' rd.rdoc_files.add 'README_INDEX.rdoc' rd.rdoc_files.add Dir['lib'] rd.rdoc_dir = 'doc' end if defined? Rake::RDocTask - -desc 'Copy the documentation over to the CodeRay website' -task :copy_doc do - cp_r 'doc/.', '../../rails/coderay/public/doc' -end diff --git a/rake_tasks/ruby-versions.rake b/rake_tasks/ruby-versions.rake deleted file mode 100644 index af408ff..0000000 --- a/rake_tasks/ruby-versions.rake +++ /dev/null @@ -1,10 +0,0 @@ -task 'ruby:version' do - puts - if defined? RUBY_DESCRIPTION - ruby_version = RUBY_DESCRIPTION - else - ruby_version = "ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE} patchlevel #{RUBY_PATCHLEVEL}) [#{RUBY_PLATFORM}]" - end - require './test/lib/term/ansicolor' - puts Term::ANSIColor.bold(Term::ANSIColor.green(ruby_version)) -end \ No newline at end of file -- cgit v1.2.1 From e423275749744fb2cf768087a07b4c839eaf4734 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 13 Jun 2013 05:33:14 +0200 Subject: remove dump/undump functionality --- bench/bench.rb | 54 +++++++++++-------------------------------- lib/coderay/encoders/debug.rb | 1 - lib/coderay/helpers/gzip.rb | 41 -------------------------------- lib/coderay/tokens.rb | 48 +------------------------------------- test/unit/tokens.rb | 9 -------- 5 files changed, 15 insertions(+), 138 deletions(-) delete mode 100644 lib/coderay/helpers/gzip.rb diff --git a/bench/bench.rb b/bench/bench.rb index 1889eed..1958c73 100644 --- a/bench/bench.rb +++ b/bench/bench.rb @@ -14,11 +14,10 @@ require 'coderay' lang = ARGV.fetch(0) do puts <<-HELP Usage: - ruby bench.rb (c|ruby|dump) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream] + ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream] SIZE defaults to 100 kB (= 100,000 bytes). SIZE = 0 means the whole input. - SIZE is ignored when dump is input. -p generates a profile (slow! use with SIZE = 1) -o shows the output @@ -48,10 +47,6 @@ if format == 'comp' end end -$dump_input = lang == 'dump' -$dump_output = format == 'dump' -require 'coderay/helpers/gzip_simple.rb' if $dump_input - def here fn = nil return MYDIR unless fn File.join here, fn @@ -66,59 +61,38 @@ N.times do data = nil File.open(here("#$filename." + lang), 'rb') { |f| data = f.read } - if $dump_input - @size = CodeRay::Tokens.load(data).text.size - else - raise 'Example file is empty.' if data.empty? - unless @size.zero? - data += data until data.size >= @size - data = data[0, @size] - end - @size = data.size + raise 'Example file is empty.' if data.empty? + unless @size.zero? + data += data until data.size >= @size + data = data[0, @size] end - + @size = data.size + options = { :tab_width => 2, # :line_numbers => :inline, :css => $style ? :style : :class, } - $hl = CodeRay.encoder(format, options) unless $dump_output + $hl = CodeRay.encoder(format, options) time = bm.report('CodeRay') do if $stream || true - if $dump_input - raise 'Can\'t stream dump.' - elsif $dump_output - raise 'Can\'t dump stream.' - end $o = $hl.encode(data, lang, options) else - if $dump_input - tokens = CodeRay::Tokens.load data - else - tokens = CodeRay.scan(data, lang) - end + tokens = CodeRay.scan(data, lang) tokens.optimize! if $optimize - if $dump_output - $o = tokens.optimize.dump - else - $o = tokens.encode($hl) - end + $o = tokens.encode($hl) end end - $file_created = here('test.' + - ($dump_output ? 'dump' : $hl.file_extension)) + $file_created = here('test.' + $hl.file_extension) File.open($file_created, 'wb') do |f| # f.write $o end - Dir.chdir(here) do - FileUtils.copy 'test.dump', 'example.dump' if $dump_output - end - + time_real = time.real - + puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000] puts $o if ARGV.include? '-o' - + end end puts "Files created: #$file_created" diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index 61520a1..f4db330 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -9,7 +9,6 @@ module Encoders # # You cannot fully restore the tokens information from the # output, because consecutive :space tokens are merged. - # Use Tokens#dump for caching purposes. # # See also: Scanners::Debug class Debug < Encoder diff --git a/lib/coderay/helpers/gzip.rb b/lib/coderay/helpers/gzip.rb deleted file mode 100644 index 245014a..0000000 --- a/lib/coderay/helpers/gzip.rb +++ /dev/null @@ -1,41 +0,0 @@ -module CodeRay - - # A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.) - module GZip - - require 'zlib' - - # The default zipping level. 7 zips good and fast. - DEFAULT_GZIP_LEVEL = 7 - - # Unzips the given string +s+. - # - # Example: - # require 'gzip_simple' - # print GZip.gunzip(File.read('adresses.gz')) - def GZip.gunzip s - Zlib::Inflate.inflate s - end - - # Zips the given string +s+. - # - # Example: - # require 'gzip_simple' - # File.open('adresses.gz', 'w') do |file - # file.write GZip.gzip('Mum: 0123 456 789', 9) - # end - # - # If you provide a +level+, you can control how strong - # the string is compressed: - # - 0: no compression, only convert to gzip format - # - 1: compress fast - # - 7: compress more, but still fast (default) - # - 8: compress more, slower - # - 9: compress best, very slow - def GZip.gzip s, level = DEFAULT_GZIP_LEVEL - Zlib::Deflate.new(level).deflate s, Zlib::FINISH - end - - end - -end diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 6957d69..9142b76 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -1,8 +1,5 @@ module CodeRay - # GZip library for writing and reading token dumps. - autoload :GZip, coderay_path('helpers', 'gzip') - # = Tokens TODO: Rewrite! # # The Tokens class represents a list of tokens returnd from @@ -45,8 +42,7 @@ module CodeRay # See how small it is? ;) # # Tokens gives you the power to handle pre-scanned code very easily: - # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string - # that you put in your DB. + # You can convert it to a webpage, a YAML file, or a .raydebug representation. # # It also allows you to generate tokens directly (without using a scanner), # to load them from a file, and still use any Encoder that CodeRay provides. @@ -157,53 +153,11 @@ module CodeRay parts end - # Dumps the object into a String that can be saved - # in files or databases. - # - # The dump is created with Marshal.dump; - # In addition, it is gzipped using GZip.gzip. - # - # The returned String object includes Undumping - # so it has an #undump method. See Tokens.load. - # - # You can configure the level of compression, - # but the default value 7 should be what you want - # in most cases as it is a good compromise between - # speed and compression rate. - # - # See GZip module. - def dump gzip_level = 7 - dump = Marshal.dump self - dump = GZip.gzip dump, gzip_level - dump.extend Undumping - end - # Return the actual number of tokens. def count size / 2 end - # Include this module to give an object an #undump - # method. - # - # The string returned by Tokens.dump includes Undumping. - module Undumping - # Calls Tokens.load with itself. - def undump - Tokens.load self - end - end - - # Undump the object using Marshal.load, then - # unzip it using GZip.gunzip. - # - # The result is commonly a Tokens object, but - # this is not guaranteed. - def Tokens.load dump - dump = GZip.gunzip dump - @dump = Marshal.load dump - end - alias text_token push def begin_group kind; push :begin_group, kind end def end_group kind; push :end_group, kind end diff --git a/test/unit/tokens.rb b/test/unit/tokens.rb index 86dc632..73b0fd5 100644 --- a/test/unit/tokens.rb +++ b/test/unit/tokens.rb @@ -18,15 +18,6 @@ class TokensTest < Test::Unit::TestCase assert_equal tokens.count, 4 end - def test_dump_undump - tokens = make_tokens - tokens2 = nil - assert_nothing_raised do - tokens2 = tokens.dump.undump - end - assert_equal tokens, tokens2 - end - def test_to_s assert_equal 'string()', make_tokens.to_s end -- cgit v1.2.1 From cb41b00d5673312f4982e914883f9cea95f2ccae Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 16 Jun 2013 01:01:12 +0200 Subject: cleanup TODOs, FIXMEs --- bin/coderay | 5 +++-- lib/coderay.rb | 1 - lib/coderay/encoders/html/numbering.rb | 2 +- lib/coderay/encoders/statistic.rb | 1 - lib/coderay/token_kinds.rb | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/bin/coderay b/bin/coderay index 9aecb88..889ae72 100755 --- a/bin/coderay +++ b/bin/coderay @@ -155,8 +155,9 @@ when 'highlight', nil puts boom.message end # puts "I don't know this plugin: #{boom.message[/Could not load plugin (.*?): /, 1]}." - rescue CodeRay::Scanners::Scanner::ScanError # FIXME: rescue Errno::EPIPE - # this is sometimes raised by pagers; ignore [TODO: wtf?] + rescue CodeRay::Scanners::Scanner::ScanError + # this is sometimes raised by pagers; ignore + # FIXME: rescue Errno::EPIPE ensure file.close if output_file end diff --git a/lib/coderay.rb b/lib/coderay.rb index 88c7cc2..24ae5a2 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -166,7 +166,6 @@ module CodeRay # # See also demo/demo_simple. def scan code, lang, options = {}, &block - # FIXME: return a proxy for direct-stream encoding TokensProxy.new code, lang, options, block end diff --git a/lib/coderay/encoders/html/numbering.rb b/lib/coderay/encoders/html/numbering.rb index 332145b..5908bf0 100644 --- a/lib/coderay/encoders/html/numbering.rb +++ b/lib/coderay/encoders/html/numbering.rb @@ -75,7 +75,7 @@ module Encoders line_number = start output.gsub!(/^.*$\n?/) do |line| line_number_text = bolding.call line_number - indent = ' ' * (max_width - line_number.to_s.size) # TODO: Optimize (10^x) + indent = ' ' * (max_width - line_number.to_s.size) line_number += 1 "#{indent}#{line_number_text}#{line}" end diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb index 2315d9e..b2f8b83 100644 --- a/lib/coderay/encoders/statistic.rb +++ b/lib/coderay/encoders/statistic.rb @@ -67,7 +67,6 @@ Token Types (%d): @type_stats['TOTAL'].count += 1 end - # TODO Hierarchy handling def begin_group kind block_token ':begin_group', kind end diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 42ea427..9154658 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -44,7 +44,7 @@ module CodeRay :important => 'important', # CSS, Taskpaper :include => 'include', # C, Groovy, Java, Python, Sass :inline => 'inline', # nested code, eg. inline string evaluation; lots of scanners - :inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why? + :inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why use inline_delimiter? :instance_variable => 'instance-variable', # Ruby :integer => 'integer', # most scanners :key => 'key', # lots of scanners, used together with :value -- cgit v1.2.1 From 462302814dc56c33f26f43516d5ddfc345dcd2a3 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 16 Jun 2013 01:01:30 +0200 Subject: cleanup Tokens documentation --- lib/coderay/tokens.rb | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 6957d69..ad59b7f 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -3,25 +3,23 @@ module CodeRay # GZip library for writing and reading token dumps. autoload :GZip, coderay_path('helpers', 'gzip') - # = Tokens TODO: Rewrite! + # The Tokens class represents a list of tokens returned from + # a Scanner. It's actually just an Array with a few helper methods. # - # The Tokens class represents a list of tokens returnd from - # a Scanner. - # - # A token is not a special object, just a two-element Array + # A token itself is not a special object, just a two-element Array # consisting of # * the _token_ _text_ (the original source of the token in a String) or # a _token_ _action_ (begin_group, end_group, begin_line, end_line) # * the _token_ _kind_ (a Symbol representing the type of the token) # - # A token looks like this: + # It looks like this: # # ['# It looks like this', :comment] # ['3.1415926', :float] # ['$^', :error] # # Some scanners also yield sub-tokens, represented by special - # token actions, namely begin_group and end_group. + # token actions, for example :begin_group and :end_group. # # The Ruby scanner, for example, splits "a string" into: # @@ -33,23 +31,17 @@ module CodeRay # [:end_group, :string] # ] # - # Tokens is the interface between Scanners and Encoders: - # The input is split and saved into a Tokens object. The Encoder - # then builds the output from this object. - # - # Thus, the syntax below becomes clear: - # - # CodeRay.scan('price = 2.59', :ruby).html - # # the Tokens object is here -------^ + # Tokens can be used to save the output of a Scanners in a simple + # Ruby object that can be send to an Encoder later: # - # See how small it is? ;) + # tokens = CodeRay.scan('price = 2.59', :ruby).tokens + # tokens.encode(:html) + # tokens.html + # CodeRay.encoder(:html).encode_tokens(tokens) # # Tokens gives you the power to handle pre-scanned code very easily: - # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string - # that you put in your DB. - # - # It also allows you to generate tokens directly (without using a scanner), - # to load them from a file, and still use any Encoder that CodeRay provides. + # You can serialize it to a JSON string and store it in a database, pass it + # around to encode it more than once, send it to other algorithms... class Tokens < Array # The Scanner instance that created the tokens. @@ -58,8 +50,7 @@ module CodeRay # Encode the tokens using encoder. # # encoder can be - # * a symbol like :html oder :statistic - # * an Encoder class + # * a plugin name like :html oder 'statistic' # * an Encoder object # # options are passed to the encoder. -- cgit v1.2.1 From dc9528456008216c265cfb6613cfa459c3c9ffad Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 16 Jun 2013 01:01:48 +0200 Subject: speedup HTML encoder numbering for Ruby 1.8.7+ --- lib/coderay/encoders/html/numbering.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/encoders/html/numbering.rb b/lib/coderay/encoders/html/numbering.rb index 5908bf0..a1b9c04 100644 --- a/lib/coderay/encoders/html/numbering.rb +++ b/lib/coderay/encoders/html/numbering.rb @@ -26,7 +26,7 @@ module Encoders "#{line}" end else - proc { |line| line.to_s } # :to_s.to_proc in Ruby 1.8.7+ + :to_s.to_proc end bold_every = options[:bold_every] -- cgit v1.2.1 From 927cc5b5c56017ee93bee208307bfa67cf90c9d7 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 16 Jun 2013 01:30:04 +0200 Subject: changelog --- Changes.textile | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes.textile b/Changes.textile index 15b0cc6..fcee29b 100644 --- a/Changes.textile +++ b/Changes.textile @@ -12,6 +12,7 @@ h2. Changes in 1.1 * Remove double-click toggle handler from HTML table output * Fixes to CSS scanner (floats, pseudoclasses) * CSS scanner uses @:id@ and @:tag@ now [#27] +* Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right? * Add .xaml file type [#121, thanks to Kozman Bálint] * @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner] * New token type @:id@ for CSS/Sass [#27] -- cgit v1.2.1 From fc21c91d7a89d98785e494ffd41e9f3adbdfed81 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Wed, 19 Jun 2013 11:50:04 +0200 Subject: use different key/value heuristic in JSON scanner --- lib/coderay/scanners/json.rb | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb index 4e0f462..3754a9b 100644 --- a/lib/coderay/scanners/json.rb +++ b/lib/coderay/scanners/json.rb @@ -14,15 +14,17 @@ module Scanners ESCAPE = / [bfnrt\\"\/] /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # :nodoc: + KEY = / (?> (?: [^\\"]+ | \\. )* ) " \s* : /x protected + def setup + @state = :initial + end + # See http://json.org/ for a definition of the JSON lexic/grammar. def scan_tokens encoder, options - - state = :initial - stack = [] - key_expected = false + state = options[:state] || @state until eos? @@ -32,18 +34,11 @@ module Scanners if match = scan(/ \s+ /x) encoder.text_token match, :space elsif match = scan(/"/) - state = key_expected ? :key : :string + state = check(/#{KEY}/o) ? :key : :string encoder.begin_group state encoder.text_token match, :delimiter elsif match = scan(/ [:,\[{\]}] /x) encoder.text_token match, :operator - case match - when ':' then key_expected = false - when ',' then key_expected = true if stack.last == :object - when '{' then stack << :object; key_expected = true - when '[' then stack << :array - when '}', ']' then stack.pop # no error recovery, but works for valid JSON - end elsif match = scan(/ true | false | null /x) encoder.text_token match, :value elsif match = scan(/ -? (?: 0 | [1-9]\d* ) /x) @@ -82,6 +77,10 @@ module Scanners end end + if options[:keep_state] + @state = state + end + if [:string, :key].include? state encoder.end_group state end -- cgit v1.2.1 From 366a02754ca0a2d4ffeb06eeddc8733cea57643e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Jun 2013 23:41:56 +0200 Subject: remove Ruby 2.0 workaround --- test/executable/suite.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/executable/suite.rb b/test/executable/suite.rb index d386f4b..997405c 100644 --- a/test/executable/suite.rb +++ b/test/executable/suite.rb @@ -14,7 +14,7 @@ class TestCodeRayExecutable < Test::Unit::TestCase ROOT_DIR = Pathname.new(File.dirname(__FILE__)) + '..' + '..' EXECUTABLE = ROOT_DIR + 'bin' + 'coderay' - RUBY_COMMAND = RUBY_VERSION < '2.0.0' ? 'ruby -w' : 'ruby' # Ruby 2 currently throws warnings for bundler + RUBY_COMMAND = 'ruby' EXE_COMMAND = if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success? # use Nailgun -- cgit v1.2.1 From 5d1812b9d0e1074eb9ec7b24d6ce0f4f7f80619a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Jun 2013 23:42:30 +0200 Subject: tags should be bold --- lib/coderay/styles/alpha.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 9a94091..f4d07f1 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -123,7 +123,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol { color:#A60 } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } -.tag { color:#070 } +.tag { color:#070; font-weight:bold } .type { color:#339; font-weight:bold } .value { color: #088 } .variable { color:#037 } -- cgit v1.2.1 From 2abfc49bdc9a9f4e86c90aa968c302ca76c20812 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Jun 2013 23:43:06 +0200 Subject: use JSON to highlight .template files (AWS CF) --- lib/coderay/helpers/file_type.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 6144e8c..19f27ac 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -120,6 +120,7 @@ module CodeRay 'sass' => :sass, 'sql' => :sql, 'taskpaper' => :taskpaper, + 'template' => :json, # AWS CloudFormation template 'tmproj' => :xml, 'xaml' => :xml, 'xhtml' => :html, -- cgit v1.2.1 From d0d1b129f09dbae8fc25c093c09f72c6396b122a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Jun 2013 23:53:32 +0200 Subject: whitespace --- lib/coderay/styles/alpha.rb | 2 +- lib/coderay/token_kinds.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index c05ffd5..c231423 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -1,6 +1,6 @@ module CodeRay module Styles - + # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index f1696df..9137a49 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,14 +1,14 @@ module CodeRay - + # A Hash of all known token kinds and their associated CSS classes. TokenKinds = Hash.new do |h, k| warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG false end - + # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity - + TokenKinds.update( # :nodoc: :debug => 'debug', # highlight for debugging (white on blue background) -- cgit v1.2.1 From 90adb394e8b4da3a733941fc764c308b34482293 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 00:15:59 +0200 Subject: add lua file type --- lib/coderay/helpers/file_type.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 19f27ac..6d4fa92 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -96,6 +96,7 @@ module CodeRay 'java' => :java, 'js' => :java_script, 'json' => :json, + 'lua' => :lua, 'mab' => :ruby, 'pas' => :delphi, 'patch' => :diff, -- cgit v1.2.1 From fbaf55d69c20897655c80e9723973e2f9490484d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:04:53 +0200 Subject: more helpful error messages from DebugLint --- lib/coderay/encoders/debug_lint.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index eeb2a92..0ac89ef 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -35,7 +35,7 @@ module Encoders end def end_group kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind super end @@ -45,7 +45,7 @@ module Encoders end def end_line kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind super end -- cgit v1.2.1 From c0dc9f2b82b84e6a7511ff9bfe3e80aaa1fef1d1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:05:41 +0200 Subject: move .map styles before .string --- lib/coderay/styles/alpha.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index c231423..a869d9e 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -99,6 +99,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } .local-variable { color:#963 } +.map .content { color:#808 } +.map .delimiter { color:#40A} +.map { background-color:hsla(200,100%,50%,0.06); } .namespace { color:#707; font-weight:bold } .octal { color:#40E } .operator { } @@ -123,9 +126,6 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .symbol { color:#A60 } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } -.map .content { color:#808 } -.map .delimiter { color:#40A} -.map { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070; font-weight:bold } .type { color:#339; font-weight:bold } .value { color: #088 } -- cgit v1.2.1 From da1425058e9b9e04bd988ddf606331d9cd0c827b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:06:33 +0200 Subject: cleanup Lua scanner, fix end_group(:map) issue - use local instead of instance variables --- lib/coderay/scanners/lua.rb | 524 ++++++++++++++++++++++---------------------- 1 file changed, 267 insertions(+), 257 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 64763dc..3bee275 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -1,265 +1,275 @@ -# -*- coding: utf-8 -*- +# encoding: utf-8 -# Scanner for the Lua[http://lua.org] programming lanuage. -# -# The language’s complete syntax is defined in -# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], -# which is what this scanner tries to conform to. -class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner +module CodeRay +module Scanners - register_for :lua - file_extension "lua" - title "Lua" - - # Keywords used in Lua. - KEYWORDS = %w[and break do else elseif end - for function goto if in - local not or repeat return - then until while - ] - - # Constants set by the Lua core. - PREDEFINED_CONSTANTS = %w[false true nil] - - # The expressions contained in this array are parts of Lua’s `basic' - # library. Although it’s not entirely necessary to load that library, - # it is highly recommended and one would have to provide own implementations - # of some of these expressions if one does not do so. They however aren’t - # keywords, neither are they constants, but nearly predefined, so they - # get tagged as `predefined' rather than anything else. + # Scanner for the Lua[http://lua.org] programming lanuage. # - # This list excludes values of form `_UPPERCASE' because the Lua manual - # requires such identifiers to be reserved by Lua anyway and they are - # highlighted directly accordingly, without the need for specific - # identifiers to be listed here. - PREDEFINED_EXPRESSIONS = %w[ - assert collectgarbage dofile error getmetatable - ipairs load loadfile next pairs pcall print - rawequal rawget rawlen rawset select setmetatable - tonumber tostring type xpcall - ] - - # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident). - add(KEYWORDS, :keyword). - add(PREDEFINED_CONSTANTS, :predefined_constant). - add(PREDEFINED_EXPRESSIONS, :predefined) - - protected - - # Scanner initialization. - def setup - @state = :initial - @brace_depth = 0 - end - - # CodeRay entry hook. Starts parsing. - def scan_tokens(encoder, options) - @encoder = encoder - @options = options - - until eos? - case state - - when :initial - if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] - @num_equals = match.count("=") # Number must match for comment end - @encoder.begin_group(:comment) - @encoder.text_token(match, :delimiter) - @state = :long_comment - - elsif match = scan(/--.*?$/) # --Lua comment - @encoder.text_token(match, :comment) - - elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] - @num_equals = match.count("=") # Number must match for comment end - @encoder.begin_group(:string) - @encoder.text_token(match, :delimiter) - @state = :long_string - - elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: - @encoder.text_token(match, :label) - - elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua - @encoder.text_token(match, :predefined) - - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) - kind = IDENT_KIND[match] - - # Extra highlighting for entities following certain keywords - if kind == :keyword and match == "function" - @state = :function_expected - elsif kind == :keyword and match == "goto" - @state = :goto_label_expected - elsif kind == :keyword and match == "local" - @state = :local_var_expected + # The language’s complete syntax is defined in + # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], + # which is what this scanner tries to conform to. + class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + + register_for :lua + file_extension "lua" + title "Lua" + + # Keywords used in Lua. + KEYWORDS = %w[and break do else elseif end + for function goto if in + local not or repeat return + then until while + ] + + # Constants set by the Lua core. + PREDEFINED_CONSTANTS = %w[false true nil] + + # The expressions contained in this array are parts of Lua’s `basic' + # library. Although it’s not entirely necessary to load that library, + # it is highly recommended and one would have to provide own implementations + # of some of these expressions if one does not do so. They however aren’t + # keywords, neither are they constants, but nearly predefined, so they + # get tagged as `predefined' rather than anything else. + # + # This list excludes values of form `_UPPERCASE' because the Lua manual + # requires such identifiers to be reserved by Lua anyway and they are + # highlighted directly accordingly, without the need for specific + # identifiers to be listed here. + PREDEFINED_EXPRESSIONS = %w[ + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall + ] + + # Automatic token kind selection for normal words. + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) + + protected + + # Scanner initialization. + def setup + @state = :initial + @brace_depth = 0 + end + + # CodeRay entry hook. Starts parsing. + def scan_tokens(encoder, options) + state = options[:state] || @state + + until eos? + case state + + when :initial + if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] + @num_equals = match.count("=") # Number must match for comment end + encoder.begin_group(:comment) + encoder.text_token(match, :delimiter) + state = :long_comment + + elsif match = scan(/--.*$/) # --Lua comment + encoder.text_token(match, :comment) + + elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] + @num_equals = match.count("=") # Number must match for comment end + encoder.begin_group(:string) + encoder.text_token(match, :delimiter) + state = :long_string + + elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: + encoder.text_token(match, :label) + + elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua + encoder.text_token(match, :predefined) + + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) + kind = IDENT_KIND[match] + + # Extra highlighting for entities following certain keywords + if kind == :keyword and match == "function" + state = :function_expected + elsif kind == :keyword and match == "goto" + state = :goto_label_expected + elsif kind == :keyword and match == "local" + state = :local_var_expected + end + + encoder.text_token(match, kind) + + elsif match = scan(/\{/) # Opening table brace { + encoder.begin_group(:map) + encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) + @brace_depth += 1 + state = :map + + elsif match = scan(/\}/) # Closing table brace } + if @brace_depth == 1 + @brace_depth = 0 + encoder.text_token(match, :delimiter) + encoder.end_group(:map) + elsif @brace_depth == 0 # Mismatched brace + encoder.text_token(match, :error) + else + @brace_depth -= 1 + encoder.text_token(match, :inline_delimiter) + encoder.end_group(:map) + state = :map + end + + elsif match = scan(/["']/) # String delimiters " and ' + encoder.begin_group(:string) + encoder.text_token(match, :delimiter) + @start_delim = match + state = :string + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + encoder.text_token(match, :float) + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + encoder.text_token(match, :integer) + + elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators + encoder.text_token(match, :operator) + + elsif match = scan(/\s+/) # Space + encoder.text_token(match, :space) + + else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. + encoder.text_token(getch, :error) + end + + # It may be that we’re scanning a full-blown subexpression of a table + # (tables can contain full expressions in parts). + # If this is the case, return to :map scanning state. + state = :map if state == :initial && @brace_depth >= 1 + + when :function_expected + if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name + encoder.text_token(match, :operator) + state = :initial + elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator + encoder.text_token(match, :ident) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() + encoder.text_token(match, :function) + state = :initial + elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + state = :initial + end + + when :goto_label_expected + if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + encoder.text_token(match, :label) + state = :initial + elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + end + + when :local_var_expected + if match = scan(/function/) # local function ... + encoder.text_token(match, :keyword) + state = :function_expected + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + encoder.text_token(match, :local_variable) + elsif match = scan(/,/) + encoder.text_token(match, :operator) + elsif match = scan(/\=/) + encoder.text_token(match, :operator) + # After encountering the equal sign, arbitrary expressions are + # allowed again, so just return to the main state for further + # parsing. + state = :initial + elsif match = scan(/\n/) + encoder.text_token(match, :space) + state = :initial + elsif match = scan(/\s+/) + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + end + + when :long_comment + if match = scan(/.*?(?=\]={#@num_equals}\])/m) + encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + encoder.text_token(rest, :error) + terminate + end + encoder.end_group(:comment) + state = :initial + + when :long_string + if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + encoder.text_token(rest, :error) + terminate + end + encoder.end_group(:string) + state = :initial + + when :string + if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + encoder.text_token(match, :content) + elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) + encoder.text_token(match, :char) + elsif match = scan(Regexp.compile(@start_delim)) + encoder.text_token(match, :delimiter) + encoder.end_group(:string) + state = :initial + elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings + encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings + encoder.end_group(:string) + state = :initial + else + encoder.text_token(getch, :error) + end + + when :map + if match = scan(/[,;]/) + encoder.text_token(match, :operator) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) + encoder.text_token(match, :key) + encoder.text_token(scan(/\s+/), :space) if check(/\s+/) + encoder.text_token(scan(/\=/), :operator) + state = :initial + elsif match = scan(/\s+/m) + encoder.text_token(match, :space) + else + # Note this clause doesn’t advance the scan pointer, it’s a kind of + # "retry with other options" (the :initial state then of course + # advances the pointer). + state = :initial + end + else + raise + end + end - - @encoder.text_token(match, kind) - - elsif match = scan(/\{/) # Opening table brace { - @encoder.begin_group(:map) - @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) - @brace_depth += 1 - @state = :map - - elsif match = scan(/\}/) # Closing table brace } - if @brace_depth == 1 - @brace_depth = 0 - @encoder.text_token(match, :delimiter) - elsif @brace_depth == 0 # Mismatched brace - @encoder.text_token(match, :error) - else - @brace_depth -= 1 - @encoder.text_token(match, :inline_delimiter) - @state = :map + + if options[:keep_state] + @state = state end - @encoder.end_group(:map) - - elsif match = scan(/["']/) # String delimiters " and ' - @encoder.begin_group(:string) - @encoder.text_token(match, :delimiter) - @start_delim = match - @state = :string - - # ↓Prefix hex number ←|→ decimal number - elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power - @encoder.text_token(match, :float) - - # ↓Prefix hex number ←|→ decimal number - elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power - @encoder.text_token(match, :integer) - - elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators - @encoder.text_token(match, :operator) - - elsif match = scan(/\s+/) # Space - @encoder.text_token(match, :space) - - else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. - @encoder.text_token(getch, :error) - end - - # It may be that we’re scanning a full-blown subexpression of a table - # (tables can contain full expressions in parts). - # If this is the case, return to :map scanning state. - @state = :map if @state == :initial && @brace_depth >= 1 - - when :function_expected - if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name - @encoder.text_token(match, :operator) - @state = :initial - elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator - @encoder.text_token(match, :ident) - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() - @encoder.text_token(match, :function) - @state = :initial - elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - @state = :initial - end - - when :goto_label_expected - if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) - @encoder.text_token(match, :label) - @state = :initial - elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - end - - when :local_var_expected - if match = scan(/function/) # local function ... - @encoder.text_token(match, :keyword) - @state = :function_expected - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) - @encoder.text_token(match, :local_variable) - elsif match = scan(/,/) - @encoder.text_token(match, :operator) - elsif match = scan(/\=/) - @encoder.text_token(match, :operator) - # After encountering the equal sign, arbitrary expressions are - # allowed again, so just return to the main state for further - # parsing. - @state = :initial - elsif match = scan(/\n/) - @encoder.text_token(match, :space) - @state = :initial - elsif match = scan(/\s+/) - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - end - - when :long_comment - if match = scan(/.*?(?=\]={#@num_equals}\])/m) - @encoder.text_token(match, :content) - - delim = scan(/\]={#@num_equals}\]/) - @encoder.text_token(delim, :delimiter) - else # No terminator found till EOF - @encoder.text_token(rest, :error) - terminate - end - @encoder.end_group(:comment) - @state = :initial - - when :long_string - if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences - @encoder.text_token(match, :content) - - delim = scan(/\]={#@num_equals}\]/) - @encoder.text_token(delim, :delimiter) - else # No terminator found till EOF - @encoder.text_token(rest, :error) - terminate - end - @encoder.end_group(:string) - @state = :initial - - when :string - if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) - @encoder.text_token(match, :content) - elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) - @encoder.text_token(match, :char) - elsif match = scan(Regexp.compile(@start_delim)) - @encoder.text_token(match, :delimiter) - @encoder.end_group(:string) - @state = :initial - elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings - @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings - @encoder.end_group(:string) - @state = :initial - else - @encoder.text_token(getch, :error) - end - - when :map - if match = scan(/[,;]/) - @encoder.text_token(match, :operator) - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) - @encoder.text_token(match, :key) - @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) - @encoder.text_token(scan(/\=/), :operator) - @state = :initial - elsif match = scan(/\s+/m) - @encoder.text_token(match, :space) - else - # Note this clause doesn’t advance the scan pointer, it’s a kind of - # "retry with other options" (the :initial state then of course - # advances the pointer). - @state = :initial + + encoder end - else - raise + end - - end - - @encoder - end - + +end end -- cgit v1.2.1 From 546b489e4b3856d361a480de5f1f02cb55c7f002 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:09:14 +0200 Subject: fix Tokens documentation --- lib/coderay/tokens.rb | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 54358d4..e7bffce 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -3,17 +3,16 @@ module CodeRay # The Tokens class represents a list of tokens returned from # a Scanner. It's actually just an Array with a few helper methods. # - # A token itself is not a special object, just a two-element Array - # consisting of + # A token itself is not a special object, just two elements in an Array: # * the _token_ _text_ (the original source of the token in a String) or # a _token_ _action_ (begin_group, end_group, begin_line, end_line) # * the _token_ _kind_ (a Symbol representing the type of the token) # # It looks like this: # - # ['# It looks like this', :comment] - # ['3.1415926', :float] - # ['$^', :error] + # ..., '# It looks like this', :comment, ... + # ..., '3.1415926', :float, ... + # ..., '$^', :error, ... # # Some scanners also yield sub-tokens, represented by special # token actions, for example :begin_group and :end_group. @@ -21,11 +20,11 @@ module CodeRay # The Ruby scanner, for example, splits "a string" into: # # [ - # [:begin_group, :string], - # ['"', :delimiter], - # ['a string', :content], - # ['"', :delimiter], - # [:end_group, :string] + # :begin_group, :string, + # '"', :delimiter, + # 'a string', :content, + # '"', :delimiter, + # :end_group, :string # ] # # Tokens can be used to save the output of a Scanners in a simple -- cgit v1.2.1 From 06cf0bab5975bc9069f6859cf58eed117fd9f474 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:16:30 +0200 Subject: changelog --- Changes.textile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes.textile b/Changes.textile index fcee29b..930fdbc 100644 --- a/Changes.textile +++ b/Changes.textile @@ -4,6 +4,7 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release h2. Changes in 1.1 +* New scanner: Lua [#21, #22, thanks to Quintus] * New scanner: Sass [#93] * New scanner: Taskpaper [#39, thanks to shimomura] * Diff scanner: Highlight inline changes in multi-line changes [#99] @@ -17,6 +18,7 @@ h2. Changes in 1.1 * @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner] * New token type @:id@ for CSS/Sass [#27] * New token type @:done@ for Taskpaper [#39] +* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany] * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] * Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05] * Fixed @:docstring@ token type style -- cgit v1.2.1 From 90c401c9001e0a670360a55d1189e9f814ae7592 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:17:49 +0200 Subject: some more code cleanups before merge --- lib/coderay/scanners/lua.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 3bee275..25bebbe 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -8,17 +8,17 @@ module Scanners # The language’s complete syntax is defined in # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], # which is what this scanner tries to conform to. - class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + class Lua < Scanner register_for :lua - file_extension "lua" - title "Lua" + file_extension 'lua' + title 'Lua' # Keywords used in Lua. KEYWORDS = %w[and break do else elseif end - for function goto if in - local not or repeat return - then until while + for function goto if in + local not or repeat return + then until while ] # Constants set by the Lua core. @@ -36,10 +36,10 @@ module Scanners # highlighted directly accordingly, without the need for specific # identifiers to be listed here. PREDEFINED_EXPRESSIONS = %w[ - assert collectgarbage dofile error getmetatable - ipairs load loadfile next pairs pcall print - rawequal rawget rawlen rawset select setmetatable - tonumber tostring type xpcall + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall ] # Automatic token kind selection for normal words. @@ -52,7 +52,7 @@ module Scanners # Scanner initialization. def setup - @state = :initial + @state = :initial @brace_depth = 0 end -- cgit v1.2.1 From fa9848b6dbd95a4a97a3e63cb99ad2d4c26516df Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:37:06 +0200 Subject: avoid empty tokens in CSS --- lib/coderay/scanners/css.rb | 2 +- lib/coderay/scanners/sass.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 732f9c5..6c52c8a 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -148,7 +148,7 @@ module Scanners encoder.text_token match[start.size..-2], :content encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content + encoder.text_token match[start.size..-1], :content if start.size < match.size end encoder.end_group :function diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 167051d..65d40b0 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -176,7 +176,7 @@ module Scanners encoder.text_token match[start.size..-2], :content encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content + encoder.text_token match[start.size..-1], :content if start.size < match.size end encoder.end_group :function -- cgit v1.2.1 From bedd4e4fb5a18270a742e71b65f88fb8e3182050 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:40:56 +0200 Subject: avoid empty tokens in Raydebug --- lib/coderay/scanners/raydebug.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index 7a21354..d39d962 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -1,11 +1,11 @@ module CodeRay module Scanners - + # = Debug Scanner # # Parses the output of the Encoders::Debug encoder. class Raydebug < Scanner - + register_for :raydebug file_extension 'raydebug' title 'CodeRay Token Dump' @@ -13,11 +13,11 @@ module Scanners protected def scan_tokens encoder, options - + opened_tokens = [] - + until eos? - + if match = scan(/\s+/) encoder.text_token match, :space @@ -26,7 +26,7 @@ module Scanners encoder.text_token kind, :class encoder.text_token '(', :operator match = self[2] - encoder.text_token match, kind.to_sym + encoder.text_token match, kind.to_sym unless match.empty? encoder.text_token match, :operator if match = scan(/\)/) elsif match = scan(/ (\w+) ([<\[]) /x) @@ -59,8 +59,8 @@ module Scanners encoder end - + end - + end end -- cgit v1.2.1 From 39c074f171eeca1c91aa89757a3f3a12fc78323b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:42:32 +0200 Subject: avoid empty tokens in YAML --- lib/coderay/scanners/yaml.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb index 96f4e93..358a3f1 100644 --- a/lib/coderay/scanners/yaml.rb +++ b/lib/coderay/scanners/yaml.rb @@ -47,7 +47,7 @@ module Scanners when !check(/(?:"[^"]*")(?=: |:$)/) && match = scan(/"/) encoder.begin_group :string encoder.text_token match, :delimiter - encoder.text_token match, :content if match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx) + encoder.text_token match, :content if (match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)) && !match.empty? encoder.text_token match, :delimiter if match = scan(/"/) encoder.end_group :string next -- cgit v1.2.1 From e7df328cb49fd09a4e5d0f02e877e3270a8703ed Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:53:53 +0200 Subject: DebugLint check open tokens at the end --- lib/coderay/encoders/debug_lint.rb | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 0ac89ef..17a0795 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -19,11 +19,6 @@ module Encoders EmptyToken = Class.new InvalidTokenStream IncorrectTokenGroupNesting = Class.new InvalidTokenStream - def initialize options = {} - super - @opened = [] - end - def text_token text, kind raise EmptyToken, 'empty token' if text.empty? super @@ -35,7 +30,8 @@ module Encoders end def end_group kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop super end @@ -45,7 +41,20 @@ module Encoders end def end_line kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + super + end + + protected + + def setup options + super + @opened = [] + end + + def finish options + raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? super end -- cgit v1.2.1 From d6cc5767c9a9bfb4c08bcd5127145b38cb1d29de Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:54:31 +0200 Subject: HTML encoder shouldn't warn about open tokens --- lib/coderay/encoders/html.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index b897f5e..20f2409 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -193,7 +193,6 @@ module Encoders def finish options unless @opened.empty? - warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG @out << '' while @opened.pop @last_opened = nil end -- cgit v1.2.1 From b059a63e7ff3cd7c26dd5664bf048aa42338fbe1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:54:56 +0200 Subject: close open tokens in Groovy scanner --- lib/coderay/scanners/groovy.rb | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index cf55daf..d04a535 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -36,9 +36,12 @@ module Scanners protected + def setup + @state = :initial + end + def scan_tokens encoder, options - - state = :initial + state = options[:state] || @state inline_block_stack = [] inline_block_paren_depth = nil string_delimiter = nil @@ -246,6 +249,16 @@ module Scanners encoder.end_group state end + if options[:keep_state] + @state = state + end + + until inline_block_stack.empty? + state, = *inline_block_stack.pop + encoder.end_group :inline + encoder.end_group state + end + encoder end -- cgit v1.2.1 From 3c3060acaf2422c23784539b270e365decc9cb2e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:55:23 +0200 Subject: close open tokens in PHP scanner --- lib/coderay/scanners/php.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb index 6c68834..40037f9 100644 --- a/lib/coderay/scanners/php.rb +++ b/lib/coderay/scanners/php.rb @@ -500,6 +500,10 @@ module Scanners end + while state = states.pop + encoder.end_group :string if [:sqstring, :dqstring].include? state + end + encoder end -- cgit v1.2.1 From 7f45168efd1abe679b261a7a91f3641984091d12 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:06:16 +0200 Subject: remove instance variables from Lua scanner --- lib/coderay/scanners/lua.rb | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 25bebbe..185c133 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -59,13 +59,15 @@ module Scanners # CodeRay entry hook. Starts parsing. def scan_tokens(encoder, options) state = options[:state] || @state + brace_depth = @brace_depth + num_equals = nil until eos? case state when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] - @num_equals = match.count("=") # Number must match for comment end + num_equals = match.count("=") # Number must match for comment end encoder.begin_group(:comment) encoder.text_token(match, :delimiter) state = :long_comment @@ -74,7 +76,7 @@ module Scanners encoder.text_token(match, :comment) elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] - @num_equals = match.count("=") # Number must match for comment end + num_equals = match.count("=") # Number must match for comment end encoder.begin_group(:string) encoder.text_token(match, :delimiter) state = :long_string @@ -101,19 +103,19 @@ module Scanners elsif match = scan(/\{/) # Opening table brace { encoder.begin_group(:map) - encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) - @brace_depth += 1 + encoder.text_token(match, brace_depth >= 1 ? :inline_delimiter : :delimiter) + brace_depth += 1 state = :map elsif match = scan(/\}/) # Closing table brace } - if @brace_depth == 1 - @brace_depth = 0 + if brace_depth == 1 + brace_depth = 0 encoder.text_token(match, :delimiter) encoder.end_group(:map) - elsif @brace_depth == 0 # Mismatched brace + elsif brace_depth == 0 # Mismatched brace encoder.text_token(match, :error) else - @brace_depth -= 1 + brace_depth -= 1 encoder.text_token(match, :inline_delimiter) encoder.end_group(:map) state = :map @@ -122,7 +124,7 @@ module Scanners elsif match = scan(/["']/) # String delimiters " and ' encoder.begin_group(:string) encoder.text_token(match, :delimiter) - @start_delim = match + start_delim = match state = :string # ↓Prefix hex number ←|→ decimal number @@ -146,7 +148,7 @@ module Scanners # It may be that we’re scanning a full-blown subexpression of a table # (tables can contain full expressions in parts). # If this is the case, return to :map scanning state. - state = :map if state == :initial && @brace_depth >= 1 + state = :map if state == :initial && brace_depth >= 1 when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @@ -198,10 +200,10 @@ module Scanners end when :long_comment - if match = scan(/.*?(?=\]={#@num_equals}\])/m) + if match = scan(/.*?(?=\]={#{num_equals}}\])/m) encoder.text_token(match, :content) - delim = scan(/\]={#@num_equals}\]/) + delim = scan(/\]={#{num_equals}}\]/) encoder.text_token(delim, :delimiter) else # No terminator found till EOF encoder.text_token(rest, :error) @@ -211,10 +213,10 @@ module Scanners state = :initial when :long_string - if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + if match = scan(/.*?(?=\]={#{num_equals}}\])/m) # Long strings do not interpret any escape sequences encoder.text_token(match, :content) - delim = scan(/\]={#@num_equals}\]/) + delim = scan(/\]={#{num_equals}}\]/) encoder.text_token(delim, :delimiter) else # No terminator found till EOF encoder.text_token(rest, :error) @@ -224,11 +226,11 @@ module Scanners state = :initial when :string - if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + if match = scan(/[^\\#{start_delim}\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) encoder.text_token(match, :content) elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) encoder.text_token(match, :char) - elsif match = scan(Regexp.compile(@start_delim)) + elsif match = scan(Regexp.compile(start_delim)) encoder.text_token(match, :delimiter) encoder.end_group(:string) state = :initial -- cgit v1.2.1 From 45bb0c576b67b7a3c0dead02078b3a16b5583154 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:06:52 +0200 Subject: close open token groups in Lua scanner --- lib/coderay/scanners/lua.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 185c133..fb1e45a 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -268,6 +268,9 @@ module Scanners @state = state end + encoder.end_group :string if [:string].include? state + brace_depth.times { encoder.end_group :map } + encoder end -- cgit v1.2.1 From dc6071129cdc1bcd15129147bbc4d92ba870f007 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:13:32 +0200 Subject: close open token groups in Sass scanner --- lib/coderay/scanners/sass.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 65d40b0..0ba383f 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -218,6 +218,14 @@ module Scanners @state = states end + while state = states.pop + if state == :sass_inline + encoder.end_group :inline + elsif state == :string + encoder.end_group :string + end + end + encoder end -- cgit v1.2.1 From 4327fcbe932ac913f7eb8e92497f97913fb398c5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:15:55 +0200 Subject: fix Sass regexp modifier --- lib/coderay/scanners/sass.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 0ba383f..e20bebe 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -195,7 +195,7 @@ module Scanners elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/) encoder.text_token match, :color - elsif match = scan(/@else if\b|#{RE::AtKeyword}/) + elsif match = scan(/@else if\b|#{RE::AtKeyword}/o) encoder.text_token match, :directive value_expected = true -- cgit v1.2.1 From 755904f7046d8a8d6208fe8367aaa3b9a19ecd65 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:30:24 +0200 Subject: avoid empty tokens in Diff scanner --- lib/coderay/scanners/diff.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index af0f755..fd1aed6 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -69,7 +69,7 @@ module Scanners state = :added elsif match = scan(/\\ .*/) encoder.text_token match, :comment - elsif match = scan(/@@(?>[^@\n]*)@@/) + elsif match = scan(/@@(?>[^@\n]+)@@/) content_scanner.state = :initial unless match?(/\n\+/) content_scanner_entry_state = nil if check(/\n|$/) -- cgit v1.2.1 From e1d5e111d968639fa03a6074cf90535ecc90d0dd Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:30:44 +0200 Subject: close correct token groups in Groovy scanner --- lib/coderay/scanners/groovy.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index d04a535..c64454f 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -226,7 +226,7 @@ module Scanners encoder.text_token match, :content # TODO: Shouldn't this be :error? elsif match = scan(/ \\ | \n /x) - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string encoder.text_token match, :error after_def = value_expected = false state = :initial @@ -246,7 +246,7 @@ module Scanners end if [:multiline_string, :string, :regexp].include? state - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string end if options[:keep_state] @@ -256,7 +256,7 @@ module Scanners until inline_block_stack.empty? state, = *inline_block_stack.pop encoder.end_group :inline - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string end encoder -- cgit v1.2.1 From 603ff7d0b14521cfd0408aa68e2e1cb6ea9086bc Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:36:46 +0200 Subject: avoid empty tokens in YAML scanner --- lib/coderay/scanners/yaml.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb index 358a3f1..32c8e2c 100644 --- a/lib/coderay/scanners/yaml.rb +++ b/lib/coderay/scanners/yaml.rb @@ -84,7 +84,7 @@ module Scanners when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/) encoder.begin_group :key encoder.text_token match[0,1], :delimiter - encoder.text_token match[1..-2], :content + encoder.text_token match[1..-2], :content if match.size > 2 encoder.text_token match[-1,1], :delimiter encoder.end_group :key key_indent = column(pos - match.size) - 1 -- cgit v1.2.1 From 253a616924000cc5d522a2d4b8030f5693c98e43 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:39:30 +0200 Subject: close open string token groups in SQL scanner --- lib/coderay/scanners/sql.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb index b757278..93aeaf3 100644 --- a/lib/coderay/scanners/sql.rb +++ b/lib/coderay/scanners/sql.rb @@ -1,8 +1,9 @@ -module CodeRay module Scanners +module CodeRay +module Scanners # by Josh Goebel class SQL < Scanner - + register_for :sql KEYWORDS = %w( @@ -149,6 +150,7 @@ module CodeRay module Scanners string_content = '' end encoder.text_token match, :error unless match.empty? + encoder.end_group :string state = :initial else raise "else case \" reached; %p not handled." % peek(1), encoder @@ -171,4 +173,5 @@ module CodeRay module Scanners end -end end \ No newline at end of file +end +end -- cgit v1.2.1 From ad5efeb8c96c976804fcfcd3f71e0d02cf73cc10 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 13:00:13 +0200 Subject: changelog --- Changes.textile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index 930fdbc..581e485 100644 --- a/Changes.textile +++ b/Changes.textile @@ -12,6 +12,8 @@ h2. Changes in 1.1 * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] * Remove double-click toggle handler from HTML table output * Fixes to CSS scanner (floats, pseudoclasses) +* Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] +* Added @:keep_state@ functionality to more scanners [#116] * CSS scanner uses @:id@ and @:tag@ now [#27] * Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right? * Add .xaml file type [#121, thanks to Kozman Bálint] @@ -20,10 +22,13 @@ h2. Changes in 1.1 * New token type @:done@ for Taskpaper [#39] * New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany] * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] -* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05] +* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05] * Fixed @:docstring@ token type style * @Plugin@ does not warn about fallback when default is defined +* @HTML@ encoder will not warn about unclosed token groups at the end of the stream * @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now +* @Debug@ encoder will not warn about errors in the token stream +* New @DebugLint@ encoder that checks for empty tokens and correct nesting h2. Changes in 1.0.9 -- cgit v1.2.1 From c8751fbc09d4a8f43b0b037e3a7dc7ddbfbcacb1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 13:15:10 +0200 Subject: fix unclosed token group in Ruby scanner --- lib/coderay/scanners/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index c282f31..80165ca 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -269,7 +269,7 @@ module Scanners end if last_state - state = last_state + state = last_state unless state.is_a?(StringState) # otherwise, a simple 'def"' results in unclosed tokens last_state = nil end -- cgit v1.2.1 From c34ad739bc7342aa40830945a5e19e90e9814e04 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 13:27:43 +0200 Subject: fix token nesting in PHP scanner --- lib/coderay/scanners/php.rb | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb index 40037f9..7a8d75d 100644 --- a/lib/coderay/scanners/php.rb +++ b/lib/coderay/scanners/php.rb @@ -265,7 +265,7 @@ module Scanners @html_scanner.tokenize match unless match.empty? end - when :php + when :php, :php_inline if match = scan(/\s+/) encoder.text_token match, :space @@ -332,7 +332,7 @@ module Scanners if states.size == 1 encoder.text_token match, :error else - states.pop + state = states.pop if states.last.is_a?(::Array) delimiter = states.last[1] states[-1] = states.last[0] @@ -340,6 +340,7 @@ module Scanners encoder.end_group :inline else encoder.text_token match, :operator + encoder.end_group :inline if state == :php_inline label_expected = true end end @@ -350,7 +351,14 @@ module Scanners elsif match = scan(RE::PHP_END) encoder.text_token match, :inline_delimiter - states = [:initial] + while state = states.pop + encoder.end_group :string if [:sqstring, :dqstring].include? state + if state.is_a? Array + encoder.end_group :inline + encoder.end_group :string if [:sqstring, :dqstring].include? state.first + end + end + states << :initial elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o) encoder.begin_group :string @@ -400,6 +408,7 @@ module Scanners elsif match = scan(/\\/) encoder.text_token match, :error else + encoder.end_group :string states.pop end @@ -459,7 +468,7 @@ module Scanners encoder.begin_group :inline states[-1] = [states.last, delimiter] delimiter = nil - states.push :php + states.push :php_inline encoder.text_token match, :delimiter else encoder.text_token match, :content @@ -469,6 +478,7 @@ module Scanners elsif match = scan(/\$/) encoder.text_token match, :content else + encoder.end_group :string states.pop end @@ -502,6 +512,10 @@ module Scanners while state = states.pop encoder.end_group :string if [:sqstring, :dqstring].include? state + if state.is_a? Array + encoder.end_group :inline + encoder.end_group :string if [:sqstring, :dqstring].include? state.first + end end encoder -- cgit v1.2.1 From 14f3a4f28341237503a51bfa0764c922bb02bdfa Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:13:15 +0200 Subject: tweak HTML CDATA token kinds --- lib/coderay/scanners/html.rb | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index fcf249a..530ce44 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -1,13 +1,13 @@ module CodeRay module Scanners - + # HTML Scanner # # Alias: +xhtml+ # # See also: Scanners::XML class HTML < Scanner - + register_for :html KINDS_NOT_LOC = [ @@ -100,15 +100,13 @@ module Scanners when :initial if match = scan(//m) - encoder.text_token match[0..-4], :content - encoder.text_token ']]>', :delimiter - else - encoder.text_token scan(/.*/m), :error + encoder.text_token match[0..-4], :plain + encoder.text_token ']]>', :inline_delimiter + elsif match = scan(/.+/) + encoder.text_token match, :error end - encoder.end_group :string elsif match = scan(/|.*)/m) encoder.text_token match, :comment elsif match = scan(/|.*)|\]>/m) -- cgit v1.2.1 From 5b0dc0f35090949d7512a85dadc8cf2871b91aac Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:15:01 +0200 Subject: scan_css in HTML scanner (tags) --- lib/coderay/scanners/html.rb | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 3ba3b79..06728e4 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -75,9 +75,14 @@ module Scanners def scan_java_script encoder, code if code && !code.empty? @java_script_scanner ||= Scanners::JavaScript.new '', :keep_tokens => true - # encoder.begin_group :inline @java_script_scanner.tokenize code, :tokens => encoder - # encoder.end_group :inline + end + end + + def scan_css encoder, code + if code && !code.empty? + @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true + @css_scanner.tokenize code, :tokens => encoder end end @@ -110,7 +115,7 @@ module Scanners elsif match = scan(/<\/[-\w.:]*>?/m) in_tag = nil encoder.text_token match, :tag - elsif match = scan(/<(?:(script)|[-\w.:]+)(>)?/m) + elsif match = scan(/<(?:(script|style)|[-\w.:]+)(>)?/m) encoder.text_token match, :tag in_tag = self[1] if self[2] @@ -206,19 +211,23 @@ module Scanners when :in_special_tag case in_tag - when 'script' + when 'script', 'style' encoder.text_token match, :space if match = scan(/[ \t]*\n/) if scan(/(\s*)|(.*))/m) code = self[2] || self[4] closing = self[3] encoder.text_token self[1], :comment else - code = scan_until(/(?=(?:\n\s*)?<\/script>)|\z/) + code = scan_until(/(?=(?:\n\s*)?<\/#{in_tag}>)|\z/) closing = false end unless code.empty? encoder.begin_group :inline - scan_java_script encoder, code + if in_tag == 'script' + scan_java_script encoder, code + else + scan_css encoder, code + end encoder.end_group :inline end encoder.text_token closing, :comment if closing -- cgit v1.2.1 From 7ada74c8f0f77815393e887db83e1d2c36ce7235 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:32:14 +0200 Subject: scan_css in HTML scanner (arguments), change token kind from :inline to :string --- Changes.textile | 1 + lib/coderay/scanners/html.rb | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Changes.textile b/Changes.textile index 930fdbc..9509b15 100644 --- a/Changes.textile +++ b/Changes.textile @@ -10,6 +10,7 @@ h2. Changes in 1.1 * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] +* HTML scanner displays style tags and attributes now * Remove double-click toggle handler from HTML table output * Fixes to CSS scanner (floats, pseudoclasses) * CSS scanner uses @:id@ and @:tag@ now [#27] diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 06728e4..f1dfba0 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -33,7 +33,8 @@ module Scanners ) IN_ATTRIBUTE = WordList::CaseIgnoring.new(nil). - add(EVENT_ATTRIBUTES, :script) + add(EVENT_ATTRIBUTES, :script). + add(['style'], :style) ATTR_NAME = /[\w.:-]+/ # :nodoc: TAG_END = /\/?>/ # :nodoc: @@ -79,10 +80,10 @@ module Scanners end end - def scan_css encoder, code + def scan_css encoder, code, state = [:initial] if code && !code.empty? @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true - @css_scanner.tokenize code, :tokens => encoder + @css_scanner.tokenize code, :tokens => encoder, :state => state end end @@ -166,17 +167,21 @@ module Scanners encoder.text_token match, :attribute_value state = :attribute elsif match = scan(/["']/) - if in_attribute == :script - encoder.begin_group :inline - encoder.text_token match, :inline_delimiter + if in_attribute == :script || in_attribute == :style + encoder.begin_group :string + encoder.text_token match, :delimiter if scan(/javascript:[ \t]*/) encoder.text_token matched, :comment end code = scan_until(match == '"' ? /(?="|\z)/ : /(?='|\z)/) - scan_java_script encoder, code + if in_attribute == :script + scan_java_script encoder, code + else + scan_css encoder, code, [:block] + end match = scan(/["']/) - encoder.text_token match, :inline_delimiter if match - encoder.end_group :inline + encoder.text_token match, :delimiter if match + encoder.end_group :string state = :attribute in_attribute = nil else -- cgit v1.2.1 From 56f4163e99689912c3797e5d3b2b97244ce65782 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:33:22 +0200 Subject: fix another CSS empty token issue --- lib/coderay/scanners/css.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 6c52c8a..9ed4618 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -145,10 +145,10 @@ module Scanners start = match[/^\w+\(/] encoder.text_token start, :delimiter if match[-1] == ?) - encoder.text_token match[start.size..-2], :content + encoder.text_token match[start.size..-2], :content if match.size > start.size + 1 encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content if start.size < match.size + encoder.text_token match[start.size..-1], :content if match.size > start.size end encoder.end_group :function -- cgit v1.2.1 From 6a2f1a2ec7d6136f1ac9dd5723bad84541dafb26 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:38:37 +0200 Subject: changelog --- Changes.textile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index 53f4109..80f1d10 100644 --- a/Changes.textile +++ b/Changes.textile @@ -10,7 +10,7 @@ h2. Changes in 1.1 * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] -* HTML scanner displays style tags and attributes now +* HTML scanner displays style tags and attributes now [#145] * Remove double-click toggle handler from HTML table output * Fixes to CSS scanner (floats, pseudoclasses) * Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] -- cgit v1.2.1 From 9fb2fd9fa3ba4b92a440f271adbb9584b236c34e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:45:31 +0200 Subject: tweak :local_variable token color --- lib/coderay/styles/alpha.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index a869d9e..b05ae53 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -98,7 +98,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .key .delimiter { color: #404 } .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } -.local-variable { color:#963 } +.local-variable { color:#950 } .map .content { color:#808 } .map .delimiter { color:#40A} .map { background-color:hsla(200,100%,50%,0.06); } -- cgit v1.2.1 From 844975b53233b4697d1385548c1295ce18a7be72 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:46:45 +0200 Subject: whitespace --- lib/coderay/styles/alpha.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index b05ae53..ff85ecc 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -3,14 +3,14 @@ module Styles # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style - + register_for :alpha - + code_background = 'hsl(0,0%,95%)' numbers_background = 'hsl(180,65%,90%)' border_color = 'silver' normal_color = 'black' - + CSS_MAIN_STYLES = <<-MAIN # :nodoc: .CodeRay { background-color: #{code_background}; @@ -145,8 +145,8 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .change .change { color: #88f } .head .head { color: #f4f } TOKENS - + end - + end end -- cgit v1.2.1 From b89caf96d1cfc304c2114d8734ebe8b91337c799 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:53:14 +0200 Subject: fix diff/json token nesting issue --- lib/coderay/scanners/json.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb index 3754a9b..b09970c 100644 --- a/lib/coderay/scanners/json.rb +++ b/lib/coderay/scanners/json.rb @@ -26,6 +26,10 @@ module Scanners def scan_tokens encoder, options state = options[:state] || @state + if [:string, :key].include? state + encoder.begin_group state + end + until eos? case state -- cgit v1.2.1 From 965b8f4d1be88e1ae75ef11e6270a7fbfb2d86bd Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:03:57 +0200 Subject: use -w in executable tests --- test/executable/suite.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/executable/suite.rb b/test/executable/suite.rb index 997405c..ac0ff1d 100644 --- a/test/executable/suite.rb +++ b/test/executable/suite.rb @@ -18,9 +18,9 @@ class TestCodeRayExecutable < Test::Unit::TestCase EXE_COMMAND = if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success? # use Nailgun - "#{RUBY_COMMAND}--ng -I%s %s" + "#{RUBY_COMMAND}--ng -w -I%s %s" else - "#{RUBY_COMMAND} -I%s %s" + "#{RUBY_COMMAND} -w -I%s %s" end % [ROOT_DIR + 'lib', EXECUTABLE] def coderay args, options = {} -- cgit v1.2.1 From a3e79c794f0998034b26ff6baa10bc801918be9d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:07:57 +0200 Subject: trying to fix TravisCI error --- lib/coderay/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4b4f085..4115d09 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' + VERSION = '1.1.0' unless defined? VERSION end -- cgit v1.2.1 From 416243353e0b4b1c449476b269a50b0f60c13bda Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:15:24 +0200 Subject: trying different ways to load CodeRay version --- lib/coderay.rb | 2 +- lib/coderay/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay.rb b/lib/coderay.rb index 24ae5a2..0c66f49 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -134,7 +134,7 @@ module CodeRay File.join CODERAY_PATH, *path end - require coderay_path('version') + require 'coderay/version' # helpers autoload :FileType, coderay_path('helpers', 'file_type') diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4115d09..4b4f085 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' unless defined? VERSION + VERSION = '1.1.0' end -- cgit v1.2.1 From a112fca72ce802b4b5fd25fdc7d76b6360cdc718 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:22:47 +0200 Subject: allow [j]ruby-head failures in Travis --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 59bb791..b3b8cf7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,8 @@ branches: - master matrix: allow_failures: - - rvm: rbx-18mode + - rvm: ruby-head + - rvm: jruby-head + - rvm: rbx-19mode - rvm: rbx-19mode script: "rake test" # test:scanners" -- cgit v1.2.1 From 64ca2ae8ad5130bdcf652aa7aa08298de00f20f4 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:27:29 +0200 Subject: how did this happen? --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b3b8cf7..6d926f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,6 @@ matrix: allow_failures: - rvm: ruby-head - rvm: jruby-head - - rvm: rbx-19mode + - rvm: rbx-18mode - rvm: rbx-19mode script: "rake test" # test:scanners" -- cgit v1.2.1