diff options
author | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2016-03-10 13:07:31 +0100 |
---|---|---|
committer | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2016-03-11 10:16:05 +0100 |
commit | e04913840c3babfdad5b4ead21a07f6a1695144e (patch) | |
tree | 4e1ace5e9f3fcc64b69be698ce624188c266bbcc | |
parent | fc290a7dcde969d956488612512d2fa4ab9d57c0 (diff) | |
download | gitlab-ce-e04913840c3babfdad5b4ead21a07f6a1695144e.tar.gz |
Add some specs for GFM AST, minor refactoringsfeature/gfm-ast
-rw-r--r-- | lib/gitlab/gfm/ast/lexer.rb | 10 | ||||
-rw-r--r-- | lib/gitlab/gfm/ast/parser.rb | 9 | ||||
-rw-r--r-- | lib/gitlab/gfm/ast/syntax/content.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/gfm/ast/syntax/markdown/code_block.rb | 14 | ||||
-rw-r--r-- | lib/gitlab/gfm/ast/syntax/node.rb | 52 | ||||
-rw-r--r-- | lib/gitlab/gfm/ast/syntax/text.rb | 10 | ||||
-rw-r--r-- | spec/lib/gitlab/gfm/ast/syntax/content_spec.rb | 29 | ||||
-rw-r--r-- | spec/lib/gitlab/gfm/ast/syntax/markdown/code_block_spec.rb | 35 | ||||
-rw-r--r-- | spec/lib/gitlab/gfm/ast/syntax/text_spec.rb | 31 |
9 files changed, 158 insertions, 38 deletions
diff --git a/lib/gitlab/gfm/ast/lexer.rb b/lib/gitlab/gfm/ast/lexer.rb index 76929655ac7..df3633fdb27 100644 --- a/lib/gitlab/gfm/ast/lexer.rb +++ b/lib/gitlab/gfm/ast/lexer.rb @@ -84,6 +84,16 @@ module Gitlab end end end + + ## + # Processes single token, and returns first lexeme that has been + # created. + # + def self.single(text, token) + lexer = new(text, [token]) + nodes = lexer.process! + nodes.first + end end end end diff --git a/lib/gitlab/gfm/ast/parser.rb b/lib/gitlab/gfm/ast/parser.rb index 217856188ee..db1bff320b0 100644 --- a/lib/gitlab/gfm/ast/parser.rb +++ b/lib/gitlab/gfm/ast/parser.rb @@ -2,16 +2,11 @@ module Gitlab module Gfm module Ast class Parser - attr_reader :tree + attr_reader :tree, :text def initialize(text) @text = text - @lexer = Lexer.new(@text, [Syntax::Content]) - @nodes = @lexer.process! - end - - def tree - @nodes.first + @tree = Lexer.single(text, Syntax::Content) end def recreate diff --git a/lib/gitlab/gfm/ast/syntax/content.rb b/lib/gitlab/gfm/ast/syntax/content.rb index 7e3c6b7a8a4..2f14e44c64b 100644 --- a/lib/gitlab/gfm/ast/syntax/content.rb +++ b/lib/gitlab/gfm/ast/syntax/content.rb @@ -6,7 +6,7 @@ module Gitlab # Main GFM content # class Content < Node - def self.allowed + def allowed [Syntax::Markdown::CodeBlock, Syntax::Text] end @@ -14,6 +14,10 @@ module Gitlab /(?<value>.+)/m end + def value + @text + end + def to_s nodes.map(&:to_s).join end diff --git a/lib/gitlab/gfm/ast/syntax/markdown/code_block.rb b/lib/gitlab/gfm/ast/syntax/markdown/code_block.rb index 4665bd59d40..401186a5a0c 100644 --- a/lib/gitlab/gfm/ast/syntax/markdown/code_block.rb +++ b/lib/gitlab/gfm/ast/syntax/markdown/code_block.rb @@ -4,16 +4,20 @@ module Gitlab module Syntax module Markdown class CodeBlock < Node + def allowed + [] + end + def to_s - @match[:start_token] + @value + @match[:end_token] + @text end - def lang - @match[:lang] + def value + @text end - def self.allowed - [] + def lang + @match[:lang] end def self.pattern diff --git a/lib/gitlab/gfm/ast/syntax/node.rb b/lib/gitlab/gfm/ast/syntax/node.rb index f52e051aac6..0003704cfa4 100644 --- a/lib/gitlab/gfm/ast/syntax/node.rb +++ b/lib/gitlab/gfm/ast/syntax/node.rb @@ -3,7 +3,7 @@ module Gitlab module Ast module Syntax class Node - attr_reader :text, :range, :parent, :value, :nodes + attr_reader :text, :range, :parent, :nodes def initialize(text, range, match, parent) @text = text @@ -16,14 +16,12 @@ module Gitlab end ## - # Process children nodes + # Nodes allowed inside this one. # - def process! - @nodes = lexer.new(@text, self.class.allowed, self).process! - end - - def index - @range.begin + # This is pipeline of lexemes, order is relevant. + # + def allowed + raise NotImplementedError end ## @@ -34,14 +32,35 @@ module Gitlab end ## - # Is this node a leaf node? + # Returns the value of this nodes, without node-specific tokens. + # + def value + raise NotImplementedError + end + + ## + # Process children nodes + # + def process! + @nodes = lexer.new(value, allowed, self).process! + end + + ## + # Position of this node in parent + # + def index + @range.begin + end + + ## + # Returns true if node is a leaf in the three. # def leaf? @nodes.empty? end ## - # Lexer for this node + # Each node can have it's own lexer. # def lexer Ast::Lexer @@ -65,18 +84,7 @@ module Gitlab end ## - # Nodes allowed inside this one. - # - # This is pipeline of lexemes, order is relevant. - # - def self.allowed - raise NotImplementedError - end - - ## - # Regexp pattern for this node - # - # Each pattern must contain at least `value` capture group. + # Regexp pattern for this token. # def self.pattern raise NotImplementedError diff --git a/lib/gitlab/gfm/ast/syntax/text.rb b/lib/gitlab/gfm/ast/syntax/text.rb index cfe634f43a4..2df68daf23c 100644 --- a/lib/gitlab/gfm/ast/syntax/text.rb +++ b/lib/gitlab/gfm/ast/syntax/text.rb @@ -6,12 +6,16 @@ module Gitlab # Text description # class Text < Node - def to_s + def allowed + [] + end + + def value @text end - def self.allowed - [] + def to_s + @text end def self.pattern diff --git a/spec/lib/gitlab/gfm/ast/syntax/content_spec.rb b/spec/lib/gitlab/gfm/ast/syntax/content_spec.rb new file mode 100644 index 00000000000..14b34fc7786 --- /dev/null +++ b/spec/lib/gitlab/gfm/ast/syntax/content_spec.rb @@ -0,0 +1,29 @@ +require 'spec_helper' + +describe Gitlab::Gfm::Ast::Syntax::Content do + describe 'token' do + let(:text) { "some multi\n\nline text" } + + it 'matches entire text' do + expect(text).to match(described_class.pattern) + end + end + + describe 'lexeme' do + let(:text) { "some text with ```ruby\nblock\n```" } + let(:lexeme) { Gitlab::Gfm::Ast::Lexer.single(text, described_class) } + + describe '#nodes' do + let(:nodes) { lexeme.nodes } + + it 'correctly instantiates children nodes' do + expect(nodes.count).to eq 2 + end + end + + describe '#to_s' do + subject { lexeme.to_s } + it { is_expected.to eq text } + end + end +end diff --git a/spec/lib/gitlab/gfm/ast/syntax/markdown/code_block_spec.rb b/spec/lib/gitlab/gfm/ast/syntax/markdown/code_block_spec.rb new file mode 100644 index 00000000000..c8a5c05310f --- /dev/null +++ b/spec/lib/gitlab/gfm/ast/syntax/markdown/code_block_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +describe Gitlab::Gfm::Ast::Syntax::Markdown::CodeBlock do + let(:text) { "```ruby\ncode block\n```" } + + describe 'token' do + it 'matches entire text' do + expect(text).to match described_class.pattern + end + end + + describe 'lexeme' do + let(:lexeme) { Gitlab::Gfm::Ast::Lexer.single(text, described_class) } + + describe '#nodes' do + subject { lexeme.nodes } + it { is_expected.to be_empty } + end + + describe '#leaf?' do + subject { lexeme.leaf? } + it { is_expected.to be true } + end + + describe '#to_s' do + subject { lexeme.to_s } + it { is_expected.to eq text } + end + + describe '#lang' do + subject { lexeme.lang } + it { is_expected.to eq 'ruby' } + end + end +end diff --git a/spec/lib/gitlab/gfm/ast/syntax/text_spec.rb b/spec/lib/gitlab/gfm/ast/syntax/text_spec.rb new file mode 100644 index 00000000000..0e532189e8c --- /dev/null +++ b/spec/lib/gitlab/gfm/ast/syntax/text_spec.rb @@ -0,0 +1,31 @@ +require 'spec_helper' + +describe Gitlab::Gfm::Ast::Syntax::Text do + describe 'token' do + let(:text) { "some multi\n\nline text" } + + it 'matches entire text' do + expect(text).to match described_class.pattern + end + end + + describe 'lexeme' do + let(:text) { "some text with ```ruby\nblock\n```" } + let(:lexeme) { Gitlab::Gfm::Ast::Lexer.single(text, described_class) } + + describe '#nodes' do + subject { lexeme.nodes } + it { is_expected.to be_empty } + end + + describe '#leaf?' do + subject { lexeme.leaf? } + it { is_expected.to be true } + end + + describe '#to_s' do + subject { lexeme.to_s } + it { is_expected.to eq text } + end + end +end |