diff options
author | Kamil Trzciński <ayufan@ayufan.eu> | 2018-03-01 15:44:51 +0000 |
---|---|---|
committer | Kamil Trzciński <ayufan@ayufan.eu> | 2018-03-01 15:44:51 +0000 |
commit | 461960262d51c719994f0f4236f79dfa1fc0f28f (patch) | |
tree | 395ae2933de16af5cce692416325ecfa68e8ec99 | |
parent | d412b90af0ff59c6c379fb83727cc23043c42cdc (diff) | |
parent | 511046f9eb8366f3b5fee44b66b763069de6829a (diff) | |
download | gitlab-ce-461960262d51c719994f0f4236f79dfa1fc0f28f.tar.gz |
Merge branch 'feature/gb/pipeline-variable-expressions' into 'master'
Pipeline variables expressions
See merge request gitlab-org/gitlab-ce!17261
19 files changed, 748 insertions, 0 deletions
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb new file mode 100644 index 00000000000..047ab66e9b3 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Base + def evaluate(**variables) + raise NotImplementedError + end + + def self.build(token) + raise NotImplementedError + end + + def self.scan(scanner) + if scanner.scan(self::PATTERN) + Expression::Token.new(scanner.matched, self) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb new file mode 100644 index 00000000000..3a2f0c6924e --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb @@ -0,0 +1,26 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Equals < Lexeme::Operator + PATTERN = /==/.freeze + + def initialize(left, right) + @left = left + @right = right + end + + def evaluate(variables = {}) + @left.evaluate(variables) == @right.evaluate(variables) + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb new file mode 100644 index 00000000000..a2778716924 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Null < Lexeme::Value + PATTERN = /null/.freeze + + def initialize(value = nil) + @value = nil + end + + def evaluate(variables = {}) + nil + end + + def self.build(_value) + self.new + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb new file mode 100644 index 00000000000..f640d0b5855 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Operator < Lexeme::Base + def self.type + :operator + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb new file mode 100644 index 00000000000..48bde213d44 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class String < Lexeme::Value + PATTERN = /("(?<string>.+?)")|('(?<string>.+?)')/.freeze + + def initialize(value) + @value = value + end + + def evaluate(variables = {}) + @value.to_s + end + + def self.build(string) + new(string.match(PATTERN)[:string]) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb new file mode 100644 index 00000000000..f2611d65faf --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Value < Lexeme::Base + def self.type + :value + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb new file mode 100644 index 00000000000..b781c15fd67 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Variable < Lexeme::Value + PATTERN = /\$(?<name>\w+)/.freeze + + def initialize(name) + @name = name + end + + def evaluate(variables = {}) + HashWithIndifferentAccess.new(variables).fetch(@name, nil) + end + + def self.build(string) + new(string.match(PATTERN)[:name]) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb new file mode 100644 index 00000000000..e1c68b7c3c2 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -0,0 +1,59 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Lexer + include ::Gitlab::Utils::StrongMemoize + + LEXEMES = [ + Expression::Lexeme::Variable, + Expression::Lexeme::String, + Expression::Lexeme::Null, + Expression::Lexeme::Equals + ].freeze + + SyntaxError = Class.new(Statement::StatementError) + + MAX_TOKENS = 100 + + def initialize(statement, max_tokens: MAX_TOKENS) + @scanner = StringScanner.new(statement) + @max_tokens = max_tokens + end + + def tokens + strong_memoize(:tokens) { tokenize } + end + + def lexemes + tokens.map(&:to_lexeme) + end + + private + + def tokenize + tokens = [] + + @max_tokens.times do + @scanner.skip(/\s+/) # ignore whitespace + + return tokens if @scanner.eos? + + lexeme = LEXEMES.find do |type| + type.scan(@scanner).tap do |token| + tokens.push(token) if token.present? + end + end + + unless lexeme.present? + raise Lexer::SyntaxError, 'Unknown lexeme found!' + end + end + + raise Lexer::SyntaxError, 'Too many tokens!' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb new file mode 100644 index 00000000000..90f94d0b763 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -0,0 +1,40 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Parser + def initialize(tokens) + @tokens = tokens.to_enum + @nodes = [] + end + + ## + # This produces a reverse descent parse tree. + # + # It currently does not support precedence of operators. + # + def tree + while token = @tokens.next + case token.type + when :operator + token.build(@nodes.pop, tree).tap do |node| + @nodes.push(node) + end + when :value + token.build.tap do |leaf| + @nodes.push(leaf) + end + end + end + rescue StopIteration + @nodes.last || Lexeme::Null.new + end + + def self.seed(statement) + new(Expression::Lexer.new(statement).tokens) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb new file mode 100644 index 00000000000..4f0e101b730 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -0,0 +1,42 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Statement + StatementError = Class.new(StandardError) + + GRAMMAR = [ + %w[variable equals string], + %w[variable equals variable], + %w[variable equals null], + %w[string equals variable], + %w[null equals variable], + %w[variable] + ].freeze + + def initialize(statement, pipeline) + @lexer = Expression::Lexer.new(statement) + + @variables = pipeline.variables.map do |variable| + [variable.key, variable.value] + end + end + + def parse_tree + raise StatementError if @lexer.lexemes.empty? + + unless GRAMMAR.find { |syntax| syntax == @lexer.lexemes } + raise StatementError, 'Unknown pipeline expression!' + end + + Expression::Parser.new(@lexer.tokens).tree + end + + def evaluate + parse_tree.evaluate(@variables.to_h) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/token.rb b/lib/gitlab/ci/pipeline/expression/token.rb new file mode 100644 index 00000000000..58211800b88 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/token.rb @@ -0,0 +1,28 @@ +module Gitlab + module Ci + module Pipeline + module Expression + class Token + attr_reader :value, :lexeme + + def initialize(value, lexeme) + @value = value + @lexeme = lexeme + end + + def build(*args) + @lexeme.build(@value, *args) + end + + def type + @lexeme.type + end + + def to_lexeme + @lexeme.name.demodulize.downcase + end + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/equals_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/equals_spec.rb new file mode 100644 index 00000000000..019a2ed184d --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/equals_spec.rb @@ -0,0 +1,39 @@ +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Expression::Lexeme::Equals do + let(:left) { double('left') } + let(:right) { double('right') } + + describe '.build' do + it 'creates a new instance of the token' do + expect(described_class.build('==', left, right)) + .to be_a(described_class) + end + end + + describe '.type' do + it 'is an operator' do + expect(described_class.type).to eq :operator + end + end + + describe '#evaluate' do + it 'returns false when left and right are not equal' do + allow(left).to receive(:evaluate).and_return(1) + allow(right).to receive(:evaluate).and_return(2) + + operator = described_class.new(left, right) + + expect(operator.evaluate(VARIABLE: 3)).to eq false + end + + it 'returns true when left and right are equal' do + allow(left).to receive(:evaluate).and_return(1) + allow(right).to receive(:evaluate).and_return(1) + + operator = described_class.new(left, right) + + expect(operator.evaluate(VARIABLE: 3)).to eq true + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/null_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/null_spec.rb new file mode 100644 index 00000000000..b5a59929e11 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/null_spec.rb @@ -0,0 +1,22 @@ +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Expression::Lexeme::Null do + describe '.build' do + it 'creates a new instance of the token' do + expect(described_class.build('null')) + .to be_a(described_class) + end + end + + describe '.type' do + it 'is a value lexeme' do + expect(described_class.type).to eq :value + end + end + + describe '#evaluate' do + it 'always evaluates to `nil`' do + expect(described_class.new('null').evaluate).to be_nil + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/string_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/string_spec.rb new file mode 100644 index 00000000000..86234dfb9e5 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/string_spec.rb @@ -0,0 +1,92 @@ +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Expression::Lexeme::String do + describe '.build' do + it 'creates a new instance of the token' do + expect(described_class.build('"my string"')) + .to be_a(described_class) + end + end + + describe '.type' do + it 'is a value lexeme' do + expect(described_class.type).to eq :value + end + end + + describe '.scan' do + context 'when using double quotes' do + it 'correctly identifies string token' do + scanner = StringScanner.new('"some string"') + + token = described_class.scan(scanner) + + expect(token).not_to be_nil + expect(token.build.evaluate).to eq 'some string' + end + end + + context 'when using single quotes' do + it 'correctly identifies string token' do + scanner = StringScanner.new("'some string 2'") + + token = described_class.scan(scanner) + + expect(token).not_to be_nil + expect(token.build.evaluate).to eq 'some string 2' + end + end + + context 'when there are mixed quotes in the string' do + it 'is a greedy scanner for double quotes' do + scanner = StringScanner.new('"some string" "and another one"') + + token = described_class.scan(scanner) + + expect(token).not_to be_nil + expect(token.build.evaluate).to eq 'some string' + end + + it 'is a greedy scanner for single quotes' do + scanner = StringScanner.new("'some string' 'and another one'") + + token = described_class.scan(scanner) + + expect(token).not_to be_nil + expect(token.build.evaluate).to eq 'some string' + end + + it 'allows to use single quotes inside double quotes' do + scanner = StringScanner.new(%("some ' string")) + + token = described_class.scan(scanner) + + expect(token).not_to be_nil + expect(token.build.evaluate).to eq "some ' string" + end + + it 'allow to use double quotes inside single quotes' do + scanner = StringScanner.new(%('some " string')) + + token = described_class.scan(scanner) + + expect(token).not_to be_nil + expect(token.build.evaluate).to eq 'some " string' + end + end + end + + describe '#evaluate' do + it 'returns string value it is is present' do + string = described_class.new('my string') + + expect(string.evaluate).to eq 'my string' + end + + it 'returns an empty string if it is empty' do + string = described_class.new('') + + expect(string.evaluate).to eq '' + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/variable_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/variable_spec.rb new file mode 100644 index 00000000000..599a5411881 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/variable_spec.rb @@ -0,0 +1,44 @@ +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Expression::Lexeme::Variable do + describe '.build' do + it 'creates a new instance of the token' do + expect(described_class.build('$VARIABLE')) + .to be_a(described_class) + end + end + + describe '.type' do + it 'is a value lexeme' do + expect(described_class.type).to eq :value + end + end + + describe '#evaluate' do + it 'returns variable value if it is defined' do + variable = described_class.new('VARIABLE') + + expect(variable.evaluate(VARIABLE: 'my variable')) + .to eq 'my variable' + end + + it 'allows to use a string as a variable key too' do + variable = described_class.new('VARIABLE') + + expect(variable.evaluate('VARIABLE' => 'my variable')) + .to eq 'my variable' + end + + it 'returns nil if it is not defined' do + variable = described_class.new('VARIABLE') + + expect(variable.evaluate(OTHER: 'variable')).to be_nil + end + + it 'returns an empty string if it is empty' do + variable = described_class.new('VARIABLE') + + expect(variable.evaluate(VARIABLE: '')).to eq '' + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb new file mode 100644 index 00000000000..230ceeb07f8 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb @@ -0,0 +1,70 @@ +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Expression::Lexer do + let(:token_class) do + Gitlab::Ci::Pipeline::Expression::Token + end + + describe '#tokens' do + it 'tokenss single value' do + tokens = described_class.new('$VARIABLE').tokens + + expect(tokens).to be_one + expect(tokens).to all(be_an_instance_of(token_class)) + end + + it 'does ignore whitespace characters' do + tokens = described_class.new("\t$VARIABLE ").tokens + + expect(tokens).to be_one + expect(tokens).to all(be_an_instance_of(token_class)) + end + + it 'tokenss multiple values of the same token' do + tokens = described_class.new("$VARIABLE1 $VARIABLE2").tokens + + expect(tokens.size).to eq 2 + expect(tokens).to all(be_an_instance_of(token_class)) + end + + it 'tokenss multiple values with different tokens' do + tokens = described_class.new('$VARIABLE "text" "value"').tokens + + expect(tokens.size).to eq 3 + expect(tokens.first.value).to eq '$VARIABLE' + expect(tokens.second.value).to eq '"text"' + expect(tokens.third.value).to eq '"value"' + end + + it 'tokenss tokens and operators' do + tokens = described_class.new('$VARIABLE == "text"').tokens + + expect(tokens.size).to eq 3 + expect(tokens.first.value).to eq '$VARIABLE' + expect(tokens.second.value).to eq '==' + expect(tokens.third.value).to eq '"text"' + end + + it 'limits statement to specified amount of tokens' do + lexer = described_class.new("$V1 $V2 $V3 $V4", max_tokens: 3) + + expect { lexer.tokens } + .to raise_error described_class::SyntaxError + end + + it 'raises syntax error in case of finding unknown tokens' do + lexer = described_class.new('$V1 123 $V2') + + expect { lexer.tokens } + .to raise_error described_class::SyntaxError + end + end + + describe '#lexemes' do + it 'returns an array of syntax lexemes' do + lexer = described_class.new('$VAR "text"') + + expect(lexer.lexemes).to eq %w[variable string] + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb new file mode 100644 index 00000000000..e8e6f585310 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb @@ -0,0 +1,26 @@ +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Expression::Parser do + describe '#tree' do + context 'when using operators' do + it 'returns a reverse descent parse tree' do + expect(described_class.seed('$VAR1 == "123" == $VAR2').tree) + .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::Equals + end + end + + context 'when using a single token' do + it 'returns a single token instance' do + expect(described_class.seed('$VAR').tree) + .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::Variable + end + end + + context 'when expression is empty' do + it 'returns a null token' do + expect(described_class.seed('').tree) + .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::Null + end + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb new file mode 100644 index 00000000000..472a58599d8 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb @@ -0,0 +1,85 @@ +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Expression::Statement do + let(:pipeline) { build(:ci_pipeline) } + + subject do + described_class.new(text, pipeline) + end + + before do + pipeline.variables.build([key: 'VARIABLE', value: 'my variable']) + end + + describe '#parse_tree' do + context 'when expression is empty' do + let(:text) { '' } + + it 'raises an error' do + expect { subject.parse_tree } + .to raise_error described_class::StatementError + end + end + + context 'when expression grammar is incorrect' do + table = [ + '$VAR "text"', # missing operator + '== "123"', # invalid right side + "'single quotes'", # single quotes string + '$VAR ==', # invalid right side + '12345', # unknown syntax + '' # empty statement + ] + + table.each do |syntax| + it "raises an error when syntax is `#{syntax}`" do + expect { described_class.new(syntax, pipeline).parse_tree } + .to raise_error described_class::StatementError + end + end + end + + context 'when expression grammar is correct' do + context 'when using an operator' do + let(:text) { '$VAR == "value"' } + + it 'returns a reverse descent parse tree' do + expect(subject.parse_tree) + .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::Equals + end + end + + context 'when using a single token' do + let(:text) { '$VARIABLE' } + + it 'returns a single token instance' do + expect(subject.parse_tree) + .to be_a Gitlab::Ci::Pipeline::Expression::Lexeme::Variable + end + end + end + end + + describe '#evaluate' do + statements = [ + ['$VARIABLE == "my variable"', true], + ["$VARIABLE == 'my variable'", true], + ['"my variable" == $VARIABLE', true], + ['$VARIABLE == null', false], + ['$VAR == null', true], + ['null == $VAR', true], + ['$VARIABLE', 'my variable'], + ['$VAR', nil] + ] + + statements.each do |expression, value| + context "when using expression `#{expression}`" do + let(:text) { expression } + + it "evaluates to `#{value.inspect}`" do + expect(subject.evaluate).to eq value + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/expression/token_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/token_spec.rb new file mode 100644 index 00000000000..6d7453f0de5 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/expression/token_spec.rb @@ -0,0 +1,45 @@ +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Expression::Token do + let(:value) { '$VARIABLE' } + let(:lexeme) { Gitlab::Ci::Pipeline::Expression::Lexeme::Variable } + + subject { described_class.new(value, lexeme) } + + describe '#value' do + it 'returns raw token value' do + expect(subject.value).to eq value + end + end + + describe '#lexeme' do + it 'returns raw token lexeme' do + expect(subject.lexeme).to eq lexeme + end + end + + describe '#build' do + it 'delegates to lexeme after adding a value' do + expect(lexeme).to receive(:build) + .with(value, 'some', 'args') + + subject.build('some', 'args') + end + + it 'allows passing only required arguments' do + expect(subject.build).to be_an_instance_of(lexeme) + end + end + + describe '#type' do + it 'delegates type query to the lexeme' do + expect(subject.type).to eq :value + end + end + + describe '#to_lexeme' do + it 'returns raw lexeme syntax component name' do + expect(subject.to_lexeme).to eq 'variable' + end + end +end |