Merge branch 'feature/gb/add-regexp-variables-expression' into 'master'

Add support for variables expression regexp syntax

Closes #43601

See merge request gitlab-org/gitlab-ce!18902

8 files changed, 294 insertions, 48 deletions

diff --git a/spec/lib/gitlab/ci/config/entry/policy_spec.rb b/spec/lib/gitlab/ci/config/entry/policy_spec.rb
index 08718c382b9..83d39b82068 100644
--- a/spec/lib/gitlab/ci/config/entry/policy_spec.rb
+++ b/spec/lib/gitlab/ci/config/entry/policy_spec.rb
@@ -111,7 +111,15 @@ describe Gitlab::Ci::Config::Entry::Policy do
     context 'when specifying invalid variables expressions token' do
       let(:config) { { variables: ['$MY_VAR == 123'] } }
 
-      it 'reports an error about invalid statement' do
+      it 'reports an error about invalid expression' do
+        expect(entry.errors).to include /invalid expression syntax/
+      end
+    end
+
+    context 'when using invalid variables expressions regexp' do
+      let(:config) { { variables: ['$MY_VAR =~ /some ( thing/'] } }
+
+      it 'reports an error about invalid expression' do
         expect(entry.errors).to include /invalid expression syntax/
       end
     end
diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/matches_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/matches_spec.rb
new file mode 100644
index 00000000000..49e5af52f4d
--- /dev/null
+++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/matches_spec.rb
@@ -0,0 +1,80 @@
+require 'fast_spec_helper'
+require_dependency 're2'
+
+describe Gitlab::Ci::Pipeline::Expression::Lexeme::Matches do
+  let(:left) { double('left') }
+  let(:right) { double('right') }
+
+  describe '.build' do
+    it 'creates a new instance of the token' do
+      expect(described_class.build('=~', left, right))
+        .to be_a(described_class)
+    end
+  end
+
+  describe '.type' do
+    it 'is an operator' do
+      expect(described_class.type).to eq :operator
+    end
+  end
+
+  describe '#evaluate' do
+    it 'returns false when left and right do not match' do
+      allow(left).to receive(:evaluate).and_return('my-string')
+      allow(right).to receive(:evaluate)
+        .and_return(Gitlab::UntrustedRegexp.new('something'))
+
+      operator = described_class.new(left, right)
+
+      expect(operator.evaluate).to eq false
+    end
+
+    it 'returns true when left and right match' do
+      allow(left).to receive(:evaluate).and_return('my-awesome-string')
+      allow(right).to receive(:evaluate)
+        .and_return(Gitlab::UntrustedRegexp.new('awesome.string$'))
+
+      operator = described_class.new(left, right)
+
+      expect(operator.evaluate).to eq true
+    end
+
+    it 'supports matching against a nil value' do
+      allow(left).to receive(:evaluate).and_return(nil)
+      allow(right).to receive(:evaluate)
+        .and_return(Gitlab::UntrustedRegexp.new('pattern'))
+
+      operator = described_class.new(left, right)
+
+      expect(operator.evaluate).to eq false
+    end
+
+    it 'supports multiline strings' do
+      allow(left).to receive(:evaluate).and_return <<~TEXT
+        My awesome contents
+
+        My-text-string!
+      TEXT
+
+      allow(right).to receive(:evaluate)
+        .and_return(Gitlab::UntrustedRegexp.new('text-string'))
+
+      operator = described_class.new(left, right)
+
+      expect(operator.evaluate).to eq true
+    end
+
+    it 'supports regexp flags' do
+      allow(left).to receive(:evaluate).and_return <<~TEXT
+        My AWESOME content
+      TEXT
+
+      allow(right).to receive(:evaluate)
+        .and_return(Gitlab::UntrustedRegexp.new('(?i)awesome'))
+
+      operator = described_class.new(left, right)
+
+      expect(operator.evaluate).to eq true
+    end
+  end
+end
diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexeme/pattern_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexeme/pattern_spec.rb
new file mode 100644
index 00000000000..3ebc2e94727
--- /dev/null
+++ b/spec/lib/gitlab/ci/pipeline/expression/lexeme/pattern_spec.rb
@@ -0,0 +1,96 @@
+require 'fast_spec_helper'
+
+describe Gitlab::Ci::Pipeline::Expression::Lexeme::Pattern do
+  describe '.build' do
+    it 'creates a new instance of the token' do
+      expect(described_class.build('/.*/'))
+        .to be_a(described_class)
+    end
+
+    it 'raises an error if pattern is invalid' do
+      expect { described_class.build('/ some ( thin/i') }
+        .to raise_error(Gitlab::Ci::Pipeline::Expression::Lexer::SyntaxError)
+    end
+  end
+
+  describe '.type' do
+    it 'is a value lexeme' do
+      expect(described_class.type).to eq :value
+    end
+  end
+
+  describe '.scan' do
+    it 'correctly identifies a pattern token' do
+      scanner = StringScanner.new('/pattern/')
+
+      token = described_class.scan(scanner)
+
+      expect(token).not_to be_nil
+      expect(token.build.evaluate)
+        .to eq Gitlab::UntrustedRegexp.new('pattern')
+    end
+
+    it 'is a greedy scanner for regexp boundaries' do
+      scanner = StringScanner.new('/some .* / pattern/')
+
+      token = described_class.scan(scanner)
+
+      expect(token).not_to be_nil
+      expect(token.build.evaluate)
+        .to eq Gitlab::UntrustedRegexp.new('some .* / pattern')
+    end
+
+    it 'does not allow to use an empty pattern' do
+      scanner = StringScanner.new(%(//))
+
+      token = described_class.scan(scanner)
+
+      expect(token).to be_nil
+    end
+
+    it 'support single flag' do
+      scanner = StringScanner.new('/pattern/i')
+
+      token = described_class.scan(scanner)
+
+      expect(token).not_to be_nil
+      expect(token.build.evaluate)
+        .to eq Gitlab::UntrustedRegexp.new('(?i)pattern')
+    end
+
+    it 'support multiple flags' do
+      scanner = StringScanner.new('/pattern/im')
+
+      token = described_class.scan(scanner)
+
+      expect(token).not_to be_nil
+      expect(token.build.evaluate)
+        .to eq Gitlab::UntrustedRegexp.new('(?im)pattern')
+    end
+
+    it 'does not support arbitrary flags' do
+      scanner = StringScanner.new('/pattern/x')
+
+      token = described_class.scan(scanner)
+
+      expect(token).to be_nil
+    end
+  end
+
+  describe '#evaluate' do
+    it 'returns a regular expression' do
+      regexp = described_class.new('/abc/')
+
+      expect(regexp.evaluate).to eq Gitlab::UntrustedRegexp.new('abc')
+    end
+
+    it 'raises error if evaluated regexp is not valid' do
+      allow(Gitlab::UntrustedRegexp).to receive(:valid?).and_return(true)
+
+      regexp = described_class.new('/invalid ( .*/')
+
+      expect { regexp.evaluate }
+        .to raise_error(Gitlab::Ci::Pipeline::Expression::RuntimeError)
+    end
+  end
+end
diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb
index 230ceeb07f8..3f11b3f7673 100644
--- a/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb
@@ -6,7 +6,7 @@ describe Gitlab::Ci::Pipeline::Expression::Lexer do
   end
 
   describe '#tokens' do
-    it 'tokenss single value' do
+    it 'returns single value' do
       tokens = described_class.new('$VARIABLE').tokens
 
       expect(tokens).to be_one
@@ -20,14 +20,14 @@ describe Gitlab::Ci::Pipeline::Expression::Lexer do
       expect(tokens).to all(be_an_instance_of(token_class))
     end
 
-    it 'tokenss multiple values of the same token' do
+    it 'returns multiple values of the same token' do
       tokens = described_class.new("$VARIABLE1 $VARIABLE2").tokens
 
       expect(tokens.size).to eq 2
       expect(tokens).to all(be_an_instance_of(token_class))
     end
 
-    it 'tokenss multiple values with different tokens' do
+    it 'returns multiple values with different tokens' do
       tokens = described_class.new('$VARIABLE "text" "value"').tokens
 
       expect(tokens.size).to eq 3
@@ -36,7 +36,7 @@ describe Gitlab::Ci::Pipeline::Expression::Lexer do
       expect(tokens.third.value).to eq '"value"'
     end
 
-    it 'tokenss tokens and operators' do
+    it 'returns tokens and operators' do
       tokens = described_class.new('$VARIABLE == "text"').tokens
 
       expect(tokens.size).to eq 3
diff --git a/spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb
index e8e6f585310..2b78b1dd4a7 100644
--- a/spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/expression/parser_spec.rb
@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'fast_spec_helper'
 
 describe Gitlab::Ci::Pipeline::Expression::Parser do
   describe '#tree' do
diff --git a/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb
index 6685bf5385b..11e73294f18 100644
--- a/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/expression/statement_spec.rb
@@ -1,4 +1,5 @@
-require 'spec_helper'
+require 'fast_spec_helper'
+require 'rspec-parameterized'
 
 describe Gitlab::Ci::Pipeline::Expression::Statement do
   subject do
@@ -36,7 +37,7 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do
         '== "123"',      # invalid left side
         '"some string"', # only string provided
         '$VAR ==',       # invalid right side
-        '12345',         # unknown syntax
+        'null',          # missing lexemes
         ''               # empty statement
       ]
 
@@ -44,7 +45,7 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do
         context "when expression grammar is #{syntax.inspect}" do
           let(:text) { syntax }
 
-          it 'aises a statement error exception' do
+          it 'raises a statement error exception' do
             expect { subject.parse_tree }
               .to raise_error described_class::StatementError
           end
@@ -82,48 +83,66 @@ describe Gitlab::Ci::Pipeline::Expression::Statement do
   end
 
   describe '#evaluate' do
-    statements = [
-      ['$PRESENT_VARIABLE == "my variable"', true],
-      ["$PRESENT_VARIABLE == 'my variable'", true],
-      ['"my variable" == $PRESENT_VARIABLE', true],
-      ['$PRESENT_VARIABLE == null', false],
-      ['$EMPTY_VARIABLE == null', false],
-      ['"" == $EMPTY_VARIABLE', true],
-      ['$EMPTY_VARIABLE', ''],
-      ['$UNDEFINED_VARIABLE == null', true],
-      ['null == $UNDEFINED_VARIABLE', true],
-      ['$PRESENT_VARIABLE', 'my variable'],
-      ['$UNDEFINED_VARIABLE', nil]
-    ]
-
-    statements.each do |expression, value|
-      context "when using expression `#{expression}`" do
-        let(:text) { expression }
-
-        it "evaluates to `#{value.inspect}`" do
-          expect(subject.evaluate).to eq value
-        end
+    using RSpec::Parameterized::TableSyntax
+
+    where(:expression, :value) do
+      '$PRESENT_VARIABLE == "my variable"' | true
+      '"my variable" == $PRESENT_VARIABLE' | true
+      '$PRESENT_VARIABLE == null'          | false
+      '$EMPTY_VARIABLE == null'            | false
+      '"" == $EMPTY_VARIABLE'              | true
+      '$EMPTY_VARIABLE'                    | ''
+      '$UNDEFINED_VARIABLE == null'        | true
+      'null == $UNDEFINED_VARIABLE'        | true
+      '$PRESENT_VARIABLE'                  | 'my variable'
+      '$UNDEFINED_VARIABLE'                | nil
+      "$PRESENT_VARIABLE =~ /var.*e$/"     | true
+      "$PRESENT_VARIABLE =~ /^var.*/"      | false
+      "$EMPTY_VARIABLE =~ /var.*/"         | false
+      "$UNDEFINED_VARIABLE =~ /var.*/"     | false
+      "$PRESENT_VARIABLE =~ /VAR.*/i"      | true
+    end
+
+    with_them do
+      let(:text) { expression }
+
+      it "evaluates to `#{params[:value].inspect}`" do
+        expect(subject.evaluate).to eq value
       end
     end
   end
 
   describe '#truthful?' do
-    statements = [
-      ['$PRESENT_VARIABLE == "my variable"', true],
-      ["$PRESENT_VARIABLE == 'no match'", false],
-      ['$UNDEFINED_VARIABLE == null', true],
-      ['$PRESENT_VARIABLE', true],
-      ['$UNDEFINED_VARIABLE', false],
-      ['$EMPTY_VARIABLE', false]
-    ]
-
-    statements.each do |expression, value|
-      context "when using expression `#{expression}`" do
-        let(:text) { expression }
-
-        it "returns `#{value.inspect}`" do
-          expect(subject.truthful?).to eq value
-        end
+    using RSpec::Parameterized::TableSyntax
+
+    where(:expression, :value) do
+      '$PRESENT_VARIABLE == "my variable"' | true
+      "$PRESENT_VARIABLE == 'no match'"    | false
+      '$UNDEFINED_VARIABLE == null'        | true
+      '$PRESENT_VARIABLE'                  | true
+      '$UNDEFINED_VARIABLE'                | false
+      '$EMPTY_VARIABLE'                    | false
+      '$INVALID = 1'                       | false
+      "$PRESENT_VARIABLE =~ /var.*/"       | true
+      "$UNDEFINED_VARIABLE =~ /var.*/"     | false
+    end
+
+    with_them do
+      let(:text) { expression }
+
+      it "returns `#{params[:value].inspect}`" do
+        expect(subject.truthful?).to eq value
+      end
+    end
+
+    context 'when evaluating expression raises an error' do
+      let(:text) { '$PRESENT_VARIABLE' }
+
+      it 'returns false' do
+        allow(subject).to receive(:evaluate)
+          .and_raise(described_class::StatementError)
+
+        expect(subject.truthful?).to be_falsey
       end
     end
   end
diff --git a/spec/lib/gitlab/ci/pipeline/expression/token_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/token_spec.rb
index 6d7453f0de5..cedfe270f9d 100644
--- a/spec/lib/gitlab/ci/pipeline/expression/token_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/expression/token_spec.rb
@@ -1,4 +1,4 @@
-require 'spec_helper'
+require 'fast_spec_helper'
 
 describe Gitlab::Ci::Pipeline::Expression::Token do
   let(:value) { '$VARIABLE' }
diff --git a/spec/lib/gitlab/untrusted_regexp_spec.rb b/spec/lib/gitlab/untrusted_regexp_spec.rb
index 0ee7fa1e570..0a6ac0aa294 100644
--- a/spec/lib/gitlab/untrusted_regexp_spec.rb
+++ b/spec/lib/gitlab/untrusted_regexp_spec.rb
@@ -1,6 +1,49 @@
-require 'spec_helper'
+require 'fast_spec_helper'
+require 'support/shared_examples/malicious_regexp_shared_examples'
 
 describe Gitlab::UntrustedRegexp do
+  describe '.valid?' do
+    it 'returns true if regexp is valid' do
+      expect(described_class.valid?('/some ( thing/'))
+        .to be false
+    end
+
+    it 'returns true if regexp is invalid' do
+      expect(described_class.valid?('/some .* thing/'))
+        .to be true
+    end
+  end
+
+  describe '.fabricate' do
+    context 'when regexp is using /regexp/ scheme with flags' do
+      it 'fabricates regexp with a single flag' do
+        regexp = described_class.fabricate('/something/i')
+
+        expect(regexp).to eq described_class.new('(?i)something')
+        expect(regexp.scan('SOMETHING')).to be_one
+      end
+
+      it 'fabricates regexp with multiple flags' do
+        regexp = described_class.fabricate('/something/im')
+
+        expect(regexp).to eq described_class.new('(?im)something')
+      end
+
+      it 'fabricates regexp without flags' do
+        regexp = described_class.fabricate('/something/')
+
+        expect(regexp).to eq described_class.new('something')
+      end
+    end
+
+    context 'when regexp is a raw pattern' do
+      it 'raises an error' do
+        expect { described_class.fabricate('some .* thing') }
+          .to raise_error(RegexpError)
+      end
+    end
+  end
+
   describe '#initialize' do
     subject { described_class.new(pattern) }