Implement pipeline expressions lexer

6 files changed, 74 insertions, 7 deletions

diff --git a/lib/gitlab/ci/pipeline/expression/lexeme.rb b/lib/gitlab/ci/pipeline/expression/lexeme.rb
index 3651f8eab95..bbf3fec7407 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme.rb
@@ -8,7 +8,9 @@ module Gitlab
           end
 
           def self.scan(scanner)
-            scanner.scan(PATTERN)
+            if scanner.scan(self::PATTERN)
+              Expression::Token.new(scanner.matched, self)
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb
index 2734ef34c06..26895174881 100644
--- a/lib/gitlab/ci/pipeline/expression/lexer.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexer.rb
@@ -2,12 +2,32 @@ module Gitlab
   module Ci
     module Pipeline
       module Expression
+        LEXEMES = [
+          Expression::Variable
+        ]
+
+        MAX_CYCLES = 5
+
         class Lexer
           def initialize(statement)
-            @statement = statement
+            @scanner = StringScanner.new(statement)
+            @tokens = []
           end
 
           def tokenize
+            @tokens.tap do
+              MAX_CYCLES.times do
+                LEXEMES.each do |lexeme|
+                  @scanner.scan(/\s+/) # ignore whitespace
+
+                  lexeme.scan(@scanner).tap do |token|
+                    @tokens.push(token) if token.present?
+                  end
+
+                  return @tokens if @scanner.eos?
+                end
+              end
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb
index aea6dc3f959..91d91195f5a 100644
--- a/lib/gitlab/ci/pipeline/expression/statement.rb
+++ b/lib/gitlab/ci/pipeline/expression/statement.rb
@@ -9,6 +9,7 @@ module Gitlab
             %w[variable equals null],
             %w[string equals variable],
             %w[null equals variable],
+            %w[variable]
           ]
 
           def initialize(pipeline, statement)
diff --git a/lib/gitlab/ci/pipeline/expression/token.rb b/lib/gitlab/ci/pipeline/expression/token.rb
new file mode 100644
index 00000000000..c800d1f0c08
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/expression/token.rb
@@ -0,0 +1,17 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Expression
+        class Token
+          def initialize(value, type)
+            @value = value
+            @type = type
+          end
+
+          def to_lexeme
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/expression/variable.rb b/lib/gitlab/ci/pipeline/expression/variable.rb
index 3ae2291fb1e..ec6c03d336a 100644
--- a/lib/gitlab/ci/pipeline/expression/variable.rb
+++ b/lib/gitlab/ci/pipeline/expression/variable.rb
@@ -2,8 +2,8 @@ module Gitlab
   module Ci
     module Pipeline
       module Expression
-        class Equality < Expression::Lexeme
-          PATTERN = /$(?<name>\w+)/.freeze
+        class Variable < Expression::Lexeme
+          PATTERN = /\$(?<name>\w+)/.freeze
 
           def initialize(value)
             @value = value
@@ -11,9 +11,6 @@ module Gitlab
 
           def evaluate(**variables)
           end
-
-          def self.build(string)
-          end
         end
       end
     end
diff --git a/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb b/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb
new file mode 100644
index 00000000000..84d54c5acf2
--- /dev/null
+++ b/spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb
@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+describe Gitlab::Ci::Pipeline::Expression::Lexer do
+  let(:token_class) do
+    Gitlab::Ci::Pipeline::Expression::Token
+  end
+
+  describe '#tokenize' do
+    it 'tokenizes single value' do
+      tokens = described_class.new('$VARIABLE').tokenize
+
+      expect(tokens).to be_one
+      expect(tokens).to all(be_an_instance_of(token_class))
+    end
+
+    it 'does ignore whitespace characters' do
+      tokens = described_class.new("\t$VARIABLE ").tokenize
+
+      expect(tokens).to be_one
+      expect(tokens).to all(be_an_instance_of(token_class))
+    end
+
+    it 'tokenizes multiple values of the same token' do
+      tokens = described_class.new("$VARIABLE1 $VARIABLE2").tokenize
+
+      expect(tokens.size).to eq 2
+      expect(tokens).to all(be_an_instance_of(token_class))
+    end
+  end
+end