Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce into feature/project-exportfeature/project-export

# Conflicts:
#	app/models/ci/pipeline.rb

19 files changed, 379 insertions, 59 deletions

diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 076e2af7d38..db1704af75e 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -3,14 +3,14 @@ module Gitlab
     Result = Struct.new(:user, :type)
 
     class << self
-      def find(login, password, project:, ip:)
+      def find_for_git_client(login, password, project:, ip:)
         raise "Must provide an IP for rate limiting" if ip.nil?
 
         result = Result.new
 
         if valid_ci_request?(login, password, project)
           result.type = :ci
-        elsif result.user = find_in_gitlab_or_ldap(login, password)
+        elsif result.user = find_with_user_password(login, password)
           result.type = :gitlab_or_ldap
         elsif result.user = oauth_access_token_check(login, password)
           result.type = :oauth
@@ -20,7 +20,7 @@ module Gitlab
         result
       end
 
-      def find_in_gitlab_or_ldap(login, password)
+      def find_with_user_password(login, password)
         user = User.by_login(login)
 
         # If no user is found, or it's an LDAP server, try LDAP.
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index 9e09d2e118d..7e3f5abba62 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -1,5 +1,3 @@
-require_relative 'shell_env'
-
 module Grack
   class AuthSpawner
     def self.call(env)
@@ -61,11 +59,6 @@ module Grack
       end
 
       @user = authenticate_user(login, password)
-
-      if @user
-        Gitlab::ShellEnv.set_env(@user)
-        @env['REMOTE_USER'] = @auth.username
-      end
     end
 
     def ci_request?(login, password)
@@ -95,7 +88,7 @@ module Grack
     end
 
     def authenticate_user(login, password)
-      user = Gitlab::Auth.find_in_gitlab_or_ldap(login, password)
+      user = Gitlab::Auth.find_with_user_password(login, password)
 
       unless user
         user = oauth_access_token_check(login, password)
diff --git a/lib/gitlab/backend/shell_env.rb b/lib/gitlab/backend/shell_env.rb
deleted file mode 100644
index 9f5adee594a..00000000000
--- a/lib/gitlab/backend/shell_env.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-module Gitlab
-  # This module provide 2 methods
-  # to set specific ENV variables for GitLab Shell
-  module ShellEnv
-    extend self
-
-    def set_env(user)
-      # Set GL_ID env variable
-      if user
-        ENV['GL_ID'] = gl_id(user)
-      end
-    end
-
-    def reset_env
-      # Reset GL_ID env variable
-      ENV['GL_ID'] = nil
-    end
-
-    def gl_id(user)
-      if user.present?
-        "user-#{user.id}"
-      else
-        # This empty string is used in the render_grack_auth_ok method
-        ""
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index ffe633d4b63..b48d3592f16 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -1,11 +1,21 @@
 module Gitlab
   module Ci
+    ##
+    # Base GitLab CI Configuration facade
+    #
     class Config
-      class LoaderError < StandardError; end
+      delegate :valid?, :errors, to: :@global
+
+      ##
+      # Temporary delegations that should be removed after refactoring
+      #
+      delegate :before_script, to: :@global
 
       def initialize(config)
-        loader = Loader.new(config)
-        @config = loader.load!
+        @config = Loader.new(config).load!
+
+        @global = Node::Global.new(@config)
+        @global.process!
       end
 
       def to_hash
diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb
new file mode 100644
index 00000000000..d60f87f3f94
--- /dev/null
+++ b/lib/gitlab/ci/config/node/configurable.rb
@@ -0,0 +1,61 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # This mixin is responsible for adding DSL, which purpose is to
+        # simplifly process of adding child nodes.
+        #
+        # This can be used only if parent node is a configuration entry that
+        # holds a hash as a configuration value, for example:
+        #
+        # job:
+        #   script: ...
+        #   artifacts: ...
+        #
+        module Configurable
+          extend ActiveSupport::Concern
+
+          def allowed_nodes
+            self.class.allowed_nodes || {}
+          end
+
+          private
+
+          def prevalidate!
+            unless @value.is_a?(Hash)
+              @errors << 'should be a configuration entry with hash value'
+            end
+          end
+
+          def create_node(key, factory)
+            factory.with(value: @value[key])
+            factory.nullify! unless @value.has_key?(key)
+            factory.create!
+          end
+
+          class_methods do
+            def allowed_nodes
+              Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }]
+            end
+
+            private
+
+            def allow_node(symbol, entry_class, metadata)
+              factory = Node::Factory.new(entry_class)
+                .with(description: metadata[:description])
+
+              define_method(symbol) do
+                raise Entry::InvalidError unless valid?
+
+                @nodes[symbol].try(:value)
+              end
+
+              (@allowed_nodes ||= {}).merge!(symbol => factory)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb
new file mode 100644
index 00000000000..52758a962f3
--- /dev/null
+++ b/lib/gitlab/ci/config/node/entry.rb
@@ -0,0 +1,77 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Base abstract class for each configuration entry node.
+        #
+        class Entry
+          class InvalidError < StandardError; end
+
+          attr_accessor :description
+
+          def initialize(value)
+            @value = value
+            @nodes = {}
+            @errors = []
+
+            prevalidate!
+          end
+
+          def process!
+            return if leaf?
+            return unless valid?
+
+            compose!
+
+            nodes.each(&:process!)
+            nodes.each(&:validate!)
+          end
+
+          def nodes
+            @nodes.values
+          end
+
+          def valid?
+            errors.none?
+          end
+
+          def leaf?
+            allowed_nodes.none?
+          end
+
+          def errors
+            @errors + nodes.map(&:errors).flatten
+          end
+
+          def allowed_nodes
+            {}
+          end
+
+          def validate!
+            raise NotImplementedError
+          end
+
+          def value
+            raise NotImplementedError
+          end
+
+          private
+
+          def prevalidate!
+          end
+
+          def compose!
+            allowed_nodes.each do |key, essence|
+              @nodes[key] = create_node(key, essence)
+            end
+          end
+
+          def create_node(key, essence)
+            raise NotImplementedError
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb
new file mode 100644
index 00000000000..787ca006f5a
--- /dev/null
+++ b/lib/gitlab/ci/config/node/factory.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Factory class responsible for fabricating node entry objects.
+        #
+        # It uses Fluent Interface pattern to set all necessary attributes.
+        #
+        class Factory
+          class InvalidFactory < StandardError; end
+
+          def initialize(entry_class)
+            @entry_class = entry_class
+            @attributes = {}
+          end
+
+          def with(attributes)
+            @attributes.merge!(attributes)
+            self
+          end
+
+          def nullify!
+            @entry_class = Node::Null
+            self
+          end
+
+          def create!
+            raise InvalidFactory unless @attributes.has_key?(:value)
+
+            @entry_class.new(@attributes[:value]).tap do |entry|
+              entry.description = @attributes[:description]
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb
new file mode 100644
index 00000000000..044603423d5
--- /dev/null
+++ b/lib/gitlab/ci/config/node/global.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # This class represents a global entry - root node for entire
+        # GitLab CI Configuration file.
+        #
+        class Global < Entry
+          include Configurable
+
+          allow_node :before_script, Script,
+            description: 'Script that will be executed before each job.'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb
new file mode 100644
index 00000000000..4f590f6bec8
--- /dev/null
+++ b/lib/gitlab/ci/config/node/null.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # This class represents a configuration entry that is not being used
+        # in configuration file.
+        #
+        # This implements Null Object pattern.
+        #
+        class Null < Entry
+          def value
+            nil
+          end
+
+          def validate!
+            nil
+          end
+
+          def method_missing(*)
+            nil
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb
new file mode 100644
index 00000000000..5072bf0db7d
--- /dev/null
+++ b/lib/gitlab/ci/config/node/script.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents a script.
+        #
+        # Each element in the value array is a command that will be executed
+        # by GitLab Runner. Currently we concatenate these commands with
+        # new line character as a separator, what is compatible with
+        # implementation in Runner.
+        #
+        class Script < Entry
+          include ValidationHelpers
+
+          def value
+            @value.join("\n")
+          end
+
+          def validate!
+            unless validate_array_of_strings(@value)
+              @errors << 'before_script should be an array of strings'
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/validation_helpers.rb
new file mode 100644
index 00000000000..3900fc89391
--- /dev/null
+++ b/lib/gitlab/ci/config/node/validation_helpers.rb
@@ -0,0 +1,38 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        module ValidationHelpers
+          private
+
+          def validate_duration(value)
+            value.is_a?(String) && ChronicDuration.parse(value)
+          rescue ChronicDuration::DurationParseError
+            false
+          end
+
+          def validate_array_of_strings(values)
+            values.is_a?(Array) && values.all? { |value| validate_string(value) }
+          end
+
+          def validate_variables(variables)
+            variables.is_a?(Hash) &&
+              variables.all? { |key, value| validate_string(key) && validate_string(value) }
+          end
+
+          def validate_string(value)
+            value.is_a?(String) || value.is_a?(Symbol)
+          end
+
+          def validate_environment(value)
+            value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex
+          end
+
+          def validate_boolean(value)
+            value.in?([true, false])
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 04fa6a3a5de..d76ecb54017 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -30,6 +30,10 @@ module Gitlab
       order
     end
 
+    def self.random
+      Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()"
+    end
+
     def true_value
       if Gitlab::Database.postgresql?
         "'t'"
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 978c3f7896d..dd3ff0ab18b 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -31,8 +31,6 @@ module Gitlab
       # Any data inserted while running this method (or after it has finished
       # running) is _not_ updated automatically.
       #
-      # This method _only_ updates rows where the column's value is set to NULL.
-      #
       # table - The name of the table.
       # column - The name of the column to update.
       # value - The value for the column.
@@ -55,10 +53,10 @@ module Gitlab
           first['count'].
           to_i
 
-        # Update in batches of 5%
+        # Update in batches of 5% until we run out of any rows to update.
         batch_size = ((total / 100.0) * 5.0).ceil
 
-        while processed < total
+        loop do
           start_row = exec_query(%Q{
             SELECT id
             FROM #{quoted_table}
@@ -66,6 +64,9 @@ module Gitlab
             LIMIT 1 OFFSET #{processed}
           }).to_hash.first
 
+          # There are no more rows to process
+          break unless start_row
+
           stop_row = exec_query(%Q{
             SELECT id
             FROM #{quoted_table}
@@ -126,6 +127,8 @@ module Gitlab
         begin
           transaction do
             update_column_in_batches(table, column, default)
+
+            change_column_null(table, column, false) unless allow_null
           end
         # We want to rescue _all_ exceptions here, even those that don't inherit
         # from StandardError.
@@ -134,8 +137,6 @@ module Gitlab
 
           raise error
         end
-
-        change_column_null(table, column, false) unless allow_null
       end
     end
   end
diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb
new file mode 100644
index 00000000000..624fd00367e
--- /dev/null
+++ b/lib/gitlab/gl_id.rb
@@ -0,0 +1,11 @@
+module Gitlab
+  module GlId
+    def self.gl_id(user)
+      if user.present?
+        "user-#{user.id}"
+      else
+        ""
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 0f115893a15..d81d26754fe 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -56,7 +56,7 @@ module Gitlab
         end
       end
 
-      # Instruments all public methods of a module.
+      # Instruments all public and private methods of a module.
       #
       # This method optionally takes a block that can be used to determine if a
       # method should be instrumented or not. The block is passed the receiving
@@ -65,7 +65,8 @@ module Gitlab
       #
       # mod - The module to instrument.
       def self.instrument_methods(mod)
-        mod.public_methods(false).each do |name|
+        methods = mod.methods(false) + mod.private_methods(false)
+        methods.each do |name|
           method = mod.method(name)
 
           if method.owner == mod.singleton_class
@@ -76,13 +77,14 @@ module Gitlab
         end
       end
 
-      # Instruments all public instance methods of a module.
+      # Instruments all public and private instance methods of a module.
       #
       # See `instrument_methods` for more information.
       #
       # mod - The module to instrument.
       def self.instrument_instance_methods(mod)
-        mod.public_instance_methods(false).each do |name|
+        methods = mod.instance_methods(false) + mod.private_instance_methods(false)
+        methods.each do |name|
           method = mod.instance_method(name)
 
           if method.owner == mod
@@ -149,13 +151,16 @@ module Gitlab
             trans = Gitlab::Metrics::Instrumentation.transaction
 
             if trans
-              start    = Time.now
-              retval   = super
-              duration = (Time.now - start) * 1000.0
+              start     = Time.now
+              cpu_start = Gitlab::Metrics::System.cpu_time
+              retval    = super
+              duration  = (Time.now - start) * 1000.0
 
               if duration >= Gitlab::Metrics.method_call_threshold
+                cpu_duration = Gitlab::Metrics::System.cpu_time - cpu_start
+
                 trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES,
-                                 { duration: duration },
+                                 { duration: duration, cpu_duration: cpu_duration },
                                  method: #{label.inspect})
               end
 
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index 6f179789d3e..3fe27779d03 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -1,8 +1,9 @@
 module Gitlab
   module Metrics
-    # Rack middleware for tracking Rails requests.
+    # Rack middleware for tracking Rails and Grape requests.
     class RackMiddleware
       CONTROLLER_KEY = 'action_controller.instance'
+      ENDPOINT_KEY   = 'api.endpoint'
 
       def initialize(app)
         @app = app
@@ -21,6 +22,8 @@ module Gitlab
         ensure
           if env[CONTROLLER_KEY]
             tag_controller(trans, env)
+          elsif env[ENDPOINT_KEY]
+            tag_endpoint(trans, env)
           end
 
           trans.finish
@@ -42,6 +45,26 @@ module Gitlab
         controller   = env[CONTROLLER_KEY]
         trans.action = "#{controller.class.name}##{controller.action_name}"
       end
+
+      def tag_endpoint(trans, env)
+        endpoint = env[ENDPOINT_KEY]
+        path = endpoint_paths_cache[endpoint.route.route_method][endpoint.route.route_path]
+        trans.action = "Grape##{endpoint.route.route_method} #{path}"
+      end
+
+      private
+
+      def endpoint_paths_cache
+        @endpoint_paths_cache ||= Hash.new do |hash, http_method|
+          hash[http_method] = Hash.new do |inner_hash, raw_path|
+            inner_hash[raw_path] = endpoint_instrumentable_path(raw_path)
+          end
+        end
+      end
+
+      def endpoint_instrumentable_path(raw_path)
+        raw_path.sub('(.:format)', '').sub('/:version', '')
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb
index fc709222a9b..0000450d9bb 100644
--- a/lib/gitlab/metrics/sampler.rb
+++ b/lib/gitlab/metrics/sampler.rb
@@ -66,7 +66,11 @@ module Gitlab
         def sample_objects
           sample = Allocations.to_hash
           counts = sample.each_with_object({}) do |(klass, count), hash|
-            hash[klass.name] = count
+            name = klass.name
+
+            next unless name
+
+            hash[name] = count
           end
 
           # Symbols aren't allocated so we'll need to add those manually.
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 1cbd6d945a0..c84c68f96f6 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -100,5 +100,13 @@ module Gitlab
     def container_registry_reference_regex
       git_reference_regex
     end
+
+    def environment_name_regex
+      @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze
+    end
+
+    def environment_name_regex_message
+      "can contain only letters, digits, '-' and '_'."
+    end
   end
 end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 388f84dbe0e..40e8299c36b 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -8,7 +8,7 @@ module Gitlab
     class << self
       def git_http_ok(repository, user)
         {
-          'GL_ID' => Gitlab::ShellEnv.gl_id(user),
+          'GL_ID' => Gitlab::GlId.gl_id(user),
           'RepoPath' => repository.path_to_repo,
         }
       end