Merge remote-tracking branch 'origin/master' into ci-commit-as-pipelineci-commit-as-pipeline

# Conflicts:
#	db/schema.rb

12 files changed, 231 insertions, 48 deletions

diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 2200208b946..8cfa1f1556b 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -23,9 +23,11 @@ module API
       end
 
       post "/allowed" do
+        Gitlab::Metrics.tag_transaction('action', 'Grape#/internal/allowed')
+
         status 200
 
-        actor = 
+        actor =
           if params[:key_id]
             Key.find_by(id: params[:key_id])
           elsif params[:user_id]
@@ -33,7 +35,7 @@ module API
           end
 
         project_path = params[:project]
-        
+
         # Check for *.wiki repositories.
         # Strip out the .wiki from the pathname before finding the
         # project. This applies the correct project permissions to
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index d1a10479e44..3e1ed3fe5c7 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -12,7 +12,7 @@ module API
       # Example Request:
       #   GET /projects/:id/repository/tags
       get ":id/repository/tags" do
-        present user_project.repo.tags.sort_by(&:name).reverse,
+        present user_project.repository.tags.sort_by(&:name).reverse,
                 with: Entities::RepoTag, project: user_project
       end
 
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 0a14bac07c0..ea6fa2dc8a8 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -11,6 +11,10 @@ module API
       #  GET /users?search=Admin
       #  GET /users?username=root
       get do
+        unless can?(current_user, :read_users_list, nil)
+          render_api_error!("Not authorized.", 403)
+        end
+
         if params[:username].present?
           @users = User.where(username: params[:username])
         else
@@ -36,10 +40,12 @@ module API
       get ":id" do
         @user = User.find(params[:id])
 
-        if current_user.is_admin?
+        if current_user && current_user.is_admin?
           present @user, with: Entities::UserFull
-        else
+        elsif can?(current_user, :read_user, @user)
           present @user, with: Entities::User
+        else
+          render_api_error!("User not found.", 404)
         end
       end
 
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index b7209c14148..ff9887cba1e 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -4,12 +4,12 @@ module Ci
 
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
-    ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache]
+    ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache]
     ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services,
                         :allow_failure, :type, :stage, :when, :artifacts, :cache,
-                        :dependencies]
+                        :dependencies, :before_script, :after_script, :variables]
 
-    attr_reader :before_script, :image, :services, :variables, :path, :cache
+    attr_reader :before_script, :after_script, :image, :services, :path, :cache
 
     def initialize(config, path = nil)
       @config = YAML.safe_load(config, [Symbol], [], true)
@@ -40,10 +40,22 @@ module Ci
       @stages || DEFAULT_STAGES
     end
 
+    def global_variables
+      @variables
+    end
+
+    def job_variables(name)
+      job = @jobs[name.to_sym]
+      return [] unless job
+
+      job.fetch(:variables, [])
+    end
+
     private
 
     def initial_parsing
       @before_script = @config[:before_script] || []
+      @after_script = @config[:after_script]
       @image = @config[:image]
       @services = @config[:services]
       @stages = @config[:stages] || @config[:types]
@@ -72,7 +84,7 @@ module Ci
       {
         stage_idx: stages.index(job[:stage]),
         stage: job[:stage],
-        commands: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}",
+        commands: [job[:before_script] || @before_script, job[:script]].flatten.join("\n"),
         tag_list: job[:tags] || [],
         name: name,
         only: job[:only],
@@ -85,23 +97,32 @@ module Ci
           artifacts: job[:artifacts],
           cache: job[:cache] || @cache,
           dependencies: job[:dependencies],
+          after_script: job[:after_script] || @after_script,
         }.compact
       }
     end
 
-    def normalize_script(script)
-      if script.is_a? Array
-        script.join("\n")
-      else
-        script
+    def validate!
+      validate_global!
+
+      @jobs.each do |name, job|
+        validate_job!(name, job)
       end
+
+      true
     end
 
-    def validate!
+    private
+
+    def validate_global!
       unless validate_array_of_strings(@before_script)
         raise ValidationError, "before_script should be an array of strings"
       end
 
+      unless @after_script.nil? || validate_array_of_strings(@after_script)
+        raise ValidationError, "after_script should be an array of strings"
+      end
+
       unless @image.nil? || @image.is_a?(String)
         raise ValidationError, "image should be a string"
       end
@@ -115,43 +136,39 @@ module Ci
       end
 
       unless @variables.nil? || validate_variables(@variables)
-        raise ValidationError, "variables should be a map of key-valued strings"
+        raise ValidationError, "variables should be a map of key-value strings"
       end
 
-      if @cache
-        if @cache[:key] && !validate_string(@cache[:key])
-          raise ValidationError, "cache:key parameter should be a string"
-        end
-
-        if @cache[:untracked] && !validate_boolean(@cache[:untracked])
-          raise ValidationError, "cache:untracked parameter should be an boolean"
-        end
+      validate_global_cache! if @cache
+    end
 
-        if @cache[:paths] && !validate_array_of_strings(@cache[:paths])
-          raise ValidationError, "cache:paths parameter should be an array of strings"
-        end
+    def validate_global_cache!
+      if @cache[:key] && !validate_string(@cache[:key])
+        raise ValidationError, "cache:key parameter should be a string"
       end
 
-      @jobs.each do |name, job|
-        validate_job!(name, job)
+      if @cache[:untracked] && !validate_boolean(@cache[:untracked])
+        raise ValidationError, "cache:untracked parameter should be an boolean"
       end
 
-      true
+      if @cache[:paths] && !validate_array_of_strings(@cache[:paths])
+        raise ValidationError, "cache:paths parameter should be an array of strings"
+      end
     end
 
     def validate_job!(name, job)
       validate_job_name!(name)
       validate_job_keys!(name, job)
       validate_job_types!(name, job)
+      validate_job_script!(name, job)
 
       validate_job_stage!(name, job) if job[:stage]
+      validate_job_variables!(name, job) if job[:variables]
       validate_job_cache!(name, job) if job[:cache]
       validate_job_artifacts!(name, job) if job[:artifacts]
       validate_job_dependencies!(name, job) if job[:dependencies]
     end
 
-    private
-
     def validate_job_name!(name)
       if name.blank? || !validate_string(name)
         raise ValidationError, "job name should be non-empty string"
@@ -167,10 +184,6 @@ module Ci
     end
 
     def validate_job_types!(name, job)
-      if !validate_string(job[:script]) && !validate_array_of_strings(job[:script])
-        raise ValidationError, "#{name} job: script should be a string or an array of a strings"
-      end
-
       if job[:image] && !validate_string(job[:image])
         raise ValidationError, "#{name} job: image should be a string"
       end
@@ -200,12 +213,33 @@ module Ci
       end
     end
 
+    def validate_job_script!(name, job)
+      if !validate_string(job[:script]) && !validate_array_of_strings(job[:script])
+        raise ValidationError, "#{name} job: script should be a string or an array of a strings"
+      end
+
+      if job[:before_script] && !validate_array_of_strings(job[:before_script])
+        raise ValidationError, "#{name} job: before_script should be an array of strings"
+      end
+
+      if job[:after_script] && !validate_array_of_strings(job[:after_script])
+        raise ValidationError, "#{name} job: after_script should be an array of strings"
+      end
+    end
+
     def validate_job_stage!(name, job)
       unless job[:stage].is_a?(String) && job[:stage].in?(stages)
         raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}"
       end
     end
 
+    def validate_job_variables!(name, job)
+      unless validate_variables(job[:variables])
+        raise ValidationError,
+          "#{name} job: variables should be a map of key-value strings"
+      end
+    end
+
     def validate_job_cache!(name, job)
       if job[:cache][:key] && !validate_string(job[:cache][:key])
         raise ValidationError, "#{name} job: cache:key parameter should be a string"
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 0b1ed510229..0f9e3ee14ee 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -16,7 +16,8 @@ module Gitlab
       end
 
       def execute
-        import_issues && import_pull_requests && import_wiki
+        import_labels && import_milestones && import_issues &&
+          import_pull_requests && import_wiki
       end
 
       private
@@ -25,6 +26,26 @@ module Gitlab
         @import_data_credentials ||= project.import_data.credentials if project.import_data
       end
 
+      def import_labels
+        client.labels(project.import_source).each do |raw_data|
+          Label.create!(LabelFormatter.new(project, raw_data).attributes)
+        end
+
+        true
+      rescue ActiveRecord::RecordInvalid => e
+        raise Projects::ImportService::Error, e.message
+      end
+
+      def import_milestones
+        client.list_milestones(project.import_source, state: :all).each do |raw_data|
+          Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes)
+        end
+
+        true
+      rescue ActiveRecord::RecordInvalid => e
+        raise Projects::ImportService::Error, e.message
+      end
+
       def import_issues
         client.list_issues(project.import_source, state: :all,
                                                   sort: :created,
@@ -33,6 +54,7 @@ module Gitlab
 
           if gh_issue.valid?
             issue = Issue.create!(gh_issue.attributes)
+            apply_labels(gh_issue.number, issue)
 
             if gh_issue.has_comments?
               import_comments(gh_issue.number, issue)
@@ -55,6 +77,7 @@ module Gitlab
             merge_request = MergeRequest.new(pull_request.attributes)
 
             if merge_request.save
+              apply_labels(pull_request.number, merge_request)
               import_comments(pull_request.number, merge_request)
               import_comments_on_diff(pull_request.number, merge_request)
             end
@@ -66,6 +89,18 @@ module Gitlab
         raise Projects::ImportService::Error, e.message
       end
 
+      def apply_labels(number, issuable)
+        issue = client.issue(project.import_source, number)
+
+        if issue.labels.count > 0
+          label_ids = issue.labels.map do |raw|
+            Label.find_by(LabelFormatter.new(project, raw).attributes).try(:id)
+          end
+
+          issuable.update_attribute(:label_ids, label_ids)
+        end
+      end
+
       def import_comments(issue_number, noteable)
         comments = client.issue_comments(project.import_source, issue_number)
         create_comments(comments, noteable)
diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb
index 1e3ba44f27c..c8173913b4e 100644
--- a/lib/gitlab/github_import/issue_formatter.rb
+++ b/lib/gitlab/github_import/issue_formatter.rb
@@ -3,7 +3,9 @@ module Gitlab
     class IssueFormatter < BaseFormatter
       def attributes
         {
+          iid: number,
           project: project,
+          milestone: milestone,
           title: raw_data.title,
           description: description,
           state: state,
@@ -54,6 +56,12 @@ module Gitlab
         @formatter.author_line(author) + body
       end
 
+      def milestone
+        if raw_data.milestone.present?
+          project.milestones.find_by(iid: raw_data.milestone.number)
+        end
+      end
+
       def state
         raw_data.state == 'closed' ? 'closed' : 'opened'
       end
diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb
new file mode 100644
index 00000000000..c2b9d40b511
--- /dev/null
+++ b/lib/gitlab/github_import/label_formatter.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module GithubImport
+    class LabelFormatter < BaseFormatter
+      def attributes
+        {
+          project: project,
+          title: title,
+          color: color
+        }
+      end
+
+      private
+
+      def color
+        "##{raw_data.color}"
+      end
+
+      def title
+        raw_data.name
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb
new file mode 100644
index 00000000000..e91a7e328cf
--- /dev/null
+++ b/lib/gitlab/github_import/milestone_formatter.rb
@@ -0,0 +1,48 @@
+module Gitlab
+  module GithubImport
+    class MilestoneFormatter < BaseFormatter
+      def attributes
+        {
+          iid: number,
+          project: project,
+          title: title,
+          description: description,
+          due_date: due_date,
+          state: state,
+          created_at: created_at,
+          updated_at: updated_at
+        }
+      end
+
+      private
+
+      def number
+        raw_data.number
+      end
+
+      def title
+        raw_data.title
+      end
+
+      def description
+        raw_data.description
+      end
+
+      def due_date
+        raw_data.due_on
+      end
+
+      def state
+        raw_data.state == 'closed' ? 'closed' : 'active'
+      end
+
+      def created_at
+        raw_data.created_at
+      end
+
+      def updated_at
+        state == 'closed' ? raw_data.closed_at : raw_data.updated_at
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb
index 4e507b090e8..d21b942ad4b 100644
--- a/lib/gitlab/github_import/pull_request_formatter.rb
+++ b/lib/gitlab/github_import/pull_request_formatter.rb
@@ -3,6 +3,7 @@ module Gitlab
     class PullRequestFormatter < BaseFormatter
       def attributes
         {
+          iid: number,
           title: raw_data.title,
           description: description,
           source_project: source_project,
@@ -10,6 +11,7 @@ module Gitlab
           target_project: target_project,
           target_branch: target_branch.name,
           state: state,
+          milestone: milestone,
           author_id: author_id,
           assignee_id: assignee_id,
           created_at: raw_data.created_at,
@@ -57,6 +59,12 @@ module Gitlab
         formatter.author_line(author) + body
       end
 
+      def milestone
+        if raw_data.milestone.present?
+          project.milestones.find_by(iid: raw_data.milestone.number)
+        end
+      end
+
       def source_project
         project
       end
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index face1921d2e..708ef79f304 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -11,6 +11,8 @@ module Gitlab
     module Instrumentation
       SERIES = 'method_calls'
 
+      PROXY_IVAR = :@__gitlab_instrumentation_proxy
+
       def self.configure
         yield self
       end
@@ -91,6 +93,18 @@ module Gitlab
         end
       end
 
+      # Returns true if a module is instrumented.
+      #
+      # mod - The module to check
+      def self.instrumented?(mod)
+        mod.instance_variable_defined?(PROXY_IVAR)
+      end
+
+      # Returns the proxy module (if any) of `mod`.
+      def self.proxy_module(mod)
+        mod.instance_variable_get(PROXY_IVAR)
+      end
+
       # Instruments a method.
       #
       # type - The type (:class or :instance) of method to instrument.
@@ -99,9 +113,8 @@ module Gitlab
       def self.instrument(type, mod, name)
         return unless Metrics.enabled?
 
-        name       = name.to_sym
-        alias_name = :"_original_#{name}"
-        target     = type == :instance ? mod : mod.singleton_class
+        name   = name.to_sym
+        target = type == :instance ? mod : mod.singleton_class
 
         if type == :instance
           target = mod
@@ -113,6 +126,12 @@ module Gitlab
           method = mod.method(name)
         end
 
+        unless instrumented?(target)
+          target.instance_variable_set(PROXY_IVAR, Module.new)
+        end
+
+        proxy_module = self.proxy_module(target)
+
         # Some code out there (e.g. the "state_machine" Gem) checks the arity of
         # a method to make sure it only passes arguments when the method expects
         # any. If we were to always overwrite a method to take an `*args`
@@ -125,17 +144,13 @@ module Gitlab
           args_signature = '*args, &block'
         end
 
-        send_signature = "__send__(#{alias_name.inspect}, #{args_signature})"
-
-        target.class_eval <<-EOF, __FILE__, __LINE__ + 1
-          alias_method #{alias_name.inspect}, #{name.inspect}
-
+        proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1
           def #{name}(#{args_signature})
             trans = Gitlab::Metrics::Instrumentation.transaction
 
             if trans
               start    = Time.now
-              retval   = #{send_signature}
+              retval   = super
               duration = (Time.now - start) * 1000.0
 
               if duration >= Gitlab::Metrics.method_call_threshold
@@ -148,10 +163,12 @@ module Gitlab
 
               retval
             else
-              #{send_signature}
+              super
             end
           end
         EOF
+
+        target.prepend(proxy_module)
       end
 
       # Small layer of indirection to make it easier to stub out the current
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index 8008b3bc895..96cad941d5c 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -9,6 +9,7 @@ module Gitlab
           return unless current_transaction
 
           current_transaction.increment(:sql_duration, event.duration)
+          current_transaction.increment(:sql_count, 1)
         end
 
         private
diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb
index 97d1edab9c1..67622f321a6 100644
--- a/lib/gitlab/push_data_builder.rb
+++ b/lib/gitlab/push_data_builder.rb
@@ -36,11 +36,12 @@ module Gitlab
           commit.hook_attrs(with_changed_files: true)
         end
 
-        type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push"
+        type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push'
 
         # Hash to be passed as post_receive_data
         data = {
           object_kind: type,
+          event_name: type,
           before: oldrev,
           after: newrev,
           ref: ref,