Merge branch 'master' into pipeline-blocking-actionspipeline-blocking-actions

* master: (26 commits)
  Fix UserBasic
  Rename `/take` to `/take_ownership`, expose `owner` in `v3`.
  Update after review
  Fix values being called at Array instead of Hash
  Fix json response in branches controller
  Improve docs and specs related to pages artifacts
  Add MR fo changelog about removing pages artifacts
  Delete artifacts for pages unless expiry date is specified
  Lint doc
  Improved team selection
  Move foreign key to separate migration
  Fix import model attributes
  Update documentation and expose ID
  Introduce tests for pipeline triggers
  Fix trigger model
  Update db/schema
  Make triggers to be user aware
  Make Pipeline Triggers to be user aware
  Update triggers API
  Remove remnants of git annex
  ...

Conflicts:
	db/schema.rb

8 files changed, 252 insertions, 24 deletions

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index d2d21f5d03a..98ef9d4118e 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -592,10 +592,6 @@ module API
       end
     end
 
-    class TriggerRequest < Grape::Entity
-      expose :id, :variables
-    end
-
     class Runner < Grape::Entity
       expose :id
       expose :description
@@ -643,7 +639,10 @@ module API
     end
 
     class Trigger < Grape::Entity
-      expose :token, :created_at, :updated_at, :deleted_at, :last_used
+      expose :id
+      expose :token, :description
+      expose :created_at, :updated_at, :deleted_at, :last_used
+      expose :owner, using: Entities::UserBasic
     end
 
     class Variable < Grape::Entity
diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb
index b7c9c5f2b7f..119e9024712 100644
--- a/lib/api/triggers.rb
+++ b/lib/api/triggers.rb
@@ -6,15 +6,15 @@ module API
       requires :id, type: String, desc: 'The ID of a project'
     end
     resource :projects do
-      desc 'Trigger a GitLab project build' do
-        success Entities::TriggerRequest
+      desc 'Trigger a GitLab project pipeline' do
+        success Entities::Pipeline
       end
       params do
         requires :ref, type: String, desc: 'The commit sha or name of a branch or tag'
         requires :token, type: String, desc: 'The unique token of trigger'
         optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
       end
-      post ":id/(ref/:ref/)trigger/builds" do
+      post ":id/(ref/:ref/)trigger/pipeline" do
         project = find_project(params[:id])
         trigger = Ci::Trigger.find_by_token(params[:token].to_s)
         not_found! unless project && trigger
@@ -29,9 +29,9 @@ module API
         # create request and trigger builds
         trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables)
         if trigger_request
-          present trigger_request, with: Entities::TriggerRequest
+          present trigger_request.pipeline, with: Entities::Pipeline
         else
-          errors = 'No builds created'
+          errors = 'No pipeline created'
           render_api_error!(errors, 400)
         end
       end
@@ -55,13 +55,13 @@ module API
         success Entities::Trigger
       end
       params do
-        requires :token, type: String, desc: 'The unique token of trigger'
+        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
       end
-      get ':id/triggers/:token' do
+      get ':id/triggers/:trigger_id' do
         authenticate!
         authorize! :admin_build, user_project
 
-        trigger = user_project.triggers.find_by(token: params[:token].to_s)
+        trigger = user_project.triggers.find(params.delete(:trigger_id))
         return not_found!('Trigger') unless trigger
 
         present trigger, with: Entities::Trigger
@@ -70,26 +70,76 @@ module API
       desc 'Create a trigger' do
         success Entities::Trigger
       end
+      params do
+        requires :description, type: String,  desc: 'The trigger description'
+      end
       post ':id/triggers' do
         authenticate!
         authorize! :admin_build, user_project
 
-        trigger = user_project.triggers.create
+        trigger = user_project.triggers.create(
+          declared_params(include_missing: false).merge(owner: current_user))
 
-        present trigger, with: Entities::Trigger
+        if trigger.valid?
+          present trigger, with: Entities::Trigger
+        else
+          render_validation_error!(trigger)
+        end
+      end
+
+      desc 'Update a trigger' do
+        success Entities::Trigger
+      end
+      params do
+        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
+        optional :description, type: String,  desc: 'The trigger description'
+      end
+      put ':id/triggers/:trigger_id' do
+        authenticate!
+        authorize! :admin_build, user_project
+
+        trigger = user_project.triggers.find(params.delete(:trigger_id))
+        return not_found!('Trigger') unless trigger
+
+        if trigger.update(declared_params(include_missing: false))
+          present trigger, with: Entities::Trigger
+        else
+          render_validation_error!(trigger)
+        end
+      end
+
+      desc 'Take ownership of trigger' do
+        success Entities::Trigger
+      end
+      params do
+        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
+      end
+      post ':id/triggers/:trigger_id/take_ownership' do
+        authenticate!
+        authorize! :admin_build, user_project
+
+        trigger = user_project.triggers.find(params.delete(:trigger_id))
+        return not_found!('Trigger') unless trigger
+
+        if trigger.update(owner: current_user)
+          status :ok
+          present trigger, with: Entities::Trigger
+        else
+          render_validation_error!(trigger)
+        end
       end
 
       desc 'Delete a trigger' do
         success Entities::Trigger
       end
       params do
-        requires :token, type: String, desc: 'The unique token of trigger'
+        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
       end
-      delete ':id/triggers/:token' do
+      delete ':id/triggers/:trigger_id' do
         authenticate!
         authorize! :admin_build, user_project
 
-        trigger = user_project.triggers.find_by(token: params[:token].to_s)
+        trigger = user_project.triggers.find(params.delete(:trigger_id))
         return not_found!('Trigger') unless trigger
 
         trigger.destroy
diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb
index 270d99a2348..69853d33bec 100644
--- a/lib/api/v3/entities.rb
+++ b/lib/api/v3/entities.rb
@@ -186,6 +186,15 @@ module API
       class Environment < ::API::Entities::EnvironmentBasic
         expose :project, using: Entities::Project
       end
+
+      class Trigger < Grape::Entity
+        expose :token, :created_at, :updated_at, :deleted_at, :last_used
+        expose :owner, using: ::API::Entities::UserBasic
+      end
+
+      class TriggerRequest < Grape::Entity
+        expose :id, :variables
+      end
     end
   end
 end
diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb
index 4051d4bca8d..1dfdb6a5956 100644
--- a/lib/api/v3/triggers.rb
+++ b/lib/api/v3/triggers.rb
@@ -7,8 +7,81 @@ module API
         requires :id, type: String, desc: 'The ID of a project'
       end
       resource :projects do
+        desc 'Trigger a GitLab project build' do
+          success ::API::V3::Entities::TriggerRequest
+        end
+        params do
+          requires :ref, type: String, desc: 'The commit sha or name of a branch or tag'
+          requires :token, type: String, desc: 'The unique token of trigger'
+          optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
+        end
+        post ":id/(ref/:ref/)trigger/builds" do
+          project = find_project(params[:id])
+          trigger = Ci::Trigger.find_by_token(params[:token].to_s)
+          not_found! unless project && trigger
+          unauthorized! unless trigger.project == project
+
+          # validate variables
+          variables = params[:variables].to_h
+          unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) }
+            render_api_error!('variables needs to be a map of key-valued strings', 400)
+          end
+
+          # create request and trigger builds
+          trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables)
+          if trigger_request
+            present trigger_request, with: ::API::V3::Entities::TriggerRequest
+          else
+            errors = 'No builds created'
+            render_api_error!(errors, 400)
+          end
+        end
+
+        desc 'Get triggers list' do
+          success ::API::V3::Entities::Trigger
+        end
+        params do
+          use :pagination
+        end
+        get ':id/triggers' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          triggers = user_project.triggers.includes(:trigger_requests)
+
+          present paginate(triggers), with: ::API::V3::Entities::Trigger
+        end
+
+        desc 'Get specific trigger of a project' do
+          success ::API::V3::Entities::Trigger
+        end
+        params do
+          requires :token, type: String, desc: 'The unique token of trigger'
+        end
+        get ':id/triggers/:token' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          trigger = user_project.triggers.find_by(token: params[:token].to_s)
+          return not_found!('Trigger') unless trigger
+
+          present trigger, with: ::API::V3::Entities::Trigger
+        end
+
+        desc 'Create a trigger' do
+          success ::API::V3::Entities::Trigger
+        end
+        post ':id/triggers' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          trigger = user_project.triggers.create
+
+          present trigger, with: ::API::V3::Entities::Trigger
+        end
+
         desc 'Delete a trigger' do
-          success ::API::Entities::Trigger
+          success ::API::V3::Entities::Trigger
         end
         params do
           requires :token, type: String, desc: 'The unique token of trigger'
@@ -22,7 +95,7 @@ module API
 
           trigger.destroy
 
-          present trigger, with: ::API::Entities::Trigger
+          present trigger, with: ::API::V3::Entities::Trigger
         end
       end
     end
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index d16d5ba4960..3c4ba5d50e6 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -180,9 +180,8 @@ module Backup
       return unless Dir.exist?(path)
 
       dir_entries = Dir.entries(path)
-      %w[annex custom_hooks].each do |entry|
-        yield(entry) if dir_entries.include?(entry)
-      end
+
+      yield('custom_hooks') if dir_entries.include?('custom_hooks')
     end
 
     def prepare
diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb
new file mode 100644
index 00000000000..0f24f9bbfde
--- /dev/null
+++ b/lib/gitlab/etag_caching/middleware.rb
@@ -0,0 +1,66 @@
+module Gitlab
+  module EtagCaching
+    class Middleware
+      RESERVED_WORDS = ProjectPathValidator::RESERVED.map { |word| "/#{word}/" }.join('|')
+      ROUTE_REGEXP = Regexp.union(
+        %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z)
+      )
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        return @app.call(env) unless enabled_for_current_route?(env)
+        Gitlab::Metrics.add_event(:etag_caching_middleware_used)
+
+        etag, cached_value_present = get_etag(env)
+        if_none_match = env['HTTP_IF_NONE_MATCH']
+
+        if if_none_match == etag
+          Gitlab::Metrics.add_event(:etag_caching_cache_hit)
+          [304, { 'ETag' => etag }, ['']]
+        else
+          track_cache_miss(if_none_match, cached_value_present)
+
+          status, headers, body = @app.call(env)
+          headers['ETag'] = etag
+          [status, headers, body]
+        end
+      end
+
+      private
+
+      def enabled_for_current_route?(env)
+        ROUTE_REGEXP.match(env['PATH_INFO'])
+      end
+
+      def get_etag(env)
+        cache_key = env['PATH_INFO']
+        store = Store.new
+        current_value = store.get(cache_key)
+        cached_value_present = current_value.present?
+
+        unless cached_value_present
+          current_value = store.touch(cache_key, only_if_missing: true)
+        end
+
+        [weak_etag_format(current_value), cached_value_present]
+      end
+
+      def weak_etag_format(value)
+        %Q{W/"#{value}"}
+      end
+
+      def track_cache_miss(if_none_match, cached_value_present)
+        if if_none_match.blank?
+          Gitlab::Metrics.add_event(:etag_caching_header_missing)
+        elsif !cached_value_present
+          Gitlab::Metrics.add_event(:etag_caching_key_not_found)
+        else
+          Gitlab::Metrics.add_event(:etag_caching_resource_changed)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb
new file mode 100644
index 00000000000..9532e432f78
--- /dev/null
+++ b/lib/gitlab/etag_caching/store.rb
@@ -0,0 +1,32 @@
+module Gitlab
+  module EtagCaching
+    class Store
+      EXPIRY_TIME = 10.minutes
+      REDIS_NAMESPACE = 'etag:'.freeze
+
+      def get(key)
+        Gitlab::Redis.with { |redis| redis.get(redis_key(key)) }
+      end
+
+      def touch(key, only_if_missing: false)
+        etag = generate_etag
+
+        Gitlab::Redis.with do |redis|
+          redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing)
+        end
+
+        etag
+      end
+
+      private
+
+      def generate_etag
+        SecureRandom.hex
+      end
+
+      def redis_key(key)
+        "#{REDIS_NAMESPACE}#{key}"
+      end
+    end
+  end
+end
diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb
index 09dfd082b3a..afc152aa02e 100644
--- a/lib/mattermost/team.rb
+++ b/lib/mattermost/team.rb
@@ -1,7 +1,7 @@
 module Mattermost
   class Team < Client
     def all
-      session_get('/api/v3/teams/all')
+      session_get('/api/v3/teams/all').values
     end
   end
 end