Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (671 commits)
  Make rubocop happy
  Use guard clause
  Improve language
  Prettify
  Use temp branch
  Pass info about who started the job and which job triggered it
  Docs: add indexes for monitoring and performance monitoring
  clearer-documentation-on-inline-diffs
  Add docs for commit diff discussion in merge requests
  sorting for tags api
  Clear BatchLoader after each spec to prevent holding onto records longer than necessary
  Include project in BatchLoader key to prevent returning blobs for the wrong project
  moved lfs_blob_ids method into ExtractsPath module
  Converted JS modules into exported modules
  spec fixes
  Bump gitlab-shell version to 5.10.3
  Clear caches before updating MR diffs
  Use new Ruby version 2.4 in GitLab QA images
  moved lfs blob fetch from extractspath file
  Update GitLab QA dependencies
  ...

13 files changed, 177 insertions, 50 deletions

diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb
index 118883f5ea5..598c76f6168 100644
--- a/lib/api/circuit_breakers.rb
+++ b/lib/api/circuit_breakers.rb
@@ -41,7 +41,7 @@ module API
             detail 'This feature was introduced in GitLab 9.5'
           end
           delete do
-            Gitlab::Git::Storage::CircuitBreaker.reset_all!
+            Gitlab::Git::Storage::FailureInfo.reset_all!
           end
         end
       end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index ce332fe85d2..928706dfda7 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -16,10 +16,13 @@ module API
 
     class UserBasic < UserSafe
       expose :state
+
       expose :avatar_url do |user, options|
         user.avatar_url(only_path: false)
       end
 
+      expose :avatar_path, if: ->(user, options) { options.fetch(:only_path, false) && user.avatar_path }
+
       expose :web_url do |user, options|
         Gitlab::Routing.url_helpers.user_url(user)
       end
@@ -88,13 +91,29 @@ module API
     end
 
     class BasicProjectDetails < ProjectIdentity
-      expose :default_branch, :tag_list
+      include ::API::ProjectsRelationBuilder
+
+      expose :default_branch
+      # Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770
+      expose :tag_list do |project|
+        # project.tags.order(:name).pluck(:name) is the most suitable option
+        # to avoid loading all the ActiveRecord objects but, if we use it here
+        # it override the preloaded associations and makes a query
+        # (fixed in https://github.com/rails/rails/pull/25976).
+        project.tags.map(&:name).sort
+      end
       expose :ssh_url_to_repo, :http_url_to_repo, :web_url
       expose :avatar_url do |project, options|
         project.avatar_url(only_path: false)
       end
       expose :star_count, :forks_count
       expose :last_activity_at
+
+      def self.preload_relation(projects_relation, options =  {})
+        projects_relation.preload(:project_feature, :route)
+                         .preload(namespace: [:route, :owner],
+                                  tags: :taggings)
+      end
     end
 
     class Project < BasicProjectDetails
@@ -146,7 +165,7 @@ module API
       expose :shared_runners_enabled
       expose :lfs_enabled?, as: :lfs_enabled
       expose :creator_id
-      expose :namespace, using: 'API::Entities::Namespace'
+      expose :namespace, using: 'API::Entities::NamespaceBasic'
       expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? }
       expose :import_status
       expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] }
@@ -156,7 +175,7 @@ module API
       expose :public_builds, as: :public_jobs
       expose :ci_config_path
       expose :shared_with_groups do |project, options|
-        SharedGroup.represent(project.project_group_links.all, options)
+        SharedGroup.represent(project.project_group_links, options)
       end
       expose :only_allow_merge_if_pipeline_succeeds
       expose :request_access_enabled
@@ -164,6 +183,18 @@ module API
       expose :printing_merge_request_link_enabled
 
       expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics
+
+      def self.preload_relation(projects_relation, options =  {})
+        super(projects_relation).preload(:group)
+                                .preload(project_group_links: :group,
+                                         fork_network: :root_project,
+                                         forked_project_link: :forked_from_project,
+                                         forked_from_project: [:route, :forks, namespace: :route, tags: :taggings])
+      end
+
+      def self.forks_counting_projects(projects_relation)
+        projects_relation + projects_relation.map(&:forked_from_project).compact
+      end
     end
 
     class ProjectStatistics < Grape::Entity
@@ -217,8 +248,21 @@ module API
     end
 
     class GroupDetail < Group
-      expose :projects, using: Entities::Project
-      expose :shared_projects, using: Entities::Project
+      expose :projects, using: Entities::Project do |group, options|
+        GroupProjectsFinder.new(
+          group: group,
+          current_user: options[:current_user],
+          options: { only_owned: true }
+        ).execute
+      end
+
+      expose :shared_projects, using: Entities::Project do |group, options|
+        GroupProjectsFinder.new(
+          group: group,
+          current_user: options[:current_user],
+          options: { only_shared: true }
+        ).execute
+      end
     end
 
     class Commit < Grape::Entity
@@ -618,9 +662,11 @@ module API
       expose :created_at
     end
 
-    class Namespace < Grape::Entity
+    class NamespaceBasic < Grape::Entity
       expose :id, :name, :path, :kind, :full_path, :parent_id
+    end
 
+    class Namespace < NamespaceBasic
       expose :members_count_with_descendants, if: -> (namespace, opts) { expose_members_count_with_descendants?(namespace, opts) } do |namespace, _|
         namespace.users_with_descendants.count
       end
@@ -680,7 +726,7 @@ module API
           if options.key?(:project_members)
             (options[:project_members] || []).find { |member| member.source_id == project.id }
           else
-            project.project_members.find_by(user_id: options[:current_user].id)
+            project.project_member(options[:current_user])
           end
         end
 
@@ -689,11 +735,25 @@ module API
             if options.key?(:group_members)
               (options[:group_members] || []).find { |member| member.source_id == project.namespace_id }
             else
-              project.group.group_members.find_by(user_id: options[:current_user].id)
+              project.group.group_member(options[:current_user])
             end
           end
         end
       end
+
+      def self.preload_relation(projects_relation, options = {})
+        relation = super(projects_relation, options)
+
+        unless options.key?(:group_members)
+          relation = relation.preload(group: [group_members: [:source, user: [notification_settings: :source]]])
+        end
+
+        unless options.key?(:project_members)
+          relation = relation.preload(project_members: [:source, user: [notification_settings: :source]])
+        end
+
+        relation
+      end
     end
 
     class LabelBasic < Grape::Entity
@@ -1006,13 +1066,9 @@ module API
         expose :type, :url, :username, :password
       end
 
-      class ArtifactFile < Grape::Entity
-        expose :filename, :size
-      end
-
       class Dependency < Grape::Entity
         expose :id, :name, :token
-        expose :artifacts_file, using: ArtifactFile, if: ->(job, _) { job.artifacts? }
+        expose :artifacts_file, using: JobArtifactFile, if: ->(job, _) { job.artifacts? }
       end
 
       class Response < Grape::Entity
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index bcf2e6dae1d..b81f07a1770 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -52,6 +52,13 @@ module API
         groups
       end
 
+      def find_group_projects(params)
+        group = find_group!(params[:id])
+        projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute
+        projects = reorder_projects(projects)
+        paginate(projects)
+      end
+
       def present_groups(params, groups)
         options = {
           with: Entities::Group,
@@ -170,11 +177,10 @@ module API
         use :pagination
       end
       get ":id/projects" do
-        group = find_group!(params[:id])
-        projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute
-        projects = reorder_projects(projects)
+        projects = find_group_projects(params)
         entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
-        present paginate(projects), with: entity, current_user: current_user
+
+        present entity.prepare_relation(projects), with: entity, current_user: current_user
       end
 
       desc 'Get a list of subgroups in this group.' do
diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb
index 95108292aac..bb70370ba77 100644
--- a/lib/api/helpers/pagination.rb
+++ b/lib/api/helpers/pagination.rb
@@ -2,6 +2,8 @@ module API
   module Helpers
     module Pagination
       def paginate(relation)
+        relation = add_default_order(relation)
+
         relation.page(params[:page]).per(params[:per_page]).tap do |data|
           add_pagination_headers(data)
         end
@@ -45,6 +47,14 @@ module API
         # Ensure there is in total at least 1 page
         [paginated_data.total_pages, 1].max
       end
+
+      def add_default_order(relation)
+        if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty?
+          relation = relation.order(:id)
+        end
+
+        relation
+      end
     end
   end
 end
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 451121a4cea..ccaaeca10d4 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -4,6 +4,7 @@ module API
     before { authenticate_by_gitlab_shell_token! }
 
     helpers ::API::Helpers::InternalHelpers
+    helpers ::Gitlab::Identifier
 
     namespace 'internal' do
       # Check if git command is allowed to project
@@ -176,17 +177,25 @@ module API
 
       post '/post_receive' do
         status 200
-
         PostReceive.perform_async(params[:gl_repository], params[:identifier],
           params[:changes])
         broadcast_message = BroadcastMessage.current&.last&.message
         reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease
 
-        {
+        output = {
           merge_request_urls: merge_request_urls,
           broadcast_message: broadcast_message,
           reference_counter_decreased: reference_counter_decreased
         }
+
+        project = Gitlab::GlRepository.parse(params[:gl_repository]).first
+        user = identify(params[:identifier])
+        redirect_message = Gitlab::Checks::ProjectMoved.fetch_redirect_message(user.id, project.id)
+        if redirect_message
+          output[:redirected_message] = redirect_message
+        end
+
+        output
       end
     end
   end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index e60e00d7956..5f943ba27d1 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -161,6 +161,8 @@ module API
         use :issue_params
       end
       post ':id/issues' do
+        authorize! :create_issue, user_project
+
         # Setting created_at time only allowed for admins and project owners
         unless current_user.admin? || user_project.owner == current_user
           params.delete(:created_at)
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 4cd7e714aa2..fa222bf2b1c 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -79,11 +79,11 @@ module API
         projects = projects.with_statistics if params[:statistics]
         projects = projects.with_issues_enabled if params[:with_issues_enabled]
         projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
+        projects = paginate(projects)
 
         if current_user
-          projects = projects.includes(:route, :taggings, namespace: :route)
-          project_members = current_user.project_members
-          group_members = current_user.group_members
+          project_members = current_user.project_members.preload(:source, user: [notification_settings: :source])
+          group_members = current_user.group_members.preload(:source, user: [notification_settings: :source])
         end
 
         options = options.reverse_merge(
@@ -95,7 +95,7 @@ module API
         )
         options[:with] = Entities::BasicProjectDetails if params[:simple]
 
-        present paginate(projects), options
+        present options[:with].prepare_relation(projects, options), options
       end
     end
 
@@ -367,15 +367,16 @@ module API
       post ":id/fork/:forked_from_id" do
         authenticated_as_admin!
 
-        forked_from_project = find_project!(params[:forked_from_id])
-        not_found!("Source Project") unless forked_from_project
+        fork_from_project = find_project!(params[:forked_from_id])
 
-        if user_project.forked_from_project.nil?
-          user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id)
+        not_found!("Source Project") unless fork_from_project
 
-          ::Projects::ForksCountService.new(forked_from_project).refresh_cache
+        result = ::Projects::ForkService.new(fork_from_project, current_user).execute(user_project)
+
+        if result
+          present user_project.reload, with: Entities::Project
         else
-          render_api_error!("Project already forked", 409)
+          render_api_error!("Project already forked", 409) if user_project.forked?
         end
       end
 
@@ -383,11 +384,11 @@ module API
       delete ":id/fork" do
         authorize! :remove_fork_project, user_project
 
-        if user_project.forked?
-          destroy_conditionally!(user_project.forked_project_link)
-        else
-          not_modified!
+        result = destroy_conditionally!(user_project) do
+          ::Projects::UnlinkForkService.new(user_project, current_user).execute
         end
+
+        result ? status(204) : not_modified!
       end
 
       desc 'Share the project with a group' do
diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb
new file mode 100644
index 00000000000..6482fd94ab8
--- /dev/null
+++ b/lib/api/projects_relation_builder.rb
@@ -0,0 +1,34 @@
+module API
+  module ProjectsRelationBuilder
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      def prepare_relation(projects_relation, options = {})
+        projects_relation = preload_relation(projects_relation, options)
+        execute_batch_counting(projects_relation)
+        projects_relation
+      end
+
+      def preload_relation(projects_relation, options =  {})
+        projects_relation
+      end
+
+      def forks_counting_projects(projects_relation)
+        projects_relation
+      end
+
+      def batch_forks_counting(projects_relation)
+        ::Projects::BatchForksCountService.new(forks_counting_projects(projects_relation)).refresh_cache
+      end
+
+      def batch_open_issues_counting(projects_relation)
+        ::Projects::BatchOpenIssuesCountService.new(projects_relation).refresh_cache
+      end
+
+      def execute_batch_counting(projects_relation)
+        batch_forks_counting(projects_relation)
+        batch_open_issues_counting(projects_relation)
+      end
+    end
+  end
+end
diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb
index b5021e8a712..614822509f0 100644
--- a/lib/api/protected_branches.rb
+++ b/lib/api/protected_branches.rb
@@ -39,10 +39,10 @@ module API
       end
       params do
         requires :name, type: String, desc: 'The name of the protected branch'
-        optional :push_access_level, type: Integer, default: Gitlab::Access::MASTER,
+        optional :push_access_level, type: Integer,
                                      values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS,
                                      desc: 'Access levels allowed to push (defaults: `40`, master access level)'
-        optional :merge_access_level, type: Integer, default: Gitlab::Access::MASTER,
+        optional :merge_access_level, type: Integer,
                                       values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS,
                                       desc: 'Access levels allowed to merge (defaults: `40`, master access level)'
       end
@@ -52,15 +52,13 @@ module API
           conflict!("Protected branch '#{params[:name]}' already exists")
         end
 
-        protected_branch_params = {
-          name: params[:name],
-          push_access_levels_attributes: [{ access_level: params[:push_access_level] }],
-          merge_access_levels_attributes: [{ access_level: params[:merge_access_level] }]
-        }
+        # Replace with `declared(params)` after updating to grape v1.0.2
+        # See https://github.com/ruby-grape/grape/pull/1710
+        # and https://gitlab.com/gitlab-org/gitlab-ce/issues/40843
+        declared_params = params.slice("name", "push_access_level", "merge_access_level", "allowed_to_push", "allowed_to_merge")
 
-        service_args = [user_project, current_user, protected_branch_params]
-
-        protected_branch = ::ProtectedBranches::CreateService.new(*service_args).execute
+        api_service = ::ProtectedBranches::ApiService.new(user_project, current_user, declared_params)
+        protected_branch = api_service.create
 
         if protected_branch.persisted?
           present protected_branch, with: Entities::ProtectedBranch, project: user_project
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 7887b886c03..4f36bbd760f 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -110,10 +110,12 @@ module API
       end
       params do
         use :pagination
+        optional :order_by, type: String, values: %w[email name commits], default: nil, desc: 'Return contributors ordered by `name` or `email` or `commits`'
+        optional :sort, type: String, values: %w[asc desc], default: nil, desc: 'Sort by asc (ascending) or desc (descending)'
       end
       get ':id/repository/contributors' do
         begin
-          contributors = ::Kaminari.paginate_array(user_project.repository.contributors)
+          contributors = ::Kaminari.paginate_array(user_project.repository.contributors(order_by: params[:order_by], sort: params[:sort]))
           present paginate(contributors), with: Entities::Contributor
         rescue
           not_found!
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index a3987c560dd..80feb629d54 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -215,18 +215,20 @@ module API
         job = authenticate_job!
         forbidden!('Job is not running!') unless job.running?
 
-        artifacts_upload_path = ArtifactUploader.artifacts_upload_path
+        artifacts_upload_path = JobArtifactUploader.artifacts_upload_path
         artifacts = uploaded_file(:file, artifacts_upload_path)
         metadata = uploaded_file(:metadata, artifacts_upload_path)
 
         bad_request!('Missing artifacts file!') unless artifacts
         file_to_large! unless artifacts.size < max_artifacts_size
 
-        job.artifacts_file = artifacts
-        job.artifacts_metadata = metadata
-        job.artifacts_expire_in = params['expire_in'] ||
+        expire_in = params['expire_in'] ||
           Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
 
+        job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, expire_in: expire_in)
+        job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, expire_in: expire_in) if metadata
+        job.artifacts_expire_in = expire_in
+
         if job.save
           present job, with: Entities::JobRequest::Response
         else
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index 0d394a7b441..5e0afc6a7e4 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -14,10 +14,15 @@ module API
         success Entities::Tag
       end
       params do
+        optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                        desc: 'Return tags sorted in updated by `asc` or `desc` order.'
+        optional :order_by, type: String, values: %w[name updated], default: 'updated',
+                            desc: 'Return tags ordered by `name` or `updated` fields.'
         use :pagination
       end
       get ':id/repository/tags' do
-        tags = ::Kaminari.paginate_array(user_project.repository.tags.sort_by(&:name).reverse)
+        tags = ::Kaminari.paginate_array(::TagsFinder.new(user_project.repository, sort: "#{params[:order_by]}_#{params[:sort]}").execute)
+
         present paginate(tags), with: Entities::Tag, project: user_project
       end
 
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 0cd89b1bcf8..e5de31ad51b 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -76,6 +76,8 @@ module API
         forbidden!("Not authorized to access /api/v4/users") unless authorized
 
         entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic
+        users = users.preload(:identities, :u2f_registrations) if entity == Entities::UserWithAdmin
+
         present paginate(users), with: entity
       end