Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (507 commits)
  Add dropdowns documentation
  Convert migration to populate latest merge request ID into a background migration
  Set 0.69.0 instead of latest for codeclimate image
  De-duplicate background migration matchers defined in spec/support/migrations_helpers.rb
  Update database_debugging.md
  Update database_debugging.md
  Move installation of apps higher
  Change to Google Kubernetes Cluster and add internal links
  Add Ingress description from official docs
  Add info on creating your own k8s cluster from the cluster page
  Add info about the installed apps in the Cluster docs
  Resolve "lock/confidential issuable sidebar custom svg icons iteration"
  Update HA README.md to clarify GitLab support does not troubleshoot DRBD.
  Update license_finder to 3.1.1
  Make sure NotesActions#noteable returns a Noteable in the update action
  Cache the number of user SSH keys
  Adjust openid_connect_spec to use `raise_error`
  Resolve "Clicking on GPG verification badge jumps to top of the page"
  Add changelog for container repository path update
  Update container repository path reference
  ...

17 files changed, 152 insertions, 48 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index c37e596eb9d..8094597d238 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -61,7 +61,10 @@ module API
       mount ::API::V3::Variables
     end
 
-    before { header['X-Frame-Options'] = 'SAMEORIGIN' }
+    before do
+      header['X-Frame-Options'] = 'SAMEORIGIN'
+      header['X-Content-Type-Options'] = 'nosniff'
+    end
 
     # The locale is set to the current user's locale when `current_user` is loaded
     after { Gitlab::I18n.use_default_locale }
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index b9c7d443f6c..c1c0d344917 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -42,6 +42,8 @@ module API
 
     # Helper Methods for Grape Endpoint
     module HelperMethods
+      include Gitlab::Utils::StrongMemoize
+
       def find_current_user!
         user = find_user_from_access_token || find_user_from_warden
         return unless user
@@ -52,9 +54,9 @@ module API
       end
 
       def access_token
-        return @access_token if defined?(@access_token)
-
-        @access_token = find_oauth_access_token || find_personal_access_token
+        strong_memoize(:access_token) do
+          find_oauth_access_token || find_personal_access_token
+        end
       end
 
       def validate_access_token!(scopes: [])
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 19152c9f395..cdef1b546a9 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -29,12 +29,11 @@ module API
         use :pagination
       end
       get ':id/repository/branches' do
-        branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name))
+        repository = user_project.repository
+        branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name))
+        merged_branch_names = repository.merged_branch_names(branches.map(&:name))
 
-        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442
-        Gitlab::GitalyClient.allow_n_plus_1_calls do
-          present paginate(branches), with: Entities::Branch, project: user_project
-        end
+        present paginate(branches), with: Entities::Branch, project: user_project, merged_branch_names: merged_branch_names
       end
 
       resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 2685dc27252..2bc4039b019 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -117,7 +117,7 @@ module API
         commit = user_project.commit(params[:sha])
 
         not_found! 'Commit' unless commit
-        notes = user_project.notes.where(commit_id: commit.id).order(:created_at)
+        notes = commit.notes.order(:created_at)
 
         present paginate(notes), with: Entities::CommitNote
       end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 398a7906dcb..16ae99b5c6c 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -242,10 +242,7 @@ module API
       end
 
       expose :merged do |repo_branch, options|
-        # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442
-        Gitlab::GitalyClient.allow_n_plus_1_calls do
-          options[:project].repository.merged_to_root_ref?(repo_branch.name)
-        end
+        options[:project].repository.merged_to_root_ref?(repo_branch, options[:merged_branch_names])
       end
 
       expose :protected do |repo_branch, options|
@@ -478,6 +475,10 @@ module API
       expose :subscribed do |merge_request, options|
         merge_request.subscribed?(options[:current_user], options[:project])
       end
+
+      expose :changes_count do |merge_request, _options|
+        merge_request.merge_request_diff.real_size
+      end
     end
 
     class MergeRequestChanges < MergeRequest
@@ -1041,6 +1042,11 @@ module API
       expose :value
     end
 
+    class PagesDomainCertificateExpiration < Grape::Entity
+      expose :expired?, as: :expired
+      expose :expiration
+    end
+
     class PagesDomainCertificate < Grape::Entity
       expose :subject
       expose :expired?, as: :expired
@@ -1048,12 +1054,23 @@ module API
       expose :certificate_text
     end
 
+    class PagesDomainBasic < Grape::Entity
+      expose :domain
+      expose :url
+      expose :certificate,
+        as: :certificate_expiration,
+        if: ->(pages_domain, _) { pages_domain.certificate? },
+        using: PagesDomainCertificateExpiration do |pages_domain|
+        pages_domain
+      end
+    end
+
     class PagesDomain < Grape::Entity
       expose :domain
       expose :url
       expose :certificate,
-             if: ->(pages_domain, _) { pages_domain.certificate? },
-             using: PagesDomainCertificate do |pages_domain|
+        if: ->(pages_domain, _) { pages_domain.certificate? },
+        using: PagesDomainCertificate do |pages_domain|
         pages_domain
       end
     end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index e817dcbbc4b..bcf2e6dae1d 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -25,22 +25,7 @@ module API
         optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
       end
 
-      def present_groups(groups, options = {})
-        options = options.reverse_merge(
-          with: Entities::Group,
-          current_user: current_user
-        )
-
-        groups = groups.with_statistics if options[:statistics]
-        present paginate(groups), options
-      end
-    end
-
-    resource :groups do
-      desc 'Get a groups list' do
-        success Entities::Group
-      end
-      params do
+      params :group_list_params do
         use :statistics_params
         optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list'
         optional :all_available, type: Boolean, desc: 'Show all group that you have access to'
@@ -50,14 +35,47 @@ module API
         optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)'
         use :pagination
       end
-      get do
-        find_params = { all_available: params[:all_available], owned: params[:owned] }
+
+      def find_groups(params)
+        find_params = {
+          all_available: params[:all_available],
+          custom_attributes: params[:custom_attributes],
+          owned: params[:owned]
+        }
+        find_params[:parent] = find_group!(params[:id]) if params[:id]
+
         groups = GroupsFinder.new(current_user, find_params).execute
         groups = groups.search(params[:search]) if params[:search].present?
         groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
         groups = groups.reorder(params[:order_by] => params[:sort])
 
-        present_groups groups, statistics: params[:statistics] && current_user.admin?
+        groups
+      end
+
+      def present_groups(params, groups)
+        options = {
+          with: Entities::Group,
+          current_user: current_user,
+          statistics: params[:statistics] && current_user.admin?
+        }
+
+        groups = groups.with_statistics if options[:statistics]
+        present paginate(groups), options
+      end
+    end
+
+    resource :groups do
+      include CustomAttributesEndpoints
+
+      desc 'Get a groups list' do
+        success Entities::Group
+      end
+      params do
+        use :group_list_params
+      end
+      get do
+        groups = find_groups(params)
+        present_groups params, groups
       end
 
       desc 'Create a group. Available only for users who can create groups.' do
@@ -159,6 +177,17 @@ module API
         present paginate(projects), with: entity, current_user: current_user
       end
 
+      desc 'Get a list of subgroups in this group.' do
+        success Entities::Group
+      end
+      params do
+        use :group_list_params
+      end
+      get ":id/subgroups" do
+        groups = find_groups(params)
+        present_groups params, groups
+      end
+
       desc 'Transfer a project to the group namespace. Available only for admin.' do
         success Entities::GroupDetail
       end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index d6df269486a..7f436b69091 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -156,6 +156,11 @@ module API
       end
     end
 
+    def authenticated_with_full_private_access!
+      authenticate!
+      forbidden! unless current_user.full_private_access?
+    end
+
     def authenticated_as_admin!
       authenticate!
       forbidden! unless current_user.admin?
@@ -191,6 +196,10 @@ module API
       not_found! unless user_project.pages_available?
     end
 
+    def require_pages_config_enabled!
+      not_found! unless Gitlab.config.pages.enabled
+    end
+
     def can?(object, action, subject = :global)
       Ability.allowed?(object, action, subject)
     end
@@ -329,6 +338,7 @@ module API
       finder_params[:archived] = params[:archived]
       finder_params[:search] = params[:search] if params[:search]
       finder_params[:user] = params.delete(:user) if params[:user]
+      finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes]
       finder_params
     end
 
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index 6bb85dd2619..0d57c822578 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -36,6 +36,18 @@ module API
         {}
       end
 
+      def fix_git_env_repository_paths(env, repository_path)
+        if obj_dir_relative = env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence
+          env['GIT_OBJECT_DIRECTORY'] = File.join(repository_path, obj_dir_relative)
+        end
+
+        if alt_obj_dirs_relative = env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE'].presence
+          env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = alt_obj_dirs_relative.map { |dir| File.join(repository_path, dir) }
+        end
+
+        env
+      end
+
       def log_user_activity(actor)
         commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS
 
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 6e78ac2c903..451121a4cea 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -19,7 +19,9 @@ module API
         status 200
 
         # Stores some Git-specific env thread-safely
-        Gitlab::Git::Env.set(parse_env)
+        env = parse_env
+        env = fix_git_env_repository_paths(env, repository_path) if project
+        Gitlab::Git::Env.set(env)
 
         actor =
           if params[:key_id]
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 0df41dcc903..74dfd9f96de 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -68,7 +68,7 @@ module API
                          desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`'
       end
       get do
-        issues = find_issues
+        issues = paginate(find_issues)
 
         options = {
           with: Entities::IssueBasic,
@@ -76,7 +76,7 @@ module API
           issuable_metadata: issuable_meta_data(issues, 'Issue')
         }
 
-        present paginate(issues), options
+        present issues, options
       end
     end
 
@@ -95,7 +95,7 @@ module API
       get ":id/issues" do
         group = find_group!(params[:id])
 
-        issues = find_issues(group_id: group.id)
+        issues = paginate(find_issues(group_id: group.id))
 
         options = {
           with: Entities::IssueBasic,
@@ -103,7 +103,7 @@ module API
           issuable_metadata: issuable_meta_data(issues, 'Issue')
         }
 
-        present paginate(issues), options
+        present issues, options
       end
     end
 
@@ -124,7 +124,7 @@ module API
       get ":id/issues" do
         project = find_project!(params[:id])
 
-        issues = find_issues(project_id: project.id)
+        issues = paginate(find_issues(project_id: project.id))
 
         options = {
           with: Entities::IssueBasic,
@@ -133,7 +133,7 @@ module API
           issuable_metadata: issuable_meta_data(issues, 'Issue')
         }
 
-        present paginate(issues), options
+        present issues, options
       end
 
       desc 'Get a single project issue' do
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
index 3c1c412ba42..a116ab3c9bd 100644
--- a/lib/api/jobs.rb
+++ b/lib/api/jobs.rb
@@ -136,7 +136,7 @@ module API
         authorize_update_builds!
 
         build = find_build!(params[:job_id])
-        authorize!(:update_build, build)
+        authorize!(:erase_build, build)
         return forbidden!('Job is not erasable!') unless build.erasable?
 
         build.erase(erased_by: current_user)
diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb
index 259f3f34068..d7b613a717e 100644
--- a/lib/api/pages_domains.rb
+++ b/lib/api/pages_domains.rb
@@ -4,7 +4,6 @@ module API
 
     before do
       authenticate!
-      require_pages_enabled!
     end
 
     after_validation do
@@ -29,10 +28,31 @@ module API
       end
     end
 
+    resource :pages do
+      before do
+        require_pages_config_enabled!
+        authenticated_with_full_private_access!
+      end
+
+      desc "Get all pages domains" do
+        success Entities::PagesDomainBasic
+      end
+      params do
+        use :pagination
+      end
+      get "domains" do
+        present paginate(PagesDomain.all), with: Entities::PagesDomainBasic
+      end
+    end
+
     params do
       requires :id, type: String, desc: 'The ID of a project'
     end
     resource :projects, requirements: { id: %r{[^/]+} } do
+      before do
+        require_pages_enabled!
+      end
+
       desc 'Get all pages domains' do
         success Entities::PagesDomain
       end
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index aab7a6c3f93..4cd7e714aa2 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -119,6 +119,8 @@ module API
     end
 
     resource :projects do
+      include CustomAttributesEndpoints
+
       desc 'Get a list of visible projects for authenticated user' do
         success Entities::BasicProjectDetails
       end
diff --git a/lib/api/services.rb b/lib/api/services.rb
index 6454e475036..bbcc851d07a 100644
--- a/lib/api/services.rb
+++ b/lib/api/services.rb
@@ -522,6 +522,12 @@ module API
           name: :webhook,
           type: String,
           desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...'
+        },
+        {
+          required: false,
+          name: :username,
+          type: String,
+          desc: 'The username to use to post the message'
         }
       ],
       'teamcity' => [
diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb
index 69cd12de72c..b201bf77667 100644
--- a/lib/api/v3/branches.rb
+++ b/lib/api/v3/branches.rb
@@ -14,9 +14,11 @@ module API
           success ::API::Entities::Branch
         end
         get ":id/repository/branches" do
-          branches = user_project.repository.branches.sort_by(&:name)
+          repository = user_project.repository
+          branches = repository.branches.sort_by(&:name)
+          merged_branch_names = repository.merged_branch_names(branches.map(&:name))
 
-          present branches, with: ::API::Entities::Branch, project: user_project
+          present branches, with: ::API::Entities::Branch, project: user_project, merged_branch_names: merged_branch_names
         end
 
         desc 'Delete a branch'
diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb
index f493fd7c7ec..fa0bef39602 100644
--- a/lib/api/v3/builds.rb
+++ b/lib/api/v3/builds.rb
@@ -169,7 +169,7 @@ module API
           authorize_update_builds!
 
           build = get_build!(params[:build_id])
-          authorize!(:update_build, build)
+          authorize!(:erase_build, build)
           return forbidden!('Build is not erasable!') unless build.erasable?
 
           build.erase(erased_by: current_user)
diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb
index ed206a6def0..be360fbfc0c 100644
--- a/lib/api/v3/commits.rb
+++ b/lib/api/v3/commits.rb
@@ -106,7 +106,7 @@ module API
           commit = user_project.commit(params[:sha])
 
           not_found! 'Commit' unless commit
-          notes = Note.where(commit_id: commit.id).order(:created_at)
+          notes = commit.notes.order(:created_at)
 
           present paginate(notes), with: ::API::Entities::CommitNote
         end