Add latest changes from gitlab-org/gitlab@master

9 files changed, 256 insertions, 10 deletions

diff --git a/lib/api/entities/note.rb b/lib/api/entities/note.rb
index 4d140454ff0..f22ab73afd0 100644
--- a/lib/api/entities/note.rb
+++ b/lib/api/entities/note.rb
@@ -23,6 +23,8 @@ module API
       expose :resolved?, as: :resolved, if: ->(note, options) { note.resolvable? }
       expose :resolved_by, using: Entities::UserBasic, if: ->(note, options) { note.resolvable? }
 
+      expose :confidential?, as: :confidential
+
       # Avoid N+1 queries as much as possible
       expose(:noteable_iid) { |note| note.noteable.iid if NOTEABLE_TYPES_WITH_IID.include?(note.noteable_type) }
 
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index ff61cceb4c9..2f603dd6bed 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -249,6 +249,10 @@ module API
       authorize! :admin_project, user_project
     end
 
+    def authorize_admin_group
+      authorize! :admin_group, user_group
+    end
+
     def authorize_read_builds!
       authorize! :read_build, user_project
     end
diff --git a/lib/api/runners.rb b/lib/api/runners.rb
index eba1b5499d0..43ee1dd1f71 100644
--- a/lib/api/runners.rb
+++ b/lib/api/runners.rb
@@ -150,10 +150,10 @@ module API
       end
       get ':id/runners' do
         runners = Ci::Runner.owned_or_instance_wide(user_project.id)
+        # scope is deprecated (for project runners), however api documentation still supports it.
+        # Not including them in `apply_filter` method as it's not supported for group runners
         runners = filter_runners(runners, params[:scope])
-        runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES)
-        runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES)
-        runners = runners.tagged_with(params[:tag_list]) if params[:tag_list]
+        runners = apply_filter(runners, params)
 
         present paginate(runners), with: Entities::Runner
       end
@@ -194,6 +194,31 @@ module API
       # rubocop: enable CodeReuse/ActiveRecord
     end
 
+    params do
+      requires :id, type: String, desc: 'The ID of a group'
+    end
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      before { authorize_admin_group }
+
+      desc 'Get runners available for group' do
+        success Entities::Runner
+      end
+      params do
+        optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES,
+                 desc: 'The type of the runners to show'
+        optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES,
+                 desc: 'The status of the runners to show'
+        optional :tag_list, type: Array[String], desc: 'The tags of the runners to show'
+        use :pagination
+      end
+      get ':id/runners' do
+        runners = Ci::Runner.belonging_to_group(user_group.id, include_ancestors: true)
+        runners = apply_filter(runners, params)
+
+        present paginate(runners), with: Entities::Runner
+      end
+    end
+
     helpers do
       def filter_runners(runners, scope, allowed_scopes: ::Ci::Runner::AVAILABLE_SCOPES)
         return runners unless scope.present?
@@ -210,6 +235,14 @@ module API
         runners.public_send(scope) # rubocop:disable GitlabSecurity/PublicSend
       end
 
+      def apply_filter(runners, params)
+        runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES)
+        runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES)
+        runners = runners.tagged_with(params[:tag_list]) if params[:tag_list]
+
+        runners
+      end
+
       def get_runner(id)
         runner = Ci::Runner.find(id)
         not_found!('Runner') unless runner
diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb
new file mode 100644
index 00000000000..c652a5bb3fc
--- /dev/null
+++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb
@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This migration creates missing services records
+    # for the projects within the given range of ids
+    class FixProjectsWithoutPrometheusService
+      # There is important inconsistency between single query timeout 15s and background migration worker minimum lease 2 minutes
+      # to address that scheduled ids range (for minimum 2 minutes processing) should be inserted in smaller portions to fit under 15s limit.
+      # https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/9064#note_279857215
+      MAX_BATCH_SIZE = 1_000
+      DEFAULTS = {
+        'active' => true,
+        'properties' => "'{}'",
+        'type' => "'PrometheusService'",
+        'template' => false,
+        'push_events' => true,
+        'issues_events' => true,
+        'merge_requests_events' => true,
+        'tag_push_events' => true,
+        'note_events' => true,
+        'category' => "'monitoring'",
+        'default' => false,
+        'wiki_page_events' => true,
+        'pipeline_events' => true,
+        'confidential_issues_events' => true,
+        'commit_events' => true,
+        'job_events' => true,
+        'confidential_note_events' => true
+      }.freeze
+
+      module Migratable
+        module Applications
+          # Migration model namespace isolated from application code.
+          class Prometheus
+            def self.statuses
+              {
+                errored: -1,
+                installed: 3,
+                updated: 5
+              }
+            end
+          end
+        end
+
+        # Migration model namespace isolated from application code.
+        class Cluster < ActiveRecord::Base
+          self.table_name = 'clusters'
+
+          enum cluster_type: {
+            instance_type: 1,
+            group_type: 2
+          }
+
+          def self.has_prometheus_application?
+            joins("INNER JOIN clusters_applications_prometheus ON clusters_applications_prometheus.cluster_id = clusters.id
+                   AND clusters_applications_prometheus.status IN (#{Applications::Prometheus.statuses[:installed]}, #{Applications::Prometheus.statuses[:updated]})").exists?
+          end
+        end
+
+        # Migration model namespace isolated from application code.
+        class PrometheusService < ActiveRecord::Base
+          self.inheritance_column = :_type_disabled
+          self.table_name = 'services'
+          default_scope { where(type: type) }
+
+          def self.type
+            'PrometheusService'
+          end
+
+          def self.template
+            find_by(template: true)
+          end
+
+          def self.values
+            (template&.attributes_for_insert || DEFAULTS).merge('template' => false, 'active' => true).values
+          end
+
+          def attributes_for_insert
+            slice(DEFAULTS.keys).transform_values do |v|
+              v.is_a?(String) ? "'#{v}'" : v
+            end
+          end
+        end
+
+        # Migration model namespace isolated from application code.
+        class Project < ActiveRecord::Base
+          self.table_name = 'projects'
+
+          scope :select_for_insert, -> {
+            select('id')
+              .select(PrometheusService.values.join(','))
+              .select("TIMEZONE('UTC', NOW()) as created_at", "TIMEZONE('UTC', NOW()) as updated_at")
+          }
+
+          scope :with_prometheus_services, ->(from_id, to_id) {
+            joins("LEFT JOIN services ON services.project_id = projects.id AND services.project_id BETWEEN #{Integer(from_id)} AND #{Integer(to_id)}
+                    AND services.type = '#{PrometheusService.type}'")
+          }
+
+          scope :with_group_prometheus_installed, -> {
+            joins("INNER JOIN cluster_groups ON cluster_groups.group_id = projects.namespace_id")
+              .joins("INNER JOIN clusters_applications_prometheus ON clusters_applications_prometheus.cluster_id = cluster_groups.cluster_id
+                      AND clusters_applications_prometheus.status IN (#{Applications::Prometheus.statuses[:installed]}, #{Applications::Prometheus.statuses[:updated]})")
+          }
+        end
+      end
+
+      def perform(from_id, to_id)
+        (from_id..to_id).each_slice(MAX_BATCH_SIZE) do |batch|
+          process_batch(batch.first, batch.last)
+        end
+      end
+
+      private
+
+      def process_batch(from_id, to_id)
+        update_inconsistent(from_id, to_id)
+        create_missing(from_id, to_id)
+      end
+
+      def create_missing(from_id, to_id)
+        result = ActiveRecord::Base.connection.select_one(create_sql(from_id, to_id))
+        return unless result
+
+        logger.info(message: "#{self.class}: created missing services for #{result['number_of_created_records']} projects in id=#{from_id}...#{to_id}")
+      end
+
+      def update_inconsistent(from_id, to_id)
+        result = ActiveRecord::Base.connection.select_one(update_sql(from_id, to_id))
+        return unless result
+
+        logger.info(message: "#{self.class}: updated inconsistent services for #{result['number_of_updated_records']} projects in id=#{from_id}...#{to_id}")
+      end
+
+      # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT
+      def create_sql(from_id, to_id)
+        <<~SQL
+          WITH created_records AS (
+            INSERT INTO services (project_id, #{DEFAULTS.keys.map { |key| %("#{key}")}.join(',')}, created_at, updated_at)
+            #{select_insert_values_sql(from_id, to_id)}
+            RETURNING *
+          )
+          SELECT COUNT(*) as number_of_created_records
+          FROM created_records
+        SQL
+      end
+
+      # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT
+      def update_sql(from_id, to_id)
+        <<~SQL
+          WITH updated_records AS (
+            UPDATE services SET active = TRUE
+            WHERE services.project_id BETWEEN #{Integer(from_id)} AND #{Integer(to_id)} AND services.properties = '{}' AND services.type = '#{Migratable::PrometheusService.type}'
+            AND #{group_cluster_condition(from_id, to_id)} AND services.active = FALSE
+            RETURNING *
+          )
+          SELECT COUNT(*) as number_of_updated_records
+          FROM updated_records
+        SQL
+      end
+
+      def group_cluster_condition(from_id, to_id)
+        return '1 = 1' if migrate_instance_cluster?
+
+        <<~SQL
+          EXISTS (
+            #{Migratable::Project.select(1).with_group_prometheus_installed.where("projects.id BETWEEN ? AND ?", Integer(from_id), Integer(to_id)).to_sql}
+          )
+        SQL
+      end
+
+      def select_insert_values_sql(from_id, to_id)
+        scope = Migratable::Project
+                  .select_for_insert
+                  .with_prometheus_services(from_id, to_id)
+                  .where("projects.id BETWEEN ? AND ? AND services.id IS NULL", Integer(from_id), Integer(to_id))
+
+        return scope.to_sql if migrate_instance_cluster?
+
+        scope.with_group_prometheus_installed.to_sql
+      end
+
+      def logger
+        @logger ||= Gitlab::BackgroundMigration::Logger.build
+      end
+
+      def migrate_instance_cluster?
+        if instance_variable_defined?('@migrate_instance_cluster')
+          @migrate_instance_cluster
+        else
+          @migrate_instance_cluster = Migratable::Cluster.instance_type.has_prometheus_application?
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
index f374bc7e26a..dc70b977452 100644
--- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
@@ -4,7 +4,7 @@
 # Entrypoint is also needed as image by default set `terraform` binary as an
 # entrypoint.
 image:
-  name: hashicorp/terraform:light
+  name: registry.gitlab.com/gitlab-org/gitlab-build-images:terraform
   entrypoint:
     - '/usr/bin/env'
     - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
@@ -18,6 +18,7 @@ cache:
     - .terraform
 
 before_script:
+  - alias convert_report="jq -r '([.resource_changes[].change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'"
   - terraform --version
   - terraform init
 
@@ -36,6 +37,7 @@ plan:
   stage: build
   script:
     - terraform plan -out=$PLAN
+    - "terraform show --json $PLAN | convert_report > tfplan.json"
   artifacts:
     name: plan
     paths:
diff --git a/lib/gitlab/lets_encrypt/challenge.rb b/lib/gitlab/lets_encrypt/challenge.rb
index 136268c974b..7775cbd0360 100644
--- a/lib/gitlab/lets_encrypt/challenge.rb
+++ b/lib/gitlab/lets_encrypt/challenge.rb
@@ -7,7 +7,7 @@ module Gitlab
         @acme_challenge = acme_challenge
       end
 
-      delegate :token, :file_content, :status, :request_validation, to: :acme_challenge
+      delegate :token, :file_content, :status, :request_validation, :error, to: :acme_challenge
 
       private
 
diff --git a/lib/gitlab/lets_encrypt/order.rb b/lib/gitlab/lets_encrypt/order.rb
index 9c2365f98a8..656964f77f9 100644
--- a/lib/gitlab/lets_encrypt/order.rb
+++ b/lib/gitlab/lets_encrypt/order.rb
@@ -8,7 +8,6 @@ module Gitlab
       end
 
       def new_challenge
-        authorization = @acme_order.authorizations.first
         challenge = authorization.http
         ::Gitlab::LetsEncrypt::Challenge.new(challenge)
       end
@@ -22,11 +21,19 @@ module Gitlab
         acme_order.finalize(csr: csr)
       end
 
+      def challenge_error
+        authorization.challenges.first&.error
+      end
+
       delegate :url, :status, :expires, :certificate, to: :acme_order
 
       private
 
       attr_reader :acme_order
+
+      def authorization
+        @acme_order.authorizations.first
+      end
     end
   end
 end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 66503621851..db531f06f11 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -17,11 +17,12 @@ module Gitlab
     end
 
     def group_name_regex
-      project_name_regex
+      @group_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9ff}_()\. ]*\z/.freeze
     end
 
     def group_name_regex_message
-      project_name_regex_message
+      "can contain only letters, digits, emojis, '_', '.', dash, space, parenthesis. " \
+      "It must start with letter, digit, emoji or '_'."
     end
 
     ##
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 7da20b49d9d..b375602a5fe 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -225,8 +225,8 @@ module Gitlab
 
       def gitaly_server_hash(repository)
         {
-          address: Gitlab::GitalyClient.address(repository.project.repository_storage),
-          token: Gitlab::GitalyClient.token(repository.project.repository_storage),
+          address: Gitlab::GitalyClient.address(repository.container.repository_storage),
+          token: Gitlab::GitalyClient.token(repository.container.repository_storage),
           features: Feature::Gitaly.server_feature_flags
         }
       end