Merge remote-tracking branch 'origin/master' into 38759-fetch-available-parameters-directly-from-gke-when-creating-a-cluster

15 files changed, 231 insertions, 62 deletions

diff --git a/app/services/application_settings/update_service.rb b/app/services/application_settings/update_service.rb
index 61589a07250..d6d3a661dab 100644
--- a/app/services/application_settings/update_service.rb
+++ b/app/services/application_settings/update_service.rb
@@ -1,7 +1,22 @@
 module ApplicationSettings
   class UpdateService < ApplicationSettings::BaseService
     def execute
+      update_terms(@params.delete(:terms))
+
       @application_setting.update(@params)
     end
+
+    private
+
+    def update_terms(terms)
+      return unless terms.present?
+
+      # Avoid creating a new terms record if the text is exactly the same.
+      terms = terms.strip
+      return if terms == @application_setting.terms
+
+      ApplicationSetting::Term.create(terms: terms)
+      @application_setting.reset_memoized_terms
+    end
   end
 end
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb
index 6ce86983287..17a53b6a8fd 100644
--- a/app/services/ci/create_pipeline_service.rb
+++ b/app/services/ci/create_pipeline_service.rb
@@ -24,6 +24,7 @@ module Ci
         ignore_skip_ci: ignore_skip_ci,
         save_incompleted: save_on_errors,
         seeds_block: block,
+        variables_attributes: params[:variables_attributes],
         project: project,
         current_user: current_user)
 
diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb
index 0b087ad73da..4291631913a 100644
--- a/app/services/ci/register_job_service.rb
+++ b/app/services/ci/register_job_service.rb
@@ -17,8 +17,10 @@ module Ci
       builds =
         if runner.shared?
           builds_for_shared_runner
+        elsif runner.group_type?
+          builds_for_group_runner
         else
-          builds_for_specific_runner
+          builds_for_project_runner
         end
 
       valid = true
@@ -75,15 +77,24 @@ module Ci
         .joins('LEFT JOIN project_features ON ci_builds.project_id = project_features.project_id')
         .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0').
 
-        # Implement fair scheduling
-        # this returns builds that are ordered by number of running builds
-        # we prefer projects that don't use shared runners at all
-        joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.project_id=project_builds.project_id")
+      # Implement fair scheduling
+      # this returns builds that are ordered by number of running builds
+      # we prefer projects that don't use shared runners at all
+      joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.project_id=project_builds.project_id")
         .order('COALESCE(project_builds.running_builds, 0) ASC', 'ci_builds.id ASC')
     end
 
-    def builds_for_specific_runner
-      new_builds.where(project: runner.projects.without_deleted.with_builds_enabled).order('created_at ASC')
+    def builds_for_project_runner
+      new_builds.where(project: runner.projects.without_deleted.with_builds_enabled).order('id ASC')
+    end
+
+    def builds_for_group_runner
+      hierarchy_groups = Gitlab::GroupHierarchy.new(runner.groups).base_and_descendants
+      projects = Project.where(namespace_id: hierarchy_groups)
+        .with_group_runners_enabled
+        .with_builds_enabled
+        .without_deleted
+      new_builds.where(project: projects).order('id ASC')
     end
 
     def running_builds_for_shared_runners
@@ -97,10 +108,6 @@ module Ci
       builds
     end
 
-    def shared_runner_build_limits_feature_enabled?
-      ENV['DISABLE_SHARED_RUNNER_BUILD_MINUTES_LIMIT'].to_s != 'true'
-    end
-
     def register_failure
       failed_attempt_counter.increment
       attempt_counter.increment
diff --git a/app/services/ci/update_build_queue_service.rb b/app/services/ci/update_build_queue_service.rb
index 152c8ae5006..41b1c144c3e 100644
--- a/app/services/ci/update_build_queue_service.rb
+++ b/app/services/ci/update_build_queue_service.rb
@@ -1,18 +1,14 @@
 module Ci
   class UpdateBuildQueueService
     def execute(build)
-      build.project.runners.each do |runner|
-        if runner.can_pick?(build)
-          runner.tick_runner_queue
-        end
-      end
+      tick_for(build, build.project.all_runners)
+    end
 
-      return unless build.project.shared_runners_enabled?
+    private
 
-      Ci::Runner.shared.each do |runner|
-        if runner.can_pick?(build)
-          runner.tick_runner_queue
-        end
+    def tick_for(build, runners)
+      runners.each do |runner|
+        runner.pick_build!(build)
       end
     end
   end
diff --git a/app/services/concerns/exclusive_lease_guard.rb b/app/services/concerns/exclusive_lease_guard.rb
new file mode 100644
index 00000000000..30be6accc32
--- /dev/null
+++ b/app/services/concerns/exclusive_lease_guard.rb
@@ -0,0 +1,52 @@
+#
+# Concern that helps with getting an exclusive lease for running a block
+# of code.
+#
+# `#try_obtain_lease` takes a block which will be run if it was able to
+# obtain  the lease. Implement `#lease_timeout` to configure the timeout
+# for the exclusive lease. Optionally override `#lease_key` to set the
+# lease key, it defaults to the class name with underscores.
+#
+module ExclusiveLeaseGuard
+  extend ActiveSupport::Concern
+
+  def try_obtain_lease
+    lease = exclusive_lease.try_obtain
+
+    unless lease
+      log_error('Cannot obtain an exclusive lease. There must be another instance already in execution.')
+      return
+    end
+
+    begin
+      yield lease
+    ensure
+      release_lease(lease)
+    end
+  end
+
+  def exclusive_lease
+    @lease ||= Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout)
+  end
+
+  def lease_key
+    @lease_key ||= self.class.name.underscore
+  end
+
+  def lease_timeout
+    raise NotImplementedError,
+          "#{self.class.name} does not implement #{__method__}"
+  end
+
+  def release_lease(uuid)
+    Gitlab::ExclusiveLease.cancel(lease_key, uuid)
+  end
+
+  def renew_lease!
+    exclusive_lease.renew
+  end
+
+  def log_error(message, extra_args = {})
+    logger.error(message)
+  end
+end
diff --git a/app/services/concerns/users/participable_service.rb b/app/services/concerns/users/participable_service.rb
new file mode 100644
index 00000000000..bf60b96938d
--- /dev/null
+++ b/app/services/concerns/users/participable_service.rb
@@ -0,0 +1,41 @@
+module Users
+  module ParticipableService
+    extend ActiveSupport::Concern
+
+    included do
+      attr_reader :noteable
+    end
+
+    def noteable_owner
+      return [] unless noteable && noteable.author.present?
+
+      [as_hash(noteable.author)]
+    end
+
+    def participants_in_noteable
+      return [] unless noteable
+
+      users = noteable.participants(current_user)
+      sorted(users)
+    end
+
+    def sorted(users)
+      users.uniq.to_a.compact.sort_by(&:username).map do |user|
+        as_hash(user)
+      end
+    end
+
+    def groups
+      current_user.authorized_groups.sort_by(&:path).map do |group|
+        count = group.users.count
+        { username: group.full_path, name: group.full_name, count: count, avatar_url: group.avatar_url }
+      end
+    end
+
+    private
+
+    def as_hash(user)
+      { username: user.username, name: user.name, avatar_url: user.avatar_url }
+    end
+  end
+end
diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb
index c037141fcde..f3bfc53dcd3 100644
--- a/app/services/git_push_service.rb
+++ b/app/services/git_push_service.rb
@@ -55,6 +55,7 @@ class GitPushService < BaseService
     execute_related_hooks
     perform_housekeeping
 
+    update_remote_mirrors
     update_caches
 
     update_signatures
@@ -119,6 +120,13 @@ class GitPushService < BaseService
 
   protected
 
+  def update_remote_mirrors
+    return unless @project.has_remote_mirror?
+
+    @project.mark_stuck_remote_mirrors_as_failed!
+    @project.update_remote_mirrors
+  end
+
   def execute_related_hooks
     # Update merge requests that may be affected by this push. A new branch
     # could cause the last commit of a merge request to change.
diff --git a/app/services/notification_recipient_service.rb b/app/services/notification_recipient_service.rb
index 83e59a649b6..5658699664d 100644
--- a/app/services/notification_recipient_service.rb
+++ b/app/services/notification_recipient_service.rb
@@ -45,6 +45,10 @@ module NotificationRecipientService
         target.project
       end
 
+      def group
+        project&.group || target.try(:group)
+      end
+
       def recipients
         @recipients ||= []
       end
@@ -67,6 +71,7 @@ module NotificationRecipientService
           user, type,
           reason: reason,
           project: project,
+          group: group,
           custom_action: custom_action,
           target: target,
           acting_user: acting_user
@@ -107,11 +112,11 @@ module NotificationRecipientService
 
         # Users with a notification setting on group or project
         user_ids += user_ids_notifiable_on(project, :custom)
-        user_ids += user_ids_notifiable_on(project.group, :custom)
+        user_ids += user_ids_notifiable_on(group, :custom)
 
         # Users with global level custom
         user_ids_with_project_level_global = user_ids_notifiable_on(project, :global)
-        user_ids_with_group_level_global   = user_ids_notifiable_on(project.group, :global)
+        user_ids_with_group_level_global   = user_ids_notifiable_on(group, :global)
 
         global_users_ids = user_ids_with_project_level_global.concat(user_ids_with_group_level_global)
         user_ids += user_ids_with_global_level_custom(global_users_ids, custom_action)
@@ -123,6 +128,10 @@ module NotificationRecipientService
         add_recipients(project_watchers, :watch, nil)
       end
 
+      def add_group_watchers
+        add_recipients(group_watchers, :watch, nil)
+      end
+
       # Get project users with WATCH notification level
       def project_watchers
         project_members_ids = user_ids_notifiable_on(project)
@@ -138,6 +147,14 @@ module NotificationRecipientService
         user_scope.where(id: user_ids_with_project_setting.concat(user_ids_with_group_setting).uniq)
       end
 
+      def group_watchers
+        user_ids_with_group_global = user_ids_notifiable_on(group, :global)
+        user_ids = user_ids_with_global_level_watch(user_ids_with_group_global)
+        user_ids_with_group_setting = select_group_members_ids(group, [], user_ids_with_group_global, user_ids)
+
+        user_scope.where(id: user_ids_with_group_setting)
+      end
+
       def add_subscribed_users
         return unless target.respond_to? :subscribers
 
@@ -281,6 +298,14 @@ module NotificationRecipientService
         note.project
       end
 
+      def group
+        if note.for_project_noteable?
+          project.group
+        else
+          target.try(:group)
+        end
+      end
+
       def build!
         # Add all users participating in the thread (author, assignee, comment authors)
         add_participants(note.author)
@@ -289,11 +314,11 @@ module NotificationRecipientService
         if note.for_project_noteable?
           # Merge project watchers
           add_project_watchers
-
-          # Merge project with custom notification
-          add_custom_notifications
+        else
+          add_group_watchers
         end
 
+        add_custom_notifications
         add_subscribed_users
       end
 
diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb
index d361d070993..d16ecdb7b9b 100644
--- a/app/services/projects/create_service.rb
+++ b/app/services/projects/create_service.rb
@@ -142,7 +142,7 @@ module Projects
 
       if @project
         @project.errors.add(:base, message)
-        @project.mark_import_as_failed(message) if @project.import?
+        @project.mark_import_as_failed(message) if @project.persisted? && @project.import?
       end
 
       @project
diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb
index 71c93660b4b..adbc498d0bf 100644
--- a/app/services/projects/destroy_service.rb
+++ b/app/services/projects/destroy_service.rb
@@ -87,7 +87,7 @@ module Projects
       new_path = removal_path(path)
 
       if mv_repository(path, new_path)
-        log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
+        log_info(%Q{Repository "#{path}" moved to "#{new_path}" for project "#{project.full_path}"})
 
         project.run_after_commit do
           # self is now project
diff --git a/app/services/projects/participants_service.rb b/app/services/projects/participants_service.rb
index e6193fcacee..eb0472c6024 100644
--- a/app/services/projects/participants_service.rb
+++ b/app/services/projects/participants_service.rb
@@ -1,6 +1,6 @@
 module Projects
   class ParticipantsService < BaseService
-    attr_reader :noteable
+    include Users::ParticipableService
 
     def execute(noteable)
       @noteable = noteable
@@ -10,36 +10,6 @@ module Projects
       participants.uniq
     end
 
-    def noteable_owner
-      return [] unless noteable && noteable.author.present?
-
-      [{
-        name: noteable.author.name,
-        username: noteable.author.username,
-        avatar_url: noteable.author.avatar_url
-      }]
-    end
-
-    def participants_in_noteable
-      return [] unless noteable
-
-      users = noteable.participants(current_user)
-      sorted(users)
-    end
-
-    def sorted(users)
-      users.uniq.to_a.compact.sort_by(&:username).map do |user|
-        { username: user.username, name: user.name, avatar_url: user.avatar_url }
-      end
-    end
-
-    def groups
-      current_user.authorized_groups.sort_by(&:path).map do |group|
-        count = group.users.count
-        { username: group.full_path, name: group.full_name, count: count, avatar_url: group.avatar_url }
-      end
-    end
-
     def all_members
       count = project.team.members.flatten.count
       [{ username: "all", name: "All Project and Group Members", count: count }]
diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb
new file mode 100644
index 00000000000..8183a2f26d7
--- /dev/null
+++ b/app/services/projects/update_remote_mirror_service.rb
@@ -0,0 +1,30 @@
+module Projects
+  class UpdateRemoteMirrorService < BaseService
+    attr_reader :errors
+
+    def execute(remote_mirror)
+      @errors = []
+
+      return success unless remote_mirror.enabled?
+
+      begin
+        repository.fetch_remote(remote_mirror.remote_name, no_tags: true)
+
+        opts = {}
+        if remote_mirror.only_protected_branches?
+          opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name)
+        end
+
+        remote_mirror.update_repository(opts)
+      rescue => e
+        errors << e.message.strip
+      end
+
+      if errors.present?
+        error(errors.join("\n\n"))
+      else
+        success
+      end
+    end
+  end
+end
diff --git a/app/services/users/migrate_to_ghost_user_service.rb b/app/services/users/migrate_to_ghost_user_service.rb
index 976017dfa82..a2833b1e051 100644
--- a/app/services/users/migrate_to_ghost_user_service.rb
+++ b/app/services/users/migrate_to_ghost_user_service.rb
@@ -49,7 +49,7 @@ module Users
       migrate_merge_requests
       migrate_notes
       migrate_abuse_reports
-      migrate_award_emojis
+      migrate_award_emoji
     end
 
     def migrate_issues
@@ -70,7 +70,7 @@ module Users
       user.reported_abuse_reports.update_all(reporter_id: ghost_user.id)
     end
 
-    def migrate_award_emojis
+    def migrate_award_emoji
       user.award_emoji.update_all(user_id: ghost_user.id)
     end
   end
diff --git a/app/services/users/respond_to_terms_service.rb b/app/services/users/respond_to_terms_service.rb
new file mode 100644
index 00000000000..06d660186cf
--- /dev/null
+++ b/app/services/users/respond_to_terms_service.rb
@@ -0,0 +1,24 @@
+module Users
+  class RespondToTermsService
+    def initialize(user, term)
+      @user, @term = user, term
+    end
+
+    def execute(accepted:)
+      agreement = @user.term_agreements.find_or_initialize_by(term: @term)
+      agreement.accepted = accepted
+
+      if agreement.save
+        store_accepted_term(accepted)
+      end
+
+      agreement
+    end
+
+    private
+
+    def store_accepted_term(accepted)
+      @user.update_column(:accepted_term_id, accepted ? @term.id : nil)
+    end
+  end
+end
diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb
index 809ce1303d8..7ec52b6ce2b 100644
--- a/app/services/web_hook_service.rb
+++ b/app/services/web_hook_service.rb
@@ -41,7 +41,7 @@ class WebHookService
       http_status: response.code,
       message: response.to_s
     }
-  rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, Net::ReadTimeout => e
+  rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError => e
     log_execution(
       trigger: hook_name,
       url: hook.url,