Add latest changes from gitlab-org/gitlab@14-0-stable-ee

17 files changed, 159 insertions, 35 deletions

diff --git a/lib/api/members.rb b/lib/api/members.rb
index 0956806da5b..70e13e8d4ae 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -97,6 +97,8 @@ module API
         end
         # rubocop: disable CodeReuse/ActiveRecord
         post ":id/members" do
+          ::Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/333434')
+
           source = find_source(source_type, params[:id])
           authorize_admin_source!(source_type, source)
 
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 36f58d43a77..580c7042f1e 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -385,7 +385,7 @@ module Gitlab
       end
 
       def can_user_login_with_non_expired_password?(user)
-        user.can?(:log_in) && !user.password_expired?
+        user.can?(:log_in) && !user.password_expired_if_applicable?
       end
     end
   end
diff --git a/lib/gitlab/auth/current_user_mode.rb b/lib/gitlab/auth/current_user_mode.rb
index a6d706c2a49..fc391543f4d 100644
--- a/lib/gitlab/auth/current_user_mode.rb
+++ b/lib/gitlab/auth/current_user_mode.rb
@@ -27,22 +27,27 @@ module Gitlab
         # will bypass the session check for a user that was already in admin mode
         #
         # If passed a block, it will surround the block execution and reset the session
-        # bypass at the end; otherwise use manually '.reset_bypass_session!'
+        # bypass at the end; otherwise you must remember to call '.reset_bypass_session!'
         def bypass_session!(admin_id)
           Gitlab::SafeRequestStore[CURRENT_REQUEST_BYPASS_SESSION_ADMIN_ID_RS_KEY] = admin_id
+          # Bypassing the session invalidates the cached value of admin_mode?
+          # Any new calls need to be re-computed.
+          uncache_admin_mode_state(admin_id)
 
           Gitlab::AppLogger.debug("Bypassing session in admin mode for: #{admin_id}")
 
-          if block_given?
-            begin
-              yield
-            ensure
-              reset_bypass_session!
-            end
+          return unless block_given?
+
+          begin
+            yield
+          ensure
+            reset_bypass_session!(admin_id)
           end
         end
 
-        def reset_bypass_session!
+        def reset_bypass_session!(admin_id = nil)
+          # Restoring the session bypass invalidates the cached value of admin_mode?
+          uncache_admin_mode_state(admin_id)
           Gitlab::SafeRequestStore.delete(CURRENT_REQUEST_BYPASS_SESSION_ADMIN_ID_RS_KEY)
         end
 
@@ -50,10 +55,21 @@ module Gitlab
           Gitlab::SafeRequestStore[CURRENT_REQUEST_BYPASS_SESSION_ADMIN_ID_RS_KEY]
         end
 
+        def uncache_admin_mode_state(admin_id = nil)
+          if admin_id
+            key = { res: :current_user_mode, user: admin_id, method: :admin_mode? }
+            Gitlab::SafeRequestStore.delete(key)
+          else
+            Gitlab::SafeRequestStore.delete_if do |key|
+              key.is_a?(Hash) && key[:res] == :current_user_mode && key[:method] == :admin_mode?
+            end
+          end
+        end
+
         # Store in the current request the provided user model (only if in admin mode)
         # and yield
         def with_current_admin(admin)
-          return yield unless self.new(admin).admin_mode?
+          return yield unless new(admin).admin_mode?
 
           Gitlab::SafeRequestStore[CURRENT_REQUEST_ADMIN_MODE_USER_RS_KEY] = admin
 
diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb
index 6639000dba8..904759919ae 100644
--- a/lib/gitlab/auth/user_access_denied_reason.rb
+++ b/lib/gitlab/auth/user_access_denied_reason.rb
@@ -23,6 +23,8 @@ module Gitlab
           "Your primary email address is not confirmed. "\
           "Please check your inbox for the confirmation instructions. "\
           "In case the link is expired, you can request a new confirmation email at #{Rails.application.routes.url_helpers.new_user_confirmation_url}"
+        when :blocked
+          "Your account has been blocked."
         when :password_expired
           "Your password expired. "\
           "Please access GitLab from a web browser to update your password."
@@ -44,6 +46,8 @@ module Gitlab
           :deactivated
         elsif !@user.confirmed?
           :unconfirmed
+        elsif @user.blocked?
+          :blocked
         elsif @user.password_expired?
           :password_expired
         else
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index c8e8f0bc1fc..e6d63969161 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -14,7 +14,7 @@ module Gitlab
           ALLOWED_KEYS = %i[tags script type image services start_in artifacts
                             cache dependencies before_script after_script
                             environment coverage retry parallel interruptible timeout
-                            release secrets].freeze
+                            release dast_configuration secrets].freeze
 
           REQUIRED_BY_NEEDS = %i[stage].freeze
 
diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
index a2b112b8e9f..5521a4a781b 100644
--- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
@@ -1,14 +1,21 @@
+# To use this template, add the following to your .gitlab-ci.yml file:
+#
+# include:
+#   template: DAST.gitlab-ci.yml
+#
+# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST:
+#
+# stages:
+#   - build
+#   - test
+#   - deploy
+#   - dast
+
 # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/
 
 # Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html).
 # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables
 
-stages:
-  - build
-  - test
-  - deploy
-  - dast
-
 variables:
   DAST_VERSION: 2
   # Setting this variable will affect all Security templates
diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
index 6834766da3d..e936364c86c 100644
--- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
@@ -46,13 +46,10 @@ dast:
           $REVIEW_DISABLED && $DAST_WEBSITE == null &&
           $DAST_API_SPECIFICATION == null
       when: never
-    - if: $CI_MERGE_REQUEST_IID &&
-          $CI_KUBERNETES_ACTIVE &&
-          $GITLAB_FEATURES =~ /\bdast\b/
-    - if: $CI_MERGE_REQUEST_IID && ($DAST_WEBSITE || $DAST_API_SPECIFICATION)
-    - if: $CI_OPEN_MERGE_REQUESTS
-      when: never
     - if: $CI_COMMIT_BRANCH &&
           $CI_KUBERNETES_ACTIVE &&
           $GITLAB_FEATURES =~ /\bdast\b/
-    - if: $CI_COMMIT_BRANCH && ($DAST_WEBSITE || $DAST_API_SPECIFICATION)
+    - if: $CI_COMMIT_BRANCH &&
+          $DAST_WEBSITE
+    - if: $CI_COMMIT_BRANCH &&
+          $DAST_API_SPECIFICATION
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index 15cc0c28296..dd5107bad9a 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -148,3 +148,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::Ci::YamlProcessor::Result.prepend_mod_with('Gitlab::Ci::YamlProcessor::Result')
diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb
index b65a303ef30..979a098d699 100644
--- a/lib/gitlab/database/migrations/observers.rb
+++ b/lib/gitlab/database/migrations/observers.rb
@@ -8,7 +8,8 @@ module Gitlab
           [
             TotalDatabaseSizeChange.new,
             QueryStatistics.new,
-            QueryLog.new
+            QueryLog.new,
+            QueryDetails.new
           ]
         end
       end
diff --git a/lib/gitlab/database/migrations/observers/query_details.rb b/lib/gitlab/database/migrations/observers/query_details.rb
new file mode 100644
index 00000000000..52b6464d449
--- /dev/null
+++ b/lib/gitlab/database/migrations/observers/query_details.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Migrations
+      module Observers
+        class QueryDetails < MigrationObserver
+          def before
+            @file_path = File.join(Instrumentation::RESULT_DIR, 'current-details.json')
+            @file = File.open(@file_path, 'wb')
+            @writer = Oj::StreamWriter.new(@file, {})
+            @writer.push_array
+            @subscriber = ActiveSupport::Notifications.subscribe('sql.active_record') do |*args|
+              record_sql_event(*args)
+            end
+          end
+
+          def after
+            ActiveSupport::Notifications.unsubscribe(@subscriber)
+            @writer.pop_all
+            @writer.flush
+            @file.close
+          end
+
+          def record(observation)
+            File.rename(@file_path, File.join(Instrumentation::RESULT_DIR, "#{observation.migration}-query-details.json"))
+          end
+
+          def record_sql_event(_name, started, finished, _unique_id, payload)
+            @writer.push_value({
+                                 start_time: started.iso8601(6),
+                                 end_time: finished.iso8601(6),
+                                 sql: payload[:sql],
+                                 binds: payload[:type_casted_binds]
+                               })
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index 6749bd6ca60..75d07a36dcd 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -36,6 +36,28 @@ module Gitlab
       end
     end
 
+    # yield to the {block} at most {count} times per {period}
+    #
+    # Defaults to once per hour.
+    #
+    # For example:
+    #
+    #   # toot the train horn at most every 20min:
+    #   throttle(locomotive.id, count: 3, period: 1.hour) { toot_train_horn }
+    #   # Brake suddenly at most once every minute:
+    #   throttle(locomotive.id, period: 1.minute) { brake_suddenly }
+    #   # Specify a uniqueness group:
+    #   throttle(locomotive.id, group: :locomotive_brake) { brake_suddenly }
+    #
+    # If a group is not specified, each block will get a separate group to itself.
+    def self.throttle(key, group: nil, period: 1.hour, count: 1, &block)
+      group ||= block.source_location.join(':')
+
+      return if new("el:throttle:#{group}:#{key}", timeout: period.to_i / count).waiting?
+
+      yield
+    end
+
     def self.cancel(key, uuid)
       return unless key.present?
 
@@ -79,6 +101,11 @@ module Gitlab
       end
     end
 
+    # This lease is waiting to obtain
+    def waiting?
+      !try_obtain
+    end
+
     # Try to renew an existing lease. Return lease UUID on success,
     # false if the lease is taken by a different UUID or inexistent.
     def renew
diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb
index d9f51a7e844..eb368af199d 100644
--- a/lib/gitlab/git/remote_mirror.rb
+++ b/lib/gitlab/git/remote_mirror.rb
@@ -5,11 +5,12 @@ module Gitlab
     class RemoteMirror
       include Gitlab::Git::WrapsGitalyErrors
 
-      attr_reader :repository, :ref_name, :only_branches_matching, :ssh_key, :known_hosts, :keep_divergent_refs
+      attr_reader :repository, :ref_name, :remote_url, :only_branches_matching, :ssh_key, :known_hosts, :keep_divergent_refs
 
-      def initialize(repository, ref_name, only_branches_matching: [], ssh_key: nil, known_hosts: nil, keep_divergent_refs: false)
+      def initialize(repository, ref_name, remote_url, only_branches_matching: [], ssh_key: nil, known_hosts: nil, keep_divergent_refs: false)
         @repository = repository
         @ref_name = ref_name
+        @remote_url = remote_url
         @only_branches_matching = only_branches_matching
         @ssh_key = ssh_key
         @known_hosts = known_hosts
@@ -20,6 +21,7 @@ module Gitlab
         wrapped_gitaly_errors do
           repository.gitaly_remote_client.update_remote_mirror(
             ref_name,
+            remote_url,
             only_branches_matching,
             ssh_key: ssh_key,
             known_hosts: known_hosts,
diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb
index 1f360385111..487127b7b74 100644
--- a/lib/gitlab/gitaly_client/remote_service.rb
+++ b/lib/gitlab/gitaly_client/remote_service.rb
@@ -55,13 +55,18 @@ module Gitlab
         encode_utf8(response.ref)
       end
 
-      def update_remote_mirror(ref_name, only_branches_matching, ssh_key: nil, known_hosts: nil, keep_divergent_refs: false)
+      def update_remote_mirror(ref_name, remote_url, only_branches_matching, ssh_key: nil, known_hosts: nil, keep_divergent_refs: false)
         req_enum = Enumerator.new do |y|
           first_request = Gitaly::UpdateRemoteMirrorRequest.new(
-            repository: @gitaly_repo,
-            ref_name: ref_name
+            repository: @gitaly_repo
           )
 
+          if remote_url
+            first_request.remote = Gitaly::UpdateRemoteMirrorRequest::Remote.new(url: remote_url)
+          else
+            first_request.ref_name = ref_name
+          end
+
           first_request.ssh_key = ssh_key if ssh_key.present?
           first_request.known_hosts = known_hosts if known_hosts.present?
           first_request.keep_divergent_refs = keep_divergent_refs
diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb
index c7f2adb27d1..2e8564b6e00 100644
--- a/lib/gitlab/lfs_token.rb
+++ b/lib/gitlab/lfs_token.rb
@@ -52,7 +52,7 @@ module Gitlab
     def valid_user?
       return true unless user?
 
-      !actor.blocked? && (!actor.allow_password_authentication? || !actor.password_expired?)
+      !actor.blocked? && !actor.password_expired_if_applicable?
     end
 
     def authentication_payload(repository_http_path)
diff --git a/lib/gitlab/pagination/keyset/paginator.rb b/lib/gitlab/pagination/keyset/paginator.rb
index 2ec4472fcd6..1c71549d86a 100644
--- a/lib/gitlab/pagination/keyset/paginator.rb
+++ b/lib/gitlab/pagination/keyset/paginator.rb
@@ -26,7 +26,7 @@ module Gitlab
         # per_page               - Number of items per page.
         # cursor_converter       - Object that serializes and de-serializes the cursor attributes. Implements dump and parse methods.
         # direction_key          - Symbol that will be the hash key of the direction within the cursor. (default: _kd => keyset direction)
-        def initialize(scope:, cursor: nil, per_page: 20, cursor_converter: Base64CursorConverter, direction_key: :_kd)
+        def initialize(scope:, cursor: nil, per_page: 20, cursor_converter: Base64CursorConverter, direction_key: :_kd, keyset_order_options: {})
           @keyset_scope = build_scope(scope)
           @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(@keyset_scope)
           @per_page = per_page
@@ -36,6 +36,7 @@ module Gitlab
           @at_last_page = false
           @at_first_page = false
           @cursor_attributes = decode_cursor_attributes(cursor)
+          @keyset_order_options = keyset_order_options
 
           set_pagination_helper_flags!
         end
@@ -45,13 +46,13 @@ module Gitlab
           @records ||= begin
             items = if paginate_backward?
                       reversed_order
-                        .apply_cursor_conditions(keyset_scope, cursor_attributes)
+                        .apply_cursor_conditions(keyset_scope, cursor_attributes, keyset_order_options)
                         .reorder(reversed_order)
                         .limit(per_page_plus_one)
                         .to_a
                     else
                       order
-                        .apply_cursor_conditions(keyset_scope, cursor_attributes)
+                        .apply_cursor_conditions(keyset_scope, cursor_attributes, keyset_order_options)
                         .limit(per_page_plus_one)
                         .to_a
                     end
@@ -120,7 +121,7 @@ module Gitlab
 
         private
 
-        attr_reader :keyset_scope, :order, :per_page, :cursor_converter, :direction_key, :cursor_attributes
+        attr_reader :keyset_scope, :order, :per_page, :cursor_converter, :direction_key, :cursor_attributes, :keyset_order_options
 
         delegate :reversed_order, to: :order
 
diff --git a/lib/gitlab/safe_request_store.rb b/lib/gitlab/safe_request_store.rb
index d146913bdb3..664afd1cc21 100644
--- a/lib/gitlab/safe_request_store.rb
+++ b/lib/gitlab/safe_request_store.rb
@@ -20,6 +20,15 @@ module Gitlab
       end
     end
 
+    # Access to the backing storage of the request store. This returns an object
+    # with `[]` and `[]=` methods that does not discard values.
+    #
+    # This can be useful if storage is needed for a delimited purpose, and the
+    # forgetful nature of the null store is undesirable.
+    def self.storage
+      store.store
+    end
+
     # This method accept an options hash to be compatible with
     # ActiveSupport::Cache::Store#write method. The options are
     # not passed to the underlying cache implementation because
@@ -27,5 +36,11 @@ module Gitlab
     def self.write(key, value, options = nil)
       store.write(key, value)
     end
+
+    def self.delete_if(&block)
+      return unless RequestStore.active?
+
+      storage.delete_if { |k, v| block.call(k) }
+    end
   end
 end
diff --git a/lib/sidebars/projects/menus/ci_cd_menu.rb b/lib/sidebars/projects/menus/ci_cd_menu.rb
index 042ad17fdfc..f85a9faacd3 100644
--- a/lib/sidebars/projects/menus/ci_cd_menu.rb
+++ b/lib/sidebars/projects/menus/ci_cd_menu.rb
@@ -61,6 +61,10 @@ module Sidebars
             pipelines#index
             pipelines#show
             pipelines#new
+            pipelines#dag
+            pipelines#failures
+            pipelines#builds
+            pipelines#test_report
           ]
         end