Merge branch 'master' into issue-discussions-refactor

* master: (66 commits)
  fix confidential border issue as well as confidential styles leaking on new MR
  Migrate force push check to Gitaly
  Add option to disable project export on instance
  Better categorize test coverage results
  Add option to disable project export on instance - db changes
  Better caching and indexing of broadcast messages
  Include the `is_admin` field in the `GET /users/:id` API when current user is an admin
  Document rspec-retry and rspec-flaky
  Fix cop description
  Retrieve and sync flaky specs report from and to S3
  Use a new RspecFlakyListener to detect flaky specs
  Fix formatting of patch_versions.md [skip ci]
  Enable Timecop safe mode
  Show error message for API 500 error in tests, and
  Fix merge request diff deserialisation when too_large was absent
  Delete correct key from `session` after authenticating using U2F
  Bumps omniauth-ldap gem version to 2.0.4
  Pending delete projects no longer return 500 error in Admins projects view
  Do not run the `ee_compat_check` job for stableish branches
  Update gitlab.po: Missing 'r' in "Fouché" that comes from "Fourcher" verb.
  ...

23 files changed, 519 insertions, 63 deletions

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 7ee25bf177c..f7e251736ab 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -497,14 +497,24 @@ module API
       expose :author, using: Entities::UserBasic
     end
 
+    class PushEventPayload < Grape::Entity
+      expose :commit_count, :action, :ref_type, :commit_from, :commit_to
+      expose :ref, :commit_title
+    end
+
     class Event < Grape::Entity
       expose :title, :project_id, :action_name
       expose :target_id, :target_iid, :target_type, :author_id
-      expose :data, :target_title
+      expose :target_title
       expose :created_at
       expose :note, using: Entities::Note, if: ->(event, options) { event.note? }
       expose :author, using: Entities::UserBasic, if: ->(event, options) { event.author }
 
+      expose :push_event_payload,
+        as: :push_data,
+        using: PushEventPayload,
+        if: -> (event, _) { event.push? }
+
       expose :author_username do |event, options|
         event.author&.username
       end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 99b8b62691f..3582ed81b0f 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -257,7 +257,15 @@ module API
       message << "  " << trace.join("\n  ")
 
       API.logger.add Logger::FATAL, message
-      rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500)
+
+      response_message =
+        if Rails.env.test?
+          message
+        else
+          '500 Internal Server Error'
+        end
+
+      rack_response({ 'message' => response_message }.to_json, 500)
     end
 
     # project helpers
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index d55a61fa638..667ba468ce6 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -29,6 +29,7 @@ module API
                                 desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com'
       optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources'
       optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.'
+      optional :project_export_enabled, type: Boolean, desc: 'Enable project export'
       optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled'
       optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects'
       optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB'
diff --git a/lib/api/users.rb b/lib/api/users.rb
index a590f2692a2..e2019d6d512 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -79,22 +79,17 @@ module API
       end
 
       desc 'Get a single user' do
-        success Entities::UserBasic
+        success Entities::User
       end
       params do
         requires :id, type: Integer, desc: 'The ID of the user'
       end
       get ":id" do
         user = User.find_by(id: params[:id])
-        not_found!('User') unless user
+        not_found!('User') unless user && can?(current_user, :read_user, user)
 
-        if current_user && current_user.admin?
-          present user, with: Entities::UserPublic
-        elsif can?(current_user, :read_user, user)
-          present user, with: Entities::User
-        else
-          render_api_error!("User not found.", 404)
-        end
+        opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : {}
+        present user, opts
       end
 
       desc 'Create a user. Available only for admins.' do
diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb
index 4a2e9c9cbb0..a9a35f2a4bd 100644
--- a/lib/api/v3/entities.rb
+++ b/lib/api/v3/entities.rb
@@ -25,14 +25,24 @@ module API
         expose(:downvote?)  { |note| false }
       end
 
+      class PushEventPayload < Grape::Entity
+        expose :commit_count, :action, :ref_type, :commit_from, :commit_to
+        expose :ref, :commit_title
+      end
+
       class Event < Grape::Entity
         expose :title, :project_id, :action_name
         expose :target_id, :target_type, :author_id
-        expose :data, :target_title
+        expose :target_title
         expose :created_at
         expose :note, using: Entities::Note, if: ->(event, options) { event.note? }
         expose :author, using: ::API::Entities::UserBasic, if: ->(event, options) { event.author }
 
+        expose :push_event_payload,
+          as: :push_data,
+          using: PushEventPayload,
+          if: -> (event, _) { event.push? }
+
         expose :author_username do |event, options|
           event.author&.username
         end
diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb
deleted file mode 100644
index 4e3c6d3c773..00000000000
--- a/lib/file_streamer.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-class FileStreamer #:nodoc:
-  attr_reader :to_path
-
-  def initialize(path)
-    @to_path = path
-  end
-
-  # Stream the file's contents if Rack::Sendfile isn't present.
-  def each
-    File.open(to_path, 'rb') do |file|
-      while chunk = file.read(16384)
-        yield chunk
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
index 0fbc6b70989..310a69a4bd4 100644
--- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
+++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
@@ -85,6 +85,8 @@ module Gitlab
           diff_hash.tap do |hash|
             diff_text = hash[:diff]
 
+            hash[:too_large] = !!hash[:too_large]
+
             if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only?
               hash[:binary] = true
               hash[:diff] = [diff_text].pack('m0')
diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb
new file mode 100644
index 00000000000..432f7c3e706
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb
@@ -0,0 +1,176 @@
+module Gitlab
+  module BackgroundMigration
+    # Class that migrates events for the new push event payloads setup. All
+    # events are copied to a shadow table, and push events will also have a row
+    # created in the push_event_payloads table.
+    class MigrateEventsToPushEventPayloads
+      class Event < ActiveRecord::Base
+        self.table_name = 'events'
+
+        serialize :data
+
+        BLANK_REF = ('0' * 40).freeze
+        TAG_REF_PREFIX = 'refs/tags/'.freeze
+        MAX_INDEX = 69
+        PUSHED = 5
+
+        def push_event?
+          action == PUSHED && data.present?
+        end
+
+        def commit_title
+          commit = commits.last
+
+          return nil unless commit && commit[:message]
+
+          index = commit[:message].index("\n")
+          message = index ? commit[:message][0..index] : commit[:message]
+
+          message.strip.truncate(70)
+        end
+
+        def commit_from_sha
+          if create?
+            nil
+          else
+            data[:before]
+          end
+        end
+
+        def commit_to_sha
+          if remove?
+            nil
+          else
+            data[:after]
+          end
+        end
+
+        def data
+          super || {}
+        end
+
+        def commits
+          data[:commits] || []
+        end
+
+        def commit_count
+          data[:total_commits_count] || 0
+        end
+
+        def ref
+          data[:ref]
+        end
+
+        def trimmed_ref_name
+          if ref_type == :tag
+            ref[10..-1]
+          else
+            ref[11..-1]
+          end
+        end
+
+        def create?
+          data[:before] == BLANK_REF
+        end
+
+        def remove?
+          data[:after] == BLANK_REF
+        end
+
+        def push_action
+          if create?
+            :created
+          elsif remove?
+            :removed
+          else
+            :pushed
+          end
+        end
+
+        def ref_type
+          if ref.start_with?(TAG_REF_PREFIX)
+            :tag
+          else
+            :branch
+          end
+        end
+      end
+
+      class EventForMigration < ActiveRecord::Base
+        self.table_name = 'events_for_migration'
+      end
+
+      class PushEventPayload < ActiveRecord::Base
+        self.table_name = 'push_event_payloads'
+
+        enum action: {
+          created: 0,
+          removed: 1,
+          pushed: 2
+        }
+
+        enum ref_type: {
+          branch: 0,
+          tag: 1
+        }
+      end
+
+      # start_id - The start ID of the range of events to process
+      # end_id - The end ID of the range to process.
+      def perform(start_id, end_id)
+        return unless migrate?
+
+        find_events(start_id, end_id).each { |event| process_event(event) }
+      end
+
+      def process_event(event)
+        replicate_event(event)
+        create_push_event_payload(event) if event.push_event?
+      end
+
+      def replicate_event(event)
+        new_attributes = event.attributes
+          .with_indifferent_access.except(:title, :data)
+
+        EventForMigration.create!(new_attributes)
+      rescue ActiveRecord::InvalidForeignKey
+        # A foreign key error means the associated event was removed. In this
+        # case we'll just skip migrating the event.
+      end
+
+      def create_push_event_payload(event)
+        commit_from = pack(event.commit_from_sha)
+        commit_to = pack(event.commit_to_sha)
+
+        PushEventPayload.create!(
+          event_id: event.id,
+          commit_count: event.commit_count,
+          ref_type: event.ref_type,
+          action: event.push_action,
+          commit_from: commit_from,
+          commit_to: commit_to,
+          ref: event.trimmed_ref_name,
+          commit_title: event.commit_title
+        )
+      rescue ActiveRecord::InvalidForeignKey
+        # A foreign key error means the associated event was removed. In this
+        # case we'll just skip migrating the event.
+      end
+
+      def find_events(start_id, end_id)
+        Event
+          .where('NOT EXISTS (SELECT true FROM events_for_migration WHERE events_for_migration.id = events.id)')
+          .where(id: start_id..end_id)
+      end
+
+      def migrate?
+        Event.table_exists? && PushEventPayload.table_exists? &&
+          EventForMigration.table_exists?
+      end
+
+      def pack(value)
+        value ? [value].pack('H*') : nil
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb
index 1e73f89158d..714464fd5e7 100644
--- a/lib/gitlab/checks/force_push.rb
+++ b/lib/gitlab/checks/force_push.rb
@@ -5,12 +5,19 @@ module Gitlab
         return false if project.empty_repo?
 
         # Created or deleted branch
-        if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev)
-          false
-        else
-          Gitlab::Git::RevList.new(
-            path_to_repo: project.repository.path_to_repo,
-            oldrev: oldrev, newrev: newrev).missed_ref.present?
+        return false if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev)
+
+        GitalyClient.migrate(:force_push) do |is_enabled|
+          if is_enabled
+            !project
+              .repository
+              .gitaly_commit_client
+              .is_ancestor(oldrev, newrev)
+          else
+            Gitlab::Git::RevList.new(
+              path_to_repo: project.repository.path_to_repo,
+              oldrev: oldrev, newrev: newrev).missed_ref.present?
+          end
         end
       end
     end
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index d7dab584a44..e001d25e7b7 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -25,6 +25,10 @@ module Gitlab
       database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
     end
 
+    def self.join_lateral_supported?
+      postgresql? && version.to_f >= 9.3
+    end
+
     def self.nulls_last_order(field, direction = 'ASC')
       order = "#{field} #{direction}"
 
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 371f8797ff2..7000b173075 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -324,6 +324,23 @@ module Gitlab
         raw_log(options).map { |c| Commit.decorate(self, c) }
       end
 
+      # Used in gitaly-ruby
+      def raw_log(options)
+        actual_ref = options[:ref] || root_ref
+        begin
+          sha = sha_from_ref(actual_ref)
+        rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
+          # Return an empty array if the ref wasn't found
+          return []
+        end
+
+        if log_using_shell?(options)
+          log_by_shell(sha, options)
+        else
+          log_by_walk(sha, options)
+        end
+      end
+
       def count_commits(options)
         gitaly_migrate(:count_commits) do |is_enabled|
           if is_enabled
@@ -733,22 +750,6 @@ module Gitlab
         sort_branches(branches, sort_by)
       end
 
-      def raw_log(options)
-        actual_ref = options[:ref] || root_ref
-        begin
-          sha = sha_from_ref(actual_ref)
-        rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
-          # Return an empty array if the ref wasn't found
-          return []
-        end
-
-        if log_using_shell?(options)
-          log_by_shell(sha, options)
-        else
-          log_by_walk(sha, options)
-        end
-      end
-
       def log_using_shell?(options)
         options[:path].present? ||
           options[:disable_walk] ||
diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb
index f5a4c5493ef..8fc937496af 100644
--- a/lib/gitlab/gitaly_client/util.rb
+++ b/lib/gitlab/gitaly_client/util.rb
@@ -5,7 +5,9 @@ module Gitlab
         def repository(repository_storage, relative_path)
           Gitaly::Repository.new(
             storage_name: repository_storage,
-            relative_path: relative_path
+            relative_path: relative_path,
+            git_object_directory: Gitlab::Git::Env['GIT_OBJECT_DIRECTORY'].to_s,
+            git_alternate_object_directories: Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES'])
           )
         end
       end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index ffd17118c91..989342389bc 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -47,12 +47,16 @@ module Gitlab
       end
 
       def remove_symlinks!
-        Dir["#{@shared.export_path}/**/*"].each do |path|
+        extracted_files.each do |path|
           FileUtils.rm(path) if File.lstat(path).symlink?
         end
 
         true
       end
+
+      def extracted_files
+        Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ /.*\/\.{1,2}$/ }
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index c5c05bfe2fb..9d9ebcb389a 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -3,18 +3,22 @@ project_tree:
   - labels:
       :priorities
   - milestones:
-    - :events
+    - events:
+      - :push_event_payload
   - issues:
-    - :events
+    - events:
+      - :push_event_payload
     - :timelogs
     - notes:
       - :author
-      - :events
+      - events:
+        - :push_event_payload
     - label_links:
       - label:
           :priorities
     - milestone:
-      - :events
+      - events:
+        - :push_event_payload
   - snippets:
     - :award_emoji
     - notes:
@@ -25,21 +29,25 @@ project_tree:
   - merge_requests:
     - notes:
       - :author
-      - :events
+      - events:
+        - :push_event_payload
     - merge_request_diff:
       - :merge_request_diff_commits
       - :merge_request_diff_files
-    - :events
+    - events:
+      - :push_event_payload
     - :timelogs
     - label_links:
       - label:
           :priorities
     - milestone:
-      - :events
+      - events:
+        - :push_event_payload
   - pipelines:
     - notes:
       - :author
-      - :events
+      - events:
+        - :push_event_payload
     - :stages
     - :statuses
   - :triggers
@@ -107,6 +115,8 @@ excluded_attributes:
   statuses:
     - :trace
     - :token
+  push_event_payload:
+    - :event_id
 
 methods:
   labels:
diff --git a/lib/gitlab/middleware/webpack_proxy.rb b/lib/gitlab/middleware/webpack_proxy.rb
index 6105d165810..6aecf63231f 100644
--- a/lib/gitlab/middleware/webpack_proxy.rb
+++ b/lib/gitlab/middleware/webpack_proxy.rb
@@ -1,6 +1,7 @@
 # This Rack middleware is intended to proxy the webpack assets directory to the
 # webpack-dev-server.  It is only intended for use in development.
 
+# :nocov:
 module Gitlab
   module Middleware
     class WebpackProxy < Rack::Proxy
@@ -22,3 +23,4 @@ module Gitlab
     end
   end
 end
+# :nocov:
diff --git a/lib/gitlab/o_auth/session.rb b/lib/gitlab/o_auth/session.rb
index f33bfd0bd0e..30739f2a2c5 100644
--- a/lib/gitlab/o_auth/session.rb
+++ b/lib/gitlab/o_auth/session.rb
@@ -1,3 +1,4 @@
+# :nocov:
 module Gitlab
   module OAuth
     module Session
@@ -15,3 +16,4 @@ module Gitlab
     end
   end
 end
+# :nocov:
diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb
index 823f697f51c..f9ab9bd466f 100644
--- a/lib/gitlab/seeder.rb
+++ b/lib/gitlab/seeder.rb
@@ -1,3 +1,4 @@
+# :nocov:
 module DeliverNever
   def deliver_later
     self
@@ -21,3 +22,4 @@ module Gitlab
     end
   end
 end
+# :nocov:
diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb
index 7e14a566696..fee1a127fd7 100644
--- a/lib/gitlab/url_blocker.rb
+++ b/lib/gitlab/url_blocker.rb
@@ -19,6 +19,8 @@ module Gitlab
           return false if internal?(uri)
 
           return true if blocked_port?(uri.port)
+          return true if blocked_user_or_hostname?(uri.user)
+          return true if blocked_user_or_hostname?(uri.hostname)
 
           server_ips = Resolv.getaddresses(uri.hostname)
           return true if (blocked_ips & server_ips).any?
@@ -37,6 +39,12 @@ module Gitlab
         port < 1024 && !VALID_PORTS.include?(port)
       end
 
+      def blocked_user_or_hostname?(value)
+        return false if value.blank?
+
+        value !~ /\A\p{Alnum}/
+      end
+
       def internal?(uri)
         internal_web?(uri) || internal_shell?(uri)
       end
diff --git a/lib/rspec_flaky/example.rb b/lib/rspec_flaky/example.rb
new file mode 100644
index 00000000000..b6e790cbbab
--- /dev/null
+++ b/lib/rspec_flaky/example.rb
@@ -0,0 +1,46 @@
+module RspecFlaky
+  # This is a wrapper class for RSpec::Core::Example
+  class Example
+    delegate :status, :exception, to: :execution_result
+
+    def initialize(rspec_example)
+      @rspec_example = rspec_example.try(:example) || rspec_example
+    end
+
+    def uid
+      @uid ||= Digest::MD5.hexdigest("#{description}-#{file}")
+    end
+
+    def example_id
+      rspec_example.id
+    end
+
+    def file
+      metadata[:file_path]
+    end
+
+    def line
+      metadata[:line_number]
+    end
+
+    def description
+      metadata[:full_description]
+    end
+
+    def attempts
+      rspec_example.try(:attempts) || 1
+    end
+
+    private
+
+    attr_reader :rspec_example
+
+    def metadata
+      rspec_example.metadata
+    end
+
+    def execution_result
+      rspec_example.execution_result
+    end
+  end
+end
diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb
new file mode 100644
index 00000000000..f81fb90e870
--- /dev/null
+++ b/lib/rspec_flaky/flaky_example.rb
@@ -0,0 +1,39 @@
+module RspecFlaky
+  # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file
+  class FlakyExample < OpenStruct
+    def initialize(example)
+      if example.respond_to?(:example_id)
+        super(
+          example_id: example.example_id,
+          file: example.file,
+          line: example.line,
+          description: example.description,
+          last_attempts_count: example.attempts,
+          flaky_reports: 1)
+      else
+        super
+      end
+    end
+
+    def first_flaky_at
+      self[:first_flaky_at] || Time.now
+    end
+
+    def last_flaky_at
+      Time.now
+    end
+
+    def last_flaky_job
+      return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID']
+
+      "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}"
+    end
+
+    def to_h
+      super.merge(
+        first_flaky_at: first_flaky_at,
+        last_flaky_at: last_flaky_at,
+        last_flaky_job: last_flaky_job)
+    end
+  end
+end
diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb
new file mode 100644
index 00000000000..ec2fbd9e36c
--- /dev/null
+++ b/lib/rspec_flaky/listener.rb
@@ -0,0 +1,75 @@
+require 'json'
+
+module RspecFlaky
+  class Listener
+    attr_reader :all_flaky_examples, :new_flaky_examples
+
+    def initialize
+      @new_flaky_examples = {}
+      @all_flaky_examples = init_all_flaky_examples
+    end
+
+    def example_passed(notification)
+      current_example = RspecFlaky::Example.new(notification.example)
+
+      return unless current_example.attempts > 1
+
+      flaky_example_hash = all_flaky_examples[current_example.uid]
+
+      all_flaky_examples[current_example.uid] =
+        if flaky_example_hash
+          FlakyExample.new(flaky_example_hash).tap do |ex|
+            ex.last_attempts_count = current_example.attempts
+            ex.flaky_reports += 1
+          end
+        else
+          FlakyExample.new(current_example).tap do |ex|
+            new_flaky_examples[current_example.uid] = ex
+          end
+        end
+    end
+
+    def dump_summary(_)
+      write_report_file(all_flaky_examples, all_flaky_examples_report_path)
+
+      if new_flaky_examples.any?
+        Rails.logger.warn "\nNew flaky examples detected:\n"
+        Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples))
+
+        write_report_file(new_flaky_examples, new_flaky_examples_report_path)
+      end
+    end
+
+    def to_report(examples)
+      Hash[examples.map { |k, ex| [k, ex.to_h] }]
+    end
+
+    private
+
+    def init_all_flaky_examples
+      return {} unless File.exist?(all_flaky_examples_report_path)
+
+      all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path))
+
+      Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }]
+    end
+
+    def write_report_file(examples, file_path)
+      return unless ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true'
+
+      report_path_dir = File.dirname(file_path)
+      FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir)
+      File.write(file_path, JSON.pretty_generate(to_report(examples)))
+    end
+
+    def all_flaky_examples_report_path
+      @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] ||
+        Rails.root.join("rspec_flaky/all-report.json")
+    end
+
+    def new_flaky_examples_report_path
+      @new_flaky_examples_report_path ||= ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] ||
+        Rails.root.join("rspec_flaky/new-report.json")
+    end
+  end
+end
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index f25e66d54c8..54f51d9d633 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -25,6 +25,39 @@ map $http_upgrade $connection_upgrade_gitlab {
     ''      close;
 }
 
+## NGINX 'combined' log format with filtered query strings
+log_format gitlab_access $remote_addr - $remote_user [$time_local] "$request_method $gitlab_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_filtered_http_referer" "$http_user_agent";
+
+## Remove private_token from the request URI
+# In:  /foo?private_token=unfiltered&authenticity_token=unfiltered&rss_token=unfiltered&...
+# Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&...
+map $request_uri $gitlab_temp_request_uri_1 {
+  default $request_uri;
+  ~(?i)^(?<start>.*)(?<temp>[\?&]private[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest";
+}
+
+## Remove authenticity_token from the request URI
+# In:  /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&...
+# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&...
+map $gitlab_temp_request_uri_1 $gitlab_temp_request_uri_2 {
+  default $gitlab_temp_request_uri_1;
+  ~(?i)^(?<start>.*)(?<temp>[\?&]authenticity[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest";
+}
+
+## Remove rss_token from the request URI
+# In:  /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&...
+# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=[FILTERED]&...
+map $gitlab_temp_request_uri_2 $gitlab_filtered_request_uri {
+  default $gitlab_temp_request_uri_2;
+  ~(?i)^(?<start>.*)(?<temp>[\?&]rss[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest";
+}
+
+## A version of the referer without the query string
+map $http_referer $gitlab_filtered_http_referer {
+  default $http_referer;
+  ~^(?<temp>.*)\? $temp;
+}
+
 ## Normal HTTP host
 server {
   ## Either remove "default_server" from the listen line below,
@@ -46,7 +79,7 @@ server {
   # set_real_ip_from YOUR_TRUSTED_ADDRESS; ## Replace this with something like 192.168.1.0/24
 
   ## Individual nginx logs for this GitLab vhost
-  access_log  /var/log/nginx/gitlab_access.log;
+  access_log  /var/log/nginx/gitlab_access.log gitlab_access;
   error_log   /var/log/nginx/gitlab_error.log;
 
   location / {
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 2b40da18bab..ed8131ef24f 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -29,6 +29,41 @@ map $http_upgrade $connection_upgrade_gitlab_ssl {
     ''      close;
 }
 
+
+## NGINX 'combined' log format with filtered query strings
+log_format gitlab_ssl_access $remote_addr - $remote_user [$time_local] "$request_method $gitlab_ssl_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_ssl_filtered_http_referer" "$http_user_agent";
+
+## Remove private_token from the request URI
+# In:  /foo?private_token=unfiltered&authenticity_token=unfiltered&rss_token=unfiltered&...
+# Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&...
+map $request_uri $gitlab_ssl_temp_request_uri_1 {
+  default $request_uri;
+  ~(?i)^(?<start>.*)(?<temp>[\?&]private[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest";
+}
+
+## Remove authenticity_token from the request URI
+# In:  /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&...
+# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&...
+map $gitlab_ssl_temp_request_uri_1 $gitlab_ssl_temp_request_uri_2 {
+  default $gitlab_ssl_temp_request_uri_1;
+  ~(?i)^(?<start>.*)(?<temp>[\?&]authenticity[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest";
+}
+
+## Remove rss_token from the request URI
+# In:  /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&...
+# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=[FILTERED]&...
+map $gitlab_ssl_temp_request_uri_2 $gitlab_ssl_filtered_request_uri {
+  default $gitlab_ssl_temp_request_uri_2;
+  ~(?i)^(?<start>.*)(?<temp>[\?&]rss[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest";
+}
+
+## A version of the referer without the query string
+map $http_referer $gitlab_ssl_filtered_http_referer {
+  default $http_referer;
+  ~^(?<temp>.*)\? $temp;
+}
+
+
 ## Redirects all HTTP traffic to the HTTPS host
 server {
   ## Either remove "default_server" from the listen line below,
@@ -40,7 +75,7 @@ server {
   server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com
   server_tokens off; ## Don't show the nginx version number, a security best practice
   return 301 https://$http_host$request_uri;
-  access_log  /var/log/nginx/gitlab_access.log;
+  access_log  /var/log/nginx/gitlab_access.log gitlab_ssl_access;
   error_log   /var/log/nginx/gitlab_error.log;
 }
 
@@ -93,7 +128,7 @@ server {
   # set_real_ip_from YOUR_TRUSTED_ADDRESS; ## Replace this with something like 192.168.1.0/24
 
   ## Individual nginx logs for this GitLab vhost
-  access_log  /var/log/nginx/gitlab_access.log;
+  access_log  /var/log/nginx/gitlab_access.log gitlab_ssl_access;
   error_log   /var/log/nginx/gitlab_error.log;
 
   location / {