Add latest changes from gitlab-org/gitlab@master

17 files changed, 89 insertions, 203 deletions

diff --git a/app/models/concerns/analytics/cycle_analytics/stage.rb b/app/models/concerns/analytics/cycle_analytics/stage.rb
index 0c603c2d5e6..0011ba10a9a 100644
--- a/app/models/concerns/analytics/cycle_analytics/stage.rb
+++ b/app/models/concerns/analytics/cycle_analytics/stage.rb
@@ -7,6 +7,7 @@ module Analytics
 
       included do
         validates :name, presence: true
+        validates :name, exclusion: { in: Gitlab::Analytics::CycleAnalytics::DefaultStages.names }, if: :custom?
         validates :start_event_identifier, presence: true
         validates :end_event_identifier, presence: true
         validate :validate_stage_event_pairs
@@ -15,6 +16,7 @@ module Analytics
         enum end_event_identifier: Gitlab::Analytics::CycleAnalytics::StageEvents.to_enum, _prefix: :end_event_identifier
 
         alias_attribute :custom_stage?, :custom
+        scope :default_stages, -> { where(custom: false) }
       end
 
       def parent=(_)
diff --git a/changelogs/unreleased/33337-fix-insights-only-projects-for-project.yml b/changelogs/unreleased/33337-fix-insights-only-projects-for-project.yml
new file mode 100644
index 00000000000..7dea4519d10
--- /dev/null
+++ b/changelogs/unreleased/33337-fix-insights-only-projects-for-project.yml
@@ -0,0 +1,5 @@
+---
+title: Make sure project insights stick on its own
+merge_request: 18082
+author:
+type: fixed
diff --git a/changelogs/unreleased/ignore-autreply.yml b/changelogs/unreleased/ignore-autreply.yml
new file mode 100644
index 00000000000..69f9885a16f
--- /dev/null
+++ b/changelogs/unreleased/ignore-autreply.yml
@@ -0,0 +1,5 @@
+---
+title: Ignore incoming emails with X-Autoreply header.
+merge_request: 18118
+author:
+type: fixed
diff --git a/changelogs/unreleased/jc-remove-cleanup-rake-tasks.yml b/changelogs/unreleased/jc-remove-cleanup-rake-tasks.yml
new file mode 100644
index 00000000000..fd2d0e323e8
--- /dev/null
+++ b/changelogs/unreleased/jc-remove-cleanup-rake-tasks.yml
@@ -0,0 +1,5 @@
+---
+title: Removing cleanup:repo, cleanup:dirs
+merge_request: 18087
+author:
+type: deprecated
diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md
index 957f583cd7d..67bf7cbd828 100644
--- a/doc/raketasks/cleanup.md
+++ b/doc/raketasks/cleanup.md
@@ -2,46 +2,6 @@
 
 ## Remove garbage from filesystem
 
-DANGER: **Danger:**
-The commands below will remove data permanently from your GitLab instance. Only use
-these commands if you are 100% certain that it is safe to delete this data.
-
-Remove namespaces(dirs) from all repository storage paths if they don't exist in GitLab database.
-
-```
-# omnibus-gitlab
-sudo gitlab-rake gitlab:cleanup:dirs
-
-# installation from source
-bundle exec rake gitlab:cleanup:dirs RAILS_ENV=production
-```
-
-DANGER: **Danger:**
-The following task does not currently work as expected.
-The use will probably mark more existing repositories as orphaned.
-For more information, see the [issue](https://gitlab.com/gitlab-org/gitlab/issues/24633).
-
-Rename repositories from all repository storage paths if they don't exist in GitLab database.
-The repositories get a `+orphaned+TIMESTAMP` suffix so that they cannot block new repositories from being created.
-
-```
-# omnibus-gitlab
-sudo gitlab-rake gitlab:cleanup:repos
-
-# installation from source
-bundle exec rake gitlab:cleanup:repos RAILS_ENV=production
-```
-
-Remove old repository copies from repositories moved to another storage.
-
-```
-# omnibus-gitlab
-sudo gitlab-rake gitlab:cleanup:moved
-
-# installation from source
-bundle exec rake gitlab:cleanup:moved RAILS_ENV=production
-```
-
 Clean up local project upload files if they don't exist in GitLab database. The
 task attempts to fix the file if it can find its project, otherwise it moves the
 file to a lost and found directory.
diff --git a/doc/user/project/merge_requests/merge_request_dependencies.md b/doc/user/project/merge_requests/merge_request_dependencies.md
index f1a02e837a5..c99e6663093 100644
--- a/doc/user/project/merge_requests/merge_request_dependencies.md
+++ b/doc/user/project/merge_requests/merge_request_dependencies.md
@@ -4,7 +4,11 @@ type: reference, concepts
 
 # Merge Request dependencies **(PREMIUM)**
 
-> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9688) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9688) in
+[GitLab Premium](https://about.gitlab.com/pricing/) 12.2.
+> - [Renamed](https://gitlab.com/gitlab-org/gitlab/merge_requests/17291) from
+"Cross-project dependencies" to "Merge Requests dependencies" in
+[GitLab Premium](https://about.gitlab.com/pricing/) 12.4.
 
 Merge request dependencies allows a required order of merging
 between merge requests to be expressed. If a merge request "depends on" another,
diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb
index 286c393005f..711645800fb 100644
--- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb
+++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb
@@ -23,6 +23,10 @@ module Gitlab
           ]
         end
 
+        def self.names
+          all.map { |stage| stage[:name] }
+        end
+
         def self.params_for_issue_stage
           {
             name: 'issue',
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 7da8b385266..847260b2e0f 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -32,7 +32,7 @@ module Gitlab
 
         mail = build_mail
 
-        ignore_auto_submitted!(mail)
+        ignore_auto_reply!(mail)
 
         mail_key = extract_mail_key(mail)
         handler = Handler.for(mail, mail_key)
@@ -96,14 +96,25 @@ module Gitlab
         end
       end
 
-      def ignore_auto_submitted!(mail)
+      def ignore_auto_reply!(mail)
+        if auto_submitted?(mail) || auto_replied?(mail)
+          raise AutoGeneratedEmailError
+        end
+      end
+
+      def auto_submitted?(mail)
         # Mail::Header#[] is case-insensitive
         auto_submitted = mail.header['Auto-Submitted']&.value
 
         # Mail::Field#value would strip leading and trailing whitespace
-        raise AutoGeneratedEmailError if
-          # See also https://tools.ietf.org/html/rfc3834
-          auto_submitted && auto_submitted != 'no'
+        # See also https://tools.ietf.org/html/rfc3834
+        auto_submitted && auto_submitted != 'no'
+      end
+
+      def auto_replied?(mail)
+        autoreply = mail.header['X-Autoreply']&.value
+
+        autoreply && autoreply == 'yes'
       end
     end
   end
diff --git a/lib/gitlab/import_export/fast_hash_serializer.rb b/lib/gitlab/import_export/fast_hash_serializer.rb
index e2d5c5ae055..5a067b5c9f3 100644
--- a/lib/gitlab/import_export/fast_hash_serializer.rb
+++ b/lib/gitlab/import_export/fast_hash_serializer.rb
@@ -135,10 +135,7 @@ module Gitlab
 
         data = []
 
-        # rubocop:disable Cop/InBatches
-        # If we put `rubocop:disable` inline after `do |batch|`,
-        # `Cop/LineBreakAroundConditionalBlock` will fail
-        record.in_batches(of: @batch_size) do |batch|
+        record.in_batches(of: @batch_size) do |batch| # rubocop:disable Cop/InBatches
           if Feature.enabled?(:export_fast_serialize_with_raw_json, default_enabled: true)
             data.append(JSONBatchRelation.new(batch, options, preloads[key]).tap(&:raw_json))
           else
@@ -146,7 +143,6 @@ module Gitlab
             data += batch.as_json(options)
           end
         end
-        # rubocop:enable Cop/InBatches
 
         data
       end
diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb
index 108d1330b47..de7870dfb8c 100644
--- a/lib/gitlab/metrics/exporter/base_exporter.rb
+++ b/lib/gitlab/metrics/exporter/base_exporter.rb
@@ -36,10 +36,11 @@ module Gitlab
         end
 
         def stop_working
-          if server # rubocop:disable Cop/LineBreakAroundConditionalBlock
+          if server
             server.shutdown
             server.listeners.each(&:close)
           end
+
           @server = nil
         end
 
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 4d854cd178d..0a0ee7b4bfa 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -3,69 +3,6 @@ require 'set'
 
 namespace :gitlab do
   namespace :cleanup do
-    desc "GitLab | Cleanup | Clean namespaces"
-    task dirs: :gitlab_environment do
-      namespaces = Set.new(Namespace.pluck(:path))
-      namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX
-
-      Gitaly::Server.all.each do |server|
-        all_dirs = Gitlab::GitalyClient::StorageService
-          .new(server.storage)
-          .list_directories(depth: 0)
-          .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) }
-
-        puts "Looking for directories to remove... "
-        all_dirs.each do |dir_path|
-          if remove?
-            begin
-              Gitlab::GitalyClient::NamespaceService.new(server.storage)
-                .remove(dir_path)
-
-              puts "Removed...#{dir_path}"
-            rescue StandardError => e
-              puts "Cannot remove #{dir_path}: #{e.message}".color(:red)
-            end
-          else
-            puts "Can be removed: #{dir_path}".color(:red)
-          end
-        end
-      end
-
-      unless remove?
-        puts "To cleanup this directories run this command with REMOVE=true".color(:yellow)
-      end
-    end
-
-    desc "GitLab | Cleanup | Clean repositories"
-    task repos: :gitlab_environment do
-      move_suffix = "+orphaned+#{Time.now.to_i}"
-
-      Gitaly::Server.all.each do |server|
-        Gitlab::GitalyClient::StorageService
-          .new(server.storage)
-          .list_directories
-          .each do |path|
-          repo_with_namespace = path.chomp('.git').chomp('.wiki')
-
-          # TODO ignoring hashed repositories for now.  But revisit to fully support
-          # possible orphaned hashed repos
-          next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX)
-          next if Project.find_by_full_path(repo_with_namespace)
-
-          new_path = path + move_suffix
-          puts path.inspect + ' -> ' + new_path.inspect
-
-          begin
-            Gitlab::GitalyClient::NamespaceService
-              .new(server.storage)
-              .rename(path, new_path)
-          rescue StandardError => e
-            puts "Error occurred while moving the repository: #{e.message}".color(:red)
-          end
-        end
-      end
-    end
-
     desc "GitLab | Cleanup | Block users that have been removed in LDAP"
     task block_removed_ldap_users: :gitlab_environment do
       warn_user_is_not_gitlab
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index a4fb4749720..4c44a32c7dd 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -255,25 +255,13 @@ EOF
 }
 
 function display_deployment_debug() {
-  kubectl get pods -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG}
+  # Get all pods that are not ready (this will return completed pods for minio and migrations jobs)
+  echoinfo "Unready Pods for release ${CI_ENVIRONMENT_SLUG}"
+  kubectl get pods -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.phase!=Running
 
-  migrations_pod=$(get_pod "migrations");
-  if [ -z "${migrations_pod}" ]; then
-    echoerr "Migrations pod not found."
-  else
-    echoinfo "Logs tail of the ${migrations_pod} pod..."
-
-    kubectl logs -n "$KUBE_NAMESPACE" "${migrations_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g"
-  fi
-
-  unicorn_pod=$(get_pod "unicorn");
-  if [ -z "${unicorn_pod}" ]; then
-    echoerr "Unicorn pod not found."
-  else
-    echoinfo "Logs tail of the ${unicorn_pod} pod..."
-
-    kubectl logs -n "$KUBE_NAMESPACE" -c unicorn "${unicorn_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g"
-  fi
+  # Get all non-completed jobs
+  echoinfo "Unsuccessful Jobs for release ${CI_ENVIRONMENT_SLUG}"
+  kubectl get jobs -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.successful!=1
 }
 
 function add_license() {
diff --git a/spec/fixtures/emails/auto_reply.eml b/spec/fixtures/emails/auto_reply.eml
index 7999c8d78b7..4d5a45df467 100644
--- a/spec/fixtures/emails/auto_reply.eml
+++ b/spec/fixtures/emails/auto_reply.eml
@@ -12,7 +12,7 @@ Mime-Version: 1.0
 Content-Type: text/plain;
  charset=ISO-8859-1
 Content-Transfer-Encoding: 7bit
-Auto-Submitted: auto-generated
+X-Autoreply: yes
 X-Sieve: CMU Sieve 2.2
 X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu,
  13 Jun 2013 14:03:48 -0700 (PDT)
diff --git a/spec/fixtures/emails/auto_submitted.eml b/spec/fixtures/emails/auto_submitted.eml
new file mode 100644
index 00000000000..7999c8d78b7
--- /dev/null
+++ b/spec/fixtures/emails/auto_submitted.eml
@@ -0,0 +1,21 @@
+Return-Path: <jake@adventuretime.ooo>
+Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400
+Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@discourse.example.com>; Thu, 13 Jun 2013 17:03:50 -0400
+Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@discourse.example.com>; Thu, 13 Jun 2013 14:03:48 -0700
+Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700
+Date: Thu, 13 Jun 2013 17:03:48 -0400
+From: Jake the Dog <jake@adventuretime.ooo>
+To: reply+636ca428858779856c226bb145ef4fad@appmail.adventuretime.ooo
+Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
+Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'
+Mime-Version: 1.0
+Content-Type: text/plain;
+ charset=ISO-8859-1
+Content-Transfer-Encoding: 7bit
+Auto-Submitted: auto-generated
+X-Sieve: CMU Sieve 2.2
+X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu,
+ 13 Jun 2013 14:03:48 -0700 (PDT)
+X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1
+
+Test reply to Discourse email digest
diff --git a/spec/lib/gitlab/email/receiver_spec.rb b/spec/lib/gitlab/email/receiver_spec.rb
index 6b5a355e598..43c73242f5f 100644
--- a/spec/lib/gitlab/email/receiver_spec.rb
+++ b/spec/lib/gitlab/email/receiver_spec.rb
@@ -40,7 +40,15 @@ describe Gitlab::Email::Receiver do
     end
   end
 
-  context "when the email was auto generated" do
+  context "when the email was auto generated with Auto-Submitted header" do
+    let(:email_raw) { fixture_file("emails/auto_submitted.eml") }
+
+    it "raises an AutoGeneratedEmailError" do
+      expect { receiver.execute }.to raise_error(Gitlab::Email::AutoGeneratedEmailError)
+    end
+  end
+
+  context "when the email was auto generated with X-Autoreply header" do
     let(:email_raw) { fixture_file("emails/auto_reply.eml") }
 
     it "raises an AutoGeneratedEmailError" do
diff --git a/spec/support/shared_examples/cycle_analytics_stage_examples.rb b/spec/support/shared_examples/cycle_analytics_stage_examples.rb
index 151f5325e84..dc2ea229171 100644
--- a/spec/support/shared_examples/cycle_analytics_stage_examples.rb
+++ b/spec/support/shared_examples/cycle_analytics_stage_examples.rb
@@ -46,6 +46,13 @@ shared_examples_for 'cycle analytics stage' do
       expect(stage).not_to be_valid
       expect(stage.errors.details[:end_event]).to eq([{ error: :not_allowed_for_the_given_start_event }])
     end
+
+    context 'disallows default stage names when creating custom stage' do
+      let(:invalid_params) { valid_params.merge(name: Gitlab::Analytics::CycleAnalytics::DefaultStages.names.first, custom: true) }
+      let(:stage) { described_class.new(invalid_params) }
+
+      it { expect(stage).not_to be_valid }
+    end
   end
 
   describe '#subject_model' do
diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb
index 4aee6d005a8..6c09bb5d9f9 100644
--- a/spec/tasks/gitlab/cleanup_rake_spec.rb
+++ b/spec/tasks/gitlab/cleanup_rake_spec.rb
@@ -5,74 +5,6 @@ describe 'gitlab:cleanup rake tasks' do
     Rake.application.rake_require 'tasks/gitlab/cleanup'
   end
 
-  describe 'cleanup namespaces and repos' do
-    let(:gitlab_shell) { Gitlab::Shell.new }
-    let(:storage) { storages.keys.first }
-    let(:storages) do
-      {
-        'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage'))
-      }
-    end
-
-    before(:all) do
-      @default_storage_hash = Gitlab.config.repositories.storages.default.to_h
-    end
-
-    before do
-      allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
-    end
-
-    after do
-      Gitlab::GitalyClient::StorageService.new(storage).delete_all_repositories
-    end
-
-    describe 'cleanup:repos' do
-      before do
-        gitlab_shell.add_namespace(storage, 'broken/project.git')
-        gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git')
-      end
-
-      it 'moves it to an orphaned path' do
-        now = Time.now
-
-        Timecop.freeze(now) do
-          run_rake_task('gitlab:cleanup:repos')
-          repo_list = Gitlab::GitalyClient::StorageService.new(storage).list_directories(depth: 0)
-
-          expect(repo_list.last).to include("broken+orphaned+#{now.to_i}")
-        end
-      end
-
-      it 'ignores @hashed repos' do
-        run_rake_task('gitlab:cleanup:repos')
-
-        expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true)
-      end
-    end
-
-    describe 'cleanup:dirs' do
-      it 'removes missing namespaces' do
-        gitlab_shell.add_namespace(storage, "namespace_1/project.git")
-        gitlab_shell.add_namespace(storage, "namespace_2/project.git")
-        allow(Namespace).to receive(:pluck).and_return(['namespace_1'])
-
-        stub_env('REMOVE', 'true')
-        run_rake_task('gitlab:cleanup:dirs')
-
-        expect(gitlab_shell.exists?(storage, 'namespace_1')).to be(true)
-        expect(gitlab_shell.exists?(storage, 'namespace_2')).to be(false)
-      end
-
-      it 'ignores @hashed directory' do
-        gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git')
-
-        run_rake_task('gitlab:cleanup:dirs')
-
-        expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true)
-      end
-    end
-  end
-
   # A single integration test that is redundant with one part of the
   # Gitlab::Cleanup::ProjectUploads spec.
   #