Add latest changes from gitlab-org/gitlab@master

4 files changed, 459 insertions, 16 deletions

diff --git a/spec/lib/banzai/filter/abstract_reference_filter_spec.rb b/spec/lib/banzai/filter/abstract_reference_filter_spec.rb
index 3e8b0ea113f..798112d0f53 100644
--- a/spec/lib/banzai/filter/abstract_reference_filter_spec.rb
+++ b/spec/lib/banzai/filter/abstract_reference_filter_spec.rb
@@ -3,30 +3,27 @@
 require 'spec_helper'
 
 describe Banzai::Filter::AbstractReferenceFilter do
-  let(:project) { create(:project) }
+  let_it_be(:project) { create(:project) }
+
+  let(:doc) { Nokogiri::HTML.fragment('') }
+  let(:filter) { described_class.new(doc, project: project) }
 
   describe '#references_per_parent' do
-    it 'returns a Hash containing references grouped per parent paths' do
-      doc = Nokogiri::HTML.fragment("#1 #{project.full_path}#2")
-      filter = described_class.new(doc, project: project)
+    let(:doc) { Nokogiri::HTML.fragment("#1 #{project.full_path}#2 #2") }
 
-      expect(filter).to receive(:object_class).exactly(4).times.and_return(Issue)
-      expect(filter).to receive(:object_sym).twice.and_return(:issue)
+    it 'returns a Hash containing references grouped per parent paths' do
+      expect(described_class).to receive(:object_class).exactly(6).times.and_return(Issue)
 
       refs = filter.references_per_parent
 
-      expect(refs).to be_an_instance_of(Hash)
-      expect(refs[project.full_path]).to eq(Set.new(%w[1 2]))
+      expect(refs).to match(a_hash_including(project.full_path => contain_exactly(1, 2)))
     end
   end
 
   describe '#parent_per_reference' do
     it 'returns a Hash containing projects grouped per parent paths' do
-      doc = Nokogiri::HTML.fragment('')
-      filter = described_class.new(doc, project: project)
-
       expect(filter).to receive(:references_per_parent)
-        .and_return({ project.full_path => Set.new(%w[1]) })
+        .and_return({ project.full_path => Set.new([1]) })
 
       expect(filter.parent_per_reference)
         .to eq({ project.full_path => project })
@@ -34,9 +31,6 @@ describe Banzai::Filter::AbstractReferenceFilter do
   end
 
   describe '#find_for_paths' do
-    let(:doc) { Nokogiri::HTML.fragment('') }
-    let(:filter) { described_class.new(doc, project: project) }
-
     context 'with RequestStore disabled' do
       it 'returns a list of Projects for a list of paths' do
         expect(filter.find_for_paths([project.full_path]))
diff --git a/spec/lib/gitlab/application_context_spec.rb b/spec/lib/gitlab/application_context_spec.rb
index 1c8fcb8d737..482bf0dc192 100644
--- a/spec/lib/gitlab/application_context_spec.rb
+++ b/spec/lib/gitlab/application_context_spec.rb
@@ -28,7 +28,7 @@ describe Gitlab::ApplicationContext do
   describe '.push' do
     it 'passes the expected context on to labkit' do
       fake_proc = duck_type(:call)
-      expected_context = hash_including(user: fake_proc, project: fake_proc, root_namespace: fake_proc)
+      expected_context = { user: fake_proc }
 
       expect(Labkit::Context).to receive(:push).with(expected_context)
 
@@ -78,5 +78,27 @@ describe Gitlab::ApplicationContext do
       expect(result(context))
         .to include(project: project.full_path, root_namespace: project.full_path_components.first)
     end
+
+    context 'only include values for which an option was specified' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:provided_options, :expected_context_keys) do
+        [:user, :namespace, :project] | [:user, :project, :root_namespace]
+        [:user, :project]             | [:user, :project, :root_namespace]
+        [:user, :namespace]           | [:user, :root_namespace]
+        [:user]                       | [:user]
+        []                            | []
+      end
+
+      with_them do
+        it do
+          # Build a hash that has all `provided_options` as keys, and `nil` as value
+          provided_values = provided_options.map { |key| [key, nil] }.to_h
+          context = described_class.new(provided_values)
+
+          expect(context.to_lazy_hash.keys).to contain_exactly(*expected_context_keys)
+        end
+      end
+    end
   end
 end
diff --git a/spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb b/spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb
new file mode 100644
index 00000000000..664e3810fc9
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb
@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, :migration, schema: 20190924152703 do
+  let(:services) { table(:services) }
+
+  # we need to define the classes due to encryption
+  class IssueTrackerData < ApplicationRecord
+    self.table_name = 'issue_tracker_data'
+
+    def self.encryption_options
+      {
+        key: Settings.attr_encrypted_db_key_base_32,
+        encode: true,
+        mode: :per_attribute_iv,
+        algorithm: 'aes-256-gcm'
+      }
+    end
+
+    attr_encrypted :project_url, encryption_options
+    attr_encrypted :issues_url, encryption_options
+    attr_encrypted :new_issue_url, encryption_options
+  end
+
+  class JiraTrackerData < ApplicationRecord
+    self.table_name = 'jira_tracker_data'
+
+    def self.encryption_options
+      {
+        key: Settings.attr_encrypted_db_key_base_32,
+        encode: true,
+        mode: :per_attribute_iv,
+        algorithm: 'aes-256-gcm'
+      }
+    end
+
+    attr_encrypted :url, encryption_options
+    attr_encrypted :api_url, encryption_options
+    attr_encrypted :username, encryption_options
+    attr_encrypted :password, encryption_options
+  end
+
+  let(:url) { 'http://base-url.tracker.com' }
+  let(:new_issue_url) { 'http://base-url.tracker.com/new_issue' }
+  let(:issues_url) { 'http://base-url.tracker.com/issues' }
+  let(:api_url) { 'http://api.tracker.com' }
+  let(:password) { 'passw1234' }
+  let(:username) { 'user9' }
+  let(:title) { 'Issue tracker' }
+  let(:description) { 'Issue tracker description' }
+
+  let(:jira_properties) do
+    {
+      'api_url' => api_url,
+      'jira_issue_transition_id' => '5',
+      'password' => password,
+      'url' => url,
+      'username' => username,
+      'title' => title,
+      'description' => description,
+      'other_field' => 'something'
+    }
+  end
+
+  let(:tracker_properties) do
+    {
+      'project_url' => url,
+      'new_issue_url' => new_issue_url,
+      'issues_url' => issues_url,
+      'title' => title,
+      'description' => description,
+      'other_field' => 'something'
+    }
+  end
+
+  let(:tracker_properties_no_url) do
+    {
+      'new_issue_url' => new_issue_url,
+      'issues_url' => issues_url,
+      'title' => title,
+      'description' => description
+    }
+  end
+
+  subject { described_class.new.perform(1, 100) }
+
+  shared_examples 'handle properties' do
+    it 'does not clear the properties' do
+      expect { subject }.not_to change { service.reload.properties}
+    end
+  end
+
+  context 'with jira service' do
+    let!(:service) do
+      services.create(id: 10, type: 'JiraService', title: nil, properties: jira_properties.to_json, category: 'issue_tracker')
+    end
+
+    it_behaves_like 'handle properties'
+
+    it 'migrates data' do
+      expect { subject }.to change { JiraTrackerData.count }.by(1)
+
+      service.reload
+      data = JiraTrackerData.find_by(service_id: service.id)
+
+      expect(data.url).to eq(url)
+      expect(data.api_url).to eq(api_url)
+      expect(data.username).to eq(username)
+      expect(data.password).to eq(password)
+      expect(service.title).to eq(title)
+      expect(service.description).to eq(description)
+    end
+  end
+
+  context 'with bugzilla service' do
+    let!(:service) do
+      services.create(id: 11, type: 'BugzillaService', title: nil, properties: tracker_properties.to_json, category: 'issue_tracker')
+    end
+
+    it_behaves_like 'handle properties'
+
+    it 'migrates data' do
+      expect { subject }.to change { IssueTrackerData.count }.by(1)
+
+      service.reload
+      data = IssueTrackerData.find_by(service_id: service.id)
+
+      expect(data.project_url).to eq(url)
+      expect(data.issues_url).to eq(issues_url)
+      expect(data.new_issue_url).to eq(new_issue_url)
+      expect(service.title).to eq(title)
+      expect(service.description).to eq(description)
+    end
+  end
+
+  context 'with youtrack service' do
+    let!(:service) do
+      services.create(id: 12, type: 'YoutrackService', title: nil, properties: tracker_properties_no_url.to_json, category: 'issue_tracker')
+    end
+
+    it_behaves_like 'handle properties'
+
+    it 'migrates data' do
+      expect { subject }.to change { IssueTrackerData.count }.by(1)
+
+      service.reload
+      data = IssueTrackerData.find_by(service_id: service.id)
+
+      expect(data.project_url).to be_nil
+      expect(data.issues_url).to eq(issues_url)
+      expect(data.new_issue_url).to eq(new_issue_url)
+      expect(service.title).to eq(title)
+      expect(service.description).to eq(description)
+    end
+  end
+
+  context 'with gitlab service with no properties' do
+    let!(:service) do
+      services.create(id: 13, type: 'GitlabIssueTrackerService', title: nil, properties: {}, category: 'issue_tracker')
+    end
+
+    it_behaves_like 'handle properties'
+
+    it 'does not migrate data' do
+      expect { subject }.not_to change { IssueTrackerData.count }
+    end
+  end
+
+  context 'with redmine service already with data fields' do
+    let!(:service) do
+      services.create(id: 14, type: 'RedmineService', title: nil, properties: tracker_properties_no_url.to_json, category: 'issue_tracker').tap do |service|
+        IssueTrackerData.create!(service_id: service.id, project_url: url, new_issue_url: new_issue_url, issues_url: issues_url)
+      end
+    end
+
+    it_behaves_like 'handle properties'
+
+    it 'does not create new data fields record' do
+      expect { subject }.not_to change { IssueTrackerData.count }
+    end
+  end
+
+  context 'with custom issue tracker which has data fields record inconsistent with properties field' do
+    let!(:service) do
+      services.create(id: 15, type: 'CustomIssueTrackerService', title: 'Existing title', properties: jira_properties.to_json, category: 'issue_tracker').tap do |service|
+        IssueTrackerData.create!(service_id: service.id, project_url: 'http://other_url', new_issue_url: 'http://other_url/new_issue', issues_url: 'http://other_url/issues')
+      end
+    end
+
+    it_behaves_like 'handle properties'
+
+    it 'does not update the data fields record' do
+      expect { subject }.not_to change { IssueTrackerData.count }
+
+      service.reload
+      data = IssueTrackerData.find_by(service_id: service.id)
+
+      expect(data.project_url).to eq('http://other_url')
+      expect(data.issues_url).to eq('http://other_url/issues')
+      expect(data.new_issue_url).to eq('http://other_url/new_issue')
+      expect(service.title).to eq('Existing title')
+    end
+  end
+
+  context 'with jira service which has data fields record inconsistent with properties field' do
+    let!(:service) do
+      services.create(id: 16, type: 'CustomIssueTrackerService', description: 'Existing description', properties: jira_properties.to_json, category: 'issue_tracker').tap do |service|
+        JiraTrackerData.create!(service_id: service.id, url: 'http://other_jira_url')
+      end
+    end
+
+    it_behaves_like 'handle properties'
+
+    it 'does not update the data fields record' do
+      expect { subject }.not_to change { JiraTrackerData.count }
+
+      service.reload
+      data = JiraTrackerData.find_by(service_id: service.id)
+
+      expect(data.url).to eq('http://other_jira_url')
+      expect(data.password).to be_nil
+      expect(data.username).to be_nil
+      expect(data.api_url).to be_nil
+      expect(service.description).to eq('Existing description')
+    end
+  end
+
+  context 'non issue tracker service' do
+    let!(:service) do
+      services.create(id: 17, title: nil, description: nil, type: 'OtherService', properties: tracker_properties.to_json)
+    end
+
+    it_behaves_like 'handle properties'
+
+    it 'does not migrate any data' do
+      expect { subject }.not_to change { IssueTrackerData.count }
+
+      service.reload
+      expect(service.title).to be_nil
+      expect(service.description).to be_nil
+    end
+  end
+
+  context 'jira service with empty properties' do
+    let!(:service) do
+      services.create(id: 18, type: 'JiraService', properties: '', category: 'issue_tracker')
+    end
+
+    it_behaves_like 'handle properties'
+
+    it 'does not migrate any data' do
+      expect { subject }.not_to change { JiraTrackerData.count }
+    end
+  end
+
+  context 'jira service with nil properties' do
+    let!(:service) do
+      services.create(id: 18, type: 'JiraService', properties: nil, category: 'issue_tracker')
+    end
+
+    it_behaves_like 'handle properties'
+
+    it 'does not migrate any data' do
+      expect { subject }.not_to change { JiraTrackerData.count }
+    end
+  end
+
+  context 'jira service with invalid properties' do
+    let!(:service) do
+      services.create(id: 18, type: 'JiraService', properties: 'invalid data', category: 'issue_tracker')
+    end
+
+    it_behaves_like 'handle properties'
+
+    it 'does not migrate any data' do
+      expect { subject }.not_to change { JiraTrackerData.count }
+    end
+  end
+
+  context 'with jira service with invalid properties, valid jira service and valid bugzilla service' do
+    let!(:jira_service_invalid) do
+      services.create(id: 19, title: 'invalid - title', description: 'invalid - description', type: 'JiraService', properties: 'invalid data', category: 'issue_tracker')
+    end
+    let!(:jira_service_valid) do
+      services.create(id: 20, type: 'JiraService', properties: jira_properties.to_json, category: 'issue_tracker')
+    end
+    let!(:bugzilla_service_valid) do
+      services.create(id: 11, type: 'BugzillaService', title: nil, properties: tracker_properties.to_json, category: 'issue_tracker')
+    end
+
+    it 'migrates data for the valid service' do
+      subject
+
+      jira_service_invalid.reload
+      expect(JiraTrackerData.find_by(service_id: jira_service_invalid.id)).to be_nil
+      expect(jira_service_invalid.title).to eq('invalid - title')
+      expect(jira_service_invalid.description).to eq('invalid - description')
+      expect(jira_service_invalid.properties).to eq('invalid data')
+
+      jira_service_valid.reload
+      data = JiraTrackerData.find_by(service_id: jira_service_valid.id)
+
+      expect(data.url).to eq(url)
+      expect(data.api_url).to eq(api_url)
+      expect(data.username).to eq(username)
+      expect(data.password).to eq(password)
+      expect(jira_service_valid.title).to eq(title)
+      expect(jira_service_valid.description).to eq(description)
+
+      bugzilla_service_valid.reload
+      data = IssueTrackerData.find_by(service_id: bugzilla_service_valid.id)
+
+      expect(data.project_url).to eq(url)
+      expect(data.issues_url).to eq(issues_url)
+      expect(data.new_issue_url).to eq(new_issue_url)
+      expect(bugzilla_service_valid.title).to eq(title)
+      expect(bugzilla_service_valid.description).to eq(description)
+    end
+  end
+end
diff --git a/spec/lib/gitlab/import_export/import_failure_service_spec.rb b/spec/lib/gitlab/import_export/import_failure_service_spec.rb
new file mode 100644
index 00000000000..0351f88afdb
--- /dev/null
+++ b/spec/lib/gitlab/import_export/import_failure_service_spec.rb
@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::ImportFailureService do
+  let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') }
+  let(:label) { create(:label) }
+  let(:subject) { described_class.new(importable) }
+  let(:relation_key) { "labels" }
+  let(:relation_index) { 0 }
+
+  describe '#log_import_failure' do
+    let(:standard_error_message) { "StandardError message" }
+    let(:exception) { StandardError.new(standard_error_message) }
+    let(:correlation_id) { 'my-correlation-id' }
+    let(:retry_count) { 2 }
+    let(:log_import_failure) do
+      subject.log_import_failure(relation_key, relation_index, exception, retry_count)
+    end
+
+    before do
+      # Import is running from the rake task, `correlation_id` is not assigned
+      allow(Labkit::Correlation::CorrelationId).to receive(:current_or_new_id).and_return(correlation_id)
+    end
+
+    context 'when importable is a group' do
+      let(:importable) { create(:group) }
+
+      it_behaves_like 'log import failure', :group_id
+    end
+
+    context 'when importable is a project' do
+      it_behaves_like 'log import failure', :project_id
+    end
+
+    context 'when ImportFailure does not support importable class' do
+      let(:importable) { create(:merge_request) }
+
+      it 'raise exception' do
+        expect { subject }.to raise_exception(ActiveRecord::AssociationNotFoundError, "Association named 'import_failures' was not found on MergeRequest; perhaps you misspelled it?")
+      end
+    end
+  end
+
+  describe '#with_retry' do
+    let(:perform_retry) do
+      subject.with_retry(relation_key, relation_index) do
+        label.save!
+      end
+    end
+
+    context 'when exceptions are retriable' do
+      where(:exception) { Gitlab::ImportExport::ImportFailureService::RETRIABLE_EXCEPTIONS }
+
+      with_them do
+        context 'when retry succeeds' do
+          before do
+            expect(label).to receive(:save!).and_raise(exception.new)
+            expect(label).to receive(:save!).and_return(true)
+          end
+
+          it 'retries and logs import failure once with correct params' do
+            expect(subject).to receive(:log_import_failure).with(relation_key, relation_index, instance_of(exception), 1).once
+
+            perform_retry
+          end
+        end
+
+        context 'when retry continues to fail with intermittent errors' do
+          let(:maximum_retry_count) do
+            Retriable.config.tries
+          end
+
+          before do
+            expect(label).to receive(:save!)
+              .exactly(maximum_retry_count).times
+              .and_raise(exception.new)
+          end
+
+          it 'retries the number of times allowed and raise exception', :aggregate_failures do
+            expect { perform_retry }.to raise_exception(exception)
+          end
+
+          it 'logs import failure each time and raise exception', :aggregate_failures do
+            maximum_retry_count.times do |index|
+              retry_count = index + 1
+
+              expect(subject).to receive(:log_import_failure).with(relation_key, relation_index, instance_of(exception), retry_count)
+            end
+
+            expect { perform_retry }.to raise_exception(exception)
+          end
+        end
+      end
+    end
+
+    context 'when exception is not retriable' do
+      let(:exception) { StandardError.new }
+
+      it 'raise the exception', :aggregate_failures do
+        expect(label).to receive(:save!).once.and_raise(exception)
+        expect(subject).not_to receive(:log_import_failure)
+        expect { perform_retry }.to raise_exception(exception)
+      end
+    end
+  end
+end