From 150effab274651b3a8d2041e64ced734d1f3a349 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 24 Sep 2019 21:06:18 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- app/assets/javascripts/lib/utils/axios_utils.js | 15 +++++ .../suppress_ajax_errors_during_navigation.js | 16 +++++ ...17_update_cs_vulnerability_confidence_column.rb | 29 +++++++++ .../geo/replication/high_availability.md | 16 +++-- jest.config.js | 3 +- .../update_vulnerability_confidence.rb | 35 +++++++++++ lib/gitlab/gl_repository/repo_type.rb | 2 + lib/gitlab/gon_helper.rb | 4 ++ spec/frontend/helpers/test_constants.js | 7 ++- .../suppress_ajax_errors_during_navigation_spec.js | 37 +++++++++++ .../frequent_items/components/app_spec.js | 7 +++ spec/javascripts/test_constants.js | 8 +-- .../update_vulnerability_confidence_spec.rb | 58 +++++++++++++++++ spec/lib/gitlab/gl_repository/repo_type_spec.rb | 30 --------- ...date_cs_vulnerability_confidence_column_spec.rb | 73 ++++++++++++++++++++++ .../shared_examples/repo_type_shared_examples.rb | 31 +++++++++ 16 files changed, 328 insertions(+), 43 deletions(-) create mode 100644 app/assets/javascripts/lib/utils/suppress_ajax_errors_during_navigation.js create mode 100644 db/post_migrate/20190909141517_update_cs_vulnerability_confidence_column.rb create mode 100644 lib/gitlab/background_migration/update_vulnerability_confidence.rb create mode 100644 spec/frontend/lib/utils/suppress_ajax_errors_during_navigation_spec.js create mode 100644 spec/lib/gitlab/background_migration/update_vulnerability_confidence_spec.rb create mode 100644 spec/migrations/update_cs_vulnerability_confidence_column_spec.rb create mode 100644 spec/support/shared_examples/repo_type_shared_examples.rb diff --git a/app/assets/javascripts/lib/utils/axios_utils.js b/app/assets/javascripts/lib/utils/axios_utils.js index c17f62c671c..a04fe609015 100644 --- a/app/assets/javascripts/lib/utils/axios_utils.js +++ b/app/assets/javascripts/lib/utils/axios_utils.js @@ -1,5 +1,6 @@ import axios from 'axios'; import csrf from './csrf'; +import suppressAjaxErrorsDuringNavigation from './suppress_ajax_errors_during_navigation'; axios.defaults.headers.common[csrf.headerKey] = csrf.token; // Used by Rails to check if it is a valid XHR request @@ -25,6 +26,20 @@ axios.interceptors.response.use( }, ); +let isUserNavigating = false; +window.addEventListener('beforeunload', () => { + isUserNavigating = true; +}); + +// Ignore AJAX errors caused by requests +// being cancelled due to browser navigation +const { gon } = window; +const featureFlagEnabled = gon && gon.features && gon.features.suppressAjaxNavigationErrors; +axios.interceptors.response.use( + response => response, + err => suppressAjaxErrorsDuringNavigation(err, isUserNavigating, featureFlagEnabled), +); + export default axios; /** diff --git a/app/assets/javascripts/lib/utils/suppress_ajax_errors_during_navigation.js b/app/assets/javascripts/lib/utils/suppress_ajax_errors_during_navigation.js new file mode 100644 index 00000000000..4c61da9b862 --- /dev/null +++ b/app/assets/javascripts/lib/utils/suppress_ajax_errors_during_navigation.js @@ -0,0 +1,16 @@ +/** + * An Axios error interceptor that suppresses AJAX errors caused + * by the request being cancelled when the user navigates to a new page + */ +export default (err, isUserNavigating, featureFlagEnabled) => { + if (featureFlagEnabled && isUserNavigating && err.code === 'ECONNABORTED') { + // If the user is navigating away from the current page, + // prevent .then() and .catch() handlers from being + // called by returning a Promise that never resolves + return new Promise(() => {}); + } + + // The error is not related to browser navigation, + // so propagate the error + return Promise.reject(err); +}; diff --git a/db/post_migrate/20190909141517_update_cs_vulnerability_confidence_column.rb b/db/post_migrate/20190909141517_update_cs_vulnerability_confidence_column.rb new file mode 100644 index 00000000000..20f5e7ae0dc --- /dev/null +++ b/db/post_migrate/20190909141517_update_cs_vulnerability_confidence_column.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. +class UpdateCsVulnerabilityConfidenceColumn < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + DOWNTIME = false + BATCH_SIZE = 1_000 + INTERVAL = 5.minutes + + # 137_424 records to be updated on GitLab.com, + # giving us an estimated runtime of 12 hours. + def up + migration = Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence + migration_name = migration.to_s.demodulize + relation = migration::Occurrence.container_scanning_reports_with_medium_confidence + queue_background_migration_jobs_by_range_at_intervals(relation, + migration_name, + INTERVAL, + batch_size: BATCH_SIZE) + end + + def down + # no-op + end +end diff --git a/doc/administration/geo/replication/high_availability.md b/doc/administration/geo/replication/high_availability.md index 2b6926c1351..8d09712d101 100644 --- a/doc/administration/geo/replication/high_availability.md +++ b/doc/administration/geo/replication/high_availability.md @@ -71,8 +71,16 @@ high availability configuration documentation for [PostgreSQL](../../high_availability/database.md#configuring-the-application-nodes) and [Redis](../../high_availability/redis.md#example-configuration-for-the-gitlab-application). -The **primary** database will require modification later, as part of -[step 2](#step-2-configure-the-main-read-only-replica-postgresql-database-on-the-secondary-node). +### Step 2: Configure the **primary** database + +1. Edit `/etc/gitlab/gitlab.rb` and add the following: + + ```ruby + ## + ## Configure the Geo primary role and the PostgreSQL role + ## + roles ['geo_primary_role', 'postgres_role'] + ``` ## Configure a **secondary** node @@ -115,9 +123,9 @@ the **primary** database. Use the following as a guide. ```ruby ## - ## Configure the PostgreSQL role + ## Configure the Geo secondary role and the PostgreSQL role ## - roles ['postgres_role'] + roles ['geo_secondary_role', 'postgres_role'] ## ## Secondary address diff --git a/jest.config.js b/jest.config.js index 34581c6532a..43d68c60137 100644 --- a/jest.config.js +++ b/jest.config.js @@ -19,7 +19,8 @@ let testMatch = ['/spec/frontend/**/*_spec.js', '/ee/spec/fron // workaround for eslint-import-resolver-jest only resolving in test files // see https://github.com/JoinColony/eslint-import-resolver-jest#note -const isESLint = module.parent.filename.includes('/eslint-import-resolver-jest/'); +const { filename: parentModuleName } = module.parent; +const isESLint = parentModuleName && parentModuleName.includes('/eslint-import-resolver-jest/'); if (isESLint) { testMatch = testMatch.map(path => path.replace('_spec.js', '')); } diff --git a/lib/gitlab/background_migration/update_vulnerability_confidence.rb b/lib/gitlab/background_migration/update_vulnerability_confidence.rb new file mode 100644 index 00000000000..f02f8151778 --- /dev/null +++ b/lib/gitlab/background_migration/update_vulnerability_confidence.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation +module Gitlab + module BackgroundMigration + class UpdateVulnerabilityConfidence + class Occurrence < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'vulnerability_occurrences' + + REPORT_TYPES = { + container_scanning: 2 + }.freeze + + CONFIDENCE_LEVELS = { + unknown: 2, + medium: 5 + }.freeze + + enum confidences: CONFIDENCE_LEVELS + enum report_type: REPORT_TYPES + + def self.container_scanning_reports_with_medium_confidence + where(report_type: self.report_types[:container_scanning], confidence: self.confidences[:medium]) + end + end + + def perform(start_id, stop_id) + Occurrence.container_scanning_reports_with_medium_confidence + .where(id: start_id..stop_id) + .update_all(confidence: Occurrence.confidences[:unknown]) + end + end + end +end diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 19915980d7f..01bc27f963b 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -40,3 +40,5 @@ module Gitlab end end end + +Gitlab::GlRepository::RepoType.prepend_if_ee('EE::Gitlab::GlRepository::RepoType') diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 92917028851..2616a19fdaa 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -38,6 +38,10 @@ module Gitlab gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url end + + # Initialize gon.features with any flags that should be + # made globally available to the frontend + push_frontend_feature_flag(:suppress_ajax_navigation_errors, default_enabled: true) end # Exposes the state of a feature flag to the frontend code. diff --git a/spec/frontend/helpers/test_constants.js b/spec/frontend/helpers/test_constants.js index 8dc4aef87e1..c97d47a6406 100644 --- a/spec/frontend/helpers/test_constants.js +++ b/spec/frontend/helpers/test_constants.js @@ -1,2 +1,7 @@ -// eslint-disable-next-line import/prefer-default-export +export const FIXTURES_PATH = `/fixtures`; export const TEST_HOST = 'http://test.host'; + +export const DUMMY_IMAGE_URL = `${FIXTURES_PATH}/static/images/one_white_pixel.png`; + +export const GREEN_BOX_IMAGE_URL = `${FIXTURES_PATH}/static/images/green_box.png`; +export const RED_BOX_IMAGE_URL = `${FIXTURES_PATH}/static/images/red_box.png`; diff --git a/spec/frontend/lib/utils/suppress_ajax_errors_during_navigation_spec.js b/spec/frontend/lib/utils/suppress_ajax_errors_during_navigation_spec.js new file mode 100644 index 00000000000..89e8459d594 --- /dev/null +++ b/spec/frontend/lib/utils/suppress_ajax_errors_during_navigation_spec.js @@ -0,0 +1,37 @@ +import suppressAjaxErrorsDuringNavigation from '~/lib/utils/suppress_ajax_errors_during_navigation'; +import waitForPromises from 'helpers/wait_for_promises'; + +describe('suppressAjaxErrorsDuringNavigation', () => { + const OTHER_ERR_CODE = 'foo'; + const NAV_ERR_CODE = 'ECONNABORTED'; + + it.each` + isFeatureFlagEnabled | isUserNavigating | code + ${false} | ${false} | ${OTHER_ERR_CODE} + ${false} | ${false} | ${NAV_ERR_CODE} + ${false} | ${true} | ${OTHER_ERR_CODE} + ${false} | ${true} | ${NAV_ERR_CODE} + ${true} | ${false} | ${OTHER_ERR_CODE} + ${true} | ${false} | ${NAV_ERR_CODE} + ${true} | ${true} | ${OTHER_ERR_CODE} + `('should return a rejected Promise', ({ isFeatureFlagEnabled, isUserNavigating, code }) => { + const err = { code }; + const actual = suppressAjaxErrorsDuringNavigation(err, isUserNavigating, isFeatureFlagEnabled); + + return expect(actual).rejects.toBe(err); + }); + + it('should return a Promise that never resolves', () => { + const err = { code: NAV_ERR_CODE }; + const actual = suppressAjaxErrorsDuringNavigation(err, true, true); + + const thenCallback = jest.fn(); + const catchCallback = jest.fn(); + actual.then(thenCallback).catch(catchCallback); + + return waitForPromises().then(() => { + expect(thenCallback).not.toHaveBeenCalled(); + expect(catchCallback).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/spec/javascripts/frequent_items/components/app_spec.js b/spec/javascripts/frequent_items/components/app_spec.js index 6814f656f5d..36dd8604d08 100644 --- a/spec/javascripts/frequent_items/components/app_spec.js +++ b/spec/javascripts/frequent_items/components/app_spec.js @@ -236,8 +236,15 @@ describe('Frequent Items App Component', () => { .then(() => { expect(vm.$el.querySelector('.loading-animation')).toBeDefined(); }) + + // This test waits for multiple ticks in order to allow the responses to + // propagate through each interceptor installed on the Axios instance. + // This shouldn't be necessary; this test should be refactored to avoid this. + // https://gitlab.com/gitlab-org/gitlab/issues/32479 + .then(vm.$nextTick) .then(vm.$nextTick) .then(vm.$nextTick) + .then(() => { expect(vm.$el.querySelectorAll('.frequent-items-list-container li').length).toBe( mockSearchedProjects.length, diff --git a/spec/javascripts/test_constants.js b/spec/javascripts/test_constants.js index c97d47a6406..51c0716b99d 100644 --- a/spec/javascripts/test_constants.js +++ b/spec/javascripts/test_constants.js @@ -1,7 +1 @@ -export const FIXTURES_PATH = `/fixtures`; -export const TEST_HOST = 'http://test.host'; - -export const DUMMY_IMAGE_URL = `${FIXTURES_PATH}/static/images/one_white_pixel.png`; - -export const GREEN_BOX_IMAGE_URL = `${FIXTURES_PATH}/static/images/green_box.png`; -export const RED_BOX_IMAGE_URL = `${FIXTURES_PATH}/static/images/red_box.png`; +export * from '../frontend/helpers/test_constants'; diff --git a/spec/lib/gitlab/background_migration/update_vulnerability_confidence_spec.rb b/spec/lib/gitlab/background_migration/update_vulnerability_confidence_spec.rb new file mode 100644 index 00000000000..1217edfecc3 --- /dev/null +++ b/spec/lib/gitlab/background_migration/update_vulnerability_confidence_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence, :migration, schema: 20190909141517 do + let(:vulnerabilities) { table(:vulnerability_occurrences) } + let(:identifiers) { table(:vulnerability_identifiers) } + let(:scanners) { table(:vulnerability_scanners) } + let(:projects) { table(:projects) } + let(:vul1) { attributes_for(:vulnerabilities_occurrence) } + let(:vul2) { attributes_for(:vulnerabilities_occurrence) } + let(:vul3) { attributes_for(:vulnerabilities_occurrence) } + + it 'updates confidence level for container scanning reports' do + projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab') + + (1..3).to_a.each do |identifier_id| + identifiers.create!(id: identifier_id, + project_id: 123, + fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c' + identifier_id.to_s, + external_type: 'SECURITY_ID', + external_id: 'SECURITY_0', + name: 'SECURITY_IDENTIFIER 0') + end + + scanners.create!(id: 6, project_id: 123, external_id: 'clair', name: 'Security Scanner') + + vulnerabilities.create!(container_scanning_vuln_params(vul1, 1)) + vulnerabilities.create!(container_scanning_vuln_params(vul2, 2)) + vulnerabilities.create!(container_scanning_vuln_params(vul3, 3).merge(report_type: 1)) + + expect(vulnerabilities.where(report_type: 2, confidence: 2).count). to eq(0) + expect(vulnerabilities.exists?(report_type: 2, confidence: 5)).to be_truthy + + described_class.new.perform(1, 3) + + expect(vulnerabilities.exists?(report_type: 2, confidence: 5)).to be_falsy + expect(vulnerabilities.where(report_type: 2, confidence: 2).count). to eq(2) + end + + def container_scanning_vuln_params(vul, primary_identifier_id) + { + id: vul[:id], + severity: 2, + confidence: 5, + report_type: 2, + project_id: 123, + scanner_id: 6, + primary_identifier_id: primary_identifier_id, + project_fingerprint: vul[:project_fingerprint], + location_fingerprint: vul[:location_fingerprint], + uuid: vul[:uuid], + name: vul[:name], + metadata_version: '1.3', + raw_metadata: vul3[:raw_metadata] + } + end +end diff --git a/spec/lib/gitlab/gl_repository/repo_type_spec.rb b/spec/lib/gitlab/gl_repository/repo_type_spec.rb index f06a2448ff7..9e09e1411ab 100644 --- a/spec/lib/gitlab/gl_repository/repo_type_spec.rb +++ b/spec/lib/gitlab/gl_repository/repo_type_spec.rb @@ -4,36 +4,6 @@ require 'spec_helper' describe Gitlab::GlRepository::RepoType do set(:project) { create(:project) } - shared_examples 'a repo type' do - describe "#identifier_for_subject" do - subject { described_class.identifier_for_subject(project) } - - it { is_expected.to eq(expected_identifier) } - end - - describe "#fetch_id" do - it "finds an id match in the identifier" do - expect(described_class.fetch_id(expected_identifier)).to eq(expected_id) - end - - it 'does not break on other identifiers' do - expect(described_class.fetch_id("wiki-noid")).to eq(nil) - end - end - - describe "#path_suffix" do - subject { described_class.path_suffix } - - it { is_expected.to eq(expected_suffix) } - end - - describe "#repository_for" do - it "finds the repository for the repo type" do - expect(described_class.repository_for(project)).to eq(expected_repository) - end - end - end - describe Gitlab::GlRepository::PROJECT do it_behaves_like 'a repo type' do let(:expected_identifier) { "project-#{project.id}" } diff --git a/spec/migrations/update_cs_vulnerability_confidence_column_spec.rb b/spec/migrations/update_cs_vulnerability_confidence_column_spec.rb new file mode 100644 index 00000000000..b8575dd9467 --- /dev/null +++ b/spec/migrations/update_cs_vulnerability_confidence_column_spec.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20190909141517_update_cs_vulnerability_confidence_column.rb') + +describe UpdateCsVulnerabilityConfidenceColumn, :migration, :sidekiq do + let(:vulnerabilities) { table(:vulnerability_occurrences) } + let(:identifiers) { table(:vulnerability_identifiers) } + let(:scanners) { table(:vulnerability_scanners) } + let(:projects) { table(:projects) } + let(:vul1) { attributes_for(:vulnerabilities_occurrence, id: 1, report_type: 2, confidence: 5) } + let(:vul2) { attributes_for(:vulnerabilities_occurrence, id: 2, report_type: 2, confidence: 5) } + + before do + stub_const("#{described_class}::BATCH_SIZE", 2) + end + + it 'updates confidence levels for container scanning reports' do + projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab') + + identifiers.create!(id: 1, + project_id: 123, + fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c2', + external_type: 'SECURITY_ID', + external_id: 'SECURITY_0', + name: 'SECURITY_IDENTIFIER 0') + + identifiers.create!(id: 2, + project_id: 123, + fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c3', + external_type: 'SECURITY_ID', + external_id: 'SECURITY_0', + name: 'SECURITY_IDENTIFIER 0') + + scanners.create!(id: 6, project_id: 123, external_id: 'clair', name: 'Security Scanner') + + vulnerabilities.create!(id: vul1[:id], + severity: 2, + confidence: 5, + report_type: 2, + project_id: 123, + scanner_id: 6, + primary_identifier_id: 1, + project_fingerprint: vul1[:project_fingerprint], + location_fingerprint: vul1[:location_fingerprint], + uuid: vul1[:uuid], + name: vul1[:name], + metadata_version: '1.3', + raw_metadata: vul1[:raw_metadata]) + + vulnerabilities.create!(id: vul2[:id], + severity: 2, + confidence: 5, + report_type: 2, + project_id: 123, + scanner_id: 6, + primary_identifier_id: 2, + project_fingerprint: vul2[:project_fingerprint], + location_fingerprint: vul2[:location_fingerprint], + uuid: vul2[:uuid], + name: vul2[:name], + metadata_version: '1.3', + raw_metadata: vul2[:raw_metadata]) + + expect(vulnerabilities.where(report_type: 2, confidence: 2).count). to eq(0) + expect(vulnerabilities.exists?(report_type: 2, confidence: 5)).to be_truthy + + migrate! + + expect(vulnerabilities.exists?(report_type: 2, confidence: 5)).to be_falsy + expect(vulnerabilities.where(report_type: 2, confidence: 2).count). to eq(2) + end +end diff --git a/spec/support/shared_examples/repo_type_shared_examples.rb b/spec/support/shared_examples/repo_type_shared_examples.rb new file mode 100644 index 00000000000..dc9e3a73346 --- /dev/null +++ b/spec/support/shared_examples/repo_type_shared_examples.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +shared_examples 'a repo type' do + describe "#identifier_for_subject" do + subject { described_class.identifier_for_subject(project) } + + it { is_expected.to eq(expected_identifier) } + end + + describe "#fetch_id" do + it "finds an id match in the identifier" do + expect(described_class.fetch_id(expected_identifier)).to eq(expected_id) + end + + it 'does not break on other identifiers' do + expect(described_class.fetch_id("wiki-noid")).to eq(nil) + end + end + + describe "#path_suffix" do + subject { described_class.path_suffix } + + it { is_expected.to eq(expected_suffix) } + end + + describe "#repository_for" do + it "finds the repository for the repo type" do + expect(described_class.repository_for(project)).to eq(expected_repository) + end + end +end -- cgit v1.2.1