diff options
32 files changed, 490 insertions, 135 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 30c21b452e0..ba8a5c290ea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -220,18 +220,6 @@ stages: paths: - log/development.log -# Review docs base -.review-docs: &review-docs - <<: *dedicated-runner - <<: *except-qa - <<: *single-script-job - variables: - <<: *single-script-job-variables - SCRIPT_NAME: trigger-build-docs - when: manual - only: - - branches - # DB migration, rollback, and seed jobs .db-migrate-reset: &db-migrate-reset <<: *dedicated-no-docs-and-no-qa-pull-cache-job @@ -273,20 +261,44 @@ package-and-qa: - //@gitlab-org/gitlab-ce - //@gitlab-org/gitlab-ee -# Trigger a docs build in gitlab-docs -# Useful to preview the docs changes live -review-docs-deploy: - <<: *review-docs - stage: build +# Review docs base +.review-docs: &review-docs + <<: *dedicated-runner + <<: *single-script-job + variables: + <<: *single-script-job-variables + SCRIPT_NAME: trigger-build-docs environment: name: review-docs/$CI_COMMIT_REF_NAME # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are secret variables # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693 - url: http://$DOCS_GITLAB_REPO_SUFFIX-$CI_COMMIT_REF_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX + url: http://$DOCS_GITLAB_REPO_SUFFIX-$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX on_stop: review-docs-cleanup + +# Trigger a manual docs build in gitlab-docs only on non docs-only branches. +# Useful to preview the docs changes live. +review-docs-deploy-manual: + <<: *review-docs + stage: build + script: + - gem install gitlab --no-ri --no-rdoc + - ./$SCRIPT_NAME deploy + when: manual + only: + - branches + <<: *except-docs-and-qa + +# Always trigger a docs build in gitlab-docs only on docs-only branches. +# Useful to preview the docs changes live. +review-docs-deploy: + <<: *review-docs + stage: post-test script: - gem install gitlab --no-ri --no-rdoc - ./$SCRIPT_NAME deploy + only: + - /(^docs[\/-].*|.*-docs$)/ + <<: *except-qa # Cleanup remote environment of gitlab-docs review-docs-cleanup: @@ -295,6 +307,7 @@ review-docs-cleanup: environment: name: review-docs/$CI_COMMIT_REF_NAME action: stop + when: manual script: - gem install gitlab --no-ri --no-rdoc - ./SCRIPT_NAME cleanup diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index f430f18ca9a..e5caa3ffa41 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -561,9 +561,9 @@ module Ci .append(key: 'CI_PIPELINE_IID', value: iid.to_s) .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) - .append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message) - .append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title) - .append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description) + .append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s) + .append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s) + .append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s) end def queued_duration diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 3df1130a6e2..f112c06e26f 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -378,6 +378,10 @@ class MergeRequest < ActiveRecord::Base end end + def non_latest_diffs + merge_request_diffs.where.not(id: merge_request_diff.id) + end + def diff_size # Calling `merge_request_diff.diffs.real_size` will also perform # highlighting, which we don't need here. @@ -619,18 +623,7 @@ class MergeRequest < ActiveRecord::Base def reload_diff(current_user = nil) return unless open? - old_diff_refs = self.diff_refs - new_diff = create_merge_request_diff - - MergeRequests::MergeRequestDiffCacheService.new.execute(self, new_diff) - - new_diff_refs = self.diff_refs - - update_diff_discussion_positions( - old_diff_refs: old_diff_refs, - new_diff_refs: new_diff_refs, - current_user: current_user - ) + MergeRequests::ReloadDiffsService.new(self, current_user).execute end def check_if_can_be_merged diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 06aa67c600f..3d72c447b4b 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -3,6 +3,7 @@ class MergeRequestDiff < ActiveRecord::Base include Importable include ManualInverseAssociation include IgnorableColumn + include EachBatch # Don't display more than 100 commits at once COMMITS_SAFE_SIZE = 100 @@ -17,8 +18,14 @@ class MergeRequestDiff < ActiveRecord::Base has_many :merge_request_diff_commits, -> { order(:merge_request_diff_id, :relative_order) } state_machine :state, initial: :empty do + event :clean do + transition any => :without_files + end + state :collected state :overflow + # Diff files have been deleted by the system + state :without_files # Deprecated states: these are no longer used but these values may still occur # in the database. state :timeout @@ -27,6 +34,7 @@ class MergeRequestDiff < ActiveRecord::Base state :overflow_diff_lines_limit end + scope :with_files, -> { without_states(:without_files, :empty) } scope :viewable, -> { without_state(:empty) } scope :by_commit_sha, ->(sha) do joins(:merge_request_diff_commits).where(merge_request_diff_commits: { sha: sha }).reorder(nil) @@ -42,6 +50,10 @@ class MergeRequestDiff < ActiveRecord::Base find_by(start_commit_sha: diff_refs.start_sha, head_commit_sha: diff_refs.head_sha, base_commit_sha: diff_refs.base_sha) end + def viewable? + collected? || without_files? || overflow? + end + # Collect information about commits and diff from repository # and save it to the database as serialized data def save_git_content @@ -170,6 +182,21 @@ class MergeRequestDiff < ActiveRecord::Base end def diffs(diff_options = nil) + if without_files? && comparison = diff_refs.compare_in(project) + # It should fetch the repository when diffs are cleaned by the system. + # We don't keep these for storage overload purposes. + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/37639 + comparison.diffs(diff_options) + else + diffs_collection(diff_options) + end + end + + # Should always return the DB persisted diffs collection + # (e.g. Gitlab::Diff::FileCollection::MergeRequestDiff. + # It's useful when trying to invalidate old caches through + # FileCollection::MergeRequestDiff#clear_cache! + def diffs_collection(diff_options = nil) Gitlab::Diff::FileCollection::MergeRequestDiff.new(self, diff_options: diff_options) end diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index d7d6aaceb27..faa831b1949 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -29,8 +29,8 @@ class ProjectAutoDevops < ActiveRecord::Base end if manual? - variables.append(key: 'STAGING_ENABLED', value: 1) - variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: 1) + variables.append(key: 'STAGING_ENABLED', value: '1') + variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1') end end end diff --git a/app/services/merge_requests/delete_non_latest_diffs_service.rb b/app/services/merge_requests/delete_non_latest_diffs_service.rb new file mode 100644 index 00000000000..40079b21189 --- /dev/null +++ b/app/services/merge_requests/delete_non_latest_diffs_service.rb @@ -0,0 +1,18 @@ +module MergeRequests + class DeleteNonLatestDiffsService + BATCH_SIZE = 10 + + def initialize(merge_request) + @merge_request = merge_request + end + + def execute + diffs = @merge_request.non_latest_diffs.with_files + + diffs.each_batch(of: BATCH_SIZE) do |relation, index| + ids = relation.pluck(:id).map { |id| [id] } + DeleteDiffFilesWorker.bulk_perform_in(index * 5.minutes, ids) + end + end + end +end diff --git a/app/services/merge_requests/merge_request_diff_cache_service.rb b/app/services/merge_requests/merge_request_diff_cache_service.rb deleted file mode 100644 index 10aa9ae609c..00000000000 --- a/app/services/merge_requests/merge_request_diff_cache_service.rb +++ /dev/null @@ -1,17 +0,0 @@ -module MergeRequests - class MergeRequestDiffCacheService - def execute(merge_request, new_diff) - # Executing the iteration we cache all the highlighted diff information - merge_request.diffs.diff_files.to_a - - # Remove cache for all diffs on this MR. Do not use the association on the - # model, as that will interfere with other actions happening when - # reloading the diff. - MergeRequestDiff.where(merge_request: merge_request).each do |merge_request_diff| - next if merge_request_diff == new_diff - - merge_request_diff.diffs.clear_cache! - end - end - end -end diff --git a/app/services/merge_requests/post_merge_service.rb b/app/services/merge_requests/post_merge_service.rb index c78e78afcd1..5b160ffba67 100644 --- a/app/services/merge_requests/post_merge_service.rb +++ b/app/services/merge_requests/post_merge_service.rb @@ -15,6 +15,7 @@ module MergeRequests execute_hooks(merge_request, 'merge') invalidate_cache_counts(merge_request, users: merge_request.assignees) merge_request.update_project_counter_caches + delete_non_latest_diffs(merge_request) end private @@ -31,6 +32,10 @@ module MergeRequests end end + def delete_non_latest_diffs(merge_request) + DeleteNonLatestDiffsService.new(merge_request).execute + end + def create_merge_event(merge_request, current_user) EventCreateService.new.merge_mr(merge_request, current_user) end diff --git a/app/services/merge_requests/reload_diffs_service.rb b/app/services/merge_requests/reload_diffs_service.rb new file mode 100644 index 00000000000..2ec7b403903 --- /dev/null +++ b/app/services/merge_requests/reload_diffs_service.rb @@ -0,0 +1,43 @@ +module MergeRequests + class ReloadDiffsService + def initialize(merge_request, current_user) + @merge_request = merge_request + @current_user = current_user + end + + def execute + old_diff_refs = merge_request.diff_refs + new_diff = merge_request.create_merge_request_diff + + clear_cache(new_diff) + update_diff_discussion_positions(old_diff_refs) + end + + private + + attr_reader :merge_request, :current_user + + def update_diff_discussion_positions(old_diff_refs) + new_diff_refs = merge_request.diff_refs + + merge_request.update_diff_discussion_positions(old_diff_refs: old_diff_refs, + new_diff_refs: new_diff_refs, + current_user: current_user) + end + + def clear_cache(new_diff) + # Executing the iteration we cache highlighted diffs for each diff file of + # MergeRequestDiff. + new_diff.diffs_collection.diff_files.to_a + + # Remove cache for all diffs on this MR. Do not use the association on the + # model, as that will interfere with other actions happening when + # reloading the diff. + MergeRequestDiff.where(merge_request: merge_request).each do |merge_request_diff| + next if merge_request_diff == new_diff + + merge_request_diff.diffs_collection.clear_cache! + end + end + end +end diff --git a/app/views/projects/merge_requests/diffs/_diffs.html.haml b/app/views/projects/merge_requests/diffs/_diffs.html.haml index 19659fe5140..bf3df0abf86 100644 --- a/app/views/projects/merge_requests/diffs/_diffs.html.haml +++ b/app/views/projects/merge_requests/diffs/_diffs.html.haml @@ -16,6 +16,6 @@ %span.ref-name= @merge_request.target_branch .text-center= link_to 'Create commit', project_new_blob_path(@project, @merge_request.source_branch), class: 'btn btn-save' - else - - diff_viewable = @merge_request_diff ? @merge_request_diff.collected? || @merge_request_diff.overflow? : true + - diff_viewable = @merge_request_diff ? @merge_request_diff.viewable? : true - if diff_viewable = render "projects/diffs/diffs", diffs: @diffs, environment: @environment, merge_request: true diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 30b6796a7d6..026f756582d 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -118,3 +118,4 @@ - web_hook - repository_update_remote_mirror - create_note_diff_file +- delete_diff_files diff --git a/app/workers/delete_diff_files_worker.rb b/app/workers/delete_diff_files_worker.rb new file mode 100644 index 00000000000..bb8fbb9c373 --- /dev/null +++ b/app/workers/delete_diff_files_worker.rb @@ -0,0 +1,17 @@ +class DeleteDiffFilesWorker + include ApplicationWorker + + def perform(merge_request_diff_id) + merge_request_diff = MergeRequestDiff.find(merge_request_diff_id) + + return if merge_request_diff.without_files? + + MergeRequestDiff.transaction do + merge_request_diff.clean! + + MergeRequestDiffFile + .where(merge_request_diff_id: merge_request_diff.id) + .delete_all + end + end +end diff --git a/changelogs/unreleased/enforce-variable-value-to-be-a-string.yml b/changelogs/unreleased/enforce-variable-value-to-be-a-string.yml new file mode 100644 index 00000000000..e2a932ee5bb --- /dev/null +++ b/changelogs/unreleased/enforce-variable-value-to-be-a-string.yml @@ -0,0 +1,5 @@ +--- +title: Fix incremental rollouts for Auto DevOps +merge_request: 20061 +author: +type: fixed diff --git a/changelogs/unreleased/osw-delete-non-latest-mr-diff-files-upon-merge.yml b/changelogs/unreleased/osw-delete-non-latest-mr-diff-files-upon-merge.yml new file mode 100644 index 00000000000..3e752125f3a --- /dev/null +++ b/changelogs/unreleased/osw-delete-non-latest-mr-diff-files-upon-merge.yml @@ -0,0 +1,5 @@ +--- +title: Delete non-latest merge request diff files upon merge +merge_request: +author: +type: other diff --git a/config/application.rb b/config/application.rb index 202e5d5e327..d9483cd806d 100644 --- a/config/application.rb +++ b/config/application.rb @@ -5,6 +5,12 @@ require 'rails/all' Bundler.require(:default, Rails.env) module Gitlab + # This method is used for smooth upgrading from the current Rails 4.x to Rails 5.0. + # https://gitlab.com/gitlab-org/gitlab-ce/issues/14286 + def self.rails5? + ENV["RAILS5"].in?(%w[1 true]) + end + class Application < Rails::Application require_dependency Rails.root.join('lib/gitlab/redis/wrapper') require_dependency Rails.root.join('lib/gitlab/redis/cache') @@ -14,6 +20,11 @@ module Gitlab require_dependency Rails.root.join('lib/gitlab/current_settings') require_dependency Rails.root.join('lib/gitlab/middleware/read_only') + # This needs to be loaded before DB connection is made + # to make sure that all connections have NO_ZERO_DATE + # setting disabled + require_dependency Rails.root.join('lib/mysql_zero_date') + # Settings in config/environments/* take precedence over those specified here. # Application configuration should go into files in config/initializers # -- all .rb files in that directory are automatically loaded. @@ -211,10 +222,4 @@ module Gitlab Gitlab::Routing.add_helpers(MilestonesRoutingHelper) end end - - # This method is used for smooth upgrading from the current Rails 4.x to Rails 5.0. - # https://gitlab.com/gitlab-org/gitlab-ce/issues/14286 - def self.rails5? - ENV["RAILS5"].in?(%w[1 true]) - end end diff --git a/config/initializers/active_record_data_types.rb b/config/initializers/active_record_data_types.rb index fda13d0c4cb..717e30b5b7e 100644 --- a/config/initializers/active_record_data_types.rb +++ b/config/initializers/active_record_data_types.rb @@ -65,7 +65,7 @@ elsif Gitlab::Database.mysql? prepend RegisterDateTimeWithTimeZone # Add the class `DateTimeWithTimeZone` so we can map `timestamp` to it. - class MysqlDateTimeWithTimeZone < MysqlDateTime + class MysqlDateTimeWithTimeZone < (Gitlab.rails5? ? ActiveRecord::Type::DateTime : MysqlDateTime) def type :datetime_with_timezone end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index d16060e8f45..3400142db36 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -76,4 +76,5 @@ - [repository_update_remote_mirror, 1] - [repository_remove_remote, 1] - [create_note_diff_file, 1] + - [delete_diff_files, 1] diff --git a/db/migrate/merge_request_diff_file_limits_to_mysql.rb b/db/migrate/merge_request_diff_file_limits_to_mysql.rb index 3958380e4b9..ca3bc7d6be9 100644 --- a/db/migrate/merge_request_diff_file_limits_to_mysql.rb +++ b/db/migrate/merge_request_diff_file_limits_to_mysql.rb @@ -4,7 +4,7 @@ class MergeRequestDiffFileLimitsToMysql < ActiveRecord::Migration def up return unless Gitlab::Database.mysql? - change_column :merge_request_diff_files, :diff, :text, limit: 2147483647 + change_column :merge_request_diff_files, :diff, :text, limit: 2147483647, default: nil end def down diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index d00e5b07f95..222aa06b800 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -4,6 +4,9 @@ module Gitlab class Collection class Item def initialize(key:, value:, public: true, file: false) + raise ArgumentError, "`value` must be of type String, while it was: #{value.class}" unless + value.is_a?(String) || value.nil? + @variable = { key: key, value: value, public: public, file: file } diff --git a/lib/mysql_zero_date.rb b/lib/mysql_zero_date.rb new file mode 100644 index 00000000000..64634f789da --- /dev/null +++ b/lib/mysql_zero_date.rb @@ -0,0 +1,18 @@ +# Disable NO_ZERO_DATE mode for mysql in rails 5. +# We use zero date as a default value +# (config/initializers/active_record_mysql_timestamp.rb), in +# Rails 5 using zero date fails by default (https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/75450216) +# and NO_ZERO_DATE has to be explicitly disabled. Disabling strict mode +# is not sufficient. + +require 'active_record/connection_adapters/abstract_mysql_adapter' + +module MysqlZeroDate + def configure_connection + super + + @connection.query "SET @@SESSION.sql_mode = REPLACE(@@SESSION.sql_mode, 'NO_ZERO_DATE', '');" # rubocop:disable Gitlab/ModuleWithInstanceVariables + end +end + +ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate) if Gitlab.rails5? diff --git a/scripts/trigger-build-docs b/scripts/trigger-build-docs index c9aaba91aa0..2a0e7f4d76e 100755 --- a/scripts/trigger-build-docs +++ b/scripts/trigger-build-docs @@ -27,7 +27,7 @@ def docs_branch # Prefix the remote branch with the slug of the project in order # to avoid name conflicts in the rare case the branch name already # exists in the docs repo and truncate to max length. - "#{slug}-#{ENV["CI_COMMIT_REF_SLUG"]}"[0...max] + "#{slug}-#{ENV["CI_ENVIRONMENT_SLUG"]}"[0...max] end # diff --git a/spec/javascripts/test_bundle.js b/spec/javascripts/test_bundle.js index aeb936b0e3c..0eff98bcc9d 100644 --- a/spec/javascripts/test_bundle.js +++ b/spec/javascripts/test_bundle.js @@ -3,7 +3,6 @@ import $ from 'jquery'; import 'vendor/jasmine-jquery'; import '~/commons'; - import Vue from 'vue'; import VueResource from 'vue-resource'; import Translate from '~/vue_shared/translate'; diff --git a/spec/lib/gitlab/ci/variables/collection/item_spec.rb b/spec/lib/gitlab/ci/variables/collection/item_spec.rb index e79f0a7f257..adb3ff4321f 100644 --- a/spec/lib/gitlab/ci/variables/collection/item_spec.rb +++ b/spec/lib/gitlab/ci/variables/collection/item_spec.rb @@ -1,19 +1,69 @@ require 'spec_helper' describe Gitlab::Ci::Variables::Collection::Item do + let(:variable_key) { 'VAR' } + let(:variable_value) { 'something' } + let(:expected_value) { variable_value } + let(:variable) do - { key: 'VAR', value: 'something', public: true } + { key: variable_key, value: variable_value, public: true } end describe '.new' do - it 'raises error if unknown key i specified' do - expect { described_class.new(key: 'VAR', value: 'abc', files: true) } - .to raise_error ArgumentError, 'unknown keyword: files' + context 'when unknown keyword is specified' do + it 'raises error' do + expect { described_class.new(key: variable_key, value: 'abc', files: true) } + .to raise_error ArgumentError, 'unknown keyword: files' + end + end + + context 'when required keywords are not specified' do + it 'raises error' do + expect { described_class.new(key: variable_key) } + .to raise_error ArgumentError, 'missing keyword: value' + end end - it 'raises error when required keywords are not specified' do - expect { described_class.new(key: 'VAR') } - .to raise_error ArgumentError, 'missing keyword: value' + shared_examples 'creates variable' do + subject { described_class.new(key: variable_key, value: variable_value) } + + it 'saves given value' do + expect(subject[:key]).to eq variable_key + expect(subject[:value]).to eq expected_value + end + end + + shared_examples 'raises error for invalid type' do + it do + expect { described_class.new(key: variable_key, value: variable_value) } + .to raise_error ArgumentError, /`value` must be of type String, while it was:/ + end + end + + it_behaves_like 'creates variable' + + context "when it's nil" do + let(:variable_value) { nil } + let(:expected_value) { nil } + + it_behaves_like 'creates variable' + end + + context "when it's an empty string" do + let(:variable_value) { '' } + let(:expected_value) { '' } + + it_behaves_like 'creates variable' + end + + context 'when provided value is not a string' do + [1, false, [], {}, Object.new].each do |val| + context "when it's #{val}" do + let(:variable_value) { val } + + it_behaves_like 'raises error for invalid type' + end + end end end diff --git a/spec/lib/gitlab/ci/variables/collection_spec.rb b/spec/lib/gitlab/ci/variables/collection_spec.rb index cb2f7718c9c..5c91816a586 100644 --- a/spec/lib/gitlab/ci/variables/collection_spec.rb +++ b/spec/lib/gitlab/ci/variables/collection_spec.rb @@ -29,7 +29,7 @@ describe Gitlab::Ci::Variables::Collection do end it 'appends an internal resource' do - collection = described_class.new([{ key: 'TEST', value: 1 }]) + collection = described_class.new([{ key: 'TEST', value: '1' }]) subject.append(collection.first) @@ -74,15 +74,15 @@ describe Gitlab::Ci::Variables::Collection do describe '#+' do it 'makes it possible to combine with an array' do - collection = described_class.new([{ key: 'TEST', value: 1 }]) + collection = described_class.new([{ key: 'TEST', value: '1' }]) variables = [{ key: 'TEST', value: 'something' }] expect((collection + variables).count).to eq 2 end it 'makes it possible to combine with another collection' do - collection = described_class.new([{ key: 'TEST', value: 1 }]) - other = described_class.new([{ key: 'TEST', value: 2 }]) + collection = described_class.new([{ key: 'TEST', value: '1' }]) + other = described_class.new([{ key: 'TEST', value: '2' }]) expect((collection + other).count).to eq 2 end @@ -90,10 +90,10 @@ describe Gitlab::Ci::Variables::Collection do describe '#to_runner_variables' do it 'creates an array of hashes in a runner-compatible format' do - collection = described_class.new([{ key: 'TEST', value: 1 }]) + collection = described_class.new([{ key: 'TEST', value: '1' }]) expect(collection.to_runner_variables) - .to eq [{ key: 'TEST', value: 1, public: true }] + .to eq [{ key: 'TEST', value: '1', public: true }] end end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 51b9b518117..6758adc59eb 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1871,7 +1871,11 @@ describe Ci::Build do end context 'when yaml_variables are undefined' do - let(:pipeline) { create(:ci_pipeline, project: project) } + let(:pipeline) do + create(:ci_pipeline, project: project, + sha: project.commit.id, + ref: project.default_branch) + end before do build.yaml_variables = nil diff --git a/spec/models/merge_request_diff_spec.rb b/spec/models/merge_request_diff_spec.rb index b4249d72fc8..48c01fc4d4e 100644 --- a/spec/models/merge_request_diff_spec.rb +++ b/spec/models/merge_request_diff_spec.rb @@ -47,6 +47,45 @@ describe MergeRequestDiff do end describe '#diffs' do + let(:merge_request) { create(:merge_request, :with_diffs) } + let!(:diff) { merge_request.merge_request_diff.reload } + + context 'when it was not cleaned by the system' do + it 'returns persisted diffs' do + expect(diff).to receive(:load_diffs) + + diff.diffs + end + end + + context 'when diff was cleaned by the system' do + before do + diff.clean! + end + + it 'returns diffs from repository if can compare with current diff refs' do + expect(diff).not_to receive(:load_diffs) + + expect(Compare) + .to receive(:new) + .with(instance_of(Gitlab::Git::Compare), merge_request.target_project, + base_sha: diff.base_commit_sha, straight: false) + .and_call_original + + diff.diffs + end + + it 'returns persisted diffs if cannot compare with diff refs' do + expect(diff).to receive(:load_diffs) + + diff.update!(head_commit_sha: 'invalid-sha') + + diff.diffs + end + end + end + + describe '#raw_diffs' do context 'when the :ignore_whitespace_change option is set' do it 'creates a new compare object instead of loading from the DB' do expect(diff_with_commits).not_to receive(:load_diffs) diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 7ae70c3afb4..2c75816654e 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -1630,28 +1630,17 @@ describe MergeRequest do end describe "#reload_diff" do - let(:discussion) { create(:diff_note_on_merge_request, project: subject.project, noteable: subject).to_discussion } - let(:commit) { subject.project.commit(sample_commit.id) } - - it "does not change existing merge request diff" do - expect(subject.merge_request_diff).not_to receive(:save_git_content) - subject.reload_diff - end + it 'calls MergeRequests::ReloadDiffsService#execute with correct params' do + user = create(:user) + service = instance_double(MergeRequests::ReloadDiffsService, execute: nil) - it "creates new merge request diff" do - expect { subject.reload_diff }.to change { subject.merge_request_diffs.count }.by(1) - end - - it "executes diff cache service" do - expect_any_instance_of(MergeRequests::MergeRequestDiffCacheService).to receive(:execute).with(subject, an_instance_of(MergeRequestDiff)) - - subject.reload_diff - end + expect(MergeRequests::ReloadDiffsService) + .to receive(:new).with(subject, user) + .and_return(service) - it "calls update_diff_discussion_positions" do - expect(subject).to receive(:update_diff_discussion_positions) + subject.reload_diff(user) - subject.reload_diff + expect(service).to have_received(:execute) end context 'when using the after_update hook to update' do diff --git a/spec/services/merge_requests/delete_non_latest_diffs_service_spec.rb b/spec/services/merge_requests/delete_non_latest_diffs_service_spec.rb new file mode 100644 index 00000000000..1c632847940 --- /dev/null +++ b/spec/services/merge_requests/delete_non_latest_diffs_service_spec.rb @@ -0,0 +1,59 @@ +require 'spec_helper' + +describe MergeRequests::DeleteNonLatestDiffsService, :clean_gitlab_redis_shared_state do + let(:merge_request) { create(:merge_request) } + + let!(:subject) { described_class.new(merge_request) } + + describe '#execute' do + before do + stub_const("#{described_class.name}::BATCH_SIZE", 2) + + 3.times { merge_request.create_merge_request_diff } + end + + it 'schedules non-latest merge request diffs removal' do + diffs = merge_request.merge_request_diffs + + expect(diffs.count).to eq(4) + + Timecop.freeze do + expect(DeleteDiffFilesWorker) + .to receive(:bulk_perform_in) + .with(5.minutes, [[diffs.first.id], [diffs.second.id]]) + expect(DeleteDiffFilesWorker) + .to receive(:bulk_perform_in) + .with(10.minutes, [[diffs.third.id]]) + + subject.execute + end + end + + it 'schedules no removal if it is already cleaned' do + merge_request.merge_request_diffs.each(&:clean!) + + expect(DeleteDiffFilesWorker).not_to receive(:bulk_perform_in) + + subject.execute + end + + it 'schedules no removal if it is empty' do + merge_request.merge_request_diffs.each { |diff| diff.update!(state: :empty) } + + expect(DeleteDiffFilesWorker).not_to receive(:bulk_perform_in) + + subject.execute + end + + it 'schedules no removal if there is no non-latest diffs' do + merge_request + .merge_request_diffs + .where.not(id: merge_request.latest_merge_request_diff_id) + .destroy_all + + expect(DeleteDiffFilesWorker).not_to receive(:bulk_perform_in) + + subject.execute + end + end +end diff --git a/spec/services/merge_requests/merge_request_diff_cache_service_spec.rb b/spec/services/merge_requests/merge_request_diff_cache_service_spec.rb deleted file mode 100644 index 57b6165cfb0..00000000000 --- a/spec/services/merge_requests/merge_request_diff_cache_service_spec.rb +++ /dev/null @@ -1,39 +0,0 @@ -require 'spec_helper' - -describe MergeRequests::MergeRequestDiffCacheService, :use_clean_rails_memory_store_caching do - let(:subject) { described_class.new } - let(:merge_request) { create(:merge_request) } - - describe '#execute' do - before do - allow_any_instance_of(Gitlab::Diff::File).to receive(:text?).and_return(true) - allow_any_instance_of(Gitlab::Diff::File).to receive(:diffable?).and_return(true) - end - - it 'retrieves the diff files to cache the highlighted result' do - new_diff = merge_request.merge_request_diff - cache_key = new_diff.diffs.cache_key - - expect(Rails.cache).to receive(:read).with(cache_key).and_call_original - expect(Rails.cache).to receive(:write).with(cache_key, anything, anything).and_call_original - - subject.execute(merge_request, new_diff) - end - - it 'clears the cache for older diffs on the merge request' do - old_diff = merge_request.merge_request_diff - old_cache_key = old_diff.diffs.cache_key - - subject.execute(merge_request, old_diff) - - new_diff = merge_request.create_merge_request_diff - new_cache_key = new_diff.diffs.cache_key - - expect(Rails.cache).to receive(:delete).with(old_cache_key).and_call_original - expect(Rails.cache).to receive(:read).with(new_cache_key).and_call_original - expect(Rails.cache).to receive(:write).with(new_cache_key, anything, anything).and_call_original - - subject.execute(merge_request, new_diff) - end - end -end diff --git a/spec/services/merge_requests/post_merge_service_spec.rb b/spec/services/merge_requests/post_merge_service_spec.rb index 70957431942..790ecce8ded 100644 --- a/spec/services/merge_requests/post_merge_service_spec.rb +++ b/spec/services/merge_requests/post_merge_service_spec.rb @@ -35,5 +35,17 @@ describe MergeRequests::PostMergeService do described_class.new(project, user, {}).execute(merge_request) end + + it 'deletes non-latest diffs' do + diff_removal_service = instance_double(MergeRequests::DeleteNonLatestDiffsService, execute: nil) + + expect(MergeRequests::DeleteNonLatestDiffsService) + .to receive(:new).with(merge_request) + .and_return(diff_removal_service) + + described_class.new(project, user, {}).execute(merge_request) + + expect(diff_removal_service).to have_received(:execute) + end end end diff --git a/spec/services/merge_requests/reload_diffs_service_spec.rb b/spec/services/merge_requests/reload_diffs_service_spec.rb new file mode 100644 index 00000000000..a0a27d247fc --- /dev/null +++ b/spec/services/merge_requests/reload_diffs_service_spec.rb @@ -0,0 +1,64 @@ +require 'spec_helper' + +describe MergeRequests::ReloadDiffsService, :use_clean_rails_memory_store_caching do + let(:current_user) { create(:user) } + let(:merge_request) { create(:merge_request) } + let(:subject) { described_class.new(merge_request, current_user) } + + describe '#execute' do + it 'creates new merge request diff' do + expect { subject.execute }.to change { merge_request.merge_request_diffs.count }.by(1) + end + + it 'calls update_diff_discussion_positions with correct params' do + old_diff_refs = merge_request.diff_refs + new_diff = merge_request.create_merge_request_diff + new_diff_refs = merge_request.diff_refs + + expect(merge_request).to receive(:create_merge_request_diff).and_return(new_diff) + expect(merge_request).to receive(:update_diff_discussion_positions) + .with(old_diff_refs: old_diff_refs, + new_diff_refs: new_diff_refs, + current_user: current_user) + + subject.execute + end + + it 'does not change existing merge request diff' do + expect(merge_request.merge_request_diff).not_to receive(:save_git_content) + + subject.execute + end + + context 'cache clearing' do + before do + allow_any_instance_of(Gitlab::Diff::File).to receive(:text?).and_return(true) + allow_any_instance_of(Gitlab::Diff::File).to receive(:diffable?).and_return(true) + end + + it 'retrieves the diff files to cache the highlighted result' do + new_diff = merge_request.create_merge_request_diff + cache_key = new_diff.diffs_collection.cache_key + + expect(merge_request).to receive(:create_merge_request_diff).and_return(new_diff) + expect(Rails.cache).to receive(:read).with(cache_key).and_call_original + expect(Rails.cache).to receive(:write).with(cache_key, anything, anything).and_call_original + + subject.execute + end + + it 'clears the cache for older diffs on the merge request' do + old_diff = merge_request.merge_request_diff + old_cache_key = old_diff.diffs_collection.cache_key + new_diff = merge_request.create_merge_request_diff + new_cache_key = new_diff.diffs_collection.cache_key + + expect(merge_request).to receive(:create_merge_request_diff).and_return(new_diff) + expect(Rails.cache).to receive(:delete).with(old_cache_key).and_call_original + expect(Rails.cache).to receive(:read).with(new_cache_key).and_call_original + expect(Rails.cache).to receive(:write).with(new_cache_key, anything, anything).and_call_original + subject.execute + end + end + end +end diff --git a/spec/workers/delete_diff_files_worker_spec.rb b/spec/workers/delete_diff_files_worker_spec.rb new file mode 100644 index 00000000000..e0edd313922 --- /dev/null +++ b/spec/workers/delete_diff_files_worker_spec.rb @@ -0,0 +1,41 @@ +require 'spec_helper' + +describe DeleteDiffFilesWorker do + describe '#perform' do + let(:merge_request) { create(:merge_request) } + let(:merge_request_diff) { merge_request.merge_request_diff } + + it 'deletes all merge request diff files' do + expect { described_class.new.perform(merge_request_diff.id) } + .to change { merge_request_diff.merge_request_diff_files.count } + .from(20).to(0) + end + + it 'updates state to without_files' do + expect { described_class.new.perform(merge_request_diff.id) } + .to change { merge_request_diff.reload.state } + .from('collected').to('without_files') + end + + it 'does nothing if diff was already marked as "without_files"' do + merge_request_diff.clean! + + expect_any_instance_of(MergeRequestDiff).not_to receive(:clean!) + + described_class.new.perform(merge_request_diff.id) + end + + it 'rollsback if something goes wrong' do + expect(MergeRequestDiffFile).to receive_message_chain(:where, :delete_all) + .and_raise + + expect { described_class.new.perform(merge_request_diff.id) } + .to raise_error + + merge_request_diff.reload + + expect(merge_request_diff.state).to eq('collected') + expect(merge_request_diff.merge_request_diff_files.count).to eq(20) + end + end +end |