diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-04-06 12:10:44 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-04-06 12:10:44 +0000 |
commit | ba174c982f40d71a87fd511b091753807174f7e7 (patch) | |
tree | b89d55c715288f2c2f76724c1b7ff4a6ebf90154 /spec | |
parent | eaea945e0355826c58c3dcf887496ea91064f85c (diff) | |
download | gitlab-ce-ba174c982f40d71a87fd511b091753807174f7e7.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'spec')
17 files changed, 549 insertions, 325 deletions
diff --git a/spec/features/issues/filtered_search/filter_issues_spec.rb b/spec/features/issues/filtered_search/filter_issues_spec.rb index 756699fb854..5e2b5921e06 100644 --- a/spec/features/issues/filtered_search/filter_issues_spec.rb +++ b/spec/features/issues/filtered_search/filter_issues_spec.rb @@ -224,14 +224,6 @@ describe 'Filter issues', :js do expect_no_issues_list expect_filtered_search_input_empty end - - it 'does show issues for bug label' do - input_filtered_search("label:!=~#{bug_label.title}") - - expect_tokens([label_token(bug_label.title)]) - expect_issues_list_count(6) - expect_filtered_search_input_empty - end end context 'label with multiple words' do diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb index 54a6ac1551b..1e13afc6033 100644 --- a/spec/features/projects/import_export/export_file_spec.rb +++ b/spec/features/projects/import_export/export_file_spec.rb @@ -38,51 +38,109 @@ describe 'Import/Export - project export integration test', :js do sign_in(user) end - it 'exports a project successfully', :sidekiq_might_not_need_inline do - visit edit_project_path(project) + shared_examples 'export file without sensitive words' do + it 'exports a project successfully', :sidekiq_inline do + export_project_and_download_file(page, project) - expect(page).to have_content('Export project') + in_directory_with_expanded_export(project) do |exit_status, tmpdir| + expect(exit_status).to eq(0) - find(:link, 'Export project').send_keys(:return) + project_json_path = File.join(tmpdir, 'project.json') + expect(File).to exist(project_json_path) - visit edit_project_path(project) + project_hash = JSON.parse(IO.read(project_json_path)) - expect(page).to have_content('Download export') + sensitive_words.each do |sensitive_word| + found = find_sensitive_attributes(sensitive_word, project_hash) - expect(project.export_status).to eq(:finished) - expect(project.export_file.path).to include('tar.gz') + expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) + end + end + end + end + + context "with legacy export" do + before do + stub_feature_flags(streaming_serializer: false) + stub_feature_flags(project_export_as_ndjson: false) + end + + it_behaves_like "export file without sensitive words" + end + + context "with streaming serializer" do + before do + stub_feature_flags(streaming_serializer: true) + stub_feature_flags(project_export_as_ndjson: false) + end + + it_behaves_like "export file without sensitive words" + end - in_directory_with_expanded_export(project) do |exit_status, tmpdir| - expect(exit_status).to eq(0) + context "with ndjson" do + before do + stub_feature_flags(streaming_serializer: true) + stub_feature_flags(project_export_as_ndjson: true) + end + + it 'exports a project successfully', :sidekiq_inline do + export_project_and_download_file(page, project) + + in_directory_with_expanded_export(project) do |exit_status, tmpdir| + expect(exit_status).to eq(0) - project_json_path = File.join(tmpdir, 'project.json') - expect(File).to exist(project_json_path) + project_json_path = File.join(tmpdir, 'tree', 'project.json') + expect(File).to exist(project_json_path) - project_hash = JSON.parse(IO.read(project_json_path)) + relations = [] + relations << JSON.parse(IO.read(project_json_path)) + Dir.glob(File.join(tmpdir, 'tree/project', '*.ndjson')) do |rb_filename| + File.foreach(rb_filename) do |line| + json = ActiveSupport::JSON.decode(line) + relations << json + end + end - sensitive_words.each do |sensitive_word| - found = find_sensitive_attributes(sensitive_word, project_hash) + relations.each do |relation_hash| + sensitive_words.each do |sensitive_word| + found = find_sensitive_attributes(sensitive_word, relation_hash) - expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) + expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) + end + end end end end + end - def failure_message(key_found, parent, sensitive_word) - <<-MSG - Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect} + def export_project_and_download_file(page, project) + visit edit_project_path(project) - If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG. + expect(page).to have_content('Export project') - Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the - correspondent hash or model as the value. + find(:link, 'Export project').send_keys(:return) - Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be - reset (to prevent duplicate column problems while importing to the same instance). + visit edit_project_path(project) - IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} - CURRENT_SPEC: #{__FILE__} - MSG - end + expect(page).to have_content('Download export') + expect(project.export_status).to eq(:finished) + expect(project.export_file.path).to include('tar.gz') + end + + def failure_message(key_found, parent, sensitive_word) + <<-MSG + Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect} + + If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG. + + Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the + correspondent hash or model as the value. + + Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be + reset (to prevent duplicate column problems while importing to the same instance). + + IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} + CURRENT_SPEC: #{__FILE__} + MSG end end diff --git a/spec/finders/environments_finder_spec.rb b/spec/finders/environments_finder_spec.rb index 7100376478a..850d160506c 100644 --- a/spec/finders/environments_finder_spec.rb +++ b/spec/finders/environments_finder_spec.rb @@ -37,11 +37,6 @@ describe EnvironmentsFinder do .to be_empty end - it 'returns environment when with_tags is set' do - expect(described_class.new(project, user, ref: 'master', commit: commit, with_tags: true).execute) - .to contain_exactly(environment, environment_two) - end - # We expect two Gitaly calls: FindCommit, CommitIsAncestor # This tests to ensure we don't call one CommitIsAncestor per environment it 'only calls Gitaly twice when multiple environments are present', :request_store do diff --git a/spec/helpers/users_helper_spec.rb b/spec/helpers/users_helper_spec.rb index 893d5cde24a..e3e599007a4 100644 --- a/spec/helpers/users_helper_spec.rb +++ b/spec/helpers/users_helper_spec.rb @@ -95,9 +95,9 @@ describe UsersHelper do end it 'includes the settings tab if the user can update themself' do - expect(helper).to receive(:can?).with(user, :read_user, user) { true } + expect(helper).to receive(:can?).with(user, :update_user, user) { true } - expect(items).to include(:profile) + expect(items).to include(:settings) end context 'when terms are enforced' do diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 0b34e887716..62adba4319e 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -1647,48 +1647,6 @@ module Gitlab it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /is not defined in prior stages/) } end - - context 'when trigger job includes artifact generated by a dependency' do - context 'when dependency is defined in previous stages' do - let(:config) do - { - build1: { stage: 'build', script: 'test' }, - test1: { stage: 'test', trigger: { - include: [{ job: 'build1', artifact: 'generated.yml' }] - } } - } - end - - it { expect { subject }.not_to raise_error } - end - - context 'when dependency is defined in later stages' do - let(:config) do - { - build1: { stage: 'build', script: 'test' }, - test1: { stage: 'test', trigger: { - include: [{ job: 'deploy1', artifact: 'generated.yml' }] - } }, - deploy1: { stage: 'deploy', script: 'test' } - } - end - - it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /is not defined in prior stages/) } - end - - context 'when dependency is not defined' do - let(:config) do - { - build1: { stage: 'build', script: 'test' }, - test1: { stage: 'test', trigger: { - include: [{ job: 'non-existent', artifact: 'generated.yml' }] - } } - } - end - - it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /undefined dependency: non-existent/) } - end - end end describe "Job Needs" do @@ -2094,34 +2052,6 @@ module Gitlab end end - describe 'with trigger:include' do - context 'when artifact and job are specified' do - let(:config) do - YAML.dump({ - build1: { stage: 'build', script: 'test' }, - test1: { stage: 'test', trigger: { - include: [{ artifact: 'generated.yml', job: 'build1' }] - } } - }) - end - - it { expect { subject }.not_to raise_error } - end - - context 'when artifact is specified without job' do - let(:config) do - YAML.dump({ - build1: { stage: 'build', script: 'test' }, - test1: { stage: 'test', trigger: { - include: [{ artifact: 'generated.yml' }] - } } - }) - end - - it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /must specify the job where to fetch the artifact from/) } - end - end - describe "Error handling" do it "fails to parse YAML" do expect do diff --git a/spec/lib/gitlab/import_export/fork_spec.rb b/spec/lib/gitlab/import_export/fork_spec.rb index 8aa28353c04..bb79331efac 100644 --- a/spec/lib/gitlab/import_export/fork_spec.rb +++ b/spec/lib/gitlab/import_export/fork_spec.rb @@ -32,6 +32,8 @@ describe 'forked project import' do end before do + stub_feature_flags(project_export_as_ndjson: false) + allow_next_instance_of(Gitlab::ImportExport) do |instance| allow(instance).to receive(:storage_path).and_return(export_path) end diff --git a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb index 50b26637cb1..707975f20b6 100644 --- a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb +++ b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb @@ -20,6 +20,10 @@ describe Gitlab::ImportExport do let(:json_fixture) { 'complex' } + before do + stub_feature_flags(project_export_as_ndjson: false) + end + it 'yields the initial tree when importing and exporting it again' do project = create(:project, creator: create(:user, :admin)) diff --git a/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb new file mode 100644 index 00000000000..bae3672474c --- /dev/null +++ b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe Gitlab::ImportExport::JSON::NdjsonWriter do + include ImportExport::CommonUtil + + let(:path) { "#{Dir.tmpdir}/ndjson_writer_spec/tree" } + let(:exportable_path) { 'projects' } + + subject { described_class.new(path) } + + after do + FileUtils.rm_rf(path) + end + + describe "#write_attributes" do + it "writes correct json to root" do + expected_hash = { "key" => "value_1", "key_1" => "value_2" } + subject.write_attributes(exportable_path, expected_hash) + + expect(consume_attributes(path, exportable_path)).to eq(expected_hash) + end + end + + describe "#write_relation" do + context "when single relation is serialized" do + it "appends json in correct file " do + relation = "relation" + value = { "key" => "value_1", "key_1" => "value_1" } + subject.write_relation(exportable_path, relation, value) + + expect(consume_relations(path, exportable_path, relation)).to eq([value]) + end + end + + context "when single relation is already serialized" do + it "raise exception" do + values = [{ "key" => "value_1", "key_1" => "value_1" }, { "key" => "value_2", "key_1" => "value_2" }] + relation = "relation" + file_path = File.join(path, exportable_path, "#{relation}.ndjson") + subject.write_relation(exportable_path, relation, values[0]) + + expect {subject.write_relation(exportable_path, relation, values[1])}.to raise_exception("The #{file_path} already exist") + end + end + end + + describe "#write_relation_array" do + it "writes json in correct files" do + values = [{ "key" => "value_1", "key_1" => "value_1" }, { "key" => "value_2", "key_1" => "value_2" }] + relations = %w(relation1 relation2) + relations.each do |relation| + subject.write_relation_array(exportable_path, relation, values.to_enum) + end + subject.close + + relations.each do |relation| + expect(consume_relations(path, exportable_path, relation)).to eq(values) + end + end + end +end diff --git a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb index 73ae6810706..175da623c1b 100644 --- a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb +++ b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb @@ -160,7 +160,7 @@ describe Gitlab::ImportExport::Project::RelationFactory do end it 'has preloaded target project' do - expect(created_object.source_project).to equal(project) + expect(created_object.target_project).to equal(project) end end diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index c910ee5430b..d859af5df02 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -3,314 +3,397 @@ require 'spec_helper' describe Gitlab::ImportExport::Project::TreeSaver do - describe 'saves the project tree into a json object' do - let(:shared) { project.import_export_shared } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } - let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } - let(:user) { create(:user) } - let!(:project) { setup_project } - - before do - project.add_maintainer(user) - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - allow_any_instance_of(MergeRequest).to receive(:source_branch_sha).and_return('ABCD') - allow_any_instance_of(MergeRequest).to receive(:target_branch_sha).and_return('DCBA') - end + let_it_be(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } + let_it_be(:exportable_path) { 'project' } - after do - FileUtils.rm_rf(export_path) - end + shared_examples 'saves project tree successfully' do |ndjson_enabled| + include ImportExport::CommonUtil - it 'saves project successfully' do - expect(project_tree_saver.save).to be true - end + subject { get_json(full_path, exportable_path, relation_name, ndjson_enabled) } - context 'JSON' do - let(:saved_project_json) do - project_tree_saver.save - project_json(project_tree_saver.full_path) - end + describe 'saves project tree attributes' do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:project) { setup_project } + let_it_be(:shared) { project.import_export_shared } + let_it_be(:project_tree_saver ) { described_class.new(project: project, current_user: user, shared: shared) } - # It is not duplicated in - # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` - context 'with description override' do - let(:params) { { description: 'Foo Bar' } } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared, params: params) } + let(:relation_name) { :projects } - it 'overrides the project description' do - expect(saved_project_json).to include({ 'description' => params[:description] }) + let_it_be(:full_path) do + if ndjson_enabled + File.join(shared.export_path, 'tree') + else + File.join(shared.export_path, Gitlab::ImportExport.project_filename) end end - it 'saves the correct json' do - expect(saved_project_json).to include({ 'description' => 'description', 'visibility_level' => 20 }) + before_all do + Feature.enable(:project_export_as_ndjson) if ndjson_enabled + project.add_maintainer(user) + project_tree_saver.save end - it 'has approvals_before_merge set' do - expect(saved_project_json['approvals_before_merge']).to eq(1) + after :all do + FileUtils.rm_rf(export_path) end - it 'has milestones' do - expect(saved_project_json['milestones']).not_to be_empty - end + context 'with project root' do + it { is_expected.to include({ 'description' => 'description', 'visibility_level' => 20 }) } - it 'has merge requests' do - expect(saved_project_json['merge_requests']).not_to be_empty - end + it { is_expected.not_to include("runners_token" => 'token') } - it 'has merge request\'s milestones' do - expect(saved_project_json['merge_requests'].first['milestone']).not_to be_empty + it 'has approvals_before_merge set' do + expect(subject['approvals_before_merge']).to eq(1) + end end - it 'has merge request\'s source branch SHA' do - expect(saved_project_json['merge_requests'].first['source_branch_sha']).to eq('ABCD') - end + context 'with milestones' do + let(:relation_name) { :milestones } - it 'has merge request\'s target branch SHA' do - expect(saved_project_json['merge_requests'].first['target_branch_sha']).to eq('DCBA') + it { is_expected.not_to be_empty } end - it 'has events' do - expect(saved_project_json['merge_requests'].first['milestone']['events']).not_to be_empty - end + context 'with merge_requests' do + let(:relation_name) { :merge_requests } - it 'has snippets' do - expect(saved_project_json['snippets']).not_to be_empty - end + it { is_expected.not_to be_empty } - it 'has snippet notes' do - expect(saved_project_json['snippets'].first['notes']).not_to be_empty - end + it 'has merge request\'s milestones' do + expect(subject.first['milestone']).not_to be_empty + end + it 'has merge request\'s source branch SHA' do + expect(subject.first['source_branch_sha']).to eq('b83d6e391c22777fca1ed3012fce84f633d7fed0') + end - it 'has releases' do - expect(saved_project_json['releases']).not_to be_empty - end + it 'has merge request\'s target branch SHA' do + expect(subject.first['target_branch_sha']).to eq('0b4bc9a49b562e85de7cc9e834518ea6828729b9') + end - it 'has no author on releases' do - expect(saved_project_json['releases'].first['author']).to be_nil - end + it 'has events' do + expect(subject.first['milestone']['events']).not_to be_empty + end - it 'has the author ID on releases' do - expect(saved_project_json['releases'].first['author_id']).not_to be_nil - end + it 'has merge requests diffs' do + expect(subject.first['merge_request_diff']).not_to be_empty + end - it 'has issues' do - expect(saved_project_json['issues']).not_to be_empty - end + it 'has merge request diff files' do + expect(subject.first['merge_request_diff']['merge_request_diff_files']).not_to be_empty + end - it 'has issue comments' do - notes = saved_project_json['issues'].first['notes'] + it 'has merge request diff commits' do + expect(subject.first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty + end - expect(notes).not_to be_empty - expect(notes.first['type']).to eq('DiscussionNote') - end + it 'has merge requests comments' do + expect(subject.first['notes']).not_to be_empty + end - it 'has issue assignees' do - expect(saved_project_json['issues'].first['issue_assignees']).not_to be_empty - end + it 'has author on merge requests comments' do + expect(subject.first['notes'].first['author']).not_to be_empty + end - it 'has author on issue comments' do - expect(saved_project_json['issues'].first['notes'].first['author']).not_to be_empty + it 'has merge request resource label events' do + expect(subject.first['resource_label_events']).not_to be_empty + end end - it 'has system note metadata on issue comments' do - metadata = saved_project_json['issues'].first['notes'].first['system_note_metadata'] + context 'with snippets' do + let(:relation_name) { :snippets } - expect(metadata['action']).to eq('description') - end + it { is_expected.not_to be_empty } - it 'has project members' do - expect(saved_project_json['project_members']).not_to be_empty + it 'has snippet notes' do + expect(subject.first['notes']).not_to be_empty + end end - it 'has merge requests diffs' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']).not_to be_empty - end + context 'with releases' do + let(:relation_name) { :releases } - it 'has merge request diff files' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_files']).not_to be_empty - end + it { is_expected.not_to be_empty } - it 'has merge request diff commits' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty - end + it 'has no author on releases' do + expect(subject.first['author']).to be_nil + end - it 'has merge requests comments' do - expect(saved_project_json['merge_requests'].first['notes']).not_to be_empty + it 'has the author ID on releases' do + expect(subject.first['author_id']).not_to be_nil + end end - it 'has author on merge requests comments' do - expect(saved_project_json['merge_requests'].first['notes'].first['author']).not_to be_empty - end + context 'with issues' do + let(:relation_name) { :issues } - it 'has system note metadata on merge requests comments' do - metadata = saved_project_json['merge_requests'].first['notes'].first['system_note_metadata'] + it { is_expected.not_to be_empty } - expect(metadata['commit_count']).to eq(1) - expect(metadata['action']).to eq('commit') - end + it 'has issue comments' do + notes = subject.first['notes'] - it 'has pipeline stages' do - expect(saved_project_json.dig('ci_pipelines', 0, 'stages')).not_to be_empty - end + expect(notes).not_to be_empty + expect(notes.first['type']).to eq('DiscussionNote') + end - it 'has pipeline statuses' do - expect(saved_project_json.dig('ci_pipelines', 0, 'stages', 0, 'statuses')).not_to be_empty - end + it 'has issue assignees' do + expect(subject.first['issue_assignees']).not_to be_empty + end - it 'has pipeline builds' do - builds_count = saved_project_json - .dig('ci_pipelines', 0, 'stages', 0, 'statuses') - .count { |hash| hash['type'] == 'Ci::Build' } + it 'has author on issue comments' do + expect(subject.first['notes'].first['author']).not_to be_empty + end - expect(builds_count).to eq(1) - end + it 'has labels associated to records' do + expect(subject.first['label_links'].first['label']).not_to be_empty + end - it 'has no when YML attributes but only the DB column' do - expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes) + it 'has project and group labels' do + label_types = subject.first['label_links'].map { |link| link['label']['type'] } - saved_project_json - end + expect(label_types).to match_array(%w(ProjectLabel GroupLabel)) + end - it 'has pipeline commits' do - expect(saved_project_json['ci_pipelines']).not_to be_empty - end + it 'has priorities associated to labels' do + priorities = subject.first['label_links'].flat_map { |link| link['label']['priorities'] } - it 'has ci pipeline notes' do - expect(saved_project_json['ci_pipelines'].first['notes']).not_to be_empty - end + expect(priorities).not_to be_empty + end - it 'has labels with no associations' do - expect(saved_project_json['labels']).not_to be_empty + it 'has issue resource label events' do + expect(subject.first['resource_label_events']).not_to be_empty + end end - it 'has labels associated to records' do - expect(saved_project_json['issues'].first['label_links'].first['label']).not_to be_empty - end + context 'with ci_pipelines' do + let(:relation_name) { :ci_pipelines } + + it { is_expected.not_to be_empty } + + it 'has pipeline stages' do + expect(subject.dig(0, 'stages')).not_to be_empty + end + + it 'has pipeline statuses' do + expect(subject.dig(0, 'stages', 0, 'statuses')).not_to be_empty + end - it 'has project and group labels' do - label_types = saved_project_json['issues'].first['label_links'].map { |link| link['label']['type'] } + it 'has pipeline builds' do + builds_count = subject.dig(0, 'stages', 0, 'statuses') + .count { |hash| hash['type'] == 'Ci::Build' } - expect(label_types).to match_array(%w(ProjectLabel GroupLabel)) + expect(builds_count).to eq(1) + end + + it 'has ci pipeline notes' do + expect(subject.first['notes']).not_to be_empty + end end - it 'has priorities associated to labels' do - priorities = saved_project_json['issues'].first['label_links'].flat_map { |link| link['label']['priorities'] } + context 'with labels' do + let(:relation_name) { :labels } - expect(priorities).not_to be_empty + it { is_expected.not_to be_empty } end - it 'has issue resource label events' do - expect(saved_project_json['issues'].first['resource_label_events']).not_to be_empty + context 'with services' do + let(:relation_name) { :services } + + it 'saves the correct service type' do + expect(subject.first['type']).to eq('CustomIssueTrackerService') + end + + it 'saves the properties for a service' do + expect(subject.first['properties']).to eq('one' => 'value') + end end - it 'has merge request resource label events' do - expect(saved_project_json['merge_requests'].first['resource_label_events']).not_to be_empty + context 'with project_feature' do + let(:relation_name) { :project_feature } + + it { is_expected.not_to be_empty } + + it 'has project feature' do + expect(subject["issues_access_level"]).to eq(ProjectFeature::DISABLED) + expect(subject["wiki_access_level"]).to eq(ProjectFeature::ENABLED) + expect(subject["builds_access_level"]).to eq(ProjectFeature::PRIVATE) + end end - it 'saves the correct service type' do - expect(saved_project_json['services'].first['type']).to eq('CustomIssueTrackerService') + context 'with custom_attributes' do + let(:relation_name) { :custom_attributes } + + it 'has custom attributes' do + expect(subject.count).to eq(2) + end end - it 'saves the properties for a service' do - expect(saved_project_json['services'].first['properties']).to eq('one' => 'value') + context 'with badges' do + let(:relation_name) { :custom_attributes } + + it 'has badges' do + expect(subject.count).to eq(2) + end end - it 'has project feature' do - project_feature = saved_project_json['project_feature'] - expect(project_feature).not_to be_empty - expect(project_feature["issues_access_level"]).to eq(ProjectFeature::DISABLED) - expect(project_feature["wiki_access_level"]).to eq(ProjectFeature::ENABLED) - expect(project_feature["builds_access_level"]).to eq(ProjectFeature::PRIVATE) + context 'with project_members' do + let(:relation_name) { :project_members } + + it { is_expected.not_to be_empty } end - it 'has custom attributes' do - expect(saved_project_json['custom_attributes'].count).to eq(2) + context 'with boards' do + let(:relation_name) { :boards } + + it { is_expected.not_to be_empty } end + end - it 'has badges' do - expect(saved_project_json['project_badges'].count).to eq(2) + describe '#saves project tree' do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let(:project) { setup_project } + let(:full_path) do + if ndjson_enabled + File.join(shared.export_path, 'tree') + else + File.join(shared.export_path, Gitlab::ImportExport.project_filename) + end end + let(:shared) { project.import_export_shared } + let(:params) { {} } - it 'does not complain about non UTF-8 characters in MR diff files' do - ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") + let(:project_tree_saver ) { described_class.new(project: project, current_user: user, shared: shared, params: params) } - expect(project_tree_saver.save).to be true + before do + stub_feature_flags(project_export_as_ndjson: ndjson_enabled) + project.add_maintainer(user) + + FileUtils.rm_rf(export_path) end - context 'group members' do + after do + FileUtils.rm_rf(export_path) + end + + context 'overrides group members' do let(:user2) { create(:user, email: 'group@member.com') } + let(:relation_name) { :project_members } + let(:member_emails) do - saved_project_json['project_members'].map do |pm| + emails = subject.map do |pm| pm['user']['email'] end + emails end before do - Group.first.add_developer(user2) + group.add_developer(user2) end - it 'does not export group members if it has no permission' do - Group.first.add_developer(user) + context 'when has no permission' do + before do + group.add_developer(user) + project_tree_saver.save + end - expect(member_emails).not_to include('group@member.com') + it 'does not export group members' do + expect(member_emails).not_to include('group@member.com') + end end - it 'does not export group members as maintainer' do - Group.first.add_maintainer(user) + context 'when has permission as maintainer' do + before do + group.add_maintainer(user) - expect(member_emails).not_to include('group@member.com') + project_tree_saver.save + end + + it 'does not export group members' do + expect(member_emails).not_to include('group@member.com') + end end - it 'exports group members as group owner' do - Group.first.add_owner(user) + context 'when has permission as group owner' do + before do + group.add_owner(user) - expect(member_emails).to include('group@member.com') + project_tree_saver.save + end + + it 'exports group members as group owner' do + expect(member_emails).to include('group@member.com') + end end context 'as admin' do let(:user) { create(:admin) } + before do + project_tree_saver.save + end + it 'exports group members as admin' do expect(member_emails).to include('group@member.com') end it 'exports group members as project members' do - member_types = saved_project_json['project_members'].map { |pm| pm['source_type'] } + member_types = subject.map { |pm| pm['source_type'] } expect(member_types).to all(eq('Project')) end end end - context 'project attributes' do - it 'does not contain the runners token' do - expect(saved_project_json).not_to include("runners_token" => 'token') + context 'with description override' do + let(:params) { { description: 'Foo Bar' } } + let(:relation_name) { :projects } + + before do + project_tree_saver.save end + + it { is_expected.to include({ 'description' => params[:description] }) } + end + + it 'saves project successfully' do + expect(project_tree_saver.save).to be true + end + + it 'does not complain about non UTF-8 characters in MR diff files' do + ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") + + expect(project_tree_saver.save).to be true end - it 'has a board and a list' do - expect(saved_project_json['boards'].first['lists']).not_to be_empty + it 'has no when YML attributes but only the DB column' do + expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes) + + project_tree_saver.save end end end + context 'with JSON' do + it_behaves_like "saves project tree successfully", false + end + + context 'with NDJSON' do + it_behaves_like "saves project tree successfully", true + end + def setup_project release = create(:release) - group = create(:group) project = create(:project, - :public, - :repository, - :issues_disabled, - :wiki_enabled, - :builds_private, - description: 'description', - releases: [release], - group: group, - approvals_before_merge: 1 - ) - allow(project).to receive(:commit).and_return(Commit.new(RepoHelpers.sample_commit, project)) + :public, + :repository, + :issues_disabled, + :wiki_enabled, + :builds_private, + description: 'description', + releases: [release], + group: group, + approvals_before_merge: 1) issue = create(:issue, assignees: [user], project: project) snippet = create(:project_snippet, project: project) @@ -331,9 +414,9 @@ describe Gitlab::ImportExport::Project::TreeSaver do mr_note = create(:note, noteable: merge_request, project: project) create(:note, noteable: snippet, project: project) create(:note_on_commit, - author: user, - project: project, - commit_id: ci_build.pipeline.sha) + author: user, + project: project, + commit_id: ci_build.pipeline.sha) create(:system_note_metadata, action: 'description', note: discussion_note) create(:system_note_metadata, commit_count: 1, action: 'commit', note: mr_note) @@ -355,8 +438,4 @@ describe Gitlab::ImportExport::Project::TreeSaver do project end - - def project_json(filename) - ::JSON.parse(IO.read(filename)) - end end diff --git a/spec/lib/gitlab/sidekiq_middleware_spec.rb b/spec/lib/gitlab/sidekiq_middleware_spec.rb index 88f83ebc2ac..32c1807ba6e 100644 --- a/spec/lib/gitlab/sidekiq_middleware_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware_spec.rb @@ -134,12 +134,12 @@ describe Gitlab::SidekiqMiddleware do let(:middleware_expected_args) { [worker_class_arg, job, queue, redis_pool] } let(:expected_middlewares) do [ - Gitlab::SidekiqStatus::ClientMiddleware, - Gitlab::SidekiqMiddleware::ClientMetrics, - Gitlab::SidekiqMiddleware::WorkerContext::Client, - Labkit::Middleware::Sidekiq::Client, - Gitlab::SidekiqMiddleware::AdminMode::Client, - Gitlab::SidekiqMiddleware::DuplicateJobs::Client + ::Gitlab::SidekiqMiddleware::WorkerContext::Client, + ::Labkit::Middleware::Sidekiq::Client, + ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client, + ::Gitlab::SidekiqStatus::ClientMiddleware, + ::Gitlab::SidekiqMiddleware::AdminMode::Client, + ::Gitlab::SidekiqMiddleware::ClientMetrics ] end diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 137795dcbc3..bfad1da1e21 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -3701,4 +3701,18 @@ describe MergeRequest do end end end + + describe '#predefined_variables' do + let(:merge_request) { create(:merge_request) } + + it 'caches all SQL-sourced data on the first call' do + control = ActiveRecord::QueryRecorder.new { merge_request.predefined_variables }.count + + expect(control).to be > 0 + + count = ActiveRecord::QueryRecorder.new { merge_request.predefined_variables }.count + + expect(count).to eq(0) + end + end end diff --git a/spec/services/projects/hashed_storage/migration_service_spec.rb b/spec/services/projects/hashed_storage/migration_service_spec.rb index f3ac26e7761..0a7975305dc 100644 --- a/spec/services/projects/hashed_storage/migration_service_spec.rb +++ b/spec/services/projects/hashed_storage/migration_service_spec.rb @@ -5,6 +5,11 @@ require 'spec_helper' describe Projects::HashedStorage::MigrationService do let(:project) { create(:project, :empty_repo, :wiki_repo, :legacy_storage) } let(:logger) { double } + let!(:project_attachment) { build(:file_uploader, project: project) } + let(:project_hashed_path) { Storage::Hashed.new(project).disk_path } + let(:project_legacy_path) { Storage::LegacyProject.new(project).disk_path } + let(:wiki_hashed_path) { "#{project_hashed_path}.wiki" } + let(:wiki_legacy_path) { "#{project_legacy_path}.wiki" } subject(:service) { described_class.new(project, project.full_path, logger: logger) } @@ -29,9 +34,24 @@ describe Projects::HashedStorage::MigrationService do service.execute end + + it 'migrates legacy repositories to hashed storage' do + legacy_attachments_path = FileUploader.absolute_base_dir(project) + hashed_project = project.dup.tap { |p| p.id = project.id } + hashed_project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments] + hashed_attachments_path = FileUploader.absolute_base_dir(hashed_project) + + expect(logger).to receive(:info).with(/Repository moved from '#{project_legacy_path}' to '#{project_hashed_path}'/) + expect(logger).to receive(:info).with(/Repository moved from '#{wiki_legacy_path}' to '#{wiki_hashed_path}'/) + expect(logger).to receive(:info).with(/Project attachments moved from '#{legacy_attachments_path}' to '#{hashed_attachments_path}'/) + + expect { service.execute }.to change { project.storage_version }.from(nil).to(2) + end end context 'attachments migration' do + let(:project) { create(:project, :empty_repo, :wiki_repo, storage_version: ::Project::HASHED_STORAGE_FEATURES[:repository]) } + let(:attachments_service) do Projects::HashedStorage::MigrateAttachmentsService.new(project: project, old_disk_path: project.full_path, @@ -51,6 +71,17 @@ describe Projects::HashedStorage::MigrationService do service.execute end + + it 'migrates legacy attachments to hashed storage' do + legacy_attachments_path = FileUploader.absolute_base_dir(project) + hashed_project = project.dup.tap { |p| p.id = project.id } + hashed_project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments] + hashed_attachments_path = FileUploader.absolute_base_dir(hashed_project) + + expect(logger).to receive(:info).with(/Project attachments moved from '#{legacy_attachments_path}' to '#{hashed_attachments_path}'/) + + expect { service.execute }.to change { project.storage_version }.from(1).to(2) + end end end end diff --git a/spec/services/projects/hashed_storage/rollback_service_spec.rb b/spec/services/projects/hashed_storage/rollback_service_spec.rb index 48d4eac9eb7..e6b7daba99e 100644 --- a/spec/services/projects/hashed_storage/rollback_service_spec.rb +++ b/spec/services/projects/hashed_storage/rollback_service_spec.rb @@ -5,6 +5,11 @@ require 'spec_helper' describe Projects::HashedStorage::RollbackService do let(:project) { create(:project, :empty_repo, :wiki_repo) } let(:logger) { double } + let!(:project_attachment) { build(:file_uploader, project: project) } + let(:project_hashed_path) { Storage::Hashed.new(project).disk_path } + let(:project_legacy_path) { Storage::LegacyProject.new(project).disk_path } + let(:wiki_hashed_path) { "#{project_hashed_path}.wiki" } + let(:wiki_legacy_path) { "#{project_legacy_path}.wiki" } subject(:service) { described_class.new(project, project.disk_path, logger: logger) } @@ -26,6 +31,20 @@ describe Projects::HashedStorage::RollbackService do service.execute end + + it 'rollbacks to legacy storage' do + hashed_attachments_path = FileUploader.absolute_base_dir(project) + legacy_project = project.dup + legacy_project.storage_version = nil + legacy_attachments_path = FileUploader.absolute_base_dir(legacy_project) + + expect(logger).to receive(:info).with(/Project attachments moved from '#{hashed_attachments_path}' to '#{legacy_attachments_path}'/) + + expect(logger).to receive(:info).with(/Repository moved from '#{project_hashed_path}' to '#{project_legacy_path}'/) + expect(logger).to receive(:info).with(/Repository moved from '#{wiki_hashed_path}' to '#{wiki_legacy_path}'/) + + expect { service.execute }.to change { project.storage_version }.from(2).to(nil) + end end context 'repository rollback' do @@ -47,6 +66,13 @@ describe Projects::HashedStorage::RollbackService do service.execute end + + it 'rollbacks to legacy storage' do + expect(logger).to receive(:info).with(/Repository moved from '#{project_hashed_path}' to '#{project_legacy_path}'/) + expect(logger).to receive(:info).with(/Repository moved from '#{wiki_hashed_path}' to '#{wiki_legacy_path}'/) + + expect { service.execute }.to change { project.storage_version }.from(1).to(nil) + end end end end diff --git a/spec/services/web_hook_service_spec.rb b/spec/services/web_hook_service_spec.rb index d8f13bc2e61..4a917ecdbb5 100644 --- a/spec/services/web_hook_service_spec.rb +++ b/spec/services/web_hook_service_spec.rb @@ -71,16 +71,6 @@ describe WebHookService do end end - it 'POSTs to the webhook URL' do - stub_full_request(project_hook.url, method: :post) - - service_instance.execute - - expect(WebMock).to have_requested(:post, stubbed_hostname(project_hook.url)).with( - headers: headers - ).once - end - it 'POSTs the data as JSON' do stub_full_request(project_hook.url, method: :post) diff --git a/spec/support/import_export/common_util.rb b/spec/support/import_export/common_util.rb index 9281937e4ba..efe14b7244c 100644 --- a/spec/support/import_export/common_util.rb +++ b/spec/support/import_export/common_util.rb @@ -26,6 +26,21 @@ module ImportExport "tmp/tests/gitlab-test/import_export" end + def get_json(path, exportable_path, key, ndjson_enabled) + if ndjson_enabled + json = if key == :projects + consume_attributes(path, exportable_path) + else + consume_relations(path, exportable_path, key) + end + else + json = project_json(path) + json = json[key.to_s] unless key == :projects + end + + json + end + def restore_then_save_project(project, import_path:, export_path:) project_restorer = get_project_restorer(project, import_path) project_saver = get_project_saver(project, export_path) @@ -50,5 +65,30 @@ module ImportExport allow(shared).to receive(:export_path).and_return(path) end end + + def consume_attributes(dir_path, exportable_path) + path = File.join(dir_path, "#{exportable_path}.json") + return unless File.exist?(path) + + ActiveSupport::JSON.decode(IO.read(path)) + end + + def consume_relations(dir_path, exportable_path, key) + path = File.join(dir_path, exportable_path, "#{key}.ndjson") + return unless File.exist?(path) + + relations = [] + + File.foreach(path) do |line| + json = ActiveSupport::JSON.decode(line) + relations << json + end + + key == :project_feature ? relations.first : relations.flatten + end + + def project_json(filename) + ActiveSupport::JSON.decode(IO.read(filename)) + end end end diff --git a/spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/import_export/project/tree_restorer_shared_examples.rb index 5f9316c1cde..5f9316c1cde 100644 --- a/spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb +++ b/spec/support/shared_examples/lib/gitlab/import_export/project/tree_restorer_shared_examples.rb |