diff options
author | John Jarvis <jarv@gitlab.com> | 2019-04-12 12:34:08 +0000 |
---|---|---|
committer | John Jarvis <jarv@gitlab.com> | 2019-04-12 12:34:08 +0000 |
commit | 2d66ee481c4b0ec58c06280e1acf20699c2fe643 (patch) | |
tree | a13d8190a8a509e317d3175abdf6bc60e6e1fafb | |
parent | 520fbcc62d94b2ef7ab12403346d477c20e43f38 (diff) | |
parent | cd51399a30d8759bf9e657123a27de431298f045 (diff) | |
download | gitlab-ce-2d66ee481c4b0ec58c06280e1acf20699c2fe643.tar.gz |
Merge branch '11-10-stable-prepare-rc7-pick-27205' into '11-10-stable-prepare-rc7'
Backport: Process at most 4 pipelines during push
See merge request gitlab-org/gitlab-ce!27308
-rw-r--r-- | app/services/git/branch_push_service.rb | 6 | ||||
-rw-r--r-- | app/services/git/tag_push_service.rb | 6 | ||||
-rw-r--r-- | app/workers/post_receive.rb | 27 | ||||
-rw-r--r-- | changelogs/unreleased/limit-amount-of-created-pipelines.yml | 5 | ||||
-rw-r--r-- | doc/ci/yaml/README.md | 9 | ||||
-rw-r--r-- | spec/workers/post_receive_spec.rb | 43 |
6 files changed, 79 insertions, 17 deletions
diff --git a/app/services/git/branch_push_service.rb b/app/services/git/branch_push_service.rb index b55aeb5f2b9..a5113300f5d 100644 --- a/app/services/git/branch_push_service.rb +++ b/app/services/git/branch_push_service.rb @@ -142,7 +142,11 @@ module Git .perform_async(project.id, current_user.id, params[:oldrev], params[:newrev], params[:ref]) EventCreateService.new.push(project, current_user, build_push_data) - Ci::CreatePipelineService.new(project, current_user, build_push_data).execute(:push, pipeline_options) + + if params.fetch(:create_pipelines, true) + Ci::CreatePipelineService.new(project, current_user, build_push_data) + .execute(:push, pipeline_options) + end project.execute_hooks(build_push_data.dup, :push_hooks) project.execute_services(build_push_data.dup, :push_hooks) diff --git a/app/services/git/tag_push_service.rb b/app/services/git/tag_push_service.rb index 9ce0fbdb206..ddf1a2d04fc 100644 --- a/app/services/git/tag_push_service.rb +++ b/app/services/git/tag_push_service.rb @@ -11,7 +11,11 @@ module Git @push_data = build_push_data EventCreateService.new.push(project, current_user, push_data) - Ci::CreatePipelineService.new(project, current_user, push_data).execute(:push, pipeline_options) + + if params.fetch(:create_pipelines, true) + Ci::CreatePipelineService.new(project, current_user, push_data) + .execute(:push, pipeline_options) + end project.execute_hooks(push_data.dup, :tag_push_hooks) project.execute_services(push_data.dup, :tag_push_hooks) diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index a5554f07699..337efa7919b 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -3,6 +3,8 @@ class PostReceive include ApplicationWorker + PIPELINE_PROCESS_LIMIT = 4 + def perform(gl_repository, identifier, changes, push_options = {}) project, repo_type = Gitlab::GlRepository.parse(gl_repository) @@ -36,23 +38,24 @@ class PostReceive return false end - post_received.changes_refs do |oldrev, newrev, ref| - if Gitlab::Git.tag_ref?(ref) - Git::TagPushService.new( - post_received.project, - @user, - oldrev: oldrev, - newrev: newrev, - ref: ref, - push_options: post_received.push_options).execute - elsif Gitlab::Git.branch_ref?(ref) - Git::BranchPushService.new( + post_received.enum_for(:changes_refs).with_index do |(oldrev, newrev, ref), index| + service_klass = + if Gitlab::Git.tag_ref?(ref) + Git::TagPushService + elsif Gitlab::Git.branch_ref?(ref) + Git::BranchPushService + end + + if service_klass + service_klass.new( post_received.project, @user, oldrev: oldrev, newrev: newrev, ref: ref, - push_options: post_received.push_options).execute + push_options: post_received.push_options, + create_pipelines: index < PIPELINE_PROCESS_LIMIT || Feature.enabled?(:git_push_create_all_pipelines, post_received.project) + ).execute end changes << Gitlab::DataBuilder::Repository.single_change(oldrev, newrev, ref) diff --git a/changelogs/unreleased/limit-amount-of-created-pipelines.yml b/changelogs/unreleased/limit-amount-of-created-pipelines.yml new file mode 100644 index 00000000000..51fdbb4d7ff --- /dev/null +++ b/changelogs/unreleased/limit-amount-of-created-pipelines.yml @@ -0,0 +1,5 @@ +--- +title: Process at most 4 pipelines during push +merge_request: 27205 +author: +type: performance diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index cf2189cd00a..d3cef5d622b 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -2697,6 +2697,15 @@ Not to be confused with [`trigger`](#trigger-premium). [Read more in the triggers documentation.](../triggers/README.md) +## Processing Git pushes + +GitLab will create at most 4 branch and tags pipelines when +doing pushing multiple changes in single `git push` invocation. + +This limitation does not affect any of the updated Merge Request pipelines, +all updated Merge Requests will have a pipeline created when using +[pipelines for merge requests](../merge_request_pipelines/index.md). + ## Skipping jobs If your commit message contains `[ci skip]` or `[skip ci]`, using any diff --git a/spec/workers/post_receive_spec.rb b/spec/workers/post_receive_spec.rb index 66958a4c116..7c39877be06 100644 --- a/spec/workers/post_receive_spec.rb +++ b/spec/workers/post_receive_spec.rb @@ -90,11 +90,22 @@ describe PostReceive do end context "gitlab-ci.yml" do - let(:changes) { "123456 789012 refs/heads/feature\n654321 210987 refs/tags/tag" } + let(:changes) do + <<-EOF.strip_heredoc + 123456 789012 refs/heads/feature + 654321 210987 refs/tags/tag + 123456 789012 refs/heads/feature2 + 123458 789013 refs/heads/feature3 + 123459 789015 refs/heads/feature4 + 654321 210987 refs/tags/tag2 + EOF + end + + let(:changes_count) { changes.lines.count } subject { described_class.new.perform(gl_repository, key_id, base64_changes) } - context "creates a Ci::Pipeline for every change" do + context "with valid .gitlab-ci.yml" do before do stub_ci_pipeline_to_return_yaml_file @@ -107,7 +118,33 @@ describe PostReceive do .and_return(true) end - it { expect { subject }.to change { Ci::Pipeline.count }.by(2) } + context 'when git_push_create_all_pipelines is disabled' do + before do + stub_feature_flags(git_push_create_all_pipelines: false) + end + + it "creates pipeline for branches and tags" do + subject + + expect(Ci::Pipeline.pluck(:ref)).to contain_exactly("feature", "tag", "feature2", "feature3") + end + + it "creates exactly #{described_class::PIPELINE_PROCESS_LIMIT} pipelines" do + expect(changes_count).to be > described_class::PIPELINE_PROCESS_LIMIT + + expect { subject }.to change { Ci::Pipeline.count }.by(described_class::PIPELINE_PROCESS_LIMIT) + end + end + + context 'when git_push_create_all_pipelines is enabled' do + before do + stub_feature_flags(git_push_create_all_pipelines: true) + end + + it "creates all pipelines" do + expect { subject }.to change { Ci::Pipeline.count }.by(changes_count) + end + end end context "does not create a Ci::Pipeline" do |