diff options
45 files changed, 468 insertions, 130 deletions
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 5d4bbc06e93..fe369ffec13 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -138,9 +138,8 @@ karma: - chrome_debug.log - coverage-javascript/ - tmp/tests/frontend/ -# see https://gitlab.com/gitlab-org/gitlab-ce/issues/64756 -# reports: -# junit: junit_karma.xml + reports: + junit: junit_karma.xml jest: extends: .dedicated-no-docs-and-no-qa-pull-cache-job @@ -163,9 +162,8 @@ jest: - coverage-frontend/ - junit_jest.xml - tmp/tests/frontend/ -# see https://gitlab.com/gitlab-org/gitlab-ce/issues/64756 -# reports: -# junit: junit_jest.xml + reports: + junit: junit_jest.xml cache: key: jest paths: diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index d0b1f1ab98f..1392768127b 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -80,9 +80,8 @@ - rspec_profiling/ - tmp/capybara/ - tmp/memory_test/ -# see https://gitlab.com/gitlab-org/gitlab-ce/issues/64756 -# reports: -# junit: junit_rspec.xml + reports: + junit: junit_rspec.xml .rspec-metadata-pg: &rspec-metadata-pg <<: *rspec-metadata diff --git a/app/assets/javascripts/pages/groups/merge_requests/index.js b/app/assets/javascripts/pages/groups/merge_requests/index.js index 12a26fd88fa..7520cfb6da0 100644 --- a/app/assets/javascripts/pages/groups/merge_requests/index.js +++ b/app/assets/javascripts/pages/groups/merge_requests/index.js @@ -1,11 +1,15 @@ import projectSelect from '~/project_select'; import initFilteredSearch from '~/pages/search/init_filtered_search'; +import issuableInitBulkUpdateSidebar from '~/issuable_init_bulk_update_sidebar'; import IssuableFilteredSearchTokenKeys from '~/filtered_search/issuable_filtered_search_token_keys'; import addExtraTokensForMergeRequests from 'ee_else_ce/filtered_search/add_extra_tokens_for_merge_requests'; import { FILTERED_SEARCH } from '~/pages/constants'; +const ISSUABLE_BULK_UPDATE_PREFIX = 'merge_request_'; + document.addEventListener('DOMContentLoaded', () => { addExtraTokensForMergeRequests(IssuableFilteredSearchTokenKeys); + issuableInitBulkUpdateSidebar.init(ISSUABLE_BULK_UPDATE_PREFIX); initFilteredSearch({ page: FILTERED_SEARCH.MERGE_REQUESTS, diff --git a/app/mailers/notify.rb b/app/mailers/notify.rb index 576caea4c10..8ef20a03541 100644 --- a/app/mailers/notify.rb +++ b/app/mailers/notify.rb @@ -78,17 +78,7 @@ class Notify < BaseMailer # # Returns a String containing the User's email address. def recipient(recipient_id, notification_group = nil) - @current_user = User.find(recipient_id) - group_notification_email = nil - - if notification_group - notification_settings = notification_group.notification_settings_for(@current_user, hierarchy_order: :asc) - group_notification_email = notification_settings.find { |n| n.notification_email.present? }&.notification_email - end - - # Return group-specific email address if present, otherwise return global - # email address - group_notification_email || @current_user.notification_email + User.find(recipient_id).notification_email_for(notification_group) end # Formats arguments into a String suitable for use as an email subject diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 635fcc86166..da70cb9a9a7 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -531,6 +531,14 @@ module Ci trace.exist? end + def has_live_trace? + trace.live_trace_exist? + end + + def has_archived_trace? + trace.archived_trace_exist? + end + def artifacts_file job_artifacts_archive&.file end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index f80e98e5bca..e132cb045e2 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -176,6 +176,10 @@ module Ci end end + def self.archived_trace_exists_for?(job_id) + where(job_id: job_id).trace.take&.file&.file&.exists? + end + private def file_format_adapter_class diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 07d00503861..43ff874ac23 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -264,7 +264,7 @@ module Ci private def cleanup_runner_queue - Gitlab::Redis::Queues.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.del(runner_queue_key) end end diff --git a/app/models/group.rb b/app/models/group.rb index 37f30552b39..26ce2957e9b 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -144,6 +144,12 @@ class Group < Namespace notification_settings(hierarchy_order: hierarchy_order).where(user: user) end + def notification_email_for(user) + # Finds the closest notification_setting with a `notification_email` + notification_settings = notification_settings_for(user, hierarchy_order: :asc) + notification_settings.find { |n| n.notification_email.present? }&.notification_email + end + def to_reference(_from = nil, full: nil) "#{self.class.reference_prefix}#{full_path}" end diff --git a/app/models/user.rb b/app/models/user.rb index 0fd3daa3383..b439d1c0c16 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1259,6 +1259,11 @@ class User < ApplicationRecord end end + def notification_email_for(notification_group) + # Return group-specific email address if present, otherwise return global notification email address + notification_group&.notification_email_for(self) || notification_email + end + def notification_settings_for(source) if notification_settings.loaded? notification_settings.find do |notification| diff --git a/app/presenters/blob_presenter.rb b/app/presenters/blob_presenter.rb index 91c9abe750b..2cf3278d240 100644 --- a/app/presenters/blob_presenter.rb +++ b/app/presenters/blob_presenter.rb @@ -4,7 +4,7 @@ class BlobPresenter < Gitlab::View::Presenter::Delegated presents :blob def highlight(plain: nil) - blob.load_all_data! if blob.respond_to?(:load_all_data!) + load_all_blob_data Gitlab::Highlight.highlight( blob.path, @@ -17,4 +17,10 @@ class BlobPresenter < Gitlab::View::Presenter::Delegated def web_url Gitlab::Routing.url_helpers.project_blob_url(blob.repository.project, File.join(blob.commit_id, blob.path)) end + + private + + def load_all_blob_data + blob.load_all_data! if blob.respond_to?(:load_all_data!) + end end diff --git a/app/presenters/blobs/unfold_presenter.rb b/app/presenters/blobs/unfold_presenter.rb index 7b13db3bb74..21a1e1309e0 100644 --- a/app/presenters/blobs/unfold_presenter.rb +++ b/app/presenters/blobs/unfold_presenter.rb @@ -16,8 +16,12 @@ module Blobs attribute :indent, Integer, default: 0 def initialize(blob, params) + # Load all blob data first as we need to ensure they're all loaded first + # so we can accurately show the rest of the diff when unfolding. + load_all_blob_data + @subject = blob - @all_lines = highlight.lines + @all_lines = blob.data.lines super(params) if full? @@ -25,10 +29,12 @@ module Blobs end end - # Converts a String array to Gitlab::Diff::Line array, with match line added + # Returns an array of Gitlab::Diff::Line with match line added def diff_lines - diff_lines = lines.map do |line| - Gitlab::Diff::Line.new(line, nil, nil, nil, nil, rich_text: line) + diff_lines = lines.map.with_index do |line, index| + full_line = limited_blob_lines[index].delete("\n") + + Gitlab::Diff::Line.new(full_line, nil, nil, nil, nil, rich_text: line) end add_match_line(diff_lines) @@ -37,11 +43,7 @@ module Blobs end def lines - strong_memoize(:lines) do - lines = @all_lines - lines = lines[since - 1..to - 1] unless full? - lines.map(&:html_safe) - end + @lines ||= limit(highlight.lines).map(&:html_safe) end def match_line_text @@ -71,5 +73,15 @@ module Blobs bottom? ? diff_lines.push(match_line) : diff_lines.unshift(match_line) end + + def limited_blob_lines + @limited_blob_lines ||= limit(@all_lines) + end + + def limit(lines) + return lines if full? + + lines[since - 1..to - 1] + end end end diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb index b5cfa1d019c..700d78361a4 100644 --- a/app/services/ci/archive_trace_service.rb +++ b/app/services/ci/archive_trace_service.rb @@ -2,8 +2,25 @@ module Ci class ArchiveTraceService - def execute(job) + def execute(job, worker_name:) + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless job.has_live_trace? + Sidekiq.logger.warn(class: worker_name, + message: 'The job does not have live trace but going to be archived.', + job_id: job.id) + return + end + job.trace.archive! + + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless job.has_archived_trace? + Sidekiq.logger.warn(class: worker_name, + message: 'The job does not have archived trace after archiving.', + job_id: job.id) + end rescue ::Gitlab::Ci::Trace::AlreadyArchivedError # It's already archived, thus we can safely ignore this exception. rescue => e @@ -11,7 +28,7 @@ module Ci # If `archive!` keeps failing for over a week, that could incur data loss. # (See more https://docs.gitlab.com/ee/administration/job_traces.html#new-live-trace-architecture) # In order to avoid interrupting the system, we do not raise an exception here. - archive_error(e, job) + archive_error(e, job, worker_name) end private @@ -22,9 +39,12 @@ module Ci "Counter of failed attempts of trace archiving") end - def archive_error(error, job) + def archive_error(error, job, worker_name) failed_archive_counter.increment - Rails.logger.error "Failed to archive trace. id: #{job.id} message: #{error.message}" # rubocop:disable Gitlab/RailsLogger + + Sidekiq.logger.warn(class: worker_name, + message: "Failed to archive trace. message: #{error.message}.", + job_id: job.id) Gitlab::Sentry .track_exception(error, diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index 5aa804666f0..a55771ed538 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -418,7 +418,9 @@ class NotificationService [pipeline.user], :watch, custom_action: :"#{pipeline.status}_pipeline", target: pipeline - ).map(&:notification_email) + ).map do |user| + user.notification_email_for(pipeline.project.group) + end if recipients.any? mailer.public_send(email_template, pipeline, recipients).deliver_later diff --git a/app/views/groups/issues.html.haml b/app/views/groups/issues.html.haml index f05e269553a..2163446425c 100644 --- a/app/views/groups/issues.html.haml +++ b/app/views/groups/issues.html.haml @@ -13,7 +13,7 @@ = render 'shared/issuable/feed_buttons' - if @can_bulk_update - = render_if_exists 'shared/issuable/bulk_update_button' + = render_if_exists 'shared/issuable/bulk_update_button', type: :issues = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", type: :issues, with_feature_enabled: 'issues', with_shared: false, include_projects_in_subgroups: true diff --git a/app/views/groups/merge_requests.html.haml b/app/views/groups/merge_requests.html.haml index 808bb1309b1..b5a2bab4799 100644 --- a/app/views/groups/merge_requests.html.haml +++ b/app/views/groups/merge_requests.html.haml @@ -1,3 +1,5 @@ +- @can_bulk_update = can?(current_user, :admin_merge_request, @group) + - page_title "Merge Requests" - if group_merge_requests_count(state: 'all').zero? @@ -7,8 +9,14 @@ = render 'shared/issuable/nav', type: :merge_requests - if current_user .nav-controls + - if @can_bulk_update + = render_if_exists 'shared/issuable/bulk_update_button', type: :merge_requests + = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", type: :merge_requests, with_feature_enabled: 'merge_requests', with_shared: false, include_projects_in_subgroups: true = render 'shared/issuable/search_bar', type: :merge_requests + - if @can_bulk_update + = render_if_exists 'shared/issuable/group_bulk_update_sidebar', group: @group, type: :merge_requests + = render 'shared/merge_requests' diff --git a/app/workers/archive_trace_worker.rb b/app/workers/archive_trace_worker.rb index 4a9becf0ca7..66f9b8d9e80 100644 --- a/app/workers/archive_trace_worker.rb +++ b/app/workers/archive_trace_worker.rb @@ -7,7 +7,7 @@ class ArchiveTraceWorker # rubocop: disable CodeReuse/ActiveRecord def perform(job_id) Ci::Build.without_archived_trace.find_by(id: job_id).try do |job| - Ci::ArchiveTraceService.new.execute(job) + Ci::ArchiveTraceService.new.execute(job, worker_name: self.class.name) end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/workers/ci/archive_traces_cron_worker.rb b/app/workers/ci/archive_traces_cron_worker.rb index f65ff239866..75e68d0233a 100644 --- a/app/workers/ci/archive_traces_cron_worker.rb +++ b/app/workers/ci/archive_traces_cron_worker.rb @@ -11,7 +11,7 @@ module Ci # This could happen when ArchiveTraceWorker sidekiq jobs were lost by receiving SIGKILL # More details in https://gitlab.com/gitlab-org/gitlab-ce/issues/36791 Ci::Build.finished.with_live_trace.find_each(batch_size: 100) do |build| - Ci::ArchiveTraceService.new.execute(build) + Ci::ArchiveTraceService.new.execute(build, worker_name: self.class.name) end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/changelogs/unreleased/57953-fix-unfolded-diff-suggestions.yml b/changelogs/unreleased/57953-fix-unfolded-diff-suggestions.yml new file mode 100644 index 00000000000..f634c0cd98a --- /dev/null +++ b/changelogs/unreleased/57953-fix-unfolded-diff-suggestions.yml @@ -0,0 +1,5 @@ +--- +title: Fix suggestion on lines that are not part of an MR +merge_request: 30606 +author: +type: fixed diff --git a/changelogs/unreleased/63485-fix-pipeline-emails-to-use-group-setting.yml b/changelogs/unreleased/63485-fix-pipeline-emails-to-use-group-setting.yml new file mode 100644 index 00000000000..c3ee3108216 --- /dev/null +++ b/changelogs/unreleased/63485-fix-pipeline-emails-to-use-group-setting.yml @@ -0,0 +1,5 @@ +--- +title: Fix pipeline emails not respecting group notification email setting +merge_request: 30907 +author: +type: fixed diff --git a/changelogs/unreleased/64974-remove-livesum-from-ruby-sampler-metrics.yml b/changelogs/unreleased/64974-remove-livesum-from-ruby-sampler-metrics.yml new file mode 100644 index 00000000000..4fa3b7783c5 --- /dev/null +++ b/changelogs/unreleased/64974-remove-livesum-from-ruby-sampler-metrics.yml @@ -0,0 +1,5 @@ +--- +title: Remove :livesum from RubySampler metrics +merge_request: 31047 +author: +type: fixed diff --git a/changelogs/unreleased/safe-archiving-for-traces.yml b/changelogs/unreleased/safe-archiving-for-traces.yml new file mode 100644 index 00000000000..2b9070bacfe --- /dev/null +++ b/changelogs/unreleased/safe-archiving-for-traces.yml @@ -0,0 +1,5 @@ +--- +title: Extra logging for new live trace architecture +merge_request: 30892 +author: +type: fixed diff --git a/changelogs/unreleased/sh-use-shared-state-cluster-pubsub.yml b/changelogs/unreleased/sh-use-shared-state-cluster-pubsub.yml new file mode 100644 index 00000000000..5e72f23d7ad --- /dev/null +++ b/changelogs/unreleased/sh-use-shared-state-cluster-pubsub.yml @@ -0,0 +1,5 @@ +--- +title: Use persistent Redis cluster for Workhorse pub/sub notifications +merge_request: 30990 +author: +type: fixed diff --git a/doc/user/group/bulk_editing/index.md b/doc/user/group/bulk_editing/index.md index 117a46da0ea..5b5f75c2dd9 100644 --- a/doc/user/group/bulk_editing/index.md +++ b/doc/user/group/bulk_editing/index.md @@ -1,22 +1,26 @@ -# Bulk editing issue milestones **(PREMIUM)** +# Bulk editing issue and merge request milestones **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/7249) in -[GitLab Ultimate](https://about.gitlab.com/pricing/) 12.1. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/7249) for issues in + [GitLab Premium](https://about.gitlab.com/pricing/) 12.1. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/12719) for merge + requests in GitLab [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. -NOTE: **Note:** -A permission level of `Reporter` or higher is required in order to manage issues. +> NOTE: **Note:** +> +> - A permission level of `Reporter` or higher is required in order to manage issues. +> - A permission level of `Developer` or higher is required in order to manage merge requests. -Milestones can be updated simultaneously across multiple issues by using the bulk editing feature. +Milestones can be updated simultaneously across multiple issues or merge requests by using the bulk editing feature.  -To bulk update group issue milestones: +To bulk update group issue or merge request milestones: -1. Navigate to the issues list. -1. Click **Edit issues**. +1. Navigate to the issues or merge requests list. +1. Click the **Edit issues** or **Edit merge requests** button. - This will open a sidebar on the right-hand side of your screen where an editable field for milestones will be displayed. - - Checkboxes will also appear beside each issue. + - Checkboxes will also appear beside each issue or merge request. 1. Check the checkbox beside each issue to be edited. 1. Select the desired milestone from the sidebar. 1. Click **Update all**. diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 7597105f36f..14c831fe671 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -78,9 +78,9 @@ Issues and merge requests are part of projects. For a given group, you can view [issues](../project/issues/index.md#issues-list) and [merge requests](../project/merge_requests/index.md#merge-requests-per-group) across all projects in that group, together in a single list view. -### Bulk editing issues +### Bulk editing issues and merge requests -For details, see [bulk editing issues](../group/bulk_editing/index.md). +For details, see [bulk editing issues and merge requests](../group/bulk_editing/index.md). ## Create a new group diff --git a/doc/user/project/bulk_editing.md b/doc/user/project/bulk_editing.md index 1783f81df3a..f4733620640 100644 --- a/doc/user/project/bulk_editing.md +++ b/doc/user/project/bulk_editing.md @@ -13,8 +13,8 @@ by using the bulk editing feature.  NOTE: **Note:** -Bulk editing of merge requests is only available at the project level. -For more details, see [bulk editing group issues](../group/bulk_editing/index.md). +Bulk editing issues and merge requests is also available at the group level. +For more details, see [bulk editing group issues and merge requests](../group/bulk_editing/index.md). To update multiple project issues or merge requests at the same time: diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index ce5857965bf..cb617080c76 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -63,7 +63,15 @@ module Gitlab end def exist? - trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present? + archived_trace_exist? || live_trace_exist? + end + + def archived_trace_exist? + trace_artifact&.exists? + end + + def live_trace_exist? + job.trace_chunks.any? || current_path.present? || old_trace.present? end def read @@ -167,7 +175,7 @@ module Gitlab def clone_file!(src_stream, temp_dir) FileUtils.mkdir_p(temp_dir) - Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path| + Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path| temp_path = File.join(dir_path, "job.log") FileUtils.touch(temp_path) size = IO.copy_stream(src_stream, temp_path) diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 8c6fd56493f..e99889f4a25 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -166,6 +166,13 @@ module Gitlab end def destroy! + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless build.has_archived_trace? + Sidekiq.logger.warn(message: 'The job does not have archived trace but going to be destroyed.', + job_id: build.id) + end + trace_chunks.fast_destroy_all @tell = @size = 0 ensure diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 79f756c8f8a..1e200db0baf 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -30,18 +30,18 @@ module Gitlab def init_metrics metrics = { - file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), - memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels), + memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels), process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), - process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels), process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) } GC.stat.keys.each do |key| - metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) + metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels) end metrics diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 46a7b5b982a..3b77fe838ae 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -221,7 +221,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis::Queues.with do |redis| + Gitlab::Redis::SharedState.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") diff --git a/spec/lib/gitlab/ci/trace/chunked_io_spec.rb b/spec/lib/gitlab/ci/trace/chunked_io_spec.rb index 546a9e7d0cc..3e5cd81f929 100644 --- a/spec/lib/gitlab/ci/trace/chunked_io_spec.rb +++ b/spec/lib/gitlab/ci/trace/chunked_io_spec.rb @@ -442,5 +442,15 @@ describe Gitlab::Ci::Trace::ChunkedIO, :clean_gitlab_redis_cache do expect(Ci::BuildTraceChunk.where(build: build).count).to eq(0) end + + context 'when the job does not have archived trace' do + it 'leaves a message in sidekiq log' do + expect(Sidekiq.logger).to receive(:warn).with( + message: 'The job does not have archived trace but going to be destroyed.', + job_id: build.id).and_call_original + + subject + end + end end end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index f8332757fcd..451e18ed91b 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -404,6 +404,7 @@ describe Gitlab::Workhorse do end it 'set and notify' do + expect(Gitlab::Redis::SharedState).to receive(:with).and_call_original expect_any_instance_of(::Redis).to receive(:publish) .with(described_class::NOTIFICATION_CHANNEL, "test-key=test-value") diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 78862de0657..c30cb70e1c1 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -692,6 +692,34 @@ describe Ci::Build do end end + describe '#has_live_trace?' do + subject { build.has_live_trace? } + + let(:build) { create(:ci_build, :trace_live) } + + it { is_expected.to be_truthy } + + context 'when build does not have live trace' do + let(:build) { create(:ci_build) } + + it { is_expected.to be_falsy } + end + end + + describe '#has_archived_trace?' do + subject { build.has_archived_trace? } + + let(:build) { create(:ci_build, :trace_artifact) } + + it { is_expected.to be_truthy } + + context 'when build does not have archived trace' do + let(:build) { create(:ci_build) } + + it { is_expected.to be_falsy } + end + end + describe '#has_job_artifacts?' do subject { build.has_job_artifacts? } diff --git a/spec/models/ci/job_artifact_spec.rb b/spec/models/ci/job_artifact_spec.rb index 1ba66565e03..1413da231e0 100644 --- a/spec/models/ci/job_artifact_spec.rb +++ b/spec/models/ci/job_artifact_spec.rb @@ -70,6 +70,31 @@ describe Ci::JobArtifact do end end + describe '.archived_trace_exists_for?' do + subject { described_class.archived_trace_exists_for?(job_id) } + + let!(:artifact) { create(:ci_job_artifact, :trace, job: job) } + let(:job) { create(:ci_build) } + + context 'when the specified job_id exists' do + let(:job_id) { job.id } + + it { is_expected.to be_truthy } + + context 'when the job does have archived trace' do + let!(:artifact) { } + + it { is_expected.to be_falsy } + end + end + + context 'when the specified job_id does not exist' do + let(:job_id) { 10000 } + + it { is_expected.to be_falsy } + end + end + describe 'callbacks' do subject { create(:ci_job_artifact, :archive) } diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index f735a89f69f..24ea059e871 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -554,7 +554,7 @@ describe Ci::Runner do end def expect_value_in_queues - Gitlab::Redis::Queues.with do |redis| + Gitlab::Redis::SharedState.with do |redis| runner_queue_key = runner.send(:runner_queue_key) expect(redis.get(runner_queue_key)) end @@ -627,7 +627,7 @@ describe Ci::Runner do end it 'cleans up the queue' do - Gitlab::Redis::Queues.with do |redis| + Gitlab::Redis::SharedState.with do |redis| expect(redis.get(queue_key)).to be_nil end end diff --git a/spec/models/group_spec.rb b/spec/models/group_spec.rb index c7fb0f51075..90e0900445e 100644 --- a/spec/models/group_spec.rb +++ b/spec/models/group_spec.rb @@ -95,6 +95,43 @@ describe Group do end end + describe '#notification_email_for' do + let(:user) { create(:user) } + let(:group) { create(:group) } + let(:subgroup) { create(:group, parent: group) } + + let(:group_notification_email) { 'user+group@example.com' } + let(:subgroup_notification_email) { 'user+subgroup@example.com' } + + subject { subgroup.notification_email_for(user) } + + context 'when both group notification emails are set' do + it 'returns subgroup notification email' do + create(:notification_setting, user: user, source: group, notification_email: group_notification_email) + create(:notification_setting, user: user, source: subgroup, notification_email: subgroup_notification_email) + + is_expected.to eq(subgroup_notification_email) + end + end + + context 'when subgroup notification email is blank' do + it 'returns parent group notification email' do + create(:notification_setting, user: user, source: group, notification_email: group_notification_email) + create(:notification_setting, user: user, source: subgroup, notification_email: '') + + is_expected.to eq(group_notification_email) + end + end + + context 'when only the parent group notification email is set' do + it 'returns parent group notification email' do + create(:notification_setting, user: user, source: group, notification_email: group_notification_email) + + is_expected.to eq(group_notification_email) + end + end + end + describe '#visibility_level_allowed_by_parent' do let(:parent) { create(:group, :internal) } let(:sub_group) { build(:group, parent_id: parent.id) } diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 5cfa64fd764..2d20f8c78cc 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -3504,4 +3504,37 @@ describe User do expect(described_class.reorder_by_name).to eq([user1, user2]) end end + + describe '#notification_email_for' do + let(:user) { create(:user) } + let(:group) { create(:group) } + + subject { user.notification_email_for(group) } + + context 'when group is nil' do + let(:group) { nil } + + it 'returns global notification email' do + is_expected.to eq(user.notification_email) + end + end + + context 'when group has no notification email set' do + it 'returns global notification email' do + create(:notification_setting, user: user, source: group, notification_email: '') + + is_expected.to eq(user.notification_email) + end + end + + context 'when group has notification email set' do + it 'returns group notification email' do + group_notification_email = 'user+group@example.com' + + create(:notification_setting, user: user, source: group, notification_email: group_notification_email) + + is_expected.to eq(group_notification_email) + end + end + end end diff --git a/spec/presenters/blobs/unfold_presenter_spec.rb b/spec/presenters/blobs/unfold_presenter_spec.rb index 7ece5f623ce..1534c572b30 100644 --- a/spec/presenters/blobs/unfold_presenter_spec.rb +++ b/spec/presenters/blobs/unfold_presenter_spec.rb @@ -54,8 +54,10 @@ describe Blobs::UnfoldPresenter do expect(lines.size).to eq(total_lines) lines.each.with_index do |line, index| - expect(line.text).to include("LC#{index + 1}") - expect(line.text).to eq(line.rich_text) + line_number = index + 1 + + expect(line.text).to eq(line_number.to_s) + expect(line.rich_text).to include("LC#{line_number}") expect(line.type).to be_nil end end diff --git a/spec/services/ci/archive_trace_service_spec.rb b/spec/services/ci/archive_trace_service_spec.rb index 44a77c29086..454db3d5a48 100644 --- a/spec/services/ci/archive_trace_service_spec.rb +++ b/spec/services/ci/archive_trace_service_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe Ci::ArchiveTraceService, '#execute' do - subject { described_class.new.execute(job) } + subject { described_class.new.execute(job, worker_name: ArchiveTraceWorker.name) } context 'when job is finished' do let(:job) { create(:ci_build, :success, :trace_live) } @@ -25,6 +25,34 @@ describe Ci::ArchiveTraceService, '#execute' do expect { subject }.not_to change { Ci::JobArtifact.trace.count } end end + + context 'when job does not have trace' do + let(:job) { create(:ci_build, :success) } + + it 'leaves a warning message in sidekiq log' do + expect(Sidekiq.logger).to receive(:warn).with( + class: ArchiveTraceWorker.name, + message: 'The job does not have live trace but going to be archived.', + job_id: job.id) + + subject + end + end + + context 'when job failed to archive trace but did not raise an exception' do + before do + allow_any_instance_of(Gitlab::Ci::Trace).to receive(:archive!) {} + end + + it 'leaves a warning message in sidekiq log' do + expect(Sidekiq.logger).to receive(:warn).with( + class: ArchiveTraceWorker.name, + message: 'The job does not have archived trace after archiving.', + job_id: job.id) + + subject + end + end end context 'when job is running' do @@ -37,10 +65,10 @@ describe Ci::ArchiveTraceService, '#execute' do issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/51502', extra: { job_id: job.id } ).once - expect(Rails.logger) - .to receive(:error) - .with("Failed to archive trace. id: #{job.id} message: Job is not finished yet") - .and_call_original + expect(Sidekiq.logger).to receive(:warn).with( + class: ArchiveTraceWorker.name, + message: "Failed to archive trace. message: Job is not finished yet.", + job_id: job.id).and_call_original expect(Gitlab::Metrics) .to receive(:counter) diff --git a/spec/services/issuable/bulk_update_service_spec.rb b/spec/services/issuable/bulk_update_service_spec.rb index b0bcd7a36ba..40da3d31408 100644 --- a/spec/services/issuable/bulk_update_service_spec.rb +++ b/spec/services/issuable/bulk_update_service_spec.rb @@ -16,22 +16,22 @@ describe Issuable::BulkUpdateService do shared_examples 'updates milestones' do it 'succeeds' do - result = bulk_update(issues, milestone_id: milestone.id) + result = bulk_update(issuables, milestone_id: milestone.id) expect(result[:success]).to be_truthy - expect(result[:count]).to eq(issues.count) + expect(result[:count]).to eq(issuables.count) end - it 'updates the issues milestone' do - bulk_update(issues, milestone_id: milestone.id) + it 'updates the issuables milestone' do + bulk_update(issuables, milestone_id: milestone.id) - issues.each do |issue| - expect(issue.reload.milestone).to eq(milestone) + issuables.each do |issuable| + expect(issuable.reload.milestone).to eq(milestone) end end end - context 'with project issues' do + context 'with project issuables' do describe 'close issues' do let(:issues) { create_list(:issue, 2, project: project) } @@ -171,7 +171,7 @@ describe Issuable::BulkUpdateService do end describe 'updating milestones' do - let(:issues) { [create(:issue, project: project)] } + let(:issuables) { [create(:issue, project: project)] } let(:milestone) { create(:milestone, project: project) } it_behaves_like 'updates milestones' @@ -360,22 +360,32 @@ describe Issuable::BulkUpdateService do end end - context 'with group issues' do + context 'with group issuables ' do let(:group) { create(:group) } - context 'updating milestone' do + describe 'updating milestones' do let(:milestone) { create(:milestone, group: group) } - let(:project1) { create(:project, :repository, group: group) } - let(:project2) { create(:project, :repository, group: group) } - let(:issue1) { create(:issue, project: project1) } - let(:issue2) { create(:issue, project: project2) } - let(:issues) { [issue1, issue2] } + let(:project) { create(:project, :repository, group: group) } before do group.add_maintainer(user) end - it_behaves_like 'updates milestones' + context 'when issues' do + let(:issue1) { create(:issue, project: project) } + let(:issue2) { create(:issue, project: project) } + let(:issuables) { [issue1, issue2] } + + it_behaves_like 'updates milestones' + end + + context 'when merge requests' do + let(:merge_request1) { create(:merge_request, source_project: project, source_branch: 'branch-1') } + let(:merge_request2) { create(:merge_request, source_project: project, source_branch: 'branch-2') } + let(:issuables) { [merge_request1, merge_request2] } + + it_behaves_like 'updates milestones' + end end end end diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb index 3e3de051732..c20de1fd079 100644 --- a/spec/services/notification_service_spec.rb +++ b/spec/services/notification_service_spec.rb @@ -2063,27 +2063,59 @@ describe NotificationService, :mailer do end context 'when the creator has custom notifications enabled' do - before do - pipeline = create_pipeline(u_custom_notification_enabled, :success) - notification.pipeline_finished(pipeline) - end + let(:pipeline) { create_pipeline(u_custom_notification_enabled, :success) } it 'emails only the creator' do + notification.pipeline_finished(pipeline) + should_only_email(u_custom_notification_enabled, kind: :bcc) end + + context 'when the creator has group notification email set' do + let(:group_notification_email) { 'user+group@example.com' } + + before do + group = create(:group) + + project.update(group: group) + create(:notification_setting, user: u_custom_notification_enabled, source: group, notification_email: group_notification_email) + end + + it 'sends to group notification email' do + notification.pipeline_finished(pipeline) + + expect(email_recipients(kind: :bcc).first).to eq(group_notification_email) + end + end end end context 'with a failed pipeline' do context 'when the creator has no custom notification set' do - before do - pipeline = create_pipeline(u_member, :failed) - notification.pipeline_finished(pipeline) - end + let(:pipeline) { create_pipeline(u_member, :failed) } it 'emails only the creator' do + notification.pipeline_finished(pipeline) + should_only_email(u_member, kind: :bcc) end + + context 'when the creator has group notification email set' do + let(:group_notification_email) { 'user+group@example.com' } + + before do + group = create(:group) + + project.update(group: group) + create(:notification_setting, user: u_member, source: group, notification_email: group_notification_email) + end + + it 'sends to group notification email' do + notification.pipeline_finished(pipeline) + + expect(email_recipients(kind: :bcc).first).to eq(group_notification_email) + end + end end context 'when the creator has watch set' do diff --git a/spec/support/shared_examples/ci_trace_shared_examples.rb b/spec/support/shared_examples/ci_trace_shared_examples.rb index ab0550e2613..68c2b6e10e2 100644 --- a/spec/support/shared_examples/ci_trace_shared_examples.rb +++ b/spec/support/shared_examples/ci_trace_shared_examples.rb @@ -720,6 +720,58 @@ shared_examples_for 'trace with enabled live trace feature' do end end + describe '#archived_trace_exist?' do + subject { trace.archived_trace_exist? } + + context 'when trace does not exist' do + it { is_expected.to be_falsy } + end + + context 'when archived trace exists' do + before do + create(:ci_job_artifact, :trace, job: build) + end + + it { is_expected.to be_truthy } + end + + context 'when live trace exists' do + before do + Gitlab::Ci::Trace::ChunkedIO.new(build) do |stream| + stream.write('abc') + end + end + + it { is_expected.to be_falsy } + end + end + + describe '#live_trace_exist?' do + subject { trace.live_trace_exist? } + + context 'when trace does not exist' do + it { is_expected.to be_falsy } + end + + context 'when archived trace exists' do + before do + create(:ci_job_artifact, :trace, job: build) + end + + it { is_expected.to be_falsy } + end + + context 'when live trace exists' do + before do + Gitlab::Ci::Trace::ChunkedIO.new(build) do |stream| + stream.write('abc') + end + end + + it { is_expected.to be_truthy } + end + end + describe '#archive!' do subject { trace.archive! } diff --git a/spec/support/shared_examples/notify_shared_examples.rb b/spec/support/shared_examples/notify_shared_examples.rb index e64c7e37a0c..4452b1c82cb 100644 --- a/spec/support/shared_examples/notify_shared_examples.rb +++ b/spec/support/shared_examples/notify_shared_examples.rb @@ -42,42 +42,17 @@ shared_examples 'an email sent from GitLab' do end shared_examples 'an email sent to a user' do - let(:group_notification_email) { 'user+group@example.com' } - it 'is sent to user\'s global notification email address' do expect(subject).to deliver_to(recipient.notification_email) end - context 'that is part of a project\'s group' do - it 'is sent to user\'s group notification email address when set' do - create(:notification_setting, user: recipient, source: project.group, notification_email: group_notification_email) - expect(subject).to deliver_to(group_notification_email) - end - - it 'is sent to user\'s global notification email address when no group email set' do - create(:notification_setting, user: recipient, source: project.group, notification_email: '') - expect(subject).to deliver_to(recipient.notification_email) - end - end - - context 'when project is in a sub-group', :nested_groups do - before do - project.update!(group: subgroup) - end - - it 'is sent to user\'s subgroup notification email address when set' do - # Set top-level group notification email address to make sure it doesn't get selected - create(:notification_setting, user: recipient, source: group, notification_email: group_notification_email) - - subgroup_notification_email = 'user+subgroup@example.com' - create(:notification_setting, user: recipient, source: subgroup, notification_email: subgroup_notification_email) + context 'with group notification email' do + it 'is sent to user\'s group notification email' do + group_notification_email = 'user+group@example.com' - expect(subject).to deliver_to(subgroup_notification_email) - end + create(:notification_setting, user: recipient, source: project.group, notification_email: group_notification_email) - it 'is sent to user\'s group notification email address when set and subgroup email address not set' do - create(:notification_setting, user: recipient, source: subgroup, notification_email: '') - expect(subject).to deliver_to(recipient.notification_email) + expect(subject).to deliver_to(group_notification_email) end end end diff --git a/spec/workers/archive_trace_worker_spec.rb b/spec/workers/archive_trace_worker_spec.rb index 368ed3f3db1..44f7be15201 100644 --- a/spec/workers/archive_trace_worker_spec.rb +++ b/spec/workers/archive_trace_worker_spec.rb @@ -11,7 +11,7 @@ describe ArchiveTraceWorker do it 'executes service' do expect_any_instance_of(Ci::ArchiveTraceService) - .to receive(:execute).with(job) + .to receive(:execute).with(job, anything) subject end diff --git a/spec/workers/ci/archive_traces_cron_worker_spec.rb b/spec/workers/ci/archive_traces_cron_worker_spec.rb index eca6cf5235f..28381fdc3be 100644 --- a/spec/workers/ci/archive_traces_cron_worker_spec.rb +++ b/spec/workers/ci/archive_traces_cron_worker_spec.rb @@ -34,7 +34,7 @@ describe Ci::ArchiveTracesCronWorker do it 'executes service' do expect_any_instance_of(Ci::ArchiveTraceService) - .to receive(:execute).with(build) + .to receive(:execute).with(build, anything) subject end @@ -60,7 +60,10 @@ describe Ci::ArchiveTracesCronWorker do end it 'puts a log' do - expect(Rails.logger).to receive(:error).with("Failed to archive trace. id: #{build.id} message: Unexpected error") + expect(Sidekiq.logger).to receive(:warn).with( + class: described_class.name, + message: "Failed to archive trace. message: Unexpected error.", + job_id: build.id) subject end diff --git a/spec/workers/stuck_ci_jobs_worker_spec.rb b/spec/workers/stuck_ci_jobs_worker_spec.rb index 72de62f1188..c3d577e2dae 100644 --- a/spec/workers/stuck_ci_jobs_worker_spec.rb +++ b/spec/workers/stuck_ci_jobs_worker_spec.rb @@ -7,8 +7,6 @@ describe StuckCiJobsWorker do let!(:runner) { create :ci_runner } let!(:job) { create :ci_build, runner: runner } - let(:trace_lease_key) { "trace:write:lock:#{job.id}" } - let(:trace_lease_uuid) { SecureRandom.uuid } let(:worker_lease_key) { StuckCiJobsWorker::EXCLUSIVE_LEASE_KEY } let(:worker_lease_uuid) { SecureRandom.uuid } @@ -16,7 +14,6 @@ describe StuckCiJobsWorker do before do stub_exclusive_lease(worker_lease_key, worker_lease_uuid) - stub_exclusive_lease(trace_lease_key, trace_lease_uuid) job.update!(status: status, updated_at: updated_at) end @@ -195,7 +192,6 @@ describe StuckCiJobsWorker do end it 'cancels exclusive leases after worker perform' do - expect_to_cancel_exclusive_lease(trace_lease_key, trace_lease_uuid) expect_to_cancel_exclusive_lease(worker_lease_key, worker_lease_uuid) worker.perform |