diff options
Diffstat (limited to 'lib/gitlab')
211 files changed, 4385 insertions, 973 deletions
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index ed5816482a9..6492ccc286a 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -103,10 +103,22 @@ module Gitlab } end + def project_creation_string_options + { + 'noone' => NO_ONE_PROJECT_ACCESS, + 'maintainer' => MAINTAINER_PROJECT_ACCESS, + 'developer' => DEVELOPER_MAINTAINER_PROJECT_ACCESS + } + end + def project_creation_values project_creation_options.values end + def project_creation_string_values + project_creation_string_options.keys + end + def project_creation_level_name(name) project_creation_options.key(name) end @@ -117,6 +129,21 @@ module Gitlab s_('SubgroupCreationlevel|Maintainers') => MAINTAINER_SUBGROUP_ACCESS } end + + def subgroup_creation_string_options + { + 'owner' => OWNER_SUBGROUP_ACCESS, + 'maintainer' => MAINTAINER_SUBGROUP_ACCESS + } + end + + def subgroup_creation_values + subgroup_creation_options.values + end + + def subgroup_creation_string_values + subgroup_creation_string_options.keys + end end def human_access diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb new file mode 100644 index 00000000000..33cbe1a62ef --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + class BaseQueryBuilder + include Gitlab::CycleAnalytics::MetricsTables + + delegate :subject_class, to: :stage + + # rubocop: disable CodeReuse/ActiveRecord + + def initialize(stage:, params: {}) + @stage = stage + @params = params + end + + def build + query = subject_class + query = filter_by_parent_model(query) + query = filter_by_time_range(query) + query = stage.start_event.apply_query_customization(query) + query = stage.end_event.apply_query_customization(query) + query.where(duration_condition) + end + + private + + attr_reader :stage, :params + + def duration_condition + stage.end_event.timestamp_projection.gteq(stage.start_event.timestamp_projection) + end + + def filter_by_parent_model(query) + if parent_class.eql?(Project) + if subject_class.eql?(Issue) + query.where(project_id: stage.parent_id) + elsif subject_class.eql?(MergeRequest) + query.where(target_project_id: stage.parent_id) + else + raise ArgumentError, "unknown subject_class: #{subject_class}" + end + else + raise ArgumentError, "unknown parent_class: #{parent_class}" + end + end + + def filter_by_time_range(query) + from = params.fetch(:from, 30.days.ago) + to = params[:to] + + query = query.where(subject_table[:created_at].gteq(from)) + query = query.where(subject_table[:created_at].lteq(to)) if to + query + end + + def subject_table + subject_class.arel_table + end + + def parent_class + stage.parent.class + end + + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb new file mode 100644 index 00000000000..0c0f737f2c9 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + # Arguments: + # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage + # params: + # current_user: an instance of User + # from: DateTime + # to: DateTime + class DataCollector + include Gitlab::Utils::StrongMemoize + + def initialize(stage:, params: {}) + @stage = stage + @params = params + end + + def records_fetcher + strong_memoize(:records_fetcher) do + RecordsFetcher.new(stage: stage, query: query, params: params) + end + end + + def median + strong_memoize(:median) do + Median.new(stage: stage, query: query) + end + end + + private + + attr_reader :stage, :params + + def query + BaseQueryBuilder.new(stage: stage, params: params).build + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb index 286c393005f..8e70236ce75 100644 --- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb +++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb @@ -23,6 +23,10 @@ module Gitlab ] end + def self.names + all.map { |stage| stage[:name] } + end + def self.params_for_issue_stage { name: 'issue', @@ -88,8 +92,8 @@ module Gitlab name: 'production', custom: false, relative_position: 7, - start_event_identifier: :merge_request_merged, - end_event_identifier: :merge_request_first_deployed_to_production + start_event_identifier: :issue_created, + end_event_identifier: :production_stage_end } end end diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb new file mode 100644 index 00000000000..41883a80338 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/median.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + class Median + include StageQueryHelpers + + def initialize(stage:, query:) + @stage = stage + @query = query + end + + def seconds + @query = @query.select(median_duration_in_seconds.as('median')) + result = execute_query(@query).first || {} + + result['median'] ? result['median'].to_i : nil + end + + private + + attr_reader :stage + + def percentile_cont + percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration) + Arel::Nodes::NamedFunction.new( + 'percentile_cont(0.5) WITHIN GROUP', + [percentile_cont_ordering] + ) + end + + def median_duration_in_seconds + Arel::Nodes::Extract.new(percentile_cont, :epoch) + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb new file mode 100644 index 00000000000..90d03142b2a --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + class RecordsFetcher + include Gitlab::Utils::StrongMemoize + include StageQueryHelpers + include Gitlab::CycleAnalytics::MetricsTables + + MAX_RECORDS = 20 + + MAPPINGS = { + Issue => { + finder_class: IssuesFinder, + serializer_class: AnalyticsIssueSerializer, + includes_for_query: { project: [:namespace], author: [] }, + columns_for_select: %I[title iid id created_at author_id project_id] + }, + MergeRequest => { + finder_class: MergeRequestsFinder, + serializer_class: AnalyticsMergeRequestSerializer, + includes_for_query: { target_project: [:namespace], author: [] }, + columns_for_select: %I[title iid id created_at author_id state target_project_id] + } + }.freeze + + delegate :subject_class, to: :stage + + def initialize(stage:, query:, params: {}) + @stage = stage + @query = query + @params = params + end + + def serialized_records + strong_memoize(:serialized_records) do + # special case (legacy): 'Test' and 'Staging' stages should show Ci::Build records + if default_test_stage? || default_staging_stage? + AnalyticsBuildSerializer.new.represent(ci_build_records.map { |e| e['build'] }) + else + records.map do |record| + project = record.project + attributes = record.attributes.merge({ + project_path: project.path, + namespace_path: project.namespace.path, + author: record.author + }) + serializer.represent(attributes) + end + end + end + end + + private + + attr_reader :stage, :query, :params + + def finder_query + MAPPINGS + .fetch(subject_class) + .fetch(:finder_class) + .new(params.fetch(:current_user), finder_params.fetch(stage.parent.class)) + .execute + end + + def columns + MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name| + subject_class.arel_table[column_name] + end + end + + # EE will override this to include Group rules + def finder_params + { + Project => { project_id: stage.parent_id } + } + end + + def default_test_stage? + stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_test_stage) + end + + def default_staging_stage? + stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_staging_stage) + end + + def serializer + MAPPINGS.fetch(subject_class).fetch(:serializer_class).new + end + + # Loading Ci::Build records instead of MergeRequest records + # rubocop: disable CodeReuse/ActiveRecord + def ci_build_records + ci_build_join = mr_metrics_table + .join(build_table) + .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + .join_sources + + q = ordered_and_limited_query + .joins(ci_build_join) + .select(build_table[:id], round_duration_to_seconds.as('total_time')) + + results = execute_query(q).to_a + + Gitlab::CycleAnalytics::Updater.update!(results, from: 'id', to: 'build', klass: ::Ci::Build.includes({ project: [:namespace], user: [], pipeline: [] })) + end + + def ordered_and_limited_query + query + .reorder(stage.end_event.timestamp_projection.desc) + .limit(MAX_RECORDS) + end + + def records + results = finder_query + .merge(ordered_and_limited_query) + .select(*columns, round_duration_to_seconds.as('total_time')) + + # using preloader instead of includes to avoid AR generating a large column list + ActiveRecord::Associations::Preloader.new.preload( + results, + MAPPINGS.fetch(subject_class).fetch(:includes_for_query) + ) + + results + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index d21f344f483..58572446de6 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -18,7 +18,8 @@ module Gitlab StageEvents::MergeRequestMerged => 104, StageEvents::CodeStageStart => 1_000, StageEvents::IssueStageEnd => 1_001, - StageEvents::PlanStageStart => 1_002 + StageEvents::PlanStageStart => 1_002, + StageEvents::ProductionStageEnd => 1_003 }.freeze EVENTS = ENUM_MAPPING.keys.freeze @@ -32,7 +33,8 @@ module Gitlab StageEvents::MergeRequestCreated ], StageEvents::IssueCreated => [ - StageEvents::IssueStageEnd + StageEvents::IssueStageEnd, + StageEvents::ProductionStageEnd ], StageEvents::MergeRequestCreated => [ StageEvents::MergeRequestMerged diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb index ff9c8a79225..6af1b90bccc 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb @@ -16,6 +16,21 @@ module Gitlab def object_type MergeRequest end + + def timestamp_projection + issue_metrics_table[:first_mentioned_in_commit_at] + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_query_customization(query) + issue_metrics_join = mr_closing_issues_table + .join(issue_metrics_table) + .on(mr_closing_issues_table[:issue_id].eq(issue_metrics_table[:issue_id])) + .join_sources + + query.joins(:merge_requests_closing_issues).joins(issue_metrics_join) + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb index a601c9797f8..8c9a80740a9 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb @@ -16,6 +16,10 @@ module Gitlab def object_type Issue end + + def timestamp_projection + issue_table[:created_at] + end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb index 7424043ef7b..fe7f2d85f8b 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb @@ -16,6 +16,16 @@ module Gitlab def object_type Issue end + + def timestamp_projection + issue_metrics_table[:first_mentioned_in_commit_at] + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_query_customization(query) + query.joins(:metrics) + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb index ceb229c552f..77e4092b9ab 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb @@ -16,6 +16,19 @@ module Gitlab def object_type Issue end + + def timestamp_projection + Arel::Nodes::NamedFunction.new('COALESCE', [ + issue_metrics_table[:first_associated_with_milestone_at], + issue_metrics_table[:first_added_to_board_at] + ]) + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_query_customization(query) + query.joins(:metrics).where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb index 8be00831b4f..7059c425b8f 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb @@ -16,6 +16,10 @@ module Gitlab def object_type MergeRequest end + + def timestamp_projection + mr_table[:created_at] + end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb index 6d7a2c023ff..3d7482eaaf0 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb @@ -16,6 +16,16 @@ module Gitlab def object_type MergeRequest end + + def timestamp_projection + mr_metrics_table[:first_deployed_to_production_at] + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_query_customization(query) + query.joins(:metrics).where(timestamp_projection.gteq(mr_table[:created_at])) + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb index 12d82fe2c62..36bb4d6fc8d 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb @@ -16,6 +16,16 @@ module Gitlab def object_type MergeRequest end + + def timestamp_projection + mr_metrics_table[:latest_build_finished_at] + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_query_customization(query) + query.joins(:metrics) + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb index 9e749b0fdfa..468d9899cc7 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb @@ -16,6 +16,16 @@ module Gitlab def object_type MergeRequest end + + def timestamp_projection + mr_metrics_table[:latest_build_started_at] + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_query_customization(query) + query.joins(:metrics) + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb index bbfb5d12992..82ecaf1cd6b 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb @@ -16,6 +16,16 @@ module Gitlab def object_type MergeRequest end + + def timestamp_projection + mr_metrics_table[:merged_at] + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_query_customization(query) + query.joins(:metrics) + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb index 803317d8b55..7ece7d62faa 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb @@ -16,6 +16,22 @@ module Gitlab def object_type Issue end + + def timestamp_projection + Arel::Nodes::NamedFunction.new('COALESCE', [ + issue_metrics_table[:first_associated_with_milestone_at], + issue_metrics_table[:first_added_to_board_at] + ]) + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_query_customization(query) + query + .joins(:metrics) + .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb new file mode 100644 index 00000000000..607371a32e8 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageEvents + class ProductionStageEnd < SimpleStageEvent + def self.name + PlanStageStart.name + end + + def self.identifier + :production_stage_end + end + + def object_type + Issue + end + + def timestamp_projection + mr_metrics_table[:first_deployed_to_production_at] + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_query_customization(query) + query.joins(merge_requests_closing_issues: { merge_request: [:metrics] }).where(mr_metrics_table[:first_deployed_to_production_at].gteq(mr_table[:created_at])) + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index a55eee048c2..aa392140eb5 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -6,6 +6,8 @@ module Gitlab module StageEvents # Base class for expressing an event that can be used for a stage. class StageEvent + include Gitlab::CycleAnalytics::MetricsTables + def initialize(params) @params = params end @@ -21,6 +23,21 @@ module Gitlab def object_type raise NotImplementedError end + + # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query. + # Example: get me all the Issue records between start event end end event + def timestamp_projection + raise NotImplementedError + end + + # Optionally a StageEvent may apply additional filtering or join other tables on the base query. + def apply_query_customization(query) + query + end + + private + + attr_reader :params end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb new file mode 100644 index 00000000000..34c726b2254 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageQueryHelpers + def execute_query(query) + ActiveRecord::Base.connection.execute(query.to_sql) + end + + def zero_interval + Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) + end + + def round_duration_to_seconds + Arel::Nodes::Extract.new(duration, :epoch) + end + + def duration + Arel::Nodes::Subtraction.new( + stage.end_event.timestamp_projection, + stage.start_event.timestamp_projection + ) + end + end + end + end +end diff --git a/lib/gitlab/artifacts/migration_helper.rb b/lib/gitlab/artifacts/migration_helper.rb new file mode 100644 index 00000000000..4f047ab3ea8 --- /dev/null +++ b/lib/gitlab/artifacts/migration_helper.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Artifacts + class MigrationHelper + def migrate_to_remote_storage(&block) + artifacts = ::Ci::JobArtifact.with_files_stored_locally + migrate(artifacts, ObjectStorage::Store::REMOTE, &block) + end + + def migrate_to_local_storage(&block) + artifacts = ::Ci::JobArtifact.with_files_stored_remotely + migrate(artifacts, ObjectStorage::Store::LOCAL, &block) + end + + private + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i + end + + def migrate(artifacts, store, &block) + artifacts.find_each(batch_size: batch_size) do |artifact| # rubocop:disable CodeReuse/ActiveRecord + artifact.file.migrate!(store) + + yield artifact if block + rescue => e + raise StandardError.new("Failed to transfer artifact of type #{artifact.file_type} and ID #{artifact.id} with error: #{e.message}") + end + end + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 53c1398d6ab..4217859f9fb 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -69,7 +69,7 @@ module Gitlab Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) - break if user && !user.active? + break if user && !user.can?(:log_in) authenticators = [] @@ -231,7 +231,7 @@ module Gitlab authentication_abilities = if token_handler.user? - full_authentication_abilities + read_write_project_authentication_abilities elsif token_handler.deploy_key_pushable?(project) read_write_authentication_abilities else @@ -272,10 +272,21 @@ module Gitlab ] end - def read_only_authentication_abilities + def read_only_project_authentication_abilities [ :read_project, - :download_code, + :download_code + ] + end + + def read_write_project_authentication_abilities + read_only_project_authentication_abilities + [ + :push_code + ] + end + + def read_only_authentication_abilities + read_only_project_authentication_abilities + [ :read_container_image ] end diff --git a/lib/gitlab/auth/current_user_mode.rb b/lib/gitlab/auth/current_user_mode.rb new file mode 100644 index 00000000000..df5039f50c1 --- /dev/null +++ b/lib/gitlab/auth/current_user_mode.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + # Keeps track of the current session user mode + # + # In order to perform administrative tasks over some interfaces, + # an administrator must have explicitly enabled admin-mode + # e.g. on web access require re-authentication + class CurrentUserMode + SESSION_STORE_KEY = :current_user_mode + ADMIN_MODE_START_TIME_KEY = 'admin_mode' + MAX_ADMIN_MODE_TIME = 6.hours + + def initialize(user) + @user = user + end + + def admin_mode? + return false unless user + + Gitlab::SafeRequestStore.fetch(request_store_key) do + user&.admin? && any_session_with_admin_mode? + end + end + + def enable_admin_mode!(password: nil, skip_password_validation: false) + return unless user&.admin? + return unless skip_password_validation || user&.valid_password?(password) + + current_session_data[ADMIN_MODE_START_TIME_KEY] = Time.now + end + + def disable_admin_mode! + current_session_data[ADMIN_MODE_START_TIME_KEY] = nil + Gitlab::SafeRequestStore.delete(request_store_key) + end + + private + + attr_reader :user + + def request_store_key + @request_store_key ||= { res: :current_user_mode, user: user.id } + end + + def current_session_data + @current_session ||= Gitlab::NamespacedSessionStore.new(SESSION_STORE_KEY) + end + + def any_session_with_admin_mode? + return true if current_session_data.initiated? && current_session_data[ADMIN_MODE_START_TIME_KEY].to_i > MAX_ADMIN_MODE_TIME.ago.to_i + + all_sessions.any? do |session| + session[ADMIN_MODE_START_TIME_KEY].to_i > MAX_ADMIN_MODE_TIME.ago.to_i + end + end + + def all_sessions + @all_sessions ||= ActiveSession.list_sessions(user).lazy.map do |session| + Gitlab::NamespacedSessionStore.new(SESSION_STORE_KEY, session.with_indifferent_access ) + end + end + end + end +end diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb index 0b7055b3256..74d359bcd28 100644 --- a/lib/gitlab/auth/ip_rate_limiter.rb +++ b/lib/gitlab/auth/ip_rate_limiter.rb @@ -24,6 +24,7 @@ module Gitlab # Allow2Ban.filter will return false if this IP has not failed too often yet @banned = Rack::Attack::Allow2Ban.filter(ip, config) do # If we return false here, the failure for this IP is ignored by Allow2Ban + # If we return true here, the count for the IP is incremented. ip_can_be_banned? end end diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb index fd09fe76c02..e73f6ca808c 100644 --- a/lib/gitlab/auth/user_access_denied_reason.rb +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -14,6 +14,9 @@ module Gitlab when :terms_not_accepted "You (#{@user.to_reference}) must accept the Terms of Service in order to perform this action. "\ "Please access GitLab from a web browser to accept these terms." + when :deactivated + "Your account has been deactivated by your administrator. "\ + "Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url}" else "Your account has been blocked." end @@ -26,6 +29,8 @@ module Gitlab :internal elsif @user.required_terms_not_accepted? :terms_not_accepted + elsif @user.deactivated? + :deactivated else :blocked end diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index 2e3a4f3b869..61e0a075018 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -78,7 +78,7 @@ module Gitlab end def self.migration_class_for(class_name) - const_get(class_name) + const_get(class_name, false) end def self.enqueued_job?(queues, migration_class) diff --git a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb index 29fa0f18448..3c142327e94 100644 --- a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb +++ b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb @@ -171,7 +171,11 @@ module Gitlab end def schedule_retry(project, retry_count) - BackgroundMigrationWorker.perform_in(RETRY_DELAY, self.class::RetryOne.name, [project.id, retry_count]) + # Constants provided to BackgroundMigrationWorker must be within the + # scope of Gitlab::BackgroundMigration + retry_class_name = self.class::RetryOne.name.sub('Gitlab::BackgroundMigration::', '') + + BackgroundMigrationWorker.perform_in(RETRY_DELAY, retry_class_name, [project.id, retry_count]) end end diff --git a/lib/gitlab/background_migration/legacy_upload_mover.rb b/lib/gitlab/background_migration/legacy_upload_mover.rb index 051c1176edb..c9e47f210be 100644 --- a/lib/gitlab/background_migration/legacy_upload_mover.rb +++ b/lib/gitlab/background_migration/legacy_upload_mover.rb @@ -92,7 +92,7 @@ module Gitlab def legacy_file_uploader strong_memoize(:legacy_file_uploader) do - uploader = upload.build_uploader + uploader = upload.retrieve_uploader uploader.retrieve_from_store!(File.basename(upload.path)) uploader end diff --git a/lib/gitlab/background_migration/migrate_pages_metadata.rb b/lib/gitlab/background_migration/migrate_pages_metadata.rb new file mode 100644 index 00000000000..68fd0c17d29 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_pages_metadata.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will insert record into project_pages_metadata + # for each existing project + class MigratePagesMetadata + def perform(start_id, stop_id) + perform_on_relation(Project.where(id: start_id..stop_id)) + end + + def perform_on_relation(relation) + successful_pages_deploy = <<~SQL + SELECT TRUE + FROM ci_builds + WHERE ci_builds.type = 'GenericCommitStatus' + AND ci_builds.status = 'success' + AND ci_builds.stage = 'deploy' + AND ci_builds.name = 'pages:deploy' + AND ci_builds.project_id = projects.id + LIMIT 1 + SQL + + select_from = relation + .select("projects.id", "COALESCE((#{successful_pages_deploy}), FALSE)") + .to_sql + + ActiveRecord::Base.connection_pool.with_connection do |connection| + connection.execute <<~SQL + INSERT INTO project_pages_metadata (project_id, deployed) + #{select_from} + ON CONFLICT (project_id) DO NOTHING + SQL + end + end + end + end +end diff --git a/lib/gitlab/badge/pipeline/template.rb b/lib/gitlab/badge/pipeline/template.rb index 2c5f9654496..0d3d44135e7 100644 --- a/lib/gitlab/badge/pipeline/template.rb +++ b/lib/gitlab/badge/pipeline/template.rb @@ -15,7 +15,7 @@ module Gitlab failed: '#e05d44', running: '#dfb317', pending: '#dfb317', - preparing: '#dfb317', + preparing: '#a7a7a7', canceled: '#9f9f9f', skipped: '#9f9f9f', unknown: '#9f9f9f' diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 24bc73e0de5..e01ffb631ba 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -104,7 +104,7 @@ module Gitlab iid: issue.iid, title: issue.title, description: description, - state: issue.state, + state_id: Issue.available_states[issue.state], author_id: gitlab_user_id(project, issue.author), milestone: milestone, created_at: issue.created_at, diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index f1a653a9d95..5382bdab7eb 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -17,6 +17,7 @@ module Gitlab i = 0 blame.each do |commit, line| commit = Commit.new(commit, project) + commit.lazy_author # preload author sha = commit.sha if prev_sha != sha diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index 4c658dc0b8d..6e48ca90054 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -23,7 +23,7 @@ module Gitlab end def request_cache(method_name, &method_key_block) - const_get(:RequestCacheExtension).module_eval do + const_get(:RequestCacheExtension, false).module_eval do cache_key_method_name = "#{method_name}_cache_key" define_method(method_name) do |*args| diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index b7886114e9c..eb5d78ebcd4 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -178,6 +178,8 @@ module Gitlab close_open_tags + # TODO: replace OpenStruct with a better type + # https://gitlab.com/gitlab-org/gitlab/issues/34305 OpenStruct.new( html: @out.force_encoding(Encoding.default_external), state: state, diff --git a/lib/gitlab/ci/ansi2json.rb b/lib/gitlab/ci/ansi2json.rb new file mode 100644 index 00000000000..79114d35916 --- /dev/null +++ b/lib/gitlab/ci/ansi2json.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Convert terminal stream to JSON +module Gitlab + module Ci + module Ansi2json + def self.convert(ansi, state = nil) + Converter.new.convert(ansi, state) + end + end + end +end diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb new file mode 100644 index 00000000000..8d25b66af9c --- /dev/null +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Ansi2json + class Converter + def convert(stream, new_state) + @lines = [] + @state = State.new(new_state, stream.size) + + append = false + truncated = false + + cur_offset = stream.tell + if cur_offset > @state.offset + @state.offset = cur_offset + truncated = true + else + stream.seek(@state.offset) + append = @state.offset > 0 + end + + start_offset = @state.offset + + @state.set_current_line!(style: Style.new(@state.inherited_style)) + + stream.each_line do |line| + s = StringScanner.new(line) + convert_line(s) + end + + # This must be assigned before flushing the current line + # or the @current_line.offset will advance to the very end + # of the trace. Instead we want @last_line_offset to always + # point to the beginning of last line. + @state.set_last_line_offset + + flush_current_line + + # TODO: replace OpenStruct with a better type + # https://gitlab.com/gitlab-org/gitlab/issues/34305 + OpenStruct.new( + lines: @lines, + state: @state.encode, + append: append, + truncated: truncated, + offset: start_offset, + size: stream.tell - start_offset, + total: stream.size + ) + end + + private + + def convert_line(scanner) + until scanner.eos? + + if scanner.scan(Gitlab::Regex.build_trace_section_regex) + handle_section(scanner) + elsif scanner.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(scanner) + elsif scanner.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif scanner.scan(/</) + @state.current_line << '<' + elsif scanner.scan(/\r?\n/) + # we advance the offset of the next current line + # so it does not start from \n + flush_current_line(advance_offset: scanner.matched_size) + else + @state.current_line << scanner.scan(/./m) + end + + @state.offset += scanner.matched_size + end + end + + def handle_sequence(scanner) + indicator = scanner[1] + commands = scanner[2].split ';' + terminator = scanner[3] + + # We are only interested in color and text style changes - triggered by + # sequences starting with '\e[' and ending with 'm'. Any other control + # sequence gets stripped (including stuff like "delete last line") + return unless indicator == '[' && terminator == 'm' + + @state.update_style(commands) + end + + def handle_section(scanner) + action = scanner[1] + timestamp = scanner[2] + section = scanner[3] + + section_name = sanitize_section_name(section) + + if action == "start" + handle_section_start(section_name, timestamp) + elsif action == "end" + handle_section_end(section_name, timestamp) + end + end + + def handle_section_start(section, timestamp) + flush_current_line unless @state.current_line.empty? + @state.open_section(section, timestamp) + end + + def handle_section_end(section, timestamp) + return unless @state.section_open?(section) + + flush_current_line unless @state.current_line.empty? + @state.close_section(section, timestamp) + + # ensure that section end is detached from the last + # line in the section + flush_current_line + end + + def flush_current_line(advance_offset: 0) + @lines << @state.current_line.to_h + + @state.set_current_line!(advance_offset: advance_offset) + end + + def sanitize_section_name(section) + section.to_s.downcase.gsub(/[^a-z0-9]/, '-') + end + end + end + end +end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb new file mode 100644 index 00000000000..173fb1df88e --- /dev/null +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Ansi2json + # Line class is responsible for keeping the internal state of + # a log line and to finally serialize it as Hash. + class Line + # Line::Segment is a portion of a line that has its own style + # and text. Multiple segments make the line content. + class Segment + attr_accessor :text, :style + + def initialize(style:) + @text = +'' + @style = style + end + + def empty? + text.empty? + end + + def to_h + # Without force encoding to UTF-8 we could get an error + # when serializing the Hash to JSON. + # Encoding::UndefinedConversionError: + # "\xE2" from ASCII-8BIT to UTF-8 + { text: text.force_encoding('UTF-8') }.tap do |result| + result[:style] = style.to_s if style.set? + end + end + end + + attr_reader :offset, :sections, :segments, :current_segment, + :section_header, :section_duration + + def initialize(offset:, style:, sections: []) + @offset = offset + @segments = [] + @sections = sections + @section_header = false + @duration = nil + @current_segment = Segment.new(style: style) + end + + def <<(data) + @current_segment.text << data + end + + def style + @current_segment.style + end + + def empty? + @segments.empty? && @current_segment.empty? + end + + def update_style(ansi_commands) + @current_segment.style.update(ansi_commands) + end + + def add_section(section) + @sections << section + end + + def set_as_section_header + @section_header = true + end + + def set_section_duration(duration) + @section_duration = Time.at(duration.to_i).strftime('%M:%S') + end + + def flush_current_segment! + return if @current_segment.empty? + + @segments << @current_segment.to_h + @current_segment = Segment.new(style: @current_segment.style) + end + + def to_h + flush_current_segment! + + { offset: offset, content: @segments }.tap do |result| + result[:section] = sections.last if sections.any? + result[:section_header] = true if @section_header + result[:section_duration] = @section_duration if @section_duration + end + end + end + end + end +end diff --git a/lib/gitlab/ci/ansi2json/parser.rb b/lib/gitlab/ci/ansi2json/parser.rb new file mode 100644 index 00000000000..d428680fb2a --- /dev/null +++ b/lib/gitlab/ci/ansi2json/parser.rb @@ -0,0 +1,200 @@ +# frozen_string_literal: true + +# This Parser translates ANSI escape codes into human readable format. +# It considers color and format changes. +# Inspired by http://en.wikipedia.org/wiki/ANSI_escape_code +module Gitlab + module Ci + module Ansi2json + class Parser + # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) + COLOR = { + 0 => 'black', # not that this is gray in the intense color table + 1 => 'red', + 2 => 'green', + 3 => 'yellow', + 4 => 'blue', + 5 => 'magenta', + 6 => 'cyan', + 7 => 'white' # not that this is gray in the dark (aka default) color table + }.freeze + + STYLE_SWITCHES = { + bold: 0x01, + italic: 0x02, + underline: 0x04, + conceal: 0x08, + cross: 0x10 + }.freeze + + def self.bold?(mask) + mask & STYLE_SWITCHES[:bold] != 0 + end + + def self.matching_formats(mask) + formats = [] + STYLE_SWITCHES.each do |text_format, flag| + formats << "term-#{text_format}" if mask & flag != 0 + end + + formats + end + + def initialize(command, ansi_stack = nil) + @command = command + @ansi_stack = ansi_stack + end + + def changes + if self.respond_to?("on_#{@command}") + send("on_#{@command}", @ansi_stack) # rubocop:disable GitlabSecurity/PublicSend + end + end + + # rubocop:disable Style/SingleLineMethods + def on_0(_) { reset: true } end + + def on_1(_) { enable: STYLE_SWITCHES[:bold] } end + + def on_3(_) { enable: STYLE_SWITCHES[:italic] } end + + def on_4(_) { enable: STYLE_SWITCHES[:underline] } end + + def on_8(_) { enable: STYLE_SWITCHES[:conceal] } end + + def on_9(_) { enable: STYLE_SWITCHES[:cross] } end + + def on_21(_) { disable: STYLE_SWITCHES[:bold] } end + + def on_22(_) { disable: STYLE_SWITCHES[:bold] } end + + def on_23(_) { disable: STYLE_SWITCHES[:italic] } end + + def on_24(_) { disable: STYLE_SWITCHES[:underline] } end + + def on_28(_) { disable: STYLE_SWITCHES[:conceal] } end + + def on_29(_) { disable: STYLE_SWITCHES[:cross] } end + + def on_30(_) { fg: fg_color(0) } end + + def on_31(_) { fg: fg_color(1) } end + + def on_32(_) { fg: fg_color(2) } end + + def on_33(_) { fg: fg_color(3) } end + + def on_34(_) { fg: fg_color(4) } end + + def on_35(_) { fg: fg_color(5) } end + + def on_36(_) { fg: fg_color(6) } end + + def on_37(_) { fg: fg_color(7) } end + + def on_38(stack) { fg: fg_color_256(stack) } end + + def on_39(_) { fg: fg_color(9) } end + + def on_40(_) { bg: bg_color(0) } end + + def on_41(_) { bg: bg_color(1) } end + + def on_42(_) { bg: bg_color(2) } end + + def on_43(_) { bg: bg_color(3) } end + + def on_44(_) { bg: bg_color(4) } end + + def on_45(_) { bg: bg_color(5) } end + + def on_46(_) { bg: bg_color(6) } end + + def on_47(_) { bg: bg_color(7) } end + + def on_48(stack) { bg: bg_color_256(stack) } end + + # TODO: all the x9 never get called? + def on_49(_) { fg: fg_color(9) } end + + def on_90(_) { fg: fg_color(0, 'l') } end + + def on_91(_) { fg: fg_color(1, 'l') } end + + def on_92(_) { fg: fg_color(2, 'l') } end + + def on_93(_) { fg: fg_color(3, 'l') } end + + def on_94(_) { fg: fg_color(4, 'l') } end + + def on_95(_) { fg: fg_color(5, 'l') } end + + def on_96(_) { fg: fg_color(6, 'l') } end + + def on_97(_) { fg: fg_color(7, 'l') } end + + def on_99(_) { fg: fg_color(9, 'l') } end + + def on_100(_) { fg: bg_color(0, 'l') } end + + def on_101(_) { fg: bg_color(1, 'l') } end + + def on_102(_) { fg: bg_color(2, 'l') } end + + def on_103(_) { fg: bg_color(3, 'l') } end + + def on_104(_) { fg: bg_color(4, 'l') } end + + def on_105(_) { fg: bg_color(5, 'l') } end + + def on_106(_) { fg: bg_color(6, 'l') } end + + def on_107(_) { fg: bg_color(7, 'l') } end + + def on_109(_) { fg: bg_color(9, 'l') } end + # rubocop:enable Style/SingleLineMethods + + def fg_color(color_index, prefix = nil) + term_color_class(color_index, ['fg', prefix]) + end + + def fg_color_256(command_stack) + xterm_color_class(command_stack, 'fg') + end + + def bg_color(color_index, prefix = nil) + term_color_class(color_index, ['bg', prefix]) + end + + def bg_color_256(command_stack) + xterm_color_class(command_stack, 'bg') + end + + def term_color_class(color_index, prefix) + color_name = COLOR[color_index] + return if color_name.nil? + + color_class(['term', prefix, color_name]) + end + + def xterm_color_class(command_stack, prefix) + # the 38 and 48 commands have to be followed by "5" and the color index + return unless command_stack.length >= 2 + return unless command_stack[0] == "5" + + command_stack.shift # ignore the "5" command + color_index = command_stack.shift.to_i + + return unless color_index >= 0 + return unless color_index <= 255 + + color_class(["xterm", prefix, color_index]) + end + + def color_class(segments) + [segments].flatten.compact.join('-') + end + end + end + end +end diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb new file mode 100644 index 00000000000..db7a9035b8b --- /dev/null +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +# In this class we keep track of the state changes that the +# Converter makes as it scans through the log stream. +module Gitlab + module Ci + module Ansi2json + class State + attr_accessor :offset, :current_line, :inherited_style, :open_sections, :last_line_offset + + def initialize(new_state, stream_size) + @offset = 0 + @inherited_style = {} + @open_sections = {} + @stream_size = stream_size + + restore_state!(new_state) + end + + def encode + state = { + offset: @last_line_offset, + style: @current_line.style.to_h, + open_sections: @open_sections + } + Base64.urlsafe_encode64(state.to_json) + end + + def open_section(section, timestamp) + @open_sections[section] = timestamp + + @current_line.add_section(section) + @current_line.set_as_section_header + end + + def close_section(section, timestamp) + return unless section_open?(section) + + duration = timestamp.to_i - @open_sections[section].to_i + @current_line.set_section_duration(duration) + + @open_sections.delete(section) + end + + def section_open?(section) + @open_sections.key?(section) + end + + def set_current_line!(style: nil, advance_offset: 0) + new_line = Line.new( + offset: @offset + advance_offset, + style: style || @current_line.style, + sections: @open_sections.keys + ) + @current_line = new_line + end + + def set_last_line_offset + @last_line_offset = @current_line.offset + end + + def update_style(commands) + @current_line.flush_current_segment! + @current_line.update_style(commands) + end + + private + + def restore_state!(encoded_state) + state = decode_state(encoded_state) + + return unless state + return if state['offset'].to_i > @stream_size + + @offset = state['offset'].to_i if state['offset'] + @open_sections = state['open_sections'] if state['open_sections'] + + if state['style'] + @inherited_style = { + fg: state.dig('style', 'fg'), + bg: state.dig('style', 'bg'), + mask: state.dig('style', 'mask') + } + end + end + + def decode_state(state) + return unless state.present? + + decoded_state = Base64.urlsafe_decode64(state) + return unless decoded_state.present? + + JSON.parse(decoded_state) + end + end + end + end +end diff --git a/lib/gitlab/ci/ansi2json/style.rb b/lib/gitlab/ci/ansi2json/style.rb new file mode 100644 index 00000000000..2739ffdfa5d --- /dev/null +++ b/lib/gitlab/ci/ansi2json/style.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Ansi2json + class Style + attr_reader :fg, :bg, :mask + + def initialize(fg: nil, bg: nil, mask: 0) + @fg = fg + @bg = bg + @mask = mask + + update_formats + end + + def update(ansi_commands) + command = ansi_commands.shift + return unless command + + if changes = Gitlab::Ci::Ansi2json::Parser.new(command, ansi_commands).changes + apply_changes(changes) + end + + update(ansi_commands) + end + + def set? + @fg || @bg || @formats.any? + end + + def reset! + @fg = nil + @bg = nil + @mask = 0 + @formats = [] + end + + def ==(other) + self.to_h == other.to_h + end + + def to_s + [@fg, @bg, @formats].flatten.compact.join(' ') + end + + def to_h + { fg: @fg, bg: @bg, mask: @mask } + end + + private + + def apply_changes(changes) + case + when changes[:reset] + reset! + when changes[:fg] + @fg = changes[:fg] + when changes[:bg] + @bg = changes[:bg] + when changes[:enable] + @mask |= changes[:enable] + when changes[:disable] + @mask &= ~changes[:disable] + else + return + end + + update_formats + end + + def update_formats + # Most terminals show bold colored text in the light color variant + # Let's mimic that here + if @fg.present? && Gitlab::Ci::Ansi2json::Parser.bold?(@mask) + @fg = @fg.sub(/fg-([a-z]{2,}+)/, 'fg-l-\1') + end + + @formats = Gitlab::Ci::Ansi2json::Parser.matching_formats(@mask) + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb index 43c46ad74af..ebeebe7fb5b 100644 --- a/lib/gitlab/ci/build/policy.rb +++ b/lib/gitlab/ci/build/policy.rb @@ -6,7 +6,7 @@ module Gitlab module Policy def self.fabricate(specs) specifications = specs.to_h.map do |spec, value| - self.const_get(spec.to_s.camelize).new(value) + self.const_get(spec.to_s.camelize, false).new(value) end specifications.compact diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb index f448d55f00a..9950e1dec55 100644 --- a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb +++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb @@ -36,7 +36,7 @@ module Gitlab Clusters::KubernetesNamespaceFinder.new( deployment_cluster, project: environment.project, - environment_slug: environment.slug, + environment_name: environment.name, allow_blank_token: true ).execute end diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb new file mode 100644 index 00000000000..62f8371283f --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule::Clause::Exists < Rules::Rule::Clause + # The maximum number of patterned glob comparisons that will be + # performed before the rule assumes that it has a match + MAX_PATTERN_COMPARISONS = 10_000 + + def initialize(globs) + globs = Array(globs) + + @top_level_only = globs.all?(&method(:top_level_glob?)) + @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?)) + end + + def satisfied_by?(pipeline, seed) + paths = worktree_paths(pipeline) + + exact_matches?(paths) || pattern_matches?(paths) + end + + private + + def worktree_paths(pipeline) + if @top_level_only + pipeline.top_level_worktree_paths + else + pipeline.all_worktree_paths + end + end + + def exact_matches?(paths) + @exact_globs.any? { |glob| paths.bsearch { |path| glob <=> path } } + end + + def pattern_matches?(paths) + comparisons = 0 + @pattern_globs.any? do |glob| + paths.any? do |path| + comparisons += 1 + comparisons > MAX_PATTERN_COMPARISONS || pattern_match?(glob, path) + end + end + end + + def pattern_match?(glob, path) + File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB) + end + + # matches glob patterns that only match files in the top level directory + def top_level_glob?(glob) + !glob.include?('/') && !glob.include?('**') + end + + # matches glob patterns that have no metacharacters for File#fnmatch? + def exact_glob?(glob) + !glob.include?('*') && !glob.include?('?') && !glob.include?('[') && !glob.include?('{') + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 668e4a5e246..9c1e6277e95 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -7,6 +7,8 @@ module Gitlab # class Config ConfigError = Class.new(StandardError) + TIMEOUT_SECONDS = 30.seconds + TIMEOUT_MESSAGE = 'Resolving config took longer than expected' RESCUE_ERRORS = [ Gitlab::Config::Loader::FormatError, @@ -17,17 +19,17 @@ module Gitlab attr_reader :root def initialize(config, project: nil, sha: nil, user: nil) - @config = Config::Extendable - .new(build_config(config, project: project, sha: sha, user: user)) - .to_hash + @context = build_context(project: project, sha: sha, user: user) + + if Feature.enabled?(:ci_limit_yaml_expansion, project, default_enabled: true) + @context.set_deadline(TIMEOUT_SECONDS) + end + + @config = expand_config(config) @root = Entry::Root.new(@config) @root.compose! - rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e - Gitlab::Sentry.track_exception(e, extra: { user: user.inspect, project: project.inspect }) - raise Config::ConfigError, e.message - rescue *rescue_errors => e raise Config::ConfigError, e.message end @@ -61,18 +63,39 @@ module Gitlab private - def build_config(config, project:, sha:, user:) + def expand_config(config) + build_config(config) + + rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e + track_exception(e) + raise Config::ConfigError, e.message + + rescue Gitlab::Ci::Config::External::Context::TimeoutError => e + track_exception(e) + raise Config::ConfigError, TIMEOUT_MESSAGE + end + + def build_config(config) initial_config = Gitlab::Config::Loader::Yaml.new(config).load! + initial_config = Config::External::Processor.new(initial_config, @context).perform + initial_config = Config::Extendable.new(initial_config).to_hash - process_external_files(initial_config, project: project, sha: sha, user: user) + if Feature.enabled?(:ci_pre_post_pipeline_stages, @context.project, default_enabled: true) + initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash + end + + initial_config end - def process_external_files(config, project:, sha:, user:) - Config::External::Processor.new(config, + def build_context(project:, sha:, user:) + Config::External::Context.new( project: project, sha: sha || project&.repository&.root_ref_sha, - user: user, - expandset: Set.new).perform + user: user) + end + + def track_exception(error) + Gitlab::Sentry.track_exception(error, extra: @context.sentry_payload) end # Overriden in EE diff --git a/lib/gitlab/ci/config/edge_stages_injector.rb b/lib/gitlab/ci/config/edge_stages_injector.rb new file mode 100644 index 00000000000..64ff9f951e4 --- /dev/null +++ b/lib/gitlab/ci/config/edge_stages_injector.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class EdgeStagesInjector + PRE_PIPELINE = '.pre' + POST_PIPELINE = '.post' + EDGES = [PRE_PIPELINE, POST_PIPELINE].freeze + + def self.wrap_stages(stages) + stages = stages.to_a - EDGES + stages.unshift PRE_PIPELINE + stages.push POST_PIPELINE + + stages + end + + def initialize(config) + @config = config.to_h.deep_dup + end + + def to_hash + if config.key?(:stages) + process(:stages) + elsif config.key?(:types) + process(:types) + else + config + end + end + + private + + attr_reader :config + + delegate :wrap_stages, to: :class + + def process(keyword) + stages = extract_stages(keyword) + return config if stages.empty? + + stages = wrap_stages(stages) + config[keyword] = stages + config + end + + def extract_stages(keyword) + stages = config[keyword] + return [] unless stages.is_a?(Array) + + stages + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb index 1f2a34ec90e..5d6d1c026e3 100644 --- a/lib/gitlab/ci/config/entry/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -8,11 +8,11 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - CLAUSES = %i[if changes].freeze - ALLOWED_KEYS = %i[if changes when start_in].freeze + CLAUSES = %i[if changes exists].freeze + ALLOWED_KEYS = %i[if changes exists when start_in].freeze ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze - attributes :if, :changes, :when, :start_in + attributes :if, :changes, :exists, :when, :start_in validations do validates :config, presence: true @@ -24,7 +24,7 @@ module Gitlab with_options allow_nil: true do validates :if, expression: true - validates :changes, array_of_strings: true + validates :changes, :exists, array_of_strings: true, length: { maximum: 50 } validates :when, allowed_values: { in: ALLOWED_WHEN } end end diff --git a/lib/gitlab/ci/config/entry/stages.rb b/lib/gitlab/ci/config/entry/stages.rb index 2d715cbc6bb..7e431f0f8bb 100644 --- a/lib/gitlab/ci/config/entry/stages.rb +++ b/lib/gitlab/ci/config/entry/stages.rb @@ -15,7 +15,7 @@ module Gitlab end def self.default - %w[build test deploy] + Config::EdgeStagesInjector.wrap_stages %w[build test deploy] end end end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb new file mode 100644 index 00000000000..bb4439cd069 --- /dev/null +++ b/lib/gitlab/ci/config/external/context.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Context + TimeoutError = Class.new(StandardError) + + attr_reader :project, :sha, :user + attr_reader :expandset, :execution_deadline + + def initialize(project: nil, sha: nil, user: nil) + @project = project + @sha = sha + @user = user + @expandset = Set.new + @execution_deadline = 0 + + yield self if block_given? + end + + def mutate(attrs = {}) + self.class.new(**attrs) do |ctx| + ctx.expandset = expandset + ctx.execution_deadline = execution_deadline + end + end + + def set_deadline(timeout_seconds) + @execution_deadline = current_monotonic_time + timeout_seconds.to_f + end + + def check_execution_time! + raise TimeoutError if execution_expired? + end + + def sentry_payload + { + user: user.inspect, + project: project.inspect + } + end + + protected + + attr_writer :expandset, :execution_deadline + + private + + def current_monotonic_time + Gitlab::Metrics::System.monotonic_time + end + + def execution_expired? + return false if execution_deadline.zero? + + current_monotonic_time > execution_deadline + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index c56d33544ba..4684a9eb981 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -12,8 +12,6 @@ module Gitlab YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze - Context = Struct.new(:project, :sha, :user, :expandset) - def initialize(params, context) @params = params @context = context @@ -69,11 +67,16 @@ module Gitlab end def validate! + validate_execution_time! validate_location! validate_content! if errors.none? validate_hash! if errors.none? end + def validate_execution_time! + context.check_execution_time! + end + def validate_location! if invalid_location_type? errors.push("Included file `#{location}` needs to be a string") @@ -95,11 +98,11 @@ module Gitlab end def expand_includes(hash) - External::Processor.new(hash, **expand_context).perform + External::Processor.new(hash, context.mutate(expand_context_attrs)).perform end - def expand_context - { project: nil, sha: nil, user: nil, expandset: context.expandset } + def expand_context_attrs + {} end end end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index cac321ec4a6..8cb1575a3e1 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -6,6 +6,7 @@ module Gitlab module External module File class Local < Base + extend ::Gitlab::Utils::Override include Gitlab::Utils::StrongMemoize def initialize(params, context) @@ -34,11 +35,13 @@ module Gitlab context.project.repository.blob_data_at(context.sha, location) end - def expand_context - super.merge( + override :expand_context_attrs + def expand_context_attrs + { project: context.project, sha: context.sha, - user: context.user) + user: context.user + } end end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index b828f77835c..c7b49b495fa 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -6,11 +6,12 @@ module Gitlab module External module File class Project < Base + extend ::Gitlab::Utils::Override include Gitlab::Utils::StrongMemoize attr_reader :project_name, :ref_name - def initialize(params, context = {}) + def initialize(params, context) @location = params[:file] @project_name = params[:project] @ref_name = params[:ref] || 'HEAD' @@ -65,11 +66,13 @@ module Gitlab end end - def expand_context - super.merge( + override :expand_context_attrs + def expand_context_attrs + { project: project, sha: sha, - user: context.user) + user: context.user + } end end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index aff5c5b9651..0143d7784fa 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,7 +7,7 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize - MAX_INCLUDES = 50 + MAX_INCLUDES = 100 FILE_CLASSES = [ External::File::Remote, @@ -21,14 +21,9 @@ module Gitlab DuplicateIncludesError = Class.new(Error) TooManyIncludesError = Class.new(Error) - def initialize(values, project:, sha:, user:, expandset:) - raise Error, 'Expanded needs to be `Set`' unless expandset.is_a?(Set) - + def initialize(values, context) @locations = Array.wrap(values.fetch(:include, [])) - @project = project - @sha = sha - @user = user - @expandset = expandset + @context = context end def process @@ -43,7 +38,9 @@ module Gitlab private - attr_reader :locations, :project, :sha, :user, :expandset + attr_reader :locations, :context + + delegate :expandset, to: :context # convert location if String to canonical form def normalize_location(location) @@ -68,11 +65,11 @@ module Gitlab end # We scope location to context, as this allows us to properly support - # relative incldues, and similarly looking relative in another project + # relative includes, and similarly looking relative in another project # does not trigger duplicate error scoped_location = location.merge( - context_project: project, - context_sha: sha) + context_project: context.project, + context_sha: context.sha) unless expandset.add?(scoped_location) raise DuplicateIncludesError, "Include `#{location.to_json}` was already included!" @@ -88,12 +85,6 @@ module Gitlab matching.first end - - def context - strong_memoize(:context) do - External::File::Base::Context.new(project, sha, user, expandset) - end - end end end end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index 4a049ecae49..de69a1b1e8f 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -7,9 +7,9 @@ module Gitlab class Processor IncludeError = Class.new(StandardError) - def initialize(values, project:, sha:, user:, expandset:) + def initialize(values, context) @values = values - @external_files = External::Mapper.new(values, project: project, sha: sha, user: user, expandset: expandset).process + @external_files = External::Mapper.new(values, context).process @content = {} rescue External::Mapper::Error, OpenSSL::SSL::SSLError => e diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index dca60eabc1c..8f8cae0b5f2 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -49,6 +49,12 @@ module Gitlab if data['failure'] status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED system_output = data['failure'] + elsif data['error'] + # For now, as an MVC, we are grouping error test cases together + # with failed ones. But we will improve this further on + # https://gitlab.com/gitlab-org/gitlab/issues/32046. + status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED + system_output = data['error'] else status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS system_output = nil diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 1f6b3853069..fc9c540088b 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -73,7 +73,9 @@ module Gitlab if bridge? ::Ci::Bridge.new(attributes) else - ::Ci::Build.new(attributes) + ::Ci::Build.new(attributes).tap do |job| + job.deployment = Seed::Deployment.new(job).to_resource + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/deployment.rb b/lib/gitlab/ci/pipeline/seed/deployment.rb new file mode 100644 index 00000000000..8c90f03cb1d --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/deployment.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Seed + class Deployment < Seed::Base + attr_reader :job, :environment + + def initialize(job) + @job = job + @environment = Seed::Environment.new(@job) + end + + def to_resource + return job.deployment if job.deployment + return unless job.starts_environment? + + deployment = ::Deployment.new(attributes) + deployment.environment = environment.to_resource + + # If there is a validation error on environment creation, such as + # the name contains invalid character, the job will fall back to a + # non-environment job. + return unless deployment.valid? && deployment.environment.persisted? + + deployment.cluster_id = + deployment.environment.deployment_platform&.cluster_id + + # Allocate IID for deployments. + # This operation must be outside of transactions of pipeline creations. + deployment.ensure_project_iid! + + deployment + end + + private + + def attributes + { + project: job.project, + user: job.user, + ref: job.ref, + tag: job.tag, + sha: job.sha, + on_stop: job.on_stop + } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb new file mode 100644 index 00000000000..2d3a1e702f9 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/environment.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Seed + class Environment < Seed::Base + attr_reader :job + + def initialize(job) + @job = job + end + + def to_resource + find_environment || ::Environment.create(attributes) + end + + private + + def find_environment + job.project.environments.find_by_name(expanded_environment_name) + end + + def expanded_environment_name + job.expanded_environment_name + end + + def attributes + { + project: job.project, + name: expanded_environment_name + } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb new file mode 100644 index 00000000000..3c00b67911f --- /dev/null +++ b/lib/gitlab/ci/status/composite.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + class Composite + include Gitlab::Utils::StrongMemoize + + # This class accepts an array of arrays/hashes/or objects + def initialize(all_statuses, with_allow_failure: true) + unless all_statuses.respond_to?(:pluck) + raise ArgumentError, "all_statuses needs to respond to `.pluck`" + end + + @status_set = Set.new + @status_key = 0 + @allow_failure_key = 1 if with_allow_failure + + consume_all_statuses(all_statuses) + end + + # The status calculation is order dependent, + # 1. In some cases we assume that that status is exact + # if the we only have given statues, + # 2. In other cases we assume that status is of that type + # based on what statuses are no longer valid based on the + # data set that we have + def status + return if none? + + strong_memoize(:status) do + if only_of?(:skipped, :ignored) + 'skipped' + elsif only_of?(:success, :skipped, :success_with_warnings, :ignored) + 'success' + elsif only_of?(:created, :success_with_warnings, :ignored) + 'created' + elsif only_of?(:preparing, :success_with_warnings, :ignored) + 'preparing' + elsif only_of?(:canceled, :success, :skipped, :success_with_warnings, :ignored) + 'canceled' + elsif only_of?(:pending, :created, :skipped, :success_with_warnings, :ignored) + 'pending' + elsif any_of?(:running, :pending) + 'running' + elsif any_of?(:manual) + 'manual' + elsif any_of?(:scheduled) + 'scheduled' + elsif any_of?(:preparing) + 'preparing' + elsif any_of?(:created) + 'running' + else + 'failed' + end + end + end + + def warnings? + @status_set.include?(:success_with_warnings) + end + + private + + def none? + @status_set.empty? + end + + def any_of?(*names) + names.any? { |name| @status_set.include?(name) } + end + + def only_of?(*names) + matching = names.count { |name| @status_set.include?(name) } + matching > 0 && + matching == @status_set.size + end + + def consume_all_statuses(all_statuses) + columns = [] + columns[@status_key] = :status + columns[@allow_failure_key] = :allow_failure if @allow_failure_key + + all_statuses + .pluck(*columns) # rubocop: disable CodeReuse/ActiveRecord + .each(&method(:consume_status)) + end + + def consume_status(description) + # convert `"status"` into `["status"]` + description = Array(description) + + status = + if success_with_warnings?(description) + :success_with_warnings + elsif ignored_status?(description) + :ignored + else + description[@status_key].to_sym + end + + @status_set.add(status) + end + + def success_with_warnings?(status) + @allow_failure_key && + status[@allow_failure_key] && + HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(status[@status_key]) + end + + def ignored_status?(status) + @allow_failure_key && + status[@allow_failure_key] && + HasStatus::EXCLUDE_IGNORED_STATUSES.include?(status[@status_key]) + end + end + end + end +end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index 2a0bf060c9b..c29dc51f076 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -20,7 +20,7 @@ module Gitlab def core_status Gitlab::Ci::Status - .const_get(@status.capitalize) + .const_get(@status.capitalize, false) .new(@subject, @user) .extend(self.class.common_helpers) end diff --git a/lib/gitlab/ci/status/preparing.rb b/lib/gitlab/ci/status/preparing.rb index 62985d0a9f9..1ebdbc482b7 100644 --- a/lib/gitlab/ci/status/preparing.rb +++ b/lib/gitlab/ci/status/preparing.rb @@ -12,20 +12,12 @@ module Gitlab s_('CiStatusLabel|preparing') end - ## - # TODO: shared with 'created' - # until we get one for 'preparing' - # def icon - 'status_created' + 'status_preparing' end - ## - # TODO: shared with 'created' - # until we get one for 'preparing' - # def favicon - 'favicon_status_created' + 'favicon_status_preparing' end end end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 1ad9dd2913e..5a7642d24ee 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -77,15 +77,10 @@ include: - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml + - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml - -# Override DAST job to exclude master branch -dast: - except: - refs: - - master diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml index f6d240b7b6d..15cdbf63cb1 100644 --- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -1,4 +1,4 @@ -build-master: +docker-build-master: # Official docker image. image: docker:latest stage: build @@ -12,7 +12,7 @@ build-master: only: - master -build: +docker-build: # Official docker image. image: docker:latest stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml new file mode 100644 index 00000000000..ae2ff9992f9 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -0,0 +1,55 @@ +.auto-deploy: + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.1.0" + +dast_environment_deploy: + extends: .auto-deploy + stage: review + script: + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy persist_environment_url + environment: + name: dast-default + url: http://dast-$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN + on_stop: stop_dast_environment + artifacts: + paths: [environment_url.txt] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bdast\b/ + kubernetes: active + except: + variables: + - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME + - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH + - $DAST_WEBSITE # we don't need to create a review app if a URL is already given + +stop_dast_environment: + extends: .auto-deploy + stage: cleanup + variables: + GIT_STRATEGY: none + script: + - auto-deploy initialize_tiller + - auto-deploy delete + environment: + name: dast-default + action: stop + needs: ["dast"] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bdast\b/ + kubernetes: active + except: + variables: + - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME + - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH + - $DAST_WEBSITE diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 7f9a7df2f31..f058468ed8e 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,9 +1,12 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/ +variables: + CS_MAJOR_VERSION: 1 + container_scanning: stage: test image: - name: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable + name: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CS_MAJOR_VERSION entrypoint: [] variables: # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 4b55ffd3771..23c65a0cb67 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -46,3 +46,4 @@ dast: except: variables: - $DAST_DISABLED + - $DAST_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 88f4b72044c..a0c2ab3aa26 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -4,7 +4,13 @@ # List of the variables: https://gitlab.com/gitlab-org/security-products/sast#settings # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables -.sast: +variables: + SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex" + SAST_MAJOR_VERSION: 2 + SAST_DISABLE_DIND: "false" + +sast: stage: test allow_failure: true artifacts: @@ -15,13 +21,6 @@ - branches variables: - $GITLAB_FEATURES =~ /\bsast\b/ - -variables: - SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SAST_DISABLE_DIND: "false" - -sast: - extends: .sast image: docker:stable variables: DOCKER_DRIVER: overlay2 @@ -84,7 +83,8 @@ sast: - $SAST_DISABLE_DIND == 'true' .analyzer: - extends: .sast + extends: sast + services: [] except: variables: - $SAST_DISABLE_DIND == 'false' @@ -94,100 +94,128 @@ sast: bandit-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit" + name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /bandit/&& + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/ brakeman-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman" + name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /brakeman/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/ eslint-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint" + name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /eslint/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/ flawfinder-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder" + name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\+\+|c\b)/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\+\+|c)\b/ gosec-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec" + name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /go/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /gosec/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bgo\b/ nodejs-scan-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan" + name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/ phpcs-security-audit-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit" + name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /php/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /php/ pmd-apex-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex" + name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /apex/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /apex/ secrets-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets" + name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_MAJOR_VERSION" + only: + variables: + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /secrets/ security-code-scan-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan" + name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /c\#/ || $CI_PROJECT_REPOSITORY_LANGUAGES =~ /visual basic/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\#|visual basic\b)/ sobelow-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow" + name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /elixir/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /sobelow/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /elixir/ spotbugs-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs" + name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /java\b/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /java\b/ tslint-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint" + name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint:$SAST_MAJOR_VERSION" only: variables: - - '$CI_PROJECT_REPOSITORY_LANGUAGES =~ /typescript/' + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /tslint/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /typescript/ diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml new file mode 100644 index 00000000000..eced181e966 --- /dev/null +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -0,0 +1,29 @@ +# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html + +stages: + - build + - test + - deploy + - performance + +performance: + stage: performance + image: docker:git + variables: + URL: https://example.com + SITESPEED_VERSION: 6.3.1 + SITESPEED_OPTIONS: '' + services: + - docker:stable-dind + script: + - mkdir gitlab-exporter + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js + - mkdir sitespeed-results + - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + - mv sitespeed-results/data/performance.json performance.json + artifacts: + paths: + - performance.json + - sitespeed-results/ + reports: + performance: performance.json diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 5b8c2d2f7c7..941f7178dac 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -4,6 +4,7 @@ module Gitlab module Ci class Trace include ::Gitlab::ExclusiveLeaseHelpers + include Checksummable LOCK_TTL = 10.minutes LOCK_RETRIES = 2 @@ -193,7 +194,7 @@ module Gitlab project: job.project, file_type: :trace, file: stream, - file_sha256: Digest::SHA256.file(path).hexdigest) + file_sha256: self.class.hexdigest(path)) end end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index e61fb50a303..20f5620dd64 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -63,10 +63,6 @@ module Gitlab end.force_encoding(Encoding.default_external) end - def html_with_state(state = nil) - ::Gitlab::Ci::Ansi2html.convert(stream, state) - end - def html(last_lines: nil) text = raw(last_lines: last_lines) buffer = StringIO.new(text) diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index 8f796748199..294ffad02ce 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -8,14 +8,50 @@ module Gitlab # watchdog threads. This lets us abstract away the Unix process # lifecycles of Unicorn, Sidekiq, Puma, Puma Cluster, etc. # - # We have three lifecycle events. + # We have the following lifecycle events. # - # - before_fork (only in forking processes) - # In forking processes (Unicorn and Puma in multiprocess mode) this - # will be called exactly once, on startup, before the workers are - # forked. This will be called in the parent process. - # - worker_start - # - before_master_restart (only in forking processes) + # - on_master_start: + # + # Unicorn/Puma Cluster: This will be called exactly once, + # on startup, before the workers are forked. This is + # called in the PARENT/MASTER process. + # + # Sidekiq/Puma Single: This is called immediately. + # + # - on_before_fork: + # + # Unicorn/Puma Cluster: This will be called exactly once, + # on startup, before the workers are forked. This is + # called in the PARENT/MASTER process. + # + # Sidekiq/Puma Single: This is not called. + # + # - on_worker_start: + # + # Unicorn/Puma Cluster: This is called in the worker process + # exactly once before processing requests. + # + # Sidekiq/Puma Single: This is called immediately. + # + # - on_before_phased_restart: + # + # Unicorn/Puma Cluster: This will be called before a graceful + # shutdown of workers starts happening. + # This is called on `master` process. + # + # Sidekiq/Puma Single: This is not called. + # + # - on_before_master_restart: + # + # Unicorn: This will be called before a new master is spun up. + # This is called on forked master before `execve` to become + # a new masterfor Unicorn. This means that this does not really + # affect old master process. + # + # Puma Cluster: This will be called before a new master is spun up. + # This is called on `master` process. + # + # Sidekiq/Puma Single: This is not called. # # Blocks will be executed in the order in which they are registered. # @@ -34,15 +70,17 @@ module Gitlab end def on_before_fork(&block) - return unless in_clustered_environment? - # Defer block execution (@before_fork_hooks ||= []) << block end - def on_master_restart(&block) - return unless in_clustered_environment? + # Read the config/initializers/cluster_events_before_phased_restart.rb + def on_before_phased_restart(&block) + # Defer block execution + (@master_phased_restart ||= []) << block + end + def on_before_master_restart(&block) # Defer block execution (@master_restart_hooks ||= []) << block end @@ -70,12 +108,21 @@ module Gitlab end end - def do_master_restart - @master_restart_hooks && @master_restart_hooks.each do |block| + def do_before_phased_restart + @master_phased_restart&.each do |block| block.call end end + def do_before_master_restart + @master_restart_hooks&.each do |block| + block.call + end + end + + # DEPRECATED + alias_method :do_master_restart, :do_before_master_restart + # Puma doesn't use singletons (which is good) but # this means we need to pass through whether the # puma server is running in single mode or cluster mode diff --git a/lib/gitlab/cluster/mixins/puma_cluster.rb b/lib/gitlab/cluster/mixins/puma_cluster.rb new file mode 100644 index 00000000000..e9157d9f1e4 --- /dev/null +++ b/lib/gitlab/cluster/mixins/puma_cluster.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Cluster + module Mixins + module PumaCluster + def self.prepended(base) + raise 'missing method Puma::Cluster#stop_workers' unless base.method_defined?(:stop_workers) + end + + def stop_workers + Gitlab::Cluster::LifecycleEvents.do_before_phased_restart + + super + end + end + end + end +end diff --git a/lib/gitlab/cluster/mixins/unicorn_http_server.rb b/lib/gitlab/cluster/mixins/unicorn_http_server.rb new file mode 100644 index 00000000000..765fd0c2baa --- /dev/null +++ b/lib/gitlab/cluster/mixins/unicorn_http_server.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Cluster + module Mixins + module UnicornHttpServer + def self.prepended(base) + raise 'missing method Unicorn::HttpServer#reexec' unless base.method_defined?(:reexec) + end + + def reexec + Gitlab::Cluster::LifecycleEvents.do_before_phased_restart + + super + end + end + end + end +end diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index 4affc52b7b0..a8440b63baa 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -3,7 +3,7 @@ module Gitlab module Cluster class PumaWorkerKillerInitializer - def self.start(puma_options, puma_per_worker_max_memory_mb: 650) + def self.start(puma_options, puma_per_worker_max_memory_mb: 850, puma_master_max_memory_mb: 550) require 'puma_worker_killer' PumaWorkerKiller.config do |config| @@ -12,10 +12,9 @@ module Gitlab # not each worker as is the case with GITLAB_UNICORN_MEMORY_MAX worker_count = puma_options[:workers] || 1 # The Puma Worker Killer checks the total RAM used by both the master - # and worker processes. Bump the limits to N+1 instead of N workers - # to account for this: + # and worker processes. # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57 - config.ram = (worker_count + 1) * puma_per_worker_max_memory_mb + config.ram = puma_master_max_memory_mb + (worker_count * puma_per_worker_max_memory_mb) config.frequency = 20 # seconds @@ -23,10 +22,9 @@ module Gitlab # of available RAM. config.percent_usage = 0.98 - # Ideally we'll never hit the maximum amount of memory. If so the worker - # is restarted already, thus periodically restarting workers shouldn't be - # needed. - config.rolling_restart_frequency = false + # Ideally we'll never hit the maximum amount of memory. Restart the workers + # regularly rather than rely on OOM behavior for periodic restarting. + config.rolling_restart_frequency = 43200 # 12 hours in seconds. observer = Gitlab::Cluster::PumaWorkerKillerObserver.new config.pre_term = observer.callback diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb index a56a89adb35..d58aba07d15 100644 --- a/lib/gitlab/config/entry/simplifiable.rb +++ b/lib/gitlab/config/entry/simplifiable.rb @@ -37,7 +37,7 @@ module Gitlab def self.entry_class(strategy) if strategy.present? - self.const_get(strategy.name) + self.const_get(strategy.name, false) else self::UnknownStrategy end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index 459bb5177b5..6aedbf64f26 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -23,6 +23,7 @@ module Gitlab .project(routes_table[:path].as("namespace_path")) query = limit_query(query, project_ids) + query = limit_query_by_date_range(query) # Load merge_requests @@ -34,7 +35,12 @@ module Gitlab def limit_query(query, project_ids) query.where(issue_table[:project_id].in(project_ids)) .where(routes_table[:source_type].eq('Namespace')) - .where(issue_table[:created_at].gteq(options[:from])) + end + + def limit_query_by_date_range(query) + query = query.where(issue_table[:created_at].gteq(options[:from])) + query = query.where(issue_table[:created_at].lteq(options[:to])) if options[:to] + query end def load_merge_requests(query) diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb index 98a30a8fc97..04f4b4f053f 100644 --- a/lib/gitlab/cycle_analytics/event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/event_fetcher.rb @@ -4,7 +4,7 @@ module Gitlab module CycleAnalytics module EventFetcher def self.[](stage_name) - CycleAnalytics.const_get("#{stage_name.to_s.camelize}EventFetcher") + CycleAnalytics.const_get("#{stage_name.to_s.camelize}EventFetcher", false) end end end diff --git a/lib/gitlab/cycle_analytics/issue_helper.rb b/lib/gitlab/cycle_analytics/issue_helper.rb index 295eca5edca..f6f85b84ed8 100644 --- a/lib/gitlab/cycle_analytics/issue_helper.rb +++ b/lib/gitlab/cycle_analytics/issue_helper.rb @@ -12,14 +12,12 @@ module Gitlab .project(routes_table[:path].as("namespace_path")) query = limit_query(query, project_ids) - - query + limit_query_by_date_range(query) end def limit_query(query, project_ids) query.where(issue_table[:project_id].in(project_ids)) .where(routes_table[:source_type].eq('Namespace')) - .where(issue_table[:created_at].gteq(options[:from])) .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) end end diff --git a/lib/gitlab/cycle_analytics/plan_helper.rb b/lib/gitlab/cycle_analytics/plan_helper.rb index a63ae58ad21..af4bf6ed3eb 100644 --- a/lib/gitlab/cycle_analytics/plan_helper.rb +++ b/lib/gitlab/cycle_analytics/plan_helper.rb @@ -14,12 +14,11 @@ module Gitlab .where(routes_table[:source_type].eq('Namespace')) query = limit_query(query) - query + limit_query_by_date_range(query) end def limit_query(query) - query.where(issue_table[:created_at].gteq(options[:from])) - .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + query.where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) end end diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb index 1bd40a7aa18..5cfd9ea4730 100644 --- a/lib/gitlab/cycle_analytics/stage.rb +++ b/lib/gitlab/cycle_analytics/stage.rb @@ -4,7 +4,7 @@ module Gitlab module CycleAnalytics module Stage def self.[](stage_name) - CycleAnalytics.const_get("#{stage_name.to_s.camelize}Stage") + CycleAnalytics.const_get("#{stage_name.to_s.camelize}Stage", false) end end end diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb index 5198dd5b4eb..ea440c441b7 100644 --- a/lib/gitlab/cycle_analytics/stage_summary.rb +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -3,16 +3,17 @@ module Gitlab module CycleAnalytics class StageSummary - def initialize(project, from:, current_user:) + def initialize(project, from:, to: nil, current_user:) @project = project @from = from + @to = to @current_user = current_user end def data - [serialize(Summary::Issue.new(project: @project, from: @from, current_user: @current_user)), - serialize(Summary::Commit.new(project: @project, from: @from)), - serialize(Summary::Deploy.new(project: @project, from: @from))] + [serialize(Summary::Issue.new(project: @project, from: @from, to: @to, current_user: @current_user)), + serialize(Summary::Commit.new(project: @project, from: @from, to: @to)), + serialize(Summary::Deploy.new(project: @project, from: @from, to: @to))] end private diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index 709221c648e..a825d48fb77 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -4,9 +4,10 @@ module Gitlab module CycleAnalytics module Summary class Base - def initialize(project:, from:) + def initialize(project:, from:, to: nil) @project = project @from = from + @to = to end def title diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index f0019b26fa2..76049c6b742 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -21,7 +21,7 @@ module Gitlab def count_commits return unless ref - gitaly_commit_client.commit_count(ref, after: @from) + gitaly_commit_client.commit_count(ref, after: @from, before: @to) end def gitaly_commit_client diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index 3b56dc2a7bc..5ff8d881143 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -4,12 +4,18 @@ module Gitlab module CycleAnalytics module Summary class Deploy < Base + include Gitlab::Utils::StrongMemoize + def title n_('Deploy', 'Deploys', value) end def value - @value ||= @project.deployments.where("created_at > ?", @from).count + strong_memoize(:value) do + query = @project.deployments.success.where("created_at >= ?", @from) + query = query.where("created_at <= ?", @to) if @to + query.count + end end end end diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb index 51695c86192..52892eb5a1a 100644 --- a/lib/gitlab/cycle_analytics/summary/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -4,9 +4,10 @@ module Gitlab module CycleAnalytics module Summary class Issue < Base - def initialize(project:, from:, current_user:) + def initialize(project:, from:, to: nil, current_user:) @project = project @from = from + @to = to @current_user = current_user end @@ -15,7 +16,7 @@ module Gitlab end def value - @value ||= IssuesFinder.new(@current_user, project_id: @project.id).execute.created_after(@from).count + @value ||= IssuesFinder.new(@current_user, project_id: @project.id, created_after: @from, created_before: @to).execute.count end end end diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index 43c159fee27..8a253893892 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -34,7 +34,9 @@ module Gitlab @mutex.synchronize do break thread if thread? - @thread = Thread.new { start_working } + if start_working + @thread = Thread.new { run_thread } + end end end @@ -57,10 +59,18 @@ module Gitlab private + # Executed in lock context before starting thread + # Needs to return success def start_working + true + end + + # Executed in separate thread + def run_thread raise NotImplementedError end + # Executed in lock context def stop_working # no-ops end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index e2911b4e6c8..f22fc41a6d8 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -35,7 +35,8 @@ module Gitlab end def ee? - ENV['CI_PROJECT_NAME'] == 'gitlab-ee' || File.exist?('../../CHANGELOG-EE.md') + # Support former project name for `dev` and support local Danger run + %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?('../../ee') end def gitlab_helper @@ -52,7 +53,7 @@ module Gitlab end def project_name - ee? ? 'gitlab-ee' : 'gitlab-ce' + ee? ? 'gitlab' : 'gitlab-foss' end def markdown_list(items) @@ -89,7 +90,7 @@ module Gitlab end CATEGORY_LABELS = { - docs: "~Documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now. + docs: "~documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now. none: "", qa: "~QA", test: "~test for `spec/features/*`", diff --git a/lib/gitlab/danger/request_helper.rb b/lib/gitlab/danger/request_helper.rb new file mode 100644 index 00000000000..06da4ed9ad3 --- /dev/null +++ b/lib/gitlab/danger/request_helper.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'net/http' +require 'json' + +module Gitlab + module Danger + module RequestHelper + HTTPError = Class.new(RuntimeError) + + # @param [String] url + def self.http_get_json(url) + rsp = Net::HTTP.get_response(URI.parse(url)) + + unless rsp.is_a?(Net::HTTPOK) + raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}" + end + + JSON.parse(rsp.body) + end + end + end +end diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb index 25de0a87c9d..dbf42912882 100644 --- a/lib/gitlab/danger/roulette.rb +++ b/lib/gitlab/danger/roulette.rb @@ -1,16 +1,11 @@ # frozen_string_literal: true -require 'net/http' -require 'json' -require 'cgi' - require_relative 'teammate' module Gitlab module Danger module Roulette ROULETTE_DATA_URL = 'https://about.gitlab.com/roulette.json' - HTTPError = Class.new(RuntimeError) # Looks up the current list of GitLab team members and parses it into a # useful form @@ -19,7 +14,7 @@ module Gitlab def team @team ||= begin - data = http_get_json(ROULETTE_DATA_URL) + data = Gitlab::Danger::RequestHelper.http_get_json(ROULETTE_DATA_URL) data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) } rescue JSON::ParserError raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}" @@ -44,6 +39,7 @@ module Gitlab # Known issue: If someone is rejected due to OOO, and then becomes not OOO, the # selection will change on next spin + # @param [Array<Teammate>] people def spin_for_person(people, random:) people.shuffle(random: random) .find(&method(:valid_person?)) @@ -51,32 +47,17 @@ module Gitlab private + # @param [Teammate] person + # @return [Boolean] def valid_person?(person) - !mr_author?(person) && !out_of_office?(person) + !mr_author?(person) && person.available? end + # @param [Teammate] person + # @return [Boolean] def mr_author?(person) person.username == gitlab.mr_author end - - def out_of_office?(person) - username = CGI.escape(person.username) - api_endpoint = "https://gitlab.com/api/v4/users/#{username}/status" - response = http_get_json(api_endpoint) - response["message"]&.match?(/OOO/i) - rescue HTTPError, JSON::ParserError - false # this is no worse than not checking for OOO - end - - def http_get_json(url) - rsp = Net::HTTP.get_response(URI.parse(url)) - - unless rsp.is_a?(Net::HTTPSuccess) - raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}" - end - - JSON.parse(rsp.body) - end end end end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index 4ad66f61c2b..5c2324836d7 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'cgi' + module Gitlab module Danger class Teammate @@ -34,8 +36,30 @@ module Gitlab has_capability?(project, category, :maintainer, labels) end + def status + api_endpoint = "https://gitlab.com/api/v4/users/#{CGI.escape(username)}/status" + @status ||= Gitlab::Danger::RequestHelper.http_get_json(api_endpoint) + rescue Gitlab::Danger::RequestHelper::HTTPError, JSON::ParserError + nil # better no status than a crashing Danger + end + + # @return [Boolean] + def available? + !out_of_office? && has_capacity? + end + private + # @return [Boolean] + def out_of_office? + status&.dig("message")&.match?(/OOO/i) || false + end + + # @return [Boolean] + def has_capacity? + status&.dig("emoji") != 'red_circle' + end + def has_capability?(project, category, kind, labels) case category when :test diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 3460e07fdc5..a83b03f540c 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -107,6 +107,14 @@ module Gitlab } end + def build_bulk(action:, ref_type:, changes:) + { + action: action, + ref_count: changes.count, + ref_type: ref_type + } + end + # This method provides a sample data generated with # existing project and commits to test webhooks def build_sample(project, user) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index bea9eb8cb31..50e23681de0 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -87,10 +87,6 @@ module Gitlab version.to_f < 10 end - def self.join_lateral_supported? - version.to_f >= 9.3 - end - def self.replication_slots_supported? version.to_f >= 9.4 end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 5a42952796c..ae29546cdac 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1018,7 +1018,7 @@ into similar problems in the future (e.g. when new tables are created). end model_class.each_batch(of: batch_size) do |relation, index| - start_id, end_id = relation.pluck('MIN(id), MAX(id)').first + start_id, end_id = relation.pluck(Arel.sql('MIN(id), MAX(id)')).first # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for # the same time, which is not helpful in most cases where we wish to diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index 5422a8631a0..dfef158cc1d 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -33,7 +33,7 @@ module Gitlab if result[:status] == :success result - elsif STEPS_ALLOWED_TO_FAIL.include?(result[:failed_step]) + elsif STEPS_ALLOWED_TO_FAIL.include?(result[:last_step]) success else raise StandardError, result[:message] @@ -42,121 +42,124 @@ module Gitlab private - def validate_application_settings + def validate_application_settings(_result) return success if application_settings log_error('No application_settings found') error(_('No application_settings found')) end - def validate_project_created - return success unless project_created? + def validate_project_created(result) + return success(result) unless project_created? log_error('Project already created') error(_('Project already created')) end - def validate_admins + def validate_admins(result) unless instance_admins.any? log_error('No active admin user found') return error(_('No active admin user found')) end - success + success(result) end - def create_group + def create_group(result) if project_created? log_info(_('Instance administrators group already exists')) - @group = application_settings.instance_administration_project.owner - return success(group: @group) + result[:group] = application_settings.instance_administration_project.owner + return success(result) end - @group = ::Groups::CreateService.new(group_owner, create_group_params).execute + result[:group] = ::Groups::CreateService.new(group_owner, create_group_params).execute - if @group.persisted? - success(group: @group) + if result[:group].persisted? + success(result) else error(_('Could not create group')) end end - def create_project + def create_project(result) if project_created? log_info('Instance administration project already exists') - @project = application_settings.instance_administration_project - return success(project: project) + result[:project] = application_settings.instance_administration_project + return success(result) end - @project = ::Projects::CreateService.new(group_owner, create_project_params).execute + result[:project] = ::Projects::CreateService.new(group_owner, create_project_params(result[:group])).execute - if project.persisted? - success(project: project) + if result[:project].persisted? + success(result) else - log_error("Could not create instance administration project. Errors: %{errors}" % { errors: project.errors.full_messages }) + log_error("Could not create instance administration project. Errors: %{errors}" % { errors: result[:project].errors.full_messages }) error(_('Could not create project')) end end - def save_project_id + def save_project_id(result) return success if project_created? - result = application_settings.update(instance_administration_project_id: @project.id) + response = application_settings.update( + instance_administration_project_id: result[:project].id + ) - if result - success + if response + success(result) else log_error("Could not save instance administration project ID, errors: %{errors}" % { errors: application_settings.errors.full_messages }) error(_('Could not save project ID')) end end - def add_group_members - members = @group.add_users(members_to_add, Gitlab::Access::MAINTAINER) + def add_group_members(result) + group = result[:group] + members = group.add_users(members_to_add(group), Gitlab::Access::MAINTAINER) errors = members.flat_map { |member| member.errors.full_messages } if errors.any? log_error('Could not add admins as members to self-monitoring project. Errors: %{errors}' % { errors: errors }) error(_('Could not add admins as members')) else - success + success(result) end end - def add_to_whitelist - return success unless prometheus_enabled? - return success unless prometheus_listen_address.present? + def add_to_whitelist(result) + return success(result) unless prometheus_enabled? + return success(result) unless prometheus_listen_address.present? uri = parse_url(internal_prometheus_listen_address_uri) return error(_('Prometheus listen_address in config/gitlab.yml is not a valid URI')) unless uri application_settings.add_to_outbound_local_requests_whitelist([uri.normalized_host]) - result = application_settings.save + response = application_settings.save - if result + if response # Expire the Gitlab::CurrentSettings cache after updating the whitelist. # This happens automatically in an after_commit hook, but in migrations, # the after_commit hook only runs at the end of the migration. Gitlab::CurrentSettings.expire_current_application_settings - success + success(result) else log_error("Could not add prometheus URL to whitelist, errors: %{errors}" % { errors: application_settings.errors.full_messages }) error(_('Could not add prometheus URL to whitelist')) end end - def add_prometheus_manual_configuration - return success unless prometheus_enabled? - return success unless prometheus_listen_address.present? + def add_prometheus_manual_configuration(result) + return success(result) unless prometheus_enabled? + return success(result) unless prometheus_listen_address.present? - service = project.find_or_initialize_service('prometheus') + service = result[:project].find_or_initialize_service('prometheus') unless service.update(prometheus_service_attributes) log_error('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}' % { errors: service.errors.full_messages }) return error(_('Could not save prometheus manual configuration')) end - success + success(result) end def application_settings @@ -196,11 +199,11 @@ module Gitlab instance_admins.first end - def members_to_add + def members_to_add(group) # Exclude admins who are already members of group because - # `@group.add_users(users)` returns an error if the users parameter contains + # `group.add_users(users)` returns an error if the users parameter contains # users who are already members of the group. - instance_admins - @group.members.collect(&:user) + instance_admins - group.members.collect(&:user) end def create_group_params @@ -217,13 +220,13 @@ module Gitlab ) end - def create_project_params + def create_project_params(group) { initialize_with_readme: true, visibility_level: VISIBILITY_LEVEL, name: PROJECT_NAME, description: "This project is automatically generated and will be used to help monitor this GitLab instance. [More information](#{docs_path})", - namespace_id: @group.id + namespace_id: group.id } end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index c46087e65de..30fe7440148 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -428,8 +428,8 @@ module Gitlab def viewer_class_from(classes) return unless diffable? - return if different_type? || external_storage_error? return unless new_file? || deleted_file? || content_changed? + return if different_type? || external_storage_error? verify_binary = !stored_externally? diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index e29bf75f341..c4288ca6408 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -3,19 +3,7 @@ module Gitlab module Diff module FileCollection - class MergeRequestDiff < Base - extend ::Gitlab::Utils::Override - - def initialize(merge_request_diff, diff_options:) - @merge_request_diff = merge_request_diff - - super(merge_request_diff, - project: merge_request_diff.project, - diff_options: diff_options, - diff_refs: merge_request_diff.diff_refs, - fallback_diff_refs: merge_request_diff.fallback_diff_refs) - end - + class MergeRequestDiff < MergeRequestDiffBase def diff_files diff_files = super diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb new file mode 100644 index 00000000000..a747a6ed475 --- /dev/null +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + module FileCollection + class MergeRequestDiffBase < Base + extend ::Gitlab::Utils::Override + + def initialize(merge_request_diff, diff_options:) + @merge_request_diff = merge_request_diff + + super(merge_request_diff, + project: merge_request_diff.project, + diff_options: diff_options, + diff_refs: merge_request_diff.diff_refs, + fallback_diff_refs: merge_request_diff.fallback_diff_refs) + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb new file mode 100644 index 00000000000..663326e01d5 --- /dev/null +++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + module FileCollection + # Builds a paginated diff file collection and collects pagination + # metadata. + # + # It doesn't handle caching yet as we're not prepared to write/read + # separate file keys (https://gitlab.com/gitlab-org/gitlab/issues/30550). + # + class MergeRequestDiffBatch < MergeRequestDiffBase + DEFAULT_BATCH_PAGE = 1 + DEFAULT_BATCH_SIZE = 20 + + attr_reader :pagination_data + + def initialize(merge_request_diff, batch_page, batch_size, diff_options:) + super(merge_request_diff, diff_options: diff_options) + + batch_page ||= DEFAULT_BATCH_PAGE + batch_size ||= DEFAULT_BATCH_SIZE + + @paginated_collection = relation.page(batch_page).per(batch_size) + @pagination_data = { + current_page: @paginated_collection.current_page, + next_page: @paginated_collection.next_page, + total_pages: @paginated_collection.total_pages + } + end + + def diff_file_paths + diff_files.map(&:file_path) + end + + override :diffs + def diffs + strong_memoize(:diffs) do + @merge_request_diff.opening_external_diff do + # Avoiding any extra queries. + collection = @paginated_collection.to_a + + # The offset collection and calculation is required so that we + # know how much has been loaded in previous batches, collapsing + # the current paginated set accordingly (collection limit calculation). + # See: https://docs.gitlab.com/ee/development/diffs.html#diff-collection-limits + # + offset_index = collection.first&.index + options = diff_options.dup + + collection = + if offset_index && offset_index > 0 + offset_collection = relation.limit(offset_index) # rubocop:disable CodeReuse/ActiveRecord + options[:offset_index] = offset_index + offset_collection + collection + else + collection + end + + Gitlab::Git::DiffCollection.new(collection.map(&:to_hash), options) + end + end + end + + private + + def relation + @merge_request_diff.merge_request_diff_files + end + end + end + end +end diff --git a/lib/gitlab/diff/lines_unfolder.rb b/lib/gitlab/diff/lines_unfolder.rb index 0bd18fe9622..6def3a074a3 100644 --- a/lib/gitlab/diff/lines_unfolder.rb +++ b/lib/gitlab/diff/lines_unfolder.rb @@ -54,7 +54,7 @@ module Gitlab def unfold_required? strong_memoize(:unfold_required) do next false unless @diff_file.text? - next false unless @position.on_text? && @position.unchanged? + next false unless @position.unfoldable? next false if @diff_file.new_file? || @diff_file.deleted_file? next false unless @position.old_line # Invalid position (MR import scenario) diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index dfa80eb4a64..8b99fd5cd42 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -79,6 +79,10 @@ module Gitlab formatter.line_age end + def unfoldable? + on_text? && unchanged? + end + def unchanged? type.nil? end @@ -118,8 +122,14 @@ module Gitlab path: file_path } + # Takes action when creating diff notes (multiple calls are + # submitted to this method). Gitlab::SafeRequestStore.fetch(key) { find_diff_file(repository) } end + + # We need to unfold diff lines according to the position in order + # to correctly calculate the line code and trace position changes. + @diff_file&.tap { |file| file.unfold_diff_lines(self) } end def diff_options @@ -152,13 +162,7 @@ module Gitlab return unless diff_refs.complete? return unless comparison = diff_refs.compare_in(repository.project) - file = comparison.diffs(diff_options).diff_files.first - - # We need to unfold diff lines according to the position in order - # to correctly calculate the line code and trace position changes. - file&.unfold_diff_lines(self) - - file + comparison.diffs(diff_options).diff_files.first end def get_formatter_class(type) diff --git a/lib/gitlab/diff/position_collection.rb b/lib/gitlab/diff/position_collection.rb new file mode 100644 index 00000000000..2112d347678 --- /dev/null +++ b/lib/gitlab/diff/position_collection.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class PositionCollection + include Enumerable + + # collection - An array of Gitlab::Diff::Position + def initialize(collection, diff_head_sha = nil) + @collection = collection + @diff_head_sha = diff_head_sha + end + + def each(&block) + filtered_positions.each(&block) + end + + def concat(positions) + tap { @collection.concat(positions) } + end + + # Doing a lightweight filter in-memory given we're not prepared for querying + # positions (https://gitlab.com/gitlab-org/gitlab/issues/33271). + def unfoldable + select do |position| + position.unfoldable? && valid_head_sha?(position) + end + end + + private + + def filtered_positions + @collection.select { |item| item.is_a?(Position) } + end + + def valid_head_sha?(position) + return true unless @diff_head_sha + + position.head_sha == @diff_head_sha + end + end + end +end diff --git a/lib/gitlab/discussions_diff/file_collection.rb b/lib/gitlab/discussions_diff/file_collection.rb index 6692dd76438..7a9d4c5c0c2 100644 --- a/lib/gitlab/discussions_diff/file_collection.rb +++ b/lib/gitlab/discussions_diff/file_collection.rb @@ -27,12 +27,14 @@ module Gitlab # - The cache content is not updated (there's no need to do so) def load_highlight ids = highlightable_collection_ids + return if ids.empty? + cached_content = read_cache(ids) uncached_ids = ids.select.each_with_index { |_, i| cached_content[i].nil? } mapping = highlighted_lines_by_ids(uncached_ids) - HighlightCache.write_multiple(mapping) + HighlightCache.write_multiple(mapping) if mapping.any? diffs = diff_files_indexed_by_id.values_at(*ids) diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb index 31bb6810391..457a3c12206 100644 --- a/lib/gitlab/downtime_check.rb +++ b/lib/gitlab/downtime_check.rb @@ -58,13 +58,13 @@ module Gitlab # Returns true if the given migration can be performed without downtime. def online?(migration) - migration.const_get(DOWNTIME_CONST) == false + migration.const_get(DOWNTIME_CONST, false) == false end # Returns the downtime reason, or nil if none was defined. def downtime_reason(migration) if migration.const_defined?(DOWNTIME_REASON_CONST) - migration.const_get(DOWNTIME_REASON_CONST) + migration.const_get(DOWNTIME_REASON_CONST, false) else nil end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 7da8b385266..847260b2e0f 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -32,7 +32,7 @@ module Gitlab mail = build_mail - ignore_auto_submitted!(mail) + ignore_auto_reply!(mail) mail_key = extract_mail_key(mail) handler = Handler.for(mail, mail_key) @@ -96,14 +96,25 @@ module Gitlab end end - def ignore_auto_submitted!(mail) + def ignore_auto_reply!(mail) + if auto_submitted?(mail) || auto_replied?(mail) + raise AutoGeneratedEmailError + end + end + + def auto_submitted?(mail) # Mail::Header#[] is case-insensitive auto_submitted = mail.header['Auto-Submitted']&.value # Mail::Field#value would strip leading and trailing whitespace - raise AutoGeneratedEmailError if - # See also https://tools.ietf.org/html/rfc3834 - auto_submitted && auto_submitted != 'no' + # See also https://tools.ietf.org/html/rfc3834 + auto_submitted && auto_submitted != 'no' + end + + def auto_replied?(mail) + autoreply = mail.header['X-Autoreply']&.value + + autoreply && autoreply == 'yes' end end end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb new file mode 100644 index 00000000000..895755376ee --- /dev/null +++ b/lib/gitlab/experimentation.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +# == Experimentation +# +# Utility module used for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant. +# The feature_toggle and environment keys are optional. If the feature_toggle is not set, a feature with the name of +# the experiment will be checked, with a default value of true. The enabled_ratio is required and should be +# the ratio for the number of users for which this experiment is enabled. For example: a ratio of 0.1 will +# enable the experiment for 10% of the users (determined by the `experimentation_subject_index`). +# +module Gitlab + module Experimentation + EXPERIMENTS = { + signup_flow: { + feature_toggle: :experimental_separate_sign_up_flow, + environment: ::Gitlab.dev_env_or_com?, + enabled_ratio: 0.1 + } + }.freeze + + # Controller concern that checks if an experimentation_subject_id cookie is present and sets it if absent. + # Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method + # to controllers and views. + # + module ControllerConcern + extend ActiveSupport::Concern + + included do + before_action :set_experimentation_subject_id_cookie + helper_method :experiment_enabled? + end + + def set_experimentation_subject_id_cookie + return if cookies[:experimentation_subject_id].present? + + cookies.permanent.signed[:experimentation_subject_id] = { + value: SecureRandom.uuid, + domain: :all, + secure: ::Gitlab.config.gitlab.https + } + end + + def experiment_enabled?(experiment_key) + Experimentation.enabled?(experiment_key, experimentation_subject_index) + end + + private + + def experimentation_subject_index + experimentation_subject_id = cookies.signed[:experimentation_subject_id] + return if experimentation_subject_id.blank? + + experimentation_subject_id.delete('-').hex % 100 + end + end + + class << self + def experiment(key) + Experiment.new(EXPERIMENTS[key].merge(key: key)) + end + + def enabled?(experiment_key, experimentation_subject_index) + return false unless EXPERIMENTS.key?(experiment_key) + + experiment = experiment(experiment_key) + + experiment.feature_toggle_enabled? && + experiment.enabled_for_environment? && + experiment.enabled_for_experimentation_subject?(experimentation_subject_index) + end + end + + Experiment = Struct.new(:key, :feature_toggle, :environment, :enabled_ratio, keyword_init: true) do + def feature_toggle_enabled? + return Feature.enabled?(key, default_enabled: true) if feature_toggle.nil? + + Feature.enabled?(feature_toggle) + end + + def enabled_for_environment? + return true if environment.nil? + + environment + end + + def enabled_for_experimentation_subject?(experimentation_subject_index) + return false if enabled_ratio.nil? || experimentation_subject_index.blank? + + experimentation_subject_index <= enabled_ratio * 100 + end + end + end +end diff --git a/lib/gitlab/file_markdown_link_builder.rb b/lib/gitlab/file_markdown_link_builder.rb index 180140e7da2..09d799b859d 100644 --- a/lib/gitlab/file_markdown_link_builder.rb +++ b/lib/gitlab/file_markdown_link_builder.rb @@ -10,14 +10,14 @@ module Gitlab return unless name = markdown_name markdown = "[#{name.gsub(']', '\\]')}](#{secure_url})" - markdown = "!#{markdown}" if image_or_video? || dangerous? + markdown = "!#{markdown}" if embeddable? || dangerous_embeddable? markdown end def markdown_name return unless filename.present? - image_or_video? ? File.basename(filename, File.extname(filename)) : filename + embeddable? ? File.basename(filename, File.extname(filename)) : filename end end end diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb index 25ee07cf940..ca78d49f99b 100644 --- a/lib/gitlab/file_type_detection.rb +++ b/lib/gitlab/file_type_detection.rb @@ -1,34 +1,69 @@ # frozen_string_literal: true -# File helpers methods. -# It needs the method filename to be defined. +# The method `filename` must be defined in classes that use this module. +# +# This module is intended to be used as a helper and not a security gate +# to validate that a file is safe, as it identifies files only by the +# file extension and not its actual contents. +# +# An example useage of this module is in `FileMarkdownLinkBuilder` that +# renders markdown depending on a file name. +# +# We use Workhorse to detect the real extension when we serve files with +# the `SendsBlob` helper methods, and ask Workhorse to set the content +# type when it serves the file: +# https://gitlab.com/gitlab-org/gitlab/blob/33e5955/app/helpers/workhorse_helper.rb#L48. +# +# Because Workhorse has access to the content when it is downloaded, if +# the type/extension doesn't match the real type, we adjust the +# `Content-Type` and `Content-Disposition` to the one we get from the detection. module Gitlab module FileTypeDetection - IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze + SAFE_IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze # We recommend using the .mp4 format over .mov. Videos in .mov format can # still be used but you really need to make sure they are served with the # proper MIME type video/mp4 and not video/quicktime or your videos won't play # on IE >= 9. # http://archive.sublimevideo.info/20150912/docs.sublimevideo.net/troubleshooting.html - VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze + SAFE_VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze + SAFE_AUDIO_EXT = %w[mp3 oga ogg spx wav].freeze + # These extension types can contain dangerous code and should only be embedded inline with # proper filtering. They should always be tagged as "Content-Disposition: attachment", not "inline". - DANGEROUS_EXT = %w[svg].freeze + DANGEROUS_IMAGE_EXT = %w[svg].freeze + DANGEROUS_VIDEO_EXT = [].freeze # None, yet + DANGEROUS_AUDIO_EXT = [].freeze # None, yet def image? - extension_match?(IMAGE_EXT) + extension_match?(SAFE_IMAGE_EXT) end def video? - extension_match?(VIDEO_EXT) + extension_match?(SAFE_VIDEO_EXT) + end + + def audio? + extension_match?(SAFE_AUDIO_EXT) + end + + def embeddable? + image? || video? || audio? + end + + def dangerous_image? + extension_match?(DANGEROUS_IMAGE_EXT) + end + + def dangerous_video? + extension_match?(DANGEROUS_VIDEO_EXT) end - def image_or_video? - image? || video? + def dangerous_audio? + extension_match?(DANGEROUS_AUDIO_EXT) end - def dangerous? - extension_match?(DANGEROUS_EXT) + def dangerous_embeddable? + dangerous_image? || dangerous_video? || dangerous_audio? end private diff --git a/lib/gitlab/git/changes.rb b/lib/gitlab/git/changes.rb new file mode 100644 index 00000000000..4e888eec44f --- /dev/null +++ b/lib/gitlab/git/changes.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class Changes + include Enumerable + + attr_reader :repository_data + + def initialize + @refs = Set.new + @items = [] + @branches_index = [] + @tags_index = [] + @repository_data = [] + end + + def includes_branches? + branches_index.any? + end + + def includes_tags? + tags_index.any? + end + + def add_branch_change(change) + @branches_index << add_change(change) + self + end + + def add_tag_change(change) + @tags_index << add_change(change) + self + end + + def each + items.each do |item| + yield item + end + end + + def refs + @refs.to_a + end + + def branch_changes + items.values_at(*branches_index) + end + + def tag_changes + items.values_at(*tags_index) + end + + private + + attr_reader :items, :branches_index, :tags_index + + def add_change(change) + # refs and repository_data are being cached when a change is added to + # the collection to remove the need to iterate through changes multiple + # times. + @refs << change[:ref] + @repository_data << build_change_repository_data(change) + @items << change + + @items.size - 1 + end + + def build_change_repository_data(change) + DataBuilder::Repository.single_change(change[:oldrev], change[:newrev], change[:ref]) + end + end + end +end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index cb9154cb1e8..b79e30bff78 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -31,6 +31,7 @@ module Gitlab @limits = self.class.limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) + @offset_index = options.fetch(:offset_index, 0) @line_count = 0 @byte_count = 0 @@ -128,7 +129,7 @@ module Gitlab def each_serialized_patch i = @array.length - @iterator.each do |raw| + @iterator.each_with_index do |raw, iterator_index| @empty = false if @enforce_limits && i >= max_files @@ -154,8 +155,12 @@ module Gitlab break end - yield @array[i] = diff - i += 1 + # We should not yield / memoize diffs before the offset index. Though, + # we still consider the limit buffers for diffs before it. + if iterator_index >= @offset_index + yield @array[i] = diff + i += 1 + end end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 4ea618f063b..b2c22898079 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -131,6 +131,18 @@ module Gitlab end end + def rename(new_relative_path) + wrapped_gitaly_errors do + gitaly_repository_client.rename(new_relative_path) + end + end + + def remove + wrapped_gitaly_errors do + gitaly_repository_client.remove + end + end + def expire_has_local_branches_cache clear_memoization(:has_local_branches) end diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index 2a8bcd015a8..5264bae47a1 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -8,7 +8,7 @@ module Gitlab def initialize(project, identifier, changes, push_options = {}) @project = project @identifier = identifier - @changes = deserialize_changes(changes) + @changes = parse_changes(changes) @push_options = push_options end @@ -16,27 +16,12 @@ module Gitlab super(identifier) end - def changes_refs - return changes unless block_given? - - changes.each do |change| - change.strip! - oldrev, newrev, ref = change.split(' ') - - yield oldrev, newrev, ref - end - end - def includes_branches? - enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| - Gitlab::Git.branch_ref?(ref) - end + changes.includes_branches? end def includes_tags? - enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| - Gitlab::Git.tag_ref?(ref) - end + changes.includes_tags? end def includes_default_branch? @@ -44,16 +29,28 @@ module Gitlab # first branch pushed will be the default. return true unless project.default_branch.present? - enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| - Gitlab::Git.branch_ref?(ref) && - Gitlab::Git.branch_name(ref) == project.default_branch + changes.branch_changes.any? do |change| + Gitlab::Git.branch_name(change[:ref]) == project.default_branch end end private - def deserialize_changes(changes) - utf8_encode_changes(changes).each_line + def parse_changes(changes) + deserialized_changes = utf8_encode_changes(changes).each_line + + Git::Changes.new.tap do |collection| + deserialized_changes.each_with_index do |raw_change, index| + oldrev, newrev, ref = raw_change.strip.split(' ') + change = { index: index, oldrev: oldrev, newrev: newrev, ref: ref } + + if Git.branch_ref?(ref) + collection.add_branch_change(change) + elsif Git.tag_ref?(ref) + collection.add_tag_change(change) + end + end + end end def utf8_encode_changes(changes) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 2ac99b1ff02..b0f29d22ad4 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -86,7 +86,7 @@ module Gitlab if name == :health_check Grpc::Health::V1::Health::Stub else - Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) + Gitaly.const_get(name.to_s.camelcase.to_sym, false).const_get(:Stub, false) end end @@ -142,13 +142,13 @@ module Gitlab # kwargs.merge(deadline: Time.now + 10) # end # - def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil) + def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout) start = Gitlab::Metrics::System.monotonic_time request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {} enforce_gitaly_request_limits(:call) - kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage) + kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage) kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend @@ -200,7 +200,7 @@ module Gitlab end private_class_method :authorization_token - def self.request_kwargs(storage, timeout, remote_storage: nil) + def self.request_kwargs(storage, timeout:, remote_storage: nil) metadata = { 'authorization' => "Bearer #{authorization_token(storage)}", 'client_name' => CLIENT_NAME @@ -216,14 +216,7 @@ module Gitlab result = { metadata: metadata } - # nil timeout indicates that we should use the default - timeout = default_timeout if timeout.nil? - - return result unless timeout > 0 - - deadline = real_time + timeout - result[:deadline] = deadline - + result[:deadline] = real_time + timeout if timeout > 0 result end @@ -357,8 +350,6 @@ module Gitlab # The default timeout on all Gitaly calls def self.default_timeout - return no_timeout if Sidekiq.server? - timeout(:gitaly_timeout_default) end @@ -370,8 +361,12 @@ module Gitlab timeout(:gitaly_timeout_medium) end - def self.no_timeout - 0 + def self.long_timeout + if Sidekiq.server? + 6.hours + else + default_timeout + end end def self.storage_metadata_file_path(storage) diff --git a/lib/gitlab/gitaly_client/attributes_bag.rb b/lib/gitlab/gitaly_client/attributes_bag.rb index 3f1a0ef4888..f935281ac2e 100644 --- a/lib/gitlab/gitaly_client/attributes_bag.rb +++ b/lib/gitlab/gitaly_client/attributes_bag.rb @@ -8,7 +8,7 @@ module Gitlab extend ActiveSupport::Concern included do - attr_accessor(*const_get(:ATTRS)) + attr_accessor(*const_get(:ATTRS, false)) end def initialize(params) @@ -26,7 +26,7 @@ module Gitlab end def attributes - self.class.const_get(:ATTRS) + self.class.const_get(:ATTRS, false) end end end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 8ccefb00d20..5cde06bb6aa 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -76,6 +76,30 @@ module Gitlab GitalyClient::BlobsStitcher.new(response) end + def get_blob_types(revision_paths, limit = -1) + return {} if revision_paths.empty? + + request_revision_paths = revision_paths.map do |rev, path| + Gitaly::GetBlobsRequest::RevisionPath.new(revision: rev, path: encode_binary(path)) + end + + request = Gitaly::GetBlobsRequest.new( + repository: @gitaly_repo, + revision_paths: request_revision_paths, + limit: limit + ) + + response = GitalyClient.call( + @gitaly_repo.storage_name, + :blob_service, + :get_blobs, + request, + timeout: GitalyClient.fast_timeout + ) + + map_blob_types(response) + end + def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil) request = Gitaly::GetNewLFSPointersRequest.new( repository: @gitaly_repo, @@ -132,6 +156,16 @@ module Gitlab end end end + + def map_blob_types(response) + types = {} + + response.each do |msg| + types[msg.path.dup.force_encoding('utf-8')] = msg.type.downcase + end + + types + end end end end diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb index a56bc35f6d7..e2293d3121a 100644 --- a/lib/gitlab/gitaly_client/cleanup_service.rb +++ b/lib/gitlab/gitaly_client/cleanup_service.rb @@ -18,7 +18,7 @@ module Gitlab :cleanup_service, :apply_bfg_object_map_stream, build_object_map_enum(io), - timeout: GitalyClient.no_timeout + timeout: GitalyClient.long_timeout ) responses.each(&blk) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index a80ce462ab0..b0559729ff3 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -140,7 +140,8 @@ module Gitlab request = Gitaly::CountCommitsRequest.new( repository: @gitaly_repo, revision: encode_binary(ref), - all: !!options[:all] + all: !!options[:all], + first_parent: !!options[:first_parent] ) request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? @@ -254,7 +255,7 @@ module Gitlab def languages(ref = nil) request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '') - response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request) + response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout) response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } } end @@ -297,18 +298,6 @@ module Gitlab Gitlab::SafeRequestStore[key] = commit end - # rubocop: disable CodeReuse/ActiveRecord - def patch(revision) - request = Gitaly::CommitPatchRequest.new( - repository: @gitaly_repo, - revision: encode_binary(revision) - ) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request, timeout: GitalyClient.medium_timeout) - - response.sum(&:data) - end - # rubocop: enable CodeReuse/ActiveRecord - def commit_stats(revision) request = Gitaly::CommitStatsRequest.new( repository: @gitaly_repo, @@ -325,6 +314,7 @@ module Gitlab follow: options[:follow], skip_merges: options[:skip_merges], all: !!options[:all], + first_parent: !!options[:first_parent], disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] @@ -360,7 +350,7 @@ module Gitlab def extract_signature(commit_id) request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id) - response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request) + response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request, timeout: GitalyClient.fast_timeout) signature = +''.b signed_text = +''.b diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb index 0e00f6e8c44..38ec910111c 100644 --- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb +++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb @@ -5,8 +5,11 @@ module Gitlab class ConflictFilesStitcher include Enumerable - def initialize(rpc_response) + attr_reader :gitaly_repo + + def initialize(rpc_response, gitaly_repo) @rpc_response = rpc_response + @gitaly_repo = gitaly_repo end def each @@ -31,7 +34,7 @@ module Gitlab def file_from_gitaly_header(header) Gitlab::Git::Conflict::File.new( - Gitlab::GitalyClient::Util.git_repository(header.repository), + Gitlab::GitalyClient::Util.git_repository(gitaly_repo), header.commit_oid, conflict_from_gitaly_file_header(header), '' diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index d16e45c964d..f7eb4b45197 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -20,9 +20,9 @@ module Gitlab our_commit_oid: @our_commit_oid, their_commit_oid: @their_commit_oid ) - response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request) + response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout) - GitalyClient::ConflictFilesStitcher.new(response) + GitalyClient::ConflictFilesStitcher.new(response, @gitaly_repo) end def conflicts? diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb index f0be3cbebd2..0be214f3035 100644 --- a/lib/gitlab/gitaly_client/namespace_service.rb +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -22,7 +22,7 @@ module Gitlab def remove(name) request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name) - gitaly_client_call(:remove_namespace, request, timeout: nil) + gitaly_client_call(:remove_namespace, request, timeout: GitalyClient.long_timeout) end def rename(from, to) diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb index d7fac26bc13..786ef0ebebe 100644 --- a/lib/gitlab/gitaly_client/object_pool_service.rb +++ b/lib/gitlab/gitaly_client/object_pool_service.rb @@ -15,13 +15,15 @@ module Gitlab object_pool: object_pool, origin: repository.gitaly_repository) - GitalyClient.call(storage, :object_pool_service, :create_object_pool, request) + GitalyClient.call(storage, :object_pool_service, :create_object_pool, + request, timeout: GitalyClient.medium_timeout) end def delete request = Gitaly::DeleteObjectPoolRequest.new(object_pool: object_pool) - GitalyClient.call(storage, :object_pool_service, :delete_object_pool, request) + GitalyClient.call(storage, :object_pool_service, :delete_object_pool, + request, timeout: GitalyClient.long_timeout) end def link_repository(repository) @@ -40,7 +42,8 @@ module Gitlab origin: repository.gitaly_repository ) - GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request) + GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, + request, timeout: GitalyClient.long_timeout) end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 33ca428a942..6e486c763da 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -19,7 +19,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -35,7 +35,7 @@ module Gitlab message: encode_binary(message.to_s) ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error elsif response.exists @@ -55,7 +55,7 @@ module Gitlab start_point: encode_binary(start_point) ) response = GitalyClient.call(@repository.storage, :operation_service, - :user_create_branch, request) + :user_create_branch, request, timeout: GitalyClient.long_timeout) if response.pre_receive_error.present? raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error) @@ -79,7 +79,8 @@ module Gitlab oldrev: encode_binary(oldrev) ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_update_branch, request) + response = GitalyClient.call(@repository.storage, :operation_service, + :user_update_branch, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -93,7 +94,8 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request) + response = GitalyClient.call(@repository.storage, :operation_service, + :user_delete_branch, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -111,7 +113,8 @@ module Gitlab first_parent_ref: encode_binary(first_parent_ref) ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request) + response = GitalyClient.call(@repository.storage, :operation_service, + :user_merge_to_ref, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -126,7 +129,8 @@ module Gitlab @repository.storage, :operation_service, :user_merge_branch, - request_enum.each + request_enum.each, + timeout: GitalyClient.long_timeout ) request_enum.push( @@ -170,7 +174,8 @@ module Gitlab @repository.storage, :operation_service, :user_ff_branch, - request + request, + timeout: GitalyClient.long_timeout ) Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) @@ -215,6 +220,7 @@ module Gitlab :operation_service, :user_rebase, request, + timeout: GitalyClient.long_timeout, remote_storage: remote_repository.storage ) @@ -236,6 +242,7 @@ module Gitlab :operation_service, :user_rebase_confirmable, request_enum.each, + timeout: GitalyClient.long_timeout, remote_storage: remote_repository.storage ) @@ -286,7 +293,8 @@ module Gitlab @repository.storage, :operation_service, :user_squash, - request + request, + timeout: GitalyClient.long_timeout ) if response.git_error.presence @@ -310,7 +318,8 @@ module Gitlab @repository.storage, :operation_service, :user_update_submodule, - request + request, + timeout: GitalyClient.long_timeout ) if response.pre_receive_error.present? @@ -352,7 +361,8 @@ module Gitlab end response = GitalyClient.call(@repository.storage, :operation_service, - :user_commit_files, req_enum, remote_storage: start_repository.storage) + :user_commit_files, req_enum, timeout: GitalyClient.long_timeout, + remote_storage: start_repository.storage) if (pre_receive_error = response.pre_receive_error.presence) raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -384,7 +394,8 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :operation_service, :user_apply_patch, chunks) + response = GitalyClient.call(@repository.storage, :operation_service, + :user_apply_patch, chunks, timeout: GitalyClient.long_timeout) Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end @@ -424,7 +435,7 @@ module Gitlab :"user_#{rpc}", request, remote_storage: start_repository.storage, - timeout: GitalyClient.medium_timeout + timeout: GitalyClient.long_timeout ) handle_cherry_pick_or_revert_response(response) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index b7d509dfa48..d1f848fae26 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -21,7 +21,7 @@ module Gitlab def remote_branches(remote_name) request = Gitaly::FindAllRemoteBranchesRequest.new(repository: @gitaly_repo, remote_name: remote_name) - response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request) + response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request, timeout: GitalyClient.medium_timeout) consume_find_all_remote_branches_response(remote_name, response) end @@ -158,7 +158,7 @@ module Gitlab start_point: encode_binary(start_point) ) - response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request) + response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request, timeout: GitalyClient.medium_timeout) case response.status when :OK @@ -182,7 +182,7 @@ module Gitlab name: encode_binary(branch_name) ) - GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) + GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request, timeout: GitalyClient.medium_timeout) end def delete_refs(refs: [], except_with_prefixes: []) @@ -192,7 +192,7 @@ module Gitlab except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.default_timeout) + response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.medium_timeout) raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? end @@ -242,7 +242,7 @@ module Gitlab def pack_refs request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :ref_service, :pack_refs, request) + GitalyClient.call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout) end private diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index f3589fea39f..d01a29e1a05 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -38,9 +38,7 @@ module Gitlab def remove_remote(name) request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name) - response = GitalyClient.call(@storage, :remote_service, :remove_remote, request) - - response.result + GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.long_timeout).result end def fetch_internal_remote(repository) @@ -51,6 +49,7 @@ module Gitlab response = GitalyClient.call(@storage, :remote_service, :fetch_internal_remote, request, + timeout: GitalyClient.medium_timeout, remote_storage: repository.storage) response.result @@ -63,7 +62,7 @@ module Gitlab ) response = GitalyClient.call(@storage, :remote_service, - :find_remote_root_ref, request) + :find_remote_root_ref, request, timeout: GitalyClient.medium_timeout) encode_utf8(response.ref) end @@ -95,7 +94,7 @@ module Gitlab end end - GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum) + GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum, timeout: GitalyClient.long_timeout) end end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index ca3e5b51ecc..d0e5e0db830 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -28,17 +28,17 @@ module Gitlab def garbage_collect(create_bitmap) request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) - GitalyClient.call(@storage, :repository_service, :garbage_collect, request) + GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) end def repack_full(create_bitmap) request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) - GitalyClient.call(@storage, :repository_service, :repack_full, request) + GitalyClient.call(@storage, :repository_service, :repack_full, request, timeout: GitalyClient.long_timeout) end def repack_incremental request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :repack_incremental, request) + GitalyClient.call(@storage, :repository_service, :repack_incremental, request, timeout: GitalyClient.long_timeout) end def repository_size @@ -86,12 +86,12 @@ module Gitlab end end - GitalyClient.call(@storage, :repository_service, :fetch_remote, request) + GitalyClient.call(@storage, :repository_service, :fetch_remote, request, timeout: GitalyClient.long_timeout) end def create_repository request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.medium_timeout) + GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) end def has_local_branches? @@ -123,7 +123,7 @@ module Gitlab :create_fork, request, remote_storage: source_repository.storage, - timeout: GitalyClient.default_timeout + timeout: GitalyClient.long_timeout ) end @@ -138,7 +138,7 @@ module Gitlab :repository_service, :create_repository_from_url, request, - timeout: GitalyClient.default_timeout + timeout: GitalyClient.long_timeout ) end @@ -189,6 +189,7 @@ module Gitlab :repository_service, :fetch_source_branch, request, + timeout: GitalyClient.long_timeout, remote_storage: source_repository.storage ) @@ -197,7 +198,7 @@ module Gitlab def fsck request = Gitaly::FsckRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.no_timeout) + response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.long_timeout) if response.error.empty? return "", 0 @@ -211,7 +212,7 @@ module Gitlab save_path, :create_bundle, Gitaly::CreateBundleRequest, - GitalyClient.no_timeout + GitalyClient.long_timeout ) end @@ -229,7 +230,7 @@ module Gitlab bundle_path, :create_repository_from_bundle, Gitaly::CreateRepositoryFromBundleRequest, - GitalyClient.no_timeout + GitalyClient.long_timeout ) end @@ -254,7 +255,7 @@ module Gitlab :repository_service, :create_repository_from_snapshot, request, - timeout: GitalyClient.no_timeout + timeout: GitalyClient.long_timeout ) end @@ -333,7 +334,7 @@ module Gitlab def search_files_by_content(ref, query) request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) - response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request) + response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout) search_results_from_response(response) end @@ -343,7 +344,19 @@ module Gitlab repository: @gitaly_repo ) - GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request) + GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request, timeout: GitalyClient.long_timeout) + end + + def rename(relative_path) + request = Gitaly::RenameRepositoryRequest.new(repository: @gitaly_repo, relative_path: relative_path) + + GitalyClient.call(@storage, :repository_service, :rename_repository, request, timeout: GitalyClient.fast_timeout) + end + + def remove + request = Gitaly::RemoveRepositoryRequest.new(repository: @gitaly_repo) + + GitalyClient.call(@storage, :repository_service, :remove_repository, request, timeout: GitalyClient.long_timeout) end private diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb deleted file mode 100644 index 4edcb0b8ba9..00000000000 --- a/lib/gitlab/gitaly_client/storage_service.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GitalyClient - class StorageService - def initialize(storage) - @storage = storage - end - - # Returns all directories in the git storage directory, lexically ordered - def list_directories(depth: 1) - request = Gitaly::ListDirectoriesRequest.new(storage_name: @storage, depth: depth) - - GitalyClient.call(@storage, :storage_service, :list_directories, request) - .flat_map(&:paths) - end - - # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation. - def delete_all_repositories - request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage) - GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request) - end - end - end -end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 7d1206e551b..43848772947 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -53,7 +53,7 @@ module Gitlab @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path'] storage['path'] = Deprecated - @hash = storage + @hash = storage.with_indifferent_access end def gitaly_address diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index ce9faad825c..15e0d7349dd 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -34,7 +34,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum, timeout: GitalyClient.medium_timeout) if error = response.duplicate_error.presence raise Gitlab::Git::Wiki::DuplicatePageError, error end @@ -61,7 +61,7 @@ module Gitlab end end - GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum) + GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum, timeout: GitalyClient.medium_timeout) end def delete_page(page_path, commit_details) @@ -187,7 +187,7 @@ module Gitlab directory: encode_binary(dir) ) - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request, timeout: GitalyClient.medium_timeout) response.reduce([]) { |memo, msg| memo << msg.data }.join end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 9d925581441..a3734ccf069 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -32,11 +32,13 @@ module Gitlab def build(release) { + name: release.name, tag: release.tag_name, description: description_for(release), created_at: release.created_at, - updated_at: release.updated_at, - released_at: release.published_at, + updated_at: release.created_at, + # Draft releases will have a null published_at + released_at: release.published_at || Time.current, project_id: project.id } end @@ -46,11 +48,7 @@ module Gitlab end def description_for(release) - if release.body.present? - release.body - else - "Release for tag #{release.tag_name}" - end + release.body.presence || "Release for tag #{release.tag_name}" end end end diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 19915980d7f..01bc27f963b 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -40,3 +40,5 @@ module Gitlab end end end + +Gitlab::GlRepository::RepoType.prepend_if_ee('EE::Gitlab::GlRepository::RepoType') diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 92917028851..f1e31a615a4 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -38,6 +38,13 @@ module Gitlab gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url end + + # Initialize gon.features with any flags that should be + # made globally available to the frontend + push_frontend_feature_flag(:suppress_ajax_navigation_errors, default_enabled: true) + + # Flag controls a GFM feature used across many routes. + push_frontend_feature_flag(:gfm_grafana_integration) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 1e7203cb82a..4da2004b74f 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -117,7 +117,7 @@ module Gitlab description: body, author_id: project.creator_id, assignee_ids: [assignee_id], - state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' + state_id: raw_issue['state'] == 'closed' ? Issue.available_states[:closed] : Issue.available_states[:opened] ) issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 4b797a0e397..dc71d0b427a 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -10,6 +10,8 @@ module Gitlab repo = commit.project.repository.raw_repository @signature_data = Gitlab::Git::Commit.extract_signature_lazily(repo, commit.sha || commit.id) + + lazy_signature end def signature_text @@ -28,18 +30,16 @@ module Gitlab !!(signature_text && signed_text) end - # rubocop: disable CodeReuse/ActiveRecord def signature return unless has_signature? return @signature if @signature - cached_signature = GpgSignature.find_by(commit_sha: @commit.sha) + cached_signature = lazy_signature&.itself return @signature = cached_signature if cached_signature.present? @signature = create_cached_signature! end - # rubocop: enable CodeReuse/ActiveRecord def update_signature!(cached_signature) using_keychain do |gpg_key| @@ -50,6 +50,14 @@ module Gitlab private + def lazy_signature + BatchLoader.for(@commit.sha).batch do |shas, loader| + GpgSignature.by_commit_sha(shas).each do |signature| + loader.call(signature.commit_sha, signature) + end + end + end + def using_keychain Gitlab::Gpg.using_tmp_keychain do # first we need to get the fingerprint from the signature to query the gpg diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb index f47a372aa19..41aef64f683 100644 --- a/lib/gitlab/graphql/docs/renderer.rb +++ b/lib/gitlab/graphql/docs/renderer.rb @@ -23,15 +23,12 @@ module Gitlab @parsed_schema = GraphQLDocs::Parser.new(schema, {}).parse end - def render - contents = @layout.render(self) - - write_file(contents) + def contents + # Render and remove an extra trailing new line + @contents ||= @layout.render(self).sub!(/\n(?=\Z)/, '') end - private - - def write_file(contents) + def write filename = File.join(@output_dir, 'index.md') FileUtils.mkdir_p(@output_dir) diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index cc22d43ab4f..33acff38ef4 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -20,6 +20,3 @@ - type[:fields].each do |field| = "| `#{field[:name]}` | #{render_field_type(field[:type][:info])} | #{field[:description]} |" \ - - - diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb index 1d31f59999c..199cd2f9b2d 100644 --- a/lib/gitlab/health_checks/base_abstract_check.rb +++ b/lib/gitlab/health_checks/base_abstract_check.rb @@ -15,10 +15,6 @@ module Gitlab raise NotImplementedError end - def liveness - HealthChecks::Result.new(true) - end - def metrics [] end diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb index e560f87bf98..e780bf8a986 100644 --- a/lib/gitlab/health_checks/gitaly_check.rb +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -5,7 +5,7 @@ module Gitlab class GitalyCheck extend BaseAbstractCheck - METRIC_PREFIX = 'gitaly_health_check' + METRIC_PREFIX = 'gitaly_health_check'.freeze class << self def readiness @@ -29,7 +29,13 @@ module Gitlab def check(storage_name) serv = Gitlab::GitalyClient::HealthCheckService.new(storage_name) result = serv.check - HealthChecks::Result.new(result[:success], result[:message], shard: storage_name) + + HealthChecks::Result.new( + name, + result[:success], + result[:message], + shard: storage_name + ) end private diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb index 184083de2bc..b697cb0d027 100644 --- a/lib/gitlab/health_checks/metric.rb +++ b/lib/gitlab/health_checks/metric.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true -module Gitlab::HealthChecks - Metric = Struct.new(:name, :value, :labels) +module Gitlab + module HealthChecks + Metric = Struct.new(:name, :value, :labels) + end end diff --git a/lib/gitlab/health_checks/probes/collection.rb b/lib/gitlab/health_checks/probes/collection.rb new file mode 100644 index 00000000000..db3ef4834c2 --- /dev/null +++ b/lib/gitlab/health_checks/probes/collection.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + module Probes + class Collection + attr_reader :checks + + # This accepts an array of objects implementing `:readiness` + # that returns `::Gitlab::HealthChecks::Result` + def initialize(*checks) + @checks = checks + end + + def execute + readiness = probe_readiness + success = all_succeeded?(readiness) + + Probes::Status.new( + success ? 200 : 503, + status(success).merge(payload(readiness)) + ) + end + + private + + def all_succeeded?(readiness) + readiness.all? do |name, probes| + probes.any?(&:success) + end + end + + def status(success) + { status: success ? 'ok' : 'failed' } + end + + def payload(readiness) + readiness.transform_values do |probes| + probes.map(&:payload) + end + end + + def probe_readiness + checks + .flat_map(&:readiness) + .compact + .group_by(&:name) + end + end + end + end +end diff --git a/lib/gitlab/health_checks/probes/status.rb b/lib/gitlab/health_checks/probes/status.rb new file mode 100644 index 00000000000..192e9366001 --- /dev/null +++ b/lib/gitlab/health_checks/probes/status.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + module Probes + Status = Struct.new(:http_status, :json) do + # We accept 2xx + def success? + http_status / 100 == 2 + end + end + end + end +end diff --git a/lib/gitlab/health_checks/prometheus_text_format.rb b/lib/gitlab/health_checks/prometheus_text_format.rb deleted file mode 100644 index 2a8f9d31cd5..00000000000 --- a/lib/gitlab/health_checks/prometheus_text_format.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - class PrometheusTextFormat - def marshal(metrics) - "#{metrics_with_type_declarations(metrics).join("\n")}\n" - end - - private - - def metrics_with_type_declarations(metrics) - type_declaration_added = {} - - metrics.flat_map do |metric| - metric_lines = [] - - unless type_declaration_added.key?(metric.name) - type_declaration_added[metric.name] = true - metric_lines << metric_type_declaration(metric) - end - - metric_lines << metric_text(metric) - end - end - - def metric_type_declaration(metric) - "# TYPE #{metric.name} gauge" - end - - def metric_text(metric) - labels = metric.labels&.map { |key, value| "#{key}=\"#{value}\"" }&.join(',') || '' - - if labels.empty? - "#{metric.name} #{metric.value}" - else - "#{metric.name}{#{labels}} #{metric.value}" - end - end - end - end -end diff --git a/lib/gitlab/health_checks/puma_check.rb b/lib/gitlab/health_checks/puma_check.rb new file mode 100644 index 00000000000..7aafe29fbae --- /dev/null +++ b/lib/gitlab/health_checks/puma_check.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + # This check can only be run on Puma `master` process + class PumaCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'puma_check' + end + + def successful?(result) + result > 0 + end + + def check + return unless defined?(::Puma) + + stats = Puma.stats + stats = JSON.parse(stats) + + # If `workers` is missing this means that + # Puma server is running in single mode + stats.fetch('workers', 1) + rescue NoMethodError + # server is not ready + 0 + end + end + end + end +end diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb index 4586b1d94a7..38a36100ec7 100644 --- a/lib/gitlab/health_checks/result.rb +++ b/lib/gitlab/health_checks/result.rb @@ -1,5 +1,15 @@ # frozen_string_literal: true -module Gitlab::HealthChecks - Result = Struct.new(:success, :message, :labels) +module Gitlab + module HealthChecks + Result = Struct.new(:name, :success, :message, :labels) do + def payload + { + status: success ? 'ok' : 'failed', + message: message, + labels: labels + }.compact + end + end + end end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index 5a1e8c2a1dd..4e0b9296819 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -7,17 +7,23 @@ module Gitlab def readiness check_result = check + return if check_result.nil? + if successful?(check_result) - HealthChecks::Result.new(true) + HealthChecks::Result.new(name, true) elsif check_result.is_a?(Timeout::Error) - HealthChecks::Result.new(false, "#{human_name} check timed out") + HealthChecks::Result.new(name, false, "#{human_name} check timed out") else - HealthChecks::Result.new(false, "unexpected #{human_name} check result: #{check_result}") + HealthChecks::Result.new(name, false, "unexpected #{human_name} check result: #{check_result}") end + rescue => e + HealthChecks::Result.new(name, false, "unexpected #{human_name} check result: #{e}") end def metrics result, elapsed = with_timing(&method(:check)) + return if result.nil? + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) # rubocop:disable Gitlab/RailsLogger [ metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), diff --git a/lib/gitlab/health_checks/unicorn_check.rb b/lib/gitlab/health_checks/unicorn_check.rb new file mode 100644 index 00000000000..a30ae015257 --- /dev/null +++ b/lib/gitlab/health_checks/unicorn_check.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + # This check can only be run on Unicorn `master` process + class UnicornCheck + extend SimpleAbstractCheck + + class << self + include Gitlab::Utils::StrongMemoize + + private + + def metric_prefix + 'unicorn_check' + end + + def successful?(result) + result > 0 + end + + def check + return unless http_servers + + http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord + end + + # Traversal of ObjectSpace is expensive, on fully loaded application + # it takes around 80ms. The instances of HttpServers are not a subject + # to change so we can cache the list of servers. + def http_servers + strong_memoize(:http_servers) do + next unless defined?(::Unicorn::HttpServer) + + ObjectSpace.each_object(::Unicorn::HttpServer).to_a + end + end + end + end + end +end diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index 1f64e440141..9d9db6cf94f 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -27,7 +27,7 @@ module Gitlab duplicated_to_id project_id relative_position - state + state_id time_estimate title updated_at @@ -46,7 +46,8 @@ module Gitlab human_time_estimate: issue.human_time_estimate, assignee_ids: issue.assignee_ids, assignee_id: issue.assignee_ids.first, # This key is deprecated - labels: issue.labels_hook_attrs + labels: issue.labels_hook_attrs, + state: issue.state } issue.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index d08848a65a8..b2ac60fe825 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -38,6 +38,10 @@ module Gitlab "lfs-objects" end + def wiki_repo_bundle_filename + "project.wiki.bundle" + end + def config_file Rails.root.join('lib/gitlab/import_export/import_export.yml') end @@ -61,3 +65,5 @@ module Gitlab end end end + +Gitlab::ImportExport.prepend_if_ee('EE::Gitlab::ImportExport') diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index c5fb39b7b52..b30258123d4 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -10,11 +10,9 @@ module Gitlab StrategyError = Class.new(StandardError) - AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action' - private - attr_reader :project, :current_user + attr_reader :project, :current_user, :lock_file public @@ -29,8 +27,9 @@ module Gitlab def execute(current_user, project) @project = project - return unless @project.export_status == :finished - + ensure_export_ready! + ensure_lock_files_path! + @lock_file = File.join(lock_files_path, SecureRandom.hex) @current_user = current_user if invalid? @@ -48,19 +47,32 @@ module Gitlab false ensure delete_after_export_lock + delete_export_file + delete_archive_path end def to_json(options = {}) @options.to_h.merge!(klass: self.class.name).to_json end - def self.lock_file_path(project) - return unless project.export_path || export_file_exists? + def ensure_export_ready! + raise StrategyError unless project.export_file_exists? + end + + def ensure_lock_files_path! + FileUtils.mkdir_p(lock_files_path) unless Dir.exist?(lock_files_path) + end + + def lock_files_path + project.import_export_shared.lock_files_path + end - lock_path = project.import_export_shared.archive_path + def archive_path + project.import_export_shared.archive_path + end - mkdir_p(lock_path) - File.join(lock_path, AFTER_EXPORT_LOCK_FILE_NAME) + def locks_present? + project.import_export_shared.locks_present? end protected @@ -69,25 +81,33 @@ module Gitlab raise NotImplementedError end + def delete_export? + true + end + private + def delete_export_file + return if locks_present? || !delete_export? + + project.remove_exports + end + + def delete_archive_path + FileUtils.rm_rf(archive_path) if File.directory?(archive_path) + end + def create_or_update_after_export_lock - FileUtils.touch(self.class.lock_file_path(project)) + FileUtils.touch(lock_file) end def delete_after_export_lock - lock_file = self.class.lock_file_path(project) - FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file) end def log_validation_errors errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } end - - def export_file_exists? - project.export_file_exists? - end end end end diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb index 1b391314a74..39a6090ad87 100644 --- a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb @@ -4,6 +4,12 @@ module Gitlab module ImportExport module AfterExportStrategies class DownloadNotificationStrategy < BaseAfterExportStrategy + protected + + def delete_export? + false + end + private def strategy_execute diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index aaa70f0b36d..fd98bc2caad 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -24,8 +24,6 @@ module Gitlab def strategy_execute handle_response_error(send_file) - - project.remove_exports end def handle_response_error(response) diff --git a/lib/gitlab/import_export/fast_hash_serializer.rb b/lib/gitlab/import_export/fast_hash_serializer.rb index a6ab4f3a3d9..5a067b5c9f3 100644 --- a/lib/gitlab/import_export/fast_hash_serializer.rb +++ b/lib/gitlab/import_export/fast_hash_serializer.rb @@ -26,6 +26,51 @@ module Gitlab class FastHashSerializer attr_reader :subject, :tree + # Usage of this class results in delayed + # serialization of relation. The serialization + # will be triggered when the `JSON.generate` + # is exected. + # + # This class uses memory-optimised, lazily + # initialised, fast to recycle relation + # serialization. + # + # The `JSON.generate` does use `#to_json`, + # that returns raw JSON content that is written + # directly to file. + class JSONBatchRelation + include Gitlab::Utils::StrongMemoize + + def initialize(relation, options, preloads) + @relation = relation + @options = options + @preloads = preloads + end + + def raw_json + strong_memoize(:raw_json) do + result = +'' + + batch = @relation + batch = batch.preload(@preloads) if @preloads + batch.each do |item| + result.concat(",") unless result.empty? + result.concat(item.to_json(@options)) + end + + result + end + end + + def to_json(options = {}) + raw_json + end + + def as_json(*) + raise NotImplementedError + end + end + BATCH_SIZE = 100 def initialize(subject, tree, batch_size: BATCH_SIZE) @@ -34,8 +79,11 @@ module Gitlab @tree = tree end - # Serializes the subject into a Hash for the given option tree - # (e.g. Project#as_json) + # With the usage of `JSONBatchRelation`, it returns partially + # serialized hash which is not easily accessible. + # It means you can only manipulate and replace top-level objects. + # All future mutations of the hash (such as `fix_project_tree`) + # should be aware of that. def execute simple_serialize.merge(serialize_includes) end @@ -85,12 +133,15 @@ module Gitlab return record.as_json(options) end - # has-many relation data = [] record.in_batches(of: @batch_size) do |batch| # rubocop:disable Cop/InBatches - batch = batch.preload(preloads[key]) if preloads&.key?(key) - data += batch.as_json(options) + if Feature.enabled?(:export_fast_serialize_with_raw_json, default_enabled: true) + data.append(JSONBatchRelation.new(batch, options, preloads[key]).tap(&:raw_json)) + else + batch = batch.preload(preloads[key]) if preloads&.key?(key) + data += batch.as_json(options) + end end data diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb index 1c62591ed5a..de1629d0e28 100644 --- a/lib/gitlab/import_export/group_project_object_builder.rb +++ b/lib/gitlab/import_export/group_project_object_builder.rb @@ -26,30 +26,60 @@ module Gitlab end def find - find_object || @klass.create(project_attributes) + find_object || klass.create(project_attributes) end private + attr_reader :klass, :attributes, :group, :project + def find_object - @klass.where(where_clause).first + klass.where(where_clause).first end def where_clause - @attributes.slice('title').map do |key, value| - scope_clause = table[:project_id].eq(@project.id) - scope_clause = scope_clause.or(table[:group_id].eq(@group.id)) if @group + where_clauses.reduce(:and) + end + + def where_clauses + [ + where_clause_base, + where_clause_for_title, + where_clause_for_klass + ].compact + end + + # Returns Arel clause `"{table_name}"."project_id" = {project.id}` + # or, if group is present: + # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}` + def where_clause_base + clause = table[:project_id].eq(project.id) + clause = clause.or(table[:group_id].eq(group.id)) if group + + clause + end - table[key].eq(value).and(scope_clause) - end.reduce(:or) + # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'` + # if attributes has 'title key, otherwise `nil`. + def where_clause_for_title + attrs_to_arel(attributes.slice('title')) + end + + # Returns Arel clause: + # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"` + # from the given Hash of attributes. + def attrs_to_arel(attrs) + attrs.map do |key, value| + table[key].eq(value) + end.reduce(:and) end def table - @table ||= @klass.arel_table + @table ||= klass.arel_table end def project_attributes - @attributes.except('group').tap do |atts| + attributes.except('group').tap do |atts| if label? atts['type'] = 'ProjectLabel' # Always create project labels elsif milestone? @@ -60,15 +90,17 @@ module Gitlab claim_iid end end + + atts['importing'] = true if klass.ancestors.include?(Importable) end end def label? - @klass == Label + klass == Label end def milestone? - @klass == Milestone + klass == Milestone end # If an existing group milestone used the IID @@ -79,7 +111,7 @@ module Gitlab def claim_iid # The milestone has to be a group milestone, as it's the only case where # we set the IID as the maximum. The rest of them are fixed. - milestone = @project.milestones.find_by(iid: @attributes['iid']) + milestone = project.milestones.find_by(iid: attributes['iid']) return unless milestone @@ -87,6 +119,15 @@ module Gitlab milestone.ensure_project_iid! milestone.save! end + + protected + + # Returns Arel clause for a particular model or `nil`. + def where_clause_for_klass + # no-op + end end end end + +Gitlab::ImportExport::GroupProjectObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::GroupProjectObjectBuilder') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 511b702553e..141e73e6a47 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -66,6 +66,7 @@ tree: - stages: - :statuses - :external_pull_request + - :merge_request - :external_pull_requests - :auto_devops - :triggers @@ -138,11 +139,14 @@ excluded_attributes: - :mirror_trigger_builds - :only_mirror_protected_branches - :pull_mirror_available_overridden + - :pull_mirror_branch_prefix - :mirror_overwrites_diverged_branches - :packages_enabled - :mirror_last_update_at - :mirror_last_successful_update_at - :emails_disabled + - :max_pages_size + - :max_artifacts_size namespaces: - :runners_token - :runners_token_encrypted @@ -166,6 +170,12 @@ excluded_attributes: - :external_diff_size issues: - :milestone_id + merge_request: + - :milestone_id + - :ref_fetched + - :merge_jid + - :rebase_jid + - :latest_merge_request_diff_id merge_requests: - :milestone_id - :ref_fetched @@ -246,7 +256,16 @@ preloads: ee: tree: project: - protected_branches: + - issues: + - designs: + - notes: + - :author + - events: + - :push_event_payload + - design_versions: + - actions: + - :design # Duplicate export of issues.designs in order to link the record to both Issue and DesignVersion + - protected_branches: - :unprotect_access_levels - protected_environments: + - protected_environments: - :deploy_access_levels diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 767f1b5de0e..62cf6c86906 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -19,9 +19,9 @@ module Gitlab def execute if import_file && check_version! && restorers.all?(&:restore) && overwrite_project - project_tree.restored_project + project else - raise Projects::ImportService::Error.new(@shared.errors.join(', ')) + raise Projects::ImportService::Error.new(shared.errors.to_sentence) end rescue => e raise Projects::ImportService::Error.new(e.message) @@ -31,70 +31,72 @@ module Gitlab private + attr_accessor :archive_file, :current_user, :project, :shared + def restorers [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer, lfs_restorer, statistics_restorer] end def import_file - Gitlab::ImportExport::FileImporter.import(project: @project, - archive_file: @archive_file, - shared: @shared) + Gitlab::ImportExport::FileImporter.import(project: project, + archive_file: archive_file, + shared: shared) end def check_version! - Gitlab::ImportExport::VersionChecker.check!(shared: @shared) + Gitlab::ImportExport::VersionChecker.check!(shared: shared) end def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, - shared: @shared, - project: @project) + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: current_user, + shared: shared, + project: project) end def avatar_restorer - Gitlab::ImportExport::AvatarRestorer.new(project: project_tree.restored_project, shared: @shared) + Gitlab::ImportExport::AvatarRestorer.new(project: project, shared: shared) end def repo_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, - shared: @shared, - project: project_tree.restored_project) + shared: shared, + project: project) end def wiki_restorer Gitlab::ImportExport::WikiRestorer.new(path_to_bundle: wiki_repo_path, - shared: @shared, - project: ProjectWiki.new(project_tree.restored_project), - wiki_enabled: @project.wiki_enabled?) + shared: shared, + project: ProjectWiki.new(project), + wiki_enabled: project.wiki_enabled?) end def uploads_restorer - Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) + Gitlab::ImportExport::UploadsRestorer.new(project: project, shared: shared) end def lfs_restorer - Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared) + Gitlab::ImportExport::LfsRestorer.new(project: project, shared: shared) end def statistics_restorer - Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared) + Gitlab::ImportExport::StatisticsRestorer.new(project: project, shared: shared) end def path_with_namespace - File.join(@project.namespace.full_path, @project.path) + File.join(project.namespace.full_path, project.path) end def repo_path - File.join(@shared.export_path, 'project.bundle') + File.join(shared.export_path, Gitlab::ImportExport.project_bundle_filename) end def wiki_repo_path - File.join(@shared.export_path, 'project.wiki.bundle') + File.join(shared.export_path, Gitlab::ImportExport.wiki_repo_bundle_filename) end def remove_import_file - upload = @project.import_export_upload + upload = project.import_export_upload return unless upload&.import_file&.file @@ -103,12 +105,10 @@ module Gitlab end def overwrite_project - project = project_tree.restored_project - - return unless can?(@current_user, :admin_namespace, project.namespace) + return unless can?(current_user, :admin_namespace, project.namespace) if overwrite_project? - ::Projects::OverwriteProjectService.new(project, @current_user) + ::Projects::OverwriteProjectService.new(project, current_user) .execute(project_to_overwrite) end @@ -116,7 +116,7 @@ module Gitlab end def original_path - @project.import_data&.data&.fetch('original_path', nil) + project.import_data&.data&.fetch('original_path', nil) end def overwrite_project? @@ -125,9 +125,11 @@ module Gitlab def project_to_overwrite strong_memoize(:project_to_overwrite) do - Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}") + Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}") end end end end end + +Gitlab::ImportExport::Importer.prepend_if_ee('EE::Gitlab::ImportExport::Importer') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 2dd18616cd6..3fa5765fd4a 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -6,19 +6,21 @@ module Gitlab # Relations which cannot be saved at project level (and have a group assigned) GROUP_MODELS = [GroupLabel, Milestone].freeze + attr_reader :user + attr_reader :shared + attr_reader :project + def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @user = user @shared = shared @project = project - @project_id = project.id @saved = true end def restore begin - json = IO.read(@path) - @tree_hash = ActiveSupport::JSON.decode(json) + @tree_hash = read_tree_hash rescue => e Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger raise Gitlab::ImportExport::Error.new('Incorrect JSON format') @@ -30,26 +32,36 @@ module Gitlab ActiveRecord::Base.uncached do ActiveRecord::Base.no_touching do + update_project_params! create_relations end end + + # ensure that we have latest version of the restore + @project.reload # rubocop:disable Cop/ActiveRecordAssociationReload + + true rescue => e @shared.error(e) false end - def restored_project - return @project unless @tree_hash + private - @restored_project ||= restore_project + def read_tree_hash + json = IO.read(@path) + ActiveSupport::JSON.decode(json) end - private - def members_mapper @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, user: @user, - project: restored_project) + project: @project) + end + + # A Hash of the imported merge request ID -> imported ID. + def merge_requests_mapping + @merge_requests_mapping ||= {} end # Loops through the tree of models defined in import_export.yml and @@ -58,7 +70,7 @@ module Gitlab # the configuration yaml file too. # Finally, it updates each attribute in the newly imported project. def create_relations - project_relations_without_project_members.each do |relation_key, relation_definition| + project_relations.each do |relation_key, relation_definition| relation_key_s = relation_key.to_s if relation_definition.present? @@ -78,10 +90,25 @@ module Gitlab remove_group_models(relation_hash) if relation_hash.is_a?(Array) - @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) + @saved = false unless @project.append_or_update_attribute(relation_key, relation_hash) - # Restore the project again, extra query that skips holding the AR objects in memory - @restored_project = Project.find(@project_id) + save_id_mappings(relation_key, relation_hash_batch, relation_hash) + + @project.reset + end + + # Older, serialized CI pipeline exports may only have a + # merge_request_id and not the full hash of the merge request. To + # import these pipelines, we need to preserve the mapping between + # the old and new the merge request ID. + def save_id_mappings(relation_key, relation_hash_batch, relation_hash) + return unless relation_key == 'merge_requests' + + relation_hash = Array(relation_hash) + + Array(relation_hash_batch).each_with_index do |raw_data, index| + merge_requests_mapping[raw_data['id']] = relation_hash[index]['id'] + end end # Remove project models that became group models as we found them at group level. @@ -93,58 +120,44 @@ module Gitlab end end - def project_relations_without_project_members - # We remove `project_members` as they are deserialized separately - project_relations.except(:project_members) + def remove_feature_dependent_sub_relations!(_relation_item) + # no-op end def project_relations - reader.attributes_finder.find_relations_tree(:project) + @project_relations ||= reader.attributes_finder.find_relations_tree(:project) end - def restore_project + def update_project_params! Gitlab::Timeless.timeless(@project) do - @project.update(project_params) - end - - @project - end + project_params = @tree_hash.reject do |key, value| + project_relations.include?(key.to_sym) + end - def project_params - @project_params ||= begin - attrs = json_params.merge(override_params).merge(visibility_level, external_label) + project_params = project_params.merge(present_project_override_params) # Cleaning all imported and overridden params - Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: attrs, - relation_class: Project, - excluded_keys: excluded_keys_for_relation(:project)) - end - end - - def override_params - @override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {} - end - - def json_params - @json_params ||= @tree_hash.reject do |key, value| - # return params that are not 1 to many or 1 to 1 relations - value.respond_to?(:each) && !Project.column_names.include?(key) + project_params = Gitlab::ImportExport::AttributeCleaner.clean( + relation_hash: project_params, + relation_class: Project, + excluded_keys: excluded_keys_for_relation(:project)) + + @project.assign_attributes(project_params) + @project.drop_visibility_level! + @project.save! end end - def visibility_level - level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level - level = @project.group.visibility_level if @project.group && level.to_i > @project.group.visibility_level - level = Gitlab::VisibilityLevel::PRIVATE if level == Gitlab::VisibilityLevel::INTERNAL && Gitlab::CurrentSettings.restricted_visibility_levels.include?(level) - - { 'visibility_level' => level } + def present_project_override_params + # we filter out the empty strings from the overrides + # keeping the default values configured + project_override_params.transform_values do |value| + value.is_a?(String) ? value.presence : value + end.compact end - def external_label - label = override_params['external_authorization_classification_label'].presence || - json_params['external_authorization_classification_label'].presence - - { 'external_authorization_classification_label' => label } + def project_override_params + @project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {} end # Given a relation hash containing one or more models and its relationships, @@ -159,17 +172,10 @@ module Gitlab return if tree_hash[relation_key].blank? tree_array = [tree_hash[relation_key]].flatten - null_iid_pipelines = [] # Avoid keeping a possible heavy object in memory once we are done with it - while relation_item = (tree_array.shift || null_iid_pipelines.shift) - if nil_iid_pipeline?(relation_key, relation_item) && tree_array.any? - # Move pipelines with NULL IIDs to the end - # so they don't clash with existing IIDs. - null_iid_pipelines << relation_item - - next - end + while relation_item = tree_array.shift + remove_feature_dependent_sub_relations!(relation_item) # The transaction at this level is less speedy than one single transaction # But we can't have it in the upper level or GC won't get rid of the AR objects @@ -216,8 +222,9 @@ module Gitlab relation_sym: relation_key.to_sym, relation_hash: relation_hash, members_mapper: members_mapper, + merge_requests_mapping: merge_requests_mapping, user: @user, - project: @restored_project, + project: @project, excluded_keys: excluded_keys_for_relation(relation_key)) end.compact @@ -231,10 +238,8 @@ module Gitlab def excluded_keys_for_relation(relation) reader.attributes_finder.find_excluded_keys(relation) end - - def nil_iid_pipeline?(relation_key, relation_item) - relation_key == 'ci_pipelines' && relation_item['iid'].nil? - end end end end + +Gitlab::ImportExport::ProjectTreeRestorer.prepend_if_ee('::EE::Gitlab::ImportExport::ProjectTreeRestorer') diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index f75f69b2c75..63c71105efe 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -20,7 +20,8 @@ module Gitlab project_tree = serialize_project_tree fix_project_tree(project_tree) - File.write(full_path, project_tree.to_json) + project_tree_json = JSON.generate(project_tree) + File.write(full_path, project_tree_json) true rescue => e @@ -30,6 +31,8 @@ module Gitlab private + # Aware that the resulting hash needs to be pure-hash and + # does not include any AR objects anymore, only objects that run `.to_json` def fix_project_tree(project_tree) if @params[:description].present? project_tree['description'] = @params[:description] diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 1e9dff405c5..cb85af91f75 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -34,13 +34,13 @@ module Gitlab PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze - BUILD_MODELS = %w[Ci::Build commit_status].freeze + BUILD_MODELS = %i[Ci::Build commit_status].freeze IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request].freeze - TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze + TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze def self.create(*args) new(*args).create @@ -55,10 +55,11 @@ module Gitlab relation_name.to_s.constantize end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: []) - @relation_name = self.class.overrides[relation_sym] || relation_sym + def initialize(relation_sym:, relation_hash:, members_mapper:, merge_requests_mapping:, user:, project:, excluded_keys: []) + @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper + @merge_requests_mapping = merge_requests_mapping @user = user @project = project @imported_object_retries = 0 @@ -92,6 +93,10 @@ module Gitlab OVERRIDES end + def self.existing_object_check + EXISTING_OBJECT_CHECK + end + private def setup_models @@ -105,7 +110,10 @@ module Gitlab update_group_references remove_duplicate_assignees - setup_pipeline if @relation_name == 'Ci::Pipeline' + if @relation_name == :'Ci::Pipeline' + update_merge_request_references + setup_pipeline + end reset_tokens! remove_encrypted_attributes! @@ -184,14 +192,36 @@ module Gitlab end def update_group_references - return unless EXISTING_OBJECT_CHECK.include?(@relation_name) + return unless self.class.existing_object_check.include?(@relation_name) return unless @relation_hash['group_id'] @relation_hash['group_id'] = @project.namespace_id end + # This code is a workaround for broken project exports that don't + # export merge requests with CI pipelines (i.e. exports that were + # generated from + # https://gitlab.com/gitlab-org/gitlab/merge_requests/17844). + # This method can be removed in GitLab 12.6. + def update_merge_request_references + # If a merge request was properly created, we don't need to fix + # up this export. + return if @relation_hash['merge_request'] + + merge_request_id = @relation_hash['merge_request_id'] + + return unless merge_request_id + + new_merge_request_id = @merge_requests_mapping[merge_request_id] + + return unless new_merge_request_id + + @relation_hash['merge_request_id'] = new_merge_request_id + parsed_relation_hash['merge_request_id'] = new_merge_request_id + end + def reset_tokens! - return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s) + return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name) # If we import/export a project to the same instance, tokens will have to be reset. # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. @@ -255,14 +285,18 @@ module Gitlab # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin - if EXISTING_OBJECT_CHECK.include?(@relation_name) + if self.class.existing_object_check.include?(@relation_name) attribute_hash = attribute_hash_for(['events']) existing_object.assign_attributes(attribute_hash) if attribute_hash.any? existing_object else - relation_class.new(parsed_relation_hash) + object = relation_class.new + + # Use #assign_attributes here to call object custom setters + object.assign_attributes(parsed_relation_hash) + object end end end @@ -284,21 +318,27 @@ module Gitlab end def legacy_trigger? - @relation_name == 'Ci::Trigger' && @relation_hash['owner_id'].nil? + @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? end def find_or_create_object! return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature + return find_or_create_merge_request! if @relation_name == :merge_request # Can't use IDs as validation exists calling `group` or `project` attributes finder_hash = parsed_relation_hash.tap do |hash| hash['group'] = @project.group if relation_class.attribute_method?('group_id') - hash['project'] = @project + hash['project'] = @project if relation_class.reflect_on_association(:project) hash.delete('project_id') end GroupProjectObjectBuilder.build(relation_class, finder_hash) end + + def find_or_create_merge_request! + @project.merge_requests.find_by(iid: parsed_relation_hash['iid']) || + relation_class.new(parsed_relation_hash) + end end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 91167a9c4fb..3123687453f 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -6,19 +6,23 @@ module Gitlab include Gitlab::ImportExport::CommandLineUtil def initialize(project:, shared:, path_to_bundle:) - @project = project + @repository = project.repository @path_to_bundle = path_to_bundle @shared = shared end def restore - return true unless File.exist?(@path_to_bundle) + return true unless File.exist?(path_to_bundle) - @project.repository.create_from_bundle(@path_to_bundle) + repository.create_from_bundle(path_to_bundle) rescue => e - @shared.error(e) + shared.error(e) false end + + private + + attr_accessor :repository, :path_to_bundle, :shared end end end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index a60618dfcec..898cd7898ba 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -5,27 +5,35 @@ module Gitlab class RepoSaver include Gitlab::ImportExport::CommandLineUtil - attr_reader :full_path + attr_reader :project, :repository, :shared def initialize(project:, shared:) @project = project @shared = shared + @repository = @project.repository end def save - return true if @project.empty_repo? # it's ok to have no repo + return true unless repository_exists? # it's ok to have no repo - @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) bundle_to_disk end private + def repository_exists? + repository.exists? && !repository.empty? + end + + def bundle_full_path + File.join(shared.export_path, ImportExport.project_bundle_filename) + end + def bundle_to_disk - mkdir_p(@shared.export_path) - @project.repository.bundle_to_disk(@full_path) + mkdir_p(shared.export_path) + repository.bundle_to_disk(bundle_full_path) rescue => e - @shared.error(e) + shared.error(e) false end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 725c1101d70..02d46a1f498 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -1,10 +1,32 @@ # frozen_string_literal: true - +# +# This class encapsulates the directories used by project import/export: +# +# 1. The project export job first generates the project metadata tree +# (e.g. `project.json) and repository bundle (e.g. `project.bundle`) +# inside a temporary `export_path` +# (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA/:randomB). +# +# 2. The job then creates a tarball (e.g. `project.tar.gz`) in +# `archive_path` (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA). +# CarrierWave moves this tarball files into its permanent location. +# +# 3. Lock files are used to indicate whether a project is in the +# `after_export` state. These are stored in a directory +# (e.g. /path/to/shared/tmp/project_exports/namespace/project/locks. The +# number of lock files present signifies how many concurrent project +# exports are running. Note that this assumes the temporary directory +# is a shared mount: +# https://gitlab.com/gitlab-org/gitlab/issues/32203 +# +# NOTE: Stale files should be cleaned up via ImportExportCleanupService. module Gitlab module ImportExport class Shared attr_reader :errors, :project + LOCKS_DIRECTORY = 'locks' + def initialize(project) @project = project @errors = [] @@ -12,20 +34,31 @@ module Gitlab end def active_export_count - Dir[File.join(archive_path, '*')].count { |name| File.directory?(name) } + Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) } end + # The path where the project metadata and repository bundle is saved def export_path @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path) end + # The path where the tarball is saved def archive_path @archive_path ||= Gitlab::ImportExport.export_path(relative_path: relative_archive_path) end + def base_path + @base_path ||= Gitlab::ImportExport.export_path(relative_path: relative_base_path) + end + + def lock_files_path + @locks_files_path ||= File.join(base_path, LOCKS_DIRECTORY) + end + def error(error) - log_error(message: error.message, caller: caller[0].dup) - log_debug(backtrace: error.backtrace&.join("\n")) + error_payload = { message: error.message } + error_payload[:error_backtrace] = Gitlab::Profiler.clean_backtrace(error.backtrace) if error.backtrace + log_error(error_payload) Gitlab::Sentry.track_acceptable_exception(error, extra: log_base_data) @@ -37,16 +70,24 @@ module Gitlab end def after_export_in_progress? - File.exist?(after_export_lock_file) + locks_present? + end + + def locks_present? + Dir.exist?(lock_files_path) && !Dir.empty?(lock_files_path) end private def relative_path - File.join(relative_archive_path, SecureRandom.hex) + @relative_path ||= File.join(relative_archive_path, SecureRandom.hex) end def relative_archive_path + @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex) + end + + def relative_base_path @project.disk_path end @@ -70,10 +111,6 @@ module Gitlab def filtered_error_message(message) Projects::ImportErrorFilter.filter_message(message) end - - def after_export_lock_file - AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project) - end end end end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index e232198150a..dca8e3a7449 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -68,7 +68,7 @@ module Gitlab yield(@project.avatar) else project_uploads_except_avatar(avatar_path).find_each(batch_size: UPLOADS_BATCH_SIZE) do |upload| - yield(upload.build_uploader) + yield(upload.retrieve_uploader) end end end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 7303bcf61a4..93ae6f6b02a 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -4,28 +4,16 @@ module Gitlab module ImportExport class WikiRepoSaver < RepoSaver def save - @wiki = ProjectWiki.new(@project) - return true unless wiki_repository_exists? # it's okay to have no Wiki + wiki = ProjectWiki.new(project) + @repository = wiki.repository - bundle_to_disk(File.join(@shared.export_path, project_filename)) - end - - def bundle_to_disk(full_path) - mkdir_p(@shared.export_path) - @wiki.repository.bundle_to_disk(full_path) - rescue => e - @shared.error(e) - false + super end private - def project_filename - "project.wiki.bundle" - end - - def wiki_repository_exists? - @wiki.repository.exists? && !@wiki.repository.empty? + def bundle_full_path + File.join(shared.export_path, ImportExport.wiki_repo_bundle_filename) end end end diff --git a/lib/gitlab/import_export/wiki_restorer.rb b/lib/gitlab/import_export/wiki_restorer.rb index 28b5e7449cd..359ba8ba769 100644 --- a/lib/gitlab/import_export/wiki_restorer.rb +++ b/lib/gitlab/import_export/wiki_restorer.rb @@ -6,19 +6,22 @@ module Gitlab def initialize(project:, shared:, path_to_bundle:, wiki_enabled:) super(project: project, shared: shared, path_to_bundle: path_to_bundle) + @project = project @wiki_enabled = wiki_enabled end def restore - @project.wiki if create_empty_wiki? + project.wiki if create_empty_wiki? super end private + attr_accessor :project, :wiki_enabled + def create_empty_wiki? - !File.exist?(@path_to_bundle) && @wiki_enabled + !File.exist?(path_to_bundle) && wiki_enabled end end end diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index 11a33a7b358..0c8b509740c 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -4,7 +4,7 @@ module Gitlab module Jira # Gitlab JIRA HTTP client to be used with jira-ruby gem, this subclasses JIRA::HTTPClient. # Uses Gitlab::HTTP to make requests to JIRA REST API. - # The parent class implementation can be found at: https://github.com/sumoheavy/jira-ruby/blob/v1.4.0/lib/jira/http_client.rb + # The parent class implementation can be found at: https://github.com/sumoheavy/jira-ruby/blob/v1.7.0/lib/jira/http_client.rb class HttpClient < JIRA::HttpClient extend ::Gitlab::Utils::Override @@ -24,7 +24,7 @@ module Gitlab password: @options.delete(:password) }.to_json - make_request(:post, @options[:context_path] + '/rest/auth/1/session', body, { 'Content-Type' => 'application/json' }) + make_request(:post, @options[:context_path] + '/rest/auth/1/session', body, 'Content-Type' => 'application/json') end override :make_request diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb index 6ae68306a9b..a3f732e1283 100644 --- a/lib/gitlab/kubernetes/helm/client_command.rb +++ b/lib/gitlab/kubernetes/helm/client_command.rb @@ -17,7 +17,8 @@ module Gitlab # This is necessary to give Tiller time to restart after upgrade. # Ideally we'd be able to use --wait but cannot because of # https://github.com/helm/helm/issues/4855 - "for i in $(seq 1 30); do #{helm_check} && break; sleep 1s; echo \"Retrying ($i)...\"; done" + + "for i in $(seq 1 30); do #{helm_check} && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s)" end def repository_command diff --git a/lib/gitlab/kubernetes/helm/reset_command.rb b/lib/gitlab/kubernetes/helm/reset_command.rb index c8349639ec3..13176360227 100644 --- a/lib/gitlab/kubernetes/helm/reset_command.rb +++ b/lib/gitlab/kubernetes/helm/reset_command.rb @@ -18,7 +18,8 @@ module Gitlab def generate_script super + [ reset_helm_command, - delete_tiller_replicaset + delete_tiller_replicaset, + delete_tiller_clusterrolebinding ].join("\n") end @@ -43,6 +44,12 @@ module Gitlab Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) end + def delete_tiller_clusterrolebinding + delete_args = %w[clusterrolebinding tiller-admin] + + Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) + end + def reset_helm_command command = %w[helm reset] + optional_tls_flags diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 64317225ec6..66c28a9b702 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -39,7 +39,9 @@ module Gitlab :get_secret, :get_service, :get_service_account, + :delete_namespace, :delete_pod, + :delete_service_account, :create_config_map, :create_namespace, :create_pod, diff --git a/lib/gitlab/legacy_github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb index fdab6b512ea..a083ae60726 100644 --- a/lib/gitlab/legacy_github_import/release_formatter.rb +++ b/lib/gitlab/legacy_github_import/release_formatter.rb @@ -10,7 +10,8 @@ module Gitlab name: raw_data.name, description: raw_data.body, created_at: raw_data.created_at, - released_at: raw_data.published_at, + # Draft releases will have a null published_at + released_at: raw_data.published_at || Time.current, updated_at: raw_data.created_at } end diff --git a/lib/gitlab/lets_encrypt.rb b/lib/gitlab/lets_encrypt.rb index 08ad2ab91b0..9d14b151f7d 100644 --- a/lib/gitlab/lets_encrypt.rb +++ b/lib/gitlab/lets_encrypt.rb @@ -5,5 +5,9 @@ module Gitlab def self.enabled? Gitlab::CurrentSettings.lets_encrypt_terms_of_service_accepted end + + def self.terms_of_service_url + ::Gitlab::LetsEncrypt::Client.new.terms_of_service_url + end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 124e34562c1..e90f3f05a33 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -34,8 +34,11 @@ module Gitlab HMACToken.new(actor).token(DEFAULT_EXPIRE_TIME) end + # When the token is an lfs one and the actor + # is blocked or the password has been changed, + # the token is no longer valid def token_valid?(token_to_check) - HMACToken.new(actor).token_valid?(token_to_check) + HMACToken.new(actor).token_valid?(token_to_check) && valid_user? end def deploy_key_pushable?(project) @@ -46,6 +49,12 @@ module Gitlab user? ? :lfs_token : :lfs_deploy_token end + def valid_user? + return true unless user? + + !actor.blocked? && (!actor.allow_password_authentication? || !actor.password_expired?) + end + def authentication_payload(repository_http_path) { username: actor_name, @@ -55,6 +64,10 @@ module Gitlab } end + def basic_encoding + ActionController::HttpAuthentication::Basic.encode_credentials(actor_name, token) + end + private # rubocop:disable Lint/UselessAccessModifier class HMACToken diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index c3169418371..297f109ff81 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -20,13 +20,17 @@ module Gitlab # @param options - dashboard_path [String] Path at which the # dashboard can be found. Nil values will # default to the system dashboard. - # @param options - group [String] Title of the group + # @param options - group [String, Group] Title of the group # to which a panel might belong. Used by - # embedded dashboards. + # embedded dashboards. If cluster dashboard, + # refers to the Group corresponding to the cluster. # @param options - title [String] Title of the panel. # Used by embedded dashboards. # @param options - y_label [String] Y-Axis label of # a panel. Used by embedded dashboards. + # @param options - cluster [Cluster] + # @param options - cluster_type [Symbol] The level of + # cluster, one of [:admin, :project, :group] # @return [Hash] def find(project, user, options = {}) service_for(options) diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb new file mode 100644 index 00000000000..7111835c85a --- /dev/null +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Exporter + class BaseExporter < Daemon + attr_reader :server + + attr_accessor :readiness_checks + + def enabled? + settings.enabled + end + + def settings + raise NotImplementedError + end + + def log_filename + raise NotImplementedError + end + + private + + def start_working + logger = WEBrick::Log.new(log_filename) + logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]" + + access_log = [ + [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] + ] + + @server = ::WEBrick::HTTPServer.new( + Port: settings.port, BindAddress: settings.address, + Logger: logger, AccessLog: access_log) + server.mount_proc '/readiness' do |req, res| + render_probe(readiness_probe, req, res) + end + server.mount_proc '/liveness' do |req, res| + render_probe(liveness_probe, req, res) + end + server.mount '/', Rack::Handler::WEBrick, rack_app + + true + end + + def run_thread + server&.start + rescue IOError + # ignore forcibily closed servers + end + + def stop_working + if server + # we close sockets if thread is not longer running + # this happens, when the process forks + if thread.alive? + server.shutdown + else + server.listeners.each(&:close) + end + end + + @server = nil + end + + def rack_app + Rack::Builder.app do + use Rack::Deflater + use ::Prometheus::Client::Rack::Exporter if ::Gitlab::Metrics.metrics_folder_present? + run -> (env) { [404, {}, ['']] } + end + end + + def readiness_probe + ::Gitlab::HealthChecks::Probes::Collection.new(*readiness_checks) + end + + def liveness_probe + ::Gitlab::HealthChecks::Probes::Collection.new + end + + def render_probe(probe, req, res) + result = probe.execute + + res.status = result.http_status + res.content_type = 'application/json; charset=utf-8' + res.body = result.json.to_json + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb new file mode 100644 index 00000000000..5ba7b29734b --- /dev/null +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'webrick' +require 'prometheus/client/rack/exporter' + +module Gitlab + module Metrics + module Exporter + class SidekiqExporter < BaseExporter + def settings + Settings.monitoring.sidekiq_exporter + end + + def log_filename + File.join(Rails.root, 'log', 'sidekiq_exporter.log') + end + + private + + # Sidekiq Exporter does not work properly in sidekiq-cluster + # mode. It tries to start the service on the same port for + # each of the cluster workers, this results in failure + # due to duplicate binding. + # + # For now we ignore this error, as metrics are still "kind of" + # valid as they are rendered from shared directory. + # + # Issue: https://gitlab.com/gitlab-org/gitlab/issues/5714 + def start_working + super + rescue Errno::EADDRINUSE => e + Sidekiq.logger.error( + class: self.class.to_s, + message: 'Cannot start sidekiq_exporter', + exception: e.message + ) + + false + end + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb new file mode 100644 index 00000000000..3940f6fa155 --- /dev/null +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'webrick' +require 'prometheus/client/rack/exporter' + +module Gitlab + module Metrics + module Exporter + class WebExporter < BaseExporter + ExporterCheck = Struct.new(:exporter) do + def readiness + Gitlab::HealthChecks::Result.new( + 'web_exporter', exporter.running) + end + end + + attr_reader :running + + # This exporter is always run on master process + def initialize + super + + self.readiness_checks = [ + WebExporter::ExporterCheck.new(self), + Gitlab::HealthChecks::PumaCheck, + Gitlab::HealthChecks::UnicornCheck + ] + end + + def settings + Gitlab.config.monitoring.web_exporter + end + + def log_filename + File.join(Rails.root, 'log', 'web_exporter.log') + end + + private + + def start_working + @running = true + super + end + + def stop_working + @running = false + wait_in_blackout_period if server && thread.alive? + super + end + + def wait_in_blackout_period + return unless blackout_seconds > 0 + + @server.logger.info( + message: 'starting blackout...', + duration_s: blackout_seconds) + + sleep(blackout_seconds) + end + + def blackout_seconds + settings['blackout_seconds'].to_i + end + end + end + end +end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 26aa0910047..46477587934 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -3,6 +3,18 @@ module Gitlab module Metrics class RequestsRackMiddleware + HTTP_METHODS = { + "delete" => %w(200 202 204 303 400 401 403 404 410 422 500 503), + "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 412 422 429 500 503), + "head" => %w(200 204 301 302 303 304 400 401 403 404 410 429 500 503), + "options" => %w(200 404), + "patch" => %w(200 202 204 400 403 404 409 416 422 500), + "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 413 415 422 429 500 503), + "propfind" => %w(404), + "put" => %w(200 202 204 400 401 403 404 405 406 409 410 415 422 500), + "report" => %w(404) + }.freeze + def initialize(app) @app = app end @@ -20,6 +32,14 @@ module Gitlab {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) end + def self.initialize_http_request_duration_seconds + HTTP_METHODS.each do |method, statuses| + statuses.each do |status| + http_request_duration_seconds.get({ method: method, status: status }) + end + end + end + def call(env) method = env['REQUEST_METHOD'].downcase started = Time.now.to_f diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index d7d848d2833..90051f85f31 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -50,6 +50,11 @@ module Gitlab def start_working @running = true + + true + end + + def run_thread sleep(sleep_interval) while running safe_sample diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index 8a24d4f3663..f788f51b1ce 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'puma/state_file' - module Gitlab module Metrics module Samplers diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb deleted file mode 100644 index 71a5406815f..00000000000 --- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require 'webrick' -require 'prometheus/client/rack/exporter' - -module Gitlab - module Metrics - class SidekiqMetricsExporter < Daemon - LOG_FILENAME = File.join(Rails.root, 'log', 'sidekiq_exporter.log') - - def enabled? - ::Gitlab::Metrics.metrics_folder_present? && settings.enabled - end - - def settings - Settings.monitoring.sidekiq_exporter - end - - private - - attr_reader :server - - def start_working - logger = WEBrick::Log.new(LOG_FILENAME) - access_log = [ - [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] - ] - - @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address, - Logger: logger, AccessLog: access_log) - server.mount "/", Rack::Handler::WEBrick, rack_app - server.start - end - - def stop_working - server.shutdown if server - @server = nil - end - - def rack_app - Rack::Builder.app do - use Rack::Deflater - use ::Prometheus::Client::Rack::Exporter - run -> (env) { [404, {}, ['']] } - end - end - end - end -end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 51f48095cb5..2a61b3de405 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -63,6 +63,21 @@ module Gitlab def self.monotonic_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) end + + def self.thread_cpu_time + # Not all OS kernels are supporting `Process::CLOCK_THREAD_CPUTIME_ID` + # Refer: https://gitlab.com/gitlab-org/gitlab/issues/30567#note_221765627 + return unless defined?(Process::CLOCK_THREAD_CPUTIME_ID) + + Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second) + end + + def self.thread_cpu_duration(start_time) + end_time = thread_cpu_time + return unless start_time && end_time + + end_time - start_time + end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index ba2a0b2ecf8..115368c8bc6 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -44,6 +44,10 @@ module Gitlab duration.in_milliseconds.to_i end + def thread_cpu_duration + System.thread_cpu_duration(@thread_cputime_start) + end + def allocated_memory @memory_after - @memory_before end @@ -53,12 +57,14 @@ module Gitlab @memory_before = System.memory_usage @started_at = System.monotonic_time + @thread_cputime_start = System.thread_cpu_time yield ensure @memory_after = System.memory_usage @finished_at = System.monotonic_time + self.class.gitlab_transaction_cputime_seconds.observe(labels, thread_cpu_duration) self.class.gitlab_transaction_duration_seconds.observe(labels, duration) self.class.gitlab_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0) @@ -142,6 +148,12 @@ module Gitlab "#{labels[:controller]}##{labels[:action]}" if labels && !labels.empty? end + define_histogram :gitlab_transaction_cputime_seconds do + docstring 'Transaction thread cputime' + base_labels BASE_LABELS + buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0] + end + define_histogram :gitlab_transaction_duration_seconds do docstring 'Transaction duration' base_labels BASE_LABELS diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index a29dc5395f3..b18f0eed1fa 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -20,6 +20,12 @@ module Gitlab 'projects/lfs_locks_api' => %w{verify create unlock} }.freeze + WHITELISTED_GIT_REVISION_ROUTES = { + 'projects/compare' => %w{create} + }.freeze + + GRAPHQL_URL = '/api/graphql' + def initialize(app, env) @app = app @env = env @@ -79,7 +85,7 @@ module Gitlab # Overridden in EE module def whitelisted_routes - grack_route? || internal_route? || lfs_route? || sidekiq_route? + grack_route? || internal_route? || lfs_route? || compare_git_revisions_route? || sidekiq_route? || graphql_query? end def grack_route? @@ -94,6 +100,13 @@ module Gitlab ReadOnly.internal_routes.any? { |path| request.path.include?(path) } end + def compare_git_revisions_route? + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.post? && request.path.end_with?('compare') + + WHITELISTED_GIT_REVISION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + end + def lfs_route? # Calling route_hash may be expensive. Only do it if we think there's a possible match unless request.path.end_with?('/info/lfs/objects/batch', @@ -108,6 +121,10 @@ module Gitlab def sidekiq_route? request.path.start_with?("#{relative_url}/admin/sidekiq") end + + def graphql_query? + request.post? && request.path.start_with?(GRAPHQL_URL) + end end end end diff --git a/lib/gitlab/pages_client.rb b/lib/gitlab/pages_client.rb deleted file mode 100644 index 30a1f9ede25..00000000000 --- a/lib/gitlab/pages_client.rb +++ /dev/null @@ -1,119 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class PagesClient - class << self - attr_reader :certificate, :token - - def call(service, rpc, request, timeout: nil) - kwargs = request_kwargs(timeout) - stub(service).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend - end - - # This function is not thread-safe. Call it from an initializer only. - def read_or_create_token - @token = read_token - rescue Errno::ENOENT - # TODO: uncomment this when omnibus knows how to write the token file for us - # https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/2466 - # - # write_token(SecureRandom.random_bytes(64)) - # - # # Read from disk in case someone else won the race and wrote the file - # # before us. If this fails again let the exception bubble up. - # @token = read_token - end - - # This function is not thread-safe. Call it from an initializer only. - def load_certificate - cert_path = config.certificate - return unless cert_path.present? - - @certificate = File.read(cert_path) - end - - def ping - request = Grpc::Health::V1::HealthCheckRequest.new - call(:health_check, :check, request, timeout: 5.seconds) - end - - private - - def request_kwargs(timeout) - encoded_token = Base64.strict_encode64(token.to_s) - metadata = { - 'authorization' => "Bearer #{encoded_token}" - } - - result = { metadata: metadata } - - return result unless timeout - - # Do not use `Time.now` for deadline calculation, since it - # will be affected by Timecop in some tests, but grpc's c-core - # uses system time instead of timecop's time, so tests will fail - # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will - # circumvent timecop - deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout - result[:deadline] = deadline - - result - end - - def stub(name) - stub_class(name).new(address, grpc_creds) - end - - def stub_class(name) - if name == :health_check - Grpc::Health::V1::Health::Stub - else - # TODO use pages namespace - Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) - end - end - - def address - addr = config.address - addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' - addr - end - - def grpc_creds - if address.start_with?('unix:') - :this_channel_is_insecure - elsif @certificate - GRPC::Core::ChannelCredentials.new(@certificate) - else - # Use system certificate pool - GRPC::Core::ChannelCredentials.new - end - end - - def config - Gitlab.config.pages.admin - end - - def read_token - File.read(token_path) - end - - def token_path - Rails.root.join('.gitlab_pages_secret').to_s - end - - def write_token(new_token) - Tempfile.open(File.basename(token_path), File.dirname(token_path), encoding: 'ascii-8bit') do |f| - f.write(new_token) - f.close - File.link(f.path, token_path) - end - rescue Errno::EACCES => ex - # TODO stop rescuing this exception in GitLab 11.0 https://gitlab.com/gitlab-org/gitlab-foss/issues/45672 - Rails.logger.error("Could not write pages admin token file: #{ex}") # rubocop:disable Gitlab/RailsLogger - rescue Errno::EEXIST - # Another process wrote the token file concurrently with us. Use their token, not ours. - end - end - end -end diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb index a9f6cfb19cb..22ece0a6a8b 100644 --- a/lib/gitlab/patch/prependable.rb +++ b/lib/gitlab/patch/prependable.rb @@ -24,7 +24,7 @@ module Gitlab super if const_defined?(:ClassMethods) - klass_methods = const_get(:ClassMethods) + klass_methods = const_get(:ClassMethods, false) base.singleton_class.prepend klass_methods base.instance_variable_set(:@_prepended_class_methods, klass_methods) end @@ -40,7 +40,7 @@ module Gitlab super if instance_variable_defined?(:@_prepended_class_methods) - const_get(:ClassMethods).prepend @_prepended_class_methods + const_get(:ClassMethods, false).prepend @_prepended_class_methods end end diff --git a/lib/gitlab/phabricator_import/base_worker.rb b/lib/gitlab/phabricator_import/base_worker.rb index b69c65e78f8..d2c2ef8db48 100644 --- a/lib/gitlab/phabricator_import/base_worker.rb +++ b/lib/gitlab/phabricator_import/base_worker.rb @@ -23,6 +23,8 @@ module Gitlab include ProjectImportOptions # This marks the project as failed after too many tries include Gitlab::ExclusiveLeaseHelpers + feature_category :importers + class << self def schedule(project_id, *args) perform_async(project_id, *args) diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 275151f7fc1..560618bb486 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -37,8 +37,7 @@ module Gitlab # - post_data: a string of raw POST data to use. Changes the HTTP verb to # POST. # - # - user: a user to authenticate as. Only works if the user has a valid - # personal access token. + # - user: a user to authenticate as. # # - private_token: instead of providing a user instance, the token can be # given as a string. Takes precedence over the user option. diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index ff9bb293b47..e04d6f250b1 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -50,7 +50,7 @@ module Gitlab content, commands = perform_substitutions(content, commands) - [content.strip, commands] + [content.rstrip, commands] end private @@ -109,7 +109,7 @@ module Gitlab [ ] (?<arg>[^\n]*) )? - (?:\n|$) + (?:\s*\n|$) ) }mix end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 7e64fe2a1f4..404e0c31871 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -135,7 +135,8 @@ module Gitlab end types Issue condition do - current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) + !quick_action_target.confidential? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) end command :confidential do @updates[:confidential] = true diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 00f817c2399..ea2b03b42c1 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -3,7 +3,8 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor - REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user epic).freeze + REFERABLES = %i(user issue label milestone + merge_request snippet commit commit_range directly_addressed_user epic).freeze attr_accessor :project, :current_user, :author def initialize(project, current_user = nil) @@ -54,9 +55,9 @@ module Gitlab def self.references_pattern return @pattern if @pattern - patterns = REFERABLES.map do |ref| - ref.to_s.classify.constantize.try(:reference_pattern) - end + patterns = REFERABLES.map do |type| + Banzai::ReferenceParser[type].reference_type.to_s.classify.constantize.try(:reference_pattern) + end.uniq @pattern = Regexp.union(patterns.compact) end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 4bfa6f7e9a5..3d1f15c72ae 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -119,6 +119,15 @@ module Gitlab def breakline_regex @breakline_regex ||= /\r\n|\r|\n/ end + + # https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html + def aws_arn_regex + /\Aarn:\S+\z/ + end + + def aws_arn_regex_message + "must be a valid Amazon Resource Name" + end end end diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index ab2549d5e68..13187836e02 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -6,6 +6,10 @@ module Gitlab def client_ip Gitlab::SafeRequestStore[:client_ip] end + + def start_thread_cpu_time + Gitlab::SafeRequestStore[:start_thread_cpu_time] + end end def initialize(app) @@ -23,6 +27,8 @@ module Gitlab Gitlab::SafeRequestStore[:client_ip] = req.ip + Gitlab::SafeRequestStore[:start_thread_cpu_time] = Gitlab::Metrics::System.thread_cpu_time + @app.call(env) end end diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index 2f3d14ecebd..5eeb8b00ff3 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -68,7 +68,7 @@ module Gitlab } relation.find_each(find_params) do |upload| - clean(upload.build_uploader, dry_run: dry_run) + clean(upload.retrieve_uploader, dry_run: dry_run) sleep sleep_time if sleep_time rescue => err logger.error "failed to sanitize #{upload_ref(upload)}: #{err.message}" diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 93e172299b9..782ac534a7b 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -2,7 +2,7 @@ module Gitlab class SearchResults - COUNT_LIMIT = 101 + COUNT_LIMIT = 100 COUNT_LIMIT_MESSAGE = "#{COUNT_LIMIT - 1}+" attr_reader :current_user, :query, :per_page diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 7dbed591b84..125d0d1cfbb 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -113,10 +113,6 @@ module Gitlab success end - # Move repository reroutes to mv_directory which is an alias for - # mv_namespace. Given the underlying implementation is a move action, - # indescriminate of what the folders might be. - # # storage - project's storage path # path - project disk path # new_path - new project disk path @@ -126,7 +122,13 @@ module Gitlab def mv_repository(storage, path, new_path) return false if path.empty? || new_path.empty? - !!mv_directory(storage, "#{path}.git", "#{new_path}.git") + Gitlab::Git::Repository.new(storage, "#{path}.git", nil, nil).rename("#{new_path}.git") + + true + rescue => e + Gitlab::Sentry.track_acceptable_exception(e, extra: { path: path, new_path: new_path, storage: storage }) + + false end # Fork repository to new path @@ -151,9 +153,13 @@ module Gitlab def remove_repository(storage, name) return false if name.empty? - !!rm_directory(storage, "#{name}.git") - rescue ArgumentError => e + Gitlab::Git::Repository.new(storage, "#{name}.git", nil, nil).remove + + true + rescue => e Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") # rubocop:disable Gitlab/RailsLogger + Gitlab::Sentry.track_acceptable_exception(e, extra: { path: name, storage: storage }) + false end @@ -265,7 +271,6 @@ module Gitlab false end - alias_method :mv_directory, :mv_namespace # Note: ShellWorker uses this alias def url_to_repo(path) Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git" @@ -292,6 +297,12 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def repository_exists?(storage, dir_name) + Gitlab::Git::Repository.new(storage, dir_name, nil, nil).exists? + rescue GRPC::Internal + false + end + def hooks_path File.join(gitlab_shell_path, 'hooks') end diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index c102fa14cfc..ffceeb68f20 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -5,7 +5,11 @@ require 'set' module Gitlab module SidekiqConfig - QUEUE_CONFIG_PATHS = %w[app/workers/all_queues.yml ee/app/workers/all_queues.yml].freeze + QUEUE_CONFIG_PATHS = begin + result = %w[app/workers/all_queues.yml] + result << 'ee/app/workers/all_queues.yml' if Gitlab.ee? + result + end.freeze # This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside # of bundler/Rails context, so we cannot use any gem or Rails methods. @@ -48,9 +52,11 @@ module Gitlab end def self.workers - @workers ||= - find_workers(Rails.root.join('app', 'workers')) + - find_workers(Rails.root.join('ee', 'app', 'workers')) + @workers ||= begin + result = find_workers(Rails.root.join('app', 'workers')) + result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee? + result + end end def self.find_workers(root) diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb new file mode 100644 index 00000000000..9d0d67a488f --- /dev/null +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -0,0 +1,263 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqDaemon + class MemoryKiller < Daemon + include ::Gitlab::Utils::StrongMemoize + + # Today 64-bit CPU support max 256T memory. It is big enough. + MAX_MEMORY_KB = 256 * 1024 * 1024 * 1024 + # RSS below `soft_limit_rss` is considered safe + SOFT_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_MAX_RSS', 2000000).to_i + # RSS above `hard_limit_rss` will be stopped + HARD_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS', MAX_MEMORY_KB).to_i + # RSS in range (soft_limit_rss, hard_limit_rss) is allowed for GRACE_BALLOON_SECONDS + GRACE_BALLOON_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', 15 * 60).to_i + # Check RSS every CHECK_INTERVAL_SECONDS, minimum 2 seconds + CHECK_INTERVAL_SECONDS = [ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2].max + # Give Sidekiq up to 30 seconds to allow existing jobs to finish after exceeding the limit + SHUTDOWN_TIMEOUT_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT', 30).to_i + # Developer/admin should always set `memory_killer_max_memory_growth_kb` explicitly + # In case not set, default to 300M. This is for extra-safe. + DEFAULT_MAX_MEMORY_GROWTH_KB = 300_000 + + # Phases of memory killer + PHASE = { + running: 1, + above_soft_limit: 2, + stop_fetching_new_jobs: 3, + shutting_down: 4, + killing_sidekiq: 5 + }.freeze + + def initialize + super + + @enabled = true + @metrics = init_metrics + end + + private + + def init_metrics + { + sidekiq_current_rss: ::Gitlab::Metrics.gauge(:sidekiq_current_rss, 'Current RSS of Sidekiq Worker'), + sidekiq_memory_killer_soft_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_soft_limit_rss, 'Current soft_limit_rss of Sidekiq Worker'), + sidekiq_memory_killer_hard_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_hard_limit_rss, 'Current hard_limit_rss of Sidekiq Worker'), + sidekiq_memory_killer_phase: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_phase, 'Current phase of Sidekiq Worker') + } + end + + def refresh_state(phase) + @phase = PHASE.fetch(phase) + @current_rss = get_rss + @soft_limit_rss = get_soft_limit_rss + @hard_limit_rss = get_hard_limit_rss + + # track the current state as prometheus gauges + @metrics[:sidekiq_memory_killer_phase].set({}, @phase) + @metrics[:sidekiq_current_rss].set({}, @current_rss) + @metrics[:sidekiq_memory_killer_soft_limit_rss].set({}, @soft_limit_rss) + @metrics[:sidekiq_memory_killer_hard_limit_rss].set({}, @hard_limit_rss) + end + + def run_thread + Sidekiq.logger.info( + class: self.class.to_s, + action: 'start', + pid: pid, + message: 'Starting Gitlab::SidekiqDaemon::MemoryKiller Daemon' + ) + + while enabled? + begin + sleep(CHECK_INTERVAL_SECONDS) + restart_sidekiq unless rss_within_range? + rescue => e + log_exception(e, __method__) + rescue Exception => e # rubocop:disable Lint/RescueException + log_exception(e, __method__ ) + raise e + end + end + ensure + Sidekiq.logger.warn( + class: self.class.to_s, + action: 'stop', + pid: pid, + message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon' + ) + end + + def log_exception(exception, method) + Sidekiq.logger.warn( + class: self.class.to_s, + pid: pid, + message: "Exception from #{method}: #{exception.message}" + ) + end + + def stop_working + @enabled = false + end + + def enabled? + @enabled + end + + def restart_sidekiq + # Tell Sidekiq to stop fetching new jobs + # We first SIGNAL and then wait given time + # We also monitor a number of running jobs and allow to restart early + refresh_state(:stop_fetching_new_jobs) + signal_and_wait(SHUTDOWN_TIMEOUT_SECONDS, 'SIGTSTP', 'stop fetching new jobs') + return unless enabled? + + # Tell sidekiq to restart itself + # Keep extra safe to wait `Sidekiq.options[:timeout] + 2` seconds before SIGKILL + refresh_state(:shutting_down) + signal_and_wait(Sidekiq.options[:timeout] + 2, 'SIGTERM', 'gracefully shut down') + return unless enabled? + + # Ideally we should never reach this condition + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't + # Kill the whole pgroup, so we can be sure no children are left behind + refresh_state(:killing_sidekiq) + signal_pgroup('SIGKILL', 'die') + end + + def rss_within_range? + refresh_state(:running) + + deadline = Gitlab::Metrics::System.monotonic_time + GRACE_BALLOON_SECONDS.seconds + loop do + return true unless enabled? + + # RSS go above hard limit should trigger forcible shutdown right away + break if @current_rss > @hard_limit_rss + + # RSS go below the soft limit + return true if @current_rss < @soft_limit_rss + + # RSS did not go below the soft limit within deadline, restart + break if Gitlab::Metrics::System.monotonic_time > deadline + + sleep(CHECK_INTERVAL_SECONDS) + + refresh_state(:above_soft_limit) + end + + # There are two chances to break from loop: + # - above hard limit, or + # - above soft limit after deadline + # When `above hard limit`, it immediately go to `stop_fetching_new_jobs` + # So ignore `above hard limit` and always set `above_soft_limit` here + refresh_state(:above_soft_limit) + log_rss_out_of_range(@current_rss, @hard_limit_rss, @soft_limit_rss) + + false + end + + def log_rss_out_of_range(current_rss, hard_limit_rss, soft_limit_rss) + Sidekiq.logger.warn( + class: self.class.to_s, + pid: pid, + message: 'Sidekiq worker RSS out of range', + current_rss: current_rss, + hard_limit_rss: hard_limit_rss, + soft_limit_rss: soft_limit_rss, + reason: out_of_range_description(current_rss, hard_limit_rss, soft_limit_rss) + ) + end + + def out_of_range_description(rss, hard_limit, soft_limit) + if rss > hard_limit + "current_rss(#{rss}) > hard_limit_rss(#{hard_limit})" + else + "current_rss(#{rss}) > soft_limit_rss(#{soft_limit}) longer than GRACE_BALLOON_SECONDS(#{GRACE_BALLOON_SECONDS})" + end + end + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) + return 0 unless status&.zero? + + output.to_i + end + + def get_soft_limit_rss + SOFT_LIMIT_RSS_KB + rss_increase_by_jobs + end + + def get_hard_limit_rss + HARD_LIMIT_RSS_KB + end + + def signal_and_wait(time, signal, explanation) + Sidekiq.logger.warn( + class: self.class.to_s, + pid: pid, + signal: signal, + explanation: explanation, + wait_time: time, + message: "Sending signal and waiting" + ) + Process.kill(signal, pid) + + deadline = Gitlab::Metrics::System.monotonic_time + time + + # we try to finish as early as all jobs finished + # so we retest that in loop + sleep(CHECK_INTERVAL_SECONDS) while enabled? && any_jobs? && Gitlab::Metrics::System.monotonic_time < deadline + end + + def signal_pgroup(signal, explanation) + if Process.getpgrp == pid + pid_or_pgrp_str = 'PGRP' + pid_to_signal = 0 + else + pid_or_pgrp_str = 'PID' + pid_to_signal = pid + end + + Sidekiq.logger.warn( + class: self.class.to_s, + signal: signal, + pid: pid, + message: "sending Sidekiq worker #{pid_or_pgrp_str}-#{pid} #{signal} (#{explanation})" + ) + Process.kill(signal, pid_to_signal) + end + + def rss_increase_by_jobs + Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord + rss_increase_by_job(job) + end + end + + def rss_increase_by_job(job) + memory_growth_kb = get_job_options(job, 'memory_killer_memory_growth_kb', 0).to_i + max_memory_growth_kb = get_job_options(job, 'memory_killer_max_memory_growth_kb', DEFAULT_MAX_MEMORY_GROWTH_KB).to_i + + return 0 if memory_growth_kb.zero? + + time_elapsed = [Gitlab::Metrics::System.monotonic_time - job[:started_at], 0].max + [memory_growth_kb * time_elapsed, max_memory_growth_kb].min + end + + def get_job_options(job, key, default) + job[:worker_class].sidekiq_options.fetch(key, default) + rescue + default + end + + def pid + Process.pid + end + + def any_jobs? + Gitlab::SidekiqDaemon::Monitor.instance.jobs.any? + end + end + end +end diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb index bbfca130425..a3d61c69ae1 100644 --- a/lib/gitlab/sidekiq_daemon/monitor.rb +++ b/lib/gitlab/sidekiq_daemon/monitor.rb @@ -14,19 +14,19 @@ module Gitlab # that should not be caught by application CancelledError = Class.new(Exception) # rubocop:disable Lint/InheritException - attr_reader :jobs_thread + attr_reader :jobs attr_reader :jobs_mutex def initialize super - @jobs_thread = {} + @jobs = {} @jobs_mutex = Mutex.new end - def within_job(jid, queue) + def within_job(worker_class, jid, queue) jobs_mutex.synchronize do - jobs_thread[jid] = Thread.current + jobs[jid] = { worker_class: worker_class, thread: Thread.current, started_at: Gitlab::Metrics::System.monotonic_time } end if cancelled?(jid) @@ -43,7 +43,7 @@ module Gitlab yield ensure jobs_mutex.synchronize do - jobs_thread.delete(jid) + jobs.delete(jid) end end @@ -61,24 +61,28 @@ module Gitlab private - def start_working - Sidekiq.logger.info( - class: self.class.to_s, - action: 'start', - message: 'Starting Monitor Daemon' - ) + def run_thread + return unless notification_channel_enabled? - while enabled? - process_messages - sleep(RECONNECT_TIME) - end + begin + Sidekiq.logger.info( + class: self.class.to_s, + action: 'start', + message: 'Starting Monitor Daemon' + ) - ensure - Sidekiq.logger.warn( - class: self.class.to_s, - action: 'stop', - message: 'Stopping Monitor Daemon' - ) + while enabled? + process_messages + sleep(RECONNECT_TIME) + end + + ensure + Sidekiq.logger.warn( + class: self.class.to_s, + action: 'stop', + message: 'Stopping Monitor Daemon' + ) + end end def stop_working @@ -156,7 +160,7 @@ module Gitlab # This is why it passes thread in block, # to ensure that we do process this thread def find_thread_unsafe(jid) - jobs_thread[jid] + jobs.dig(jid, :thread) end def find_thread_with_lock(jid) @@ -179,6 +183,10 @@ module Gitlab def self.cancel_job_key(jid) "sidekiq:cancel:#{jid}" end + + def notification_channel_enabled? + ENV.fetch("SIDEKIQ_MONITOR_WORKER", 0).to_i.nonzero? + end end end end diff --git a/lib/gitlab/sidekiq_logging/exception_handler.rb b/lib/gitlab/sidekiq_logging/exception_handler.rb new file mode 100644 index 00000000000..fba74b6c9ed --- /dev/null +++ b/lib/gitlab/sidekiq_logging/exception_handler.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqLogging + class ExceptionHandler + def call(job_exception, context) + data = { + error_class: job_exception.class.name, + error_message: job_exception.message + } + + if context.is_a?(Hash) + data.merge!(context) + # correlation_id, jid, and class are available inside the job + # Hash, so promote these arguments to the root tree so that + # can be searched alongside other Sidekiq log messages. + job_data = data.delete(:job) + data.merge!(job_data) if job_data.present? + end + + data[:error_backtrace] = Gitlab::Profiler.clean_backtrace(job_exception.backtrace) if job_exception.backtrace.present? + + Sidekiq.logger.warn(data) + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 48b1524f9c7..853fb2777c3 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -58,8 +58,7 @@ module Gitlab payload['message'] = "#{message}: fail: #{payload['duration']} sec" payload['job_status'] = 'fail' payload['error_message'] = job_exception.message - payload['error'] = job_exception.class - payload['error_backtrace'] = backtrace_cleaner.clean(job_exception.backtrace) + payload['error_class'] = job_exception.class.name else payload['message'] = "#{message}: done: #{payload['duration']} sec" payload['job_status'] = 'done' @@ -71,10 +70,11 @@ module Gitlab end def add_time_keys!(time, payload) - payload['duration'] = time[:duration].round(3) - payload['system_s'] = time[:stime].round(3) - payload['user_s'] = time[:utime].round(3) - payload['child_s'] = time[:ctime].round(3) if time[:ctime] > 0 + payload['duration'] = time[:duration].round(6) + + # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0) + # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime + payload['cpu_s'] = time[:cputime].round(6) if time[:cputime] > 0 payload['completed_at'] = Time.now.utc end @@ -99,42 +99,32 @@ module Gitlab end def elapsed_by_absolute_time(start) - (Time.now.utc - start).to_f.round(3) + (Time.now.utc - start).to_f.round(6) end def elapsed(t0) t1 = get_time { duration: t1[:now] - t0[:now], - stime: t1[:times][:stime] - t0[:times][:stime], - utime: t1[:times][:utime] - t0[:times][:utime], - ctime: ctime(t1[:times]) - ctime(t0[:times]) + cputime: t1[:thread_cputime] - t0[:thread_cputime] } end def get_time { now: current_time, - times: Process.times + thread_cputime: defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 } end - def ctime(times) - times[:cstime] + times[:cutime] - end - def current_time Gitlab::Metrics::System.monotonic_time end - def backtrace_cleaner - @backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new - end - def format_time(timestamp) return timestamp if timestamp.is_a?(String) - Time.at(timestamp).utc.iso8601(3) + Time.at(timestamp).utc.iso8601(6) end def limited_job_args(args) diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb index 368f37a5d8c..8af353d8674 100644 --- a/lib/gitlab/sidekiq_middleware/metrics.rb +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -19,10 +19,16 @@ module Gitlab @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) end + job_thread_cputime_start = get_thread_cputime + realtime = Benchmark.realtime do yield end + job_thread_cputime_end = get_thread_cputime + job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start + @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) + @metrics[:sidekiq_jobs_completion_seconds].observe(labels, realtime) rescue Exception # rubocop: disable Lint/RescueException @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) @@ -35,6 +41,7 @@ module Gitlab def init_metrics { + sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), @@ -47,6 +54,10 @@ module Gitlab queue: queue } end + + def get_thread_cputime + defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 + end end end end diff --git a/lib/gitlab/sidekiq_middleware/monitor.rb b/lib/gitlab/sidekiq_middleware/monitor.rb index 00965bf5506..ed825dbfd60 100644 --- a/lib/gitlab/sidekiq_middleware/monitor.rb +++ b/lib/gitlab/sidekiq_middleware/monitor.rb @@ -4,7 +4,7 @@ module Gitlab module SidekiqMiddleware class Monitor def call(worker, job, queue) - Gitlab::SidekiqDaemon::Monitor.instance.within_job(job['jid'], queue) do + Gitlab::SidekiqDaemon::Monitor.instance.within_job(worker.class, job['jid'], queue) do yield end rescue Gitlab::SidekiqDaemon::Monitor::CancelledError diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb index b1bfaa6cb59..9ce1bcfb37c 100644 --- a/lib/gitlab/slash_commands/presenters/access.rb +++ b/lib/gitlab/slash_commands/presenters/access.rb @@ -15,6 +15,15 @@ module Gitlab MESSAGE end + def deactivated + ephemeral_response(text: <<~MESSAGE) + You are not allowed to perform the given chatops command since + your account has been deactivated by your administrator. + + Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url} + MESSAGE + end + def not_found ephemeral_response(text: "404 not found! GitLab couldn't find what you were looking for! :boom:") end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index ac3b219e0c7..e955ccd35da 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -4,19 +4,19 @@ module Gitlab class SnippetSearchResults < SearchResults include SnippetsHelper - attr_reader :limit_snippets + attr_reader :current_user - def initialize(limit_snippets, query) - @limit_snippets = limit_snippets + def initialize(current_user, query) + @current_user = current_user @query = query end def objects(scope, page = nil) case scope when 'snippet_titles' - snippet_titles.page(page).per(per_page) + paginated_objects(snippet_titles, page) when 'snippet_blobs' - snippet_blobs.page(page).per(per_page) + paginated_objects(snippet_blobs, page) else super(scope, nil, false) end @@ -25,38 +25,53 @@ module Gitlab def formatted_count(scope) case scope when 'snippet_titles' - snippet_titles_count.to_s + formatted_limited_count(limited_snippet_titles_count) when 'snippet_blobs' - snippet_blobs_count.to_s + formatted_limited_count(limited_snippet_blobs_count) else super end end - def snippet_titles_count - @snippet_titles_count ||= snippet_titles.count + def limited_snippet_titles_count + @limited_snippet_titles_count ||= limited_count(snippet_titles) end - def snippet_blobs_count - @snippet_blobs_count ||= snippet_blobs.count + def limited_snippet_blobs_count + @limited_snippet_blobs_count ||= limited_count(snippet_blobs) end private # rubocop: disable CodeReuse/ActiveRecord - def snippet_titles - limit_snippets.search(query).order('updated_at DESC').includes(:author) + def snippets + SnippetsFinder.new(current_user, finder_params) + .execute + .includes(:author) + .reorder(updated_at: :desc) end # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord + def snippet_titles + snippets.search(query) + end + def snippet_blobs - limit_snippets.search_code(query).order('updated_at DESC').includes(:author) + snippets.search_code(query) end - # rubocop: enable CodeReuse/ActiveRecord def default_scope 'snippet_blobs' end + + def paginated_objects(relation, page) + relation.page(page).per(per_page) + end + + def finder_params + {} + end end end + +Gitlab::SnippetSearchResults.prepend_if_ee('::EE::Gitlab::SnippetSearchResults') diff --git a/lib/gitlab/submodule_links.rb b/lib/gitlab/submodule_links.rb index 18fd604a3b0..b0ee0877f30 100644 --- a/lib/gitlab/submodule_links.rb +++ b/lib/gitlab/submodule_links.rb @@ -6,6 +6,7 @@ module Gitlab def initialize(repository) @repository = repository + @cache_store = {} end def for(submodule, sha) @@ -18,8 +19,9 @@ module Gitlab attr_reader :repository def submodule_urls_for(sha) - strong_memoize(:"submodule_urls_for_#{sha}") do - repository.submodule_urls_for(sha) + @cache_store.fetch(sha) do + submodule_urls = repository.submodule_urls_for(sha) + @cache_store[sha] = submodule_urls end end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 78177c6d306..2470685bc00 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -6,6 +6,21 @@ module Gitlab module Tracking SNOWPLOW_NAMESPACE = 'gl' + module ControllerConcern + extend ActiveSupport::Concern + + protected + + def track_event(action = action_name, **args) + category = args.delete(:category) || self.class.name + Gitlab::Tracking.event(category, action.to_s, **args) + end + + def track_self_describing_event(schema_url, event_data_json, **args) + Gitlab::Tracking.self_describing_event(schema_url, event_data_json, **args) + end + end + class << self def enabled? Gitlab::CurrentSettings.snowplow_enabled? @@ -17,6 +32,13 @@ module Gitlab snowplow.track_struct_event(category, action, label, property, value, context, Time.now.to_i) end + def self_describing_event(schema_url, event_data_json, context: nil) + return unless enabled? + + event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, event_data_json) + snowplow.track_self_describing_event(event_json, context, Time.now.to_i) + end + def snowplow_options(group) additional_features = Feature.enabled?(:additional_snowplow_tracking, group) { @@ -33,7 +55,7 @@ module Gitlab def snowplow @snowplow ||= SnowplowTracker::Tracker.new( - SnowplowTracker::Emitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname), + SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'https'), SnowplowTracker::Subject.new, SNOWPLOW_NAMESPACE, Gitlab::CurrentSettings.snowplow_site_id diff --git a/lib/gitlab/tracking/incident_management.rb b/lib/gitlab/tracking/incident_management.rb new file mode 100644 index 00000000000..bd8d1669dd3 --- /dev/null +++ b/lib/gitlab/tracking/incident_management.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module IncidentManagement + class << self + def track_from_params(incident_params) + return if incident_params.blank? + + incident_params.each do |k, v| + prefix = ['', '0'].include?(v.to_s) ? 'disabled' : 'enabled' + + key = tracking_keys.dig(k, :name) + label = tracking_keys.dig(k, :label) + + next if key.blank? + + details = label ? { label: label, property: v } : {} + + ::Gitlab::Tracking.event('IncidentManagement::Settings', "#{prefix}_#{key}", **details ) + end + end + + def tracking_keys + { + create_issue: { + name: 'issue_auto_creation_on_alerts' + }, + issue_template_key: { + name: 'issue_template_on_alerts', + label: 'Template name' + }, + send_email: { + name: 'sending_emails' + } + }.with_indifferent_access.freeze + end + end + end + end +end diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb new file mode 100644 index 00000000000..4ff064007f1 --- /dev/null +++ b/lib/gitlab/uploads/migration_helper.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Uploads + class MigrationHelper + attr_reader :logger + + CATEGORIES = [%w(AvatarUploader Project :avatar), + %w(AvatarUploader Group :avatar), + %w(AvatarUploader User :avatar), + %w(AttachmentUploader Note :attachment), + %w(AttachmentUploader Appearance :logo), + %w(AttachmentUploader Appearance :header_logo), + %w(FaviconUploader Appearance :favicon), + %w(FileUploader Project), + %w(PersonalFileUploader Snippet), + %w(NamespaceFileUploader Snippet), + %w(FileUploader MergeRequest)].freeze + + def initialize(args, logger) + prepare_variables(args, logger) + end + + def migrate_to_remote_storage + @to_store = ObjectStorage::Store::REMOTE + + uploads.each_batch(of: batch_size, &method(:enqueue_batch)) + end + + def migrate_to_local_storage + @to_store = ObjectStorage::Store::LOCAL + + uploads(ObjectStorage::Store::REMOTE).each_batch(of: batch_size, &method(:enqueue_batch)) + end + + private + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 200).to_i + end + + def prepare_variables(args, logger) + @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym + @uploader_class = args.uploader_class.constantize + @model_class = args.model_class.constantize + @logger = logger + end + + def enqueue_batch(batch, index) + job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, + @model_class, + @mounted_as, + @to_store) + logger.info(message: "[Uploads migration] Enqueued upload migration job", index: index, job_id: job) + rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e + # continue for the next batch + logger.warn(message: "[Uploads migration] Could not enqueue batch", ids: batch.ids, reason: e.message) # rubocop:disable CodeReuse/ActiveRecord + end + + # rubocop:disable CodeReuse/ActiveRecord + def uploads(store_type = [nil, ObjectStorage::Store::LOCAL]) + Upload.class_eval { include EachBatch } unless Upload < EachBatch + + Upload + .where(store: store_type, + uploader: @uploader_class.to_s, + model_type: @model_class.base_class.sti_name) + end + # rubocop:enable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 4285b2675c5..0adca34440c 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -125,6 +125,11 @@ module Gitlab # If the addr can't be resolved or the url is invalid (i.e http://1.1.1.1.1) # we block the url raise BlockedUrlError, "Host cannot be resolved or invalid" + rescue ArgumentError => error + # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters. + raise unless error.message.include?('hostname too long') + + raise BlockedUrlError, "Host is too long (maximum is 1024 characters)" end def validate_local_request( diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 42cf1ec1f0e..038067eeae4 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -81,3 +81,5 @@ module Gitlab end end end + +::Gitlab::UrlBuilder.prepend_if_ee('EE::Gitlab::UrlBuilder') diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index c6c2876033d..cb492b69fec 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -17,7 +17,6 @@ module Gitlab .merge(features_usage_data) .merge(components_usage_data) .merge(cycle_analytics_usage_data) - .merge(usage_counters) end def to_json(force_refresh: false) @@ -38,7 +37,7 @@ module Gitlab usage_data end - # rubocop:disable Metrics/AbcSize + # rubocop: disable Metrics/AbcSize # rubocop: disable CodeReuse/ActiveRecord def system_usage_data { @@ -97,13 +96,16 @@ module Gitlab todos: count(Todo), uploads: count(Upload), web_hooks: count(WebHook) - }.merge(services_usage) - .merge(approximate_counts) - }.tap do |data| - data[:counts][:user_preferences] = user_preferences_usage - end + }.merge( + services_usage, + approximate_counts, + usage_counters, + user_preferences_usage + ) + } end # rubocop: enable CodeReuse/ActiveRecord + # rubocop: enable Metrics/AbcSize def cycle_analytics_usage_data Gitlab::CycleAnalytics::UsageData.new.to_json @@ -116,6 +118,7 @@ module Gitlab def features_usage_data_ce { container_registry_enabled: Gitlab.config.registry.enabled, + dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled, gitlab_shared_runners_enabled: Gitlab.config.gitlab_ci.shared_runners_enabled, gravatar_enabled: Gitlab::CurrentSettings.gravatar_enabled?, influxdb_metrics_enabled: Gitlab::Metrics.influx_metrics_enabled?, @@ -136,15 +139,15 @@ module Gitlab # @return [Array<#totals>] An array of objects that respond to `#totals` def usage_data_counters [ - Gitlab::UsageDataCounters::WikiPageCounter, - Gitlab::UsageDataCounters::WebIdeCounter, - Gitlab::UsageDataCounters::NoteCounter, - Gitlab::UsageDataCounters::SnippetCounter, - Gitlab::UsageDataCounters::SearchCounter, - Gitlab::UsageDataCounters::CycleAnalyticsCounter, - Gitlab::UsageDataCounters::ProductivityAnalyticsCounter, - Gitlab::UsageDataCounters::SourceCodeCounter, - Gitlab::UsageDataCounters::MergeRequestCounter + Gitlab::UsageDataCounters::WikiPageCounter, + Gitlab::UsageDataCounters::WebIdeCounter, + Gitlab::UsageDataCounters::NoteCounter, + Gitlab::UsageDataCounters::SnippetCounter, + Gitlab::UsageDataCounters::SearchCounter, + Gitlab::UsageDataCounters::CycleAnalyticsCounter, + Gitlab::UsageDataCounters::ProductivityAnalyticsCounter, + Gitlab::UsageDataCounters::SourceCodeCounter, + Gitlab::UsageDataCounters::MergeRequestCounter ] end @@ -186,7 +189,7 @@ module Gitlab .find_in_batches(batch_size: BATCH_SIZE) do |services| counts = services.group_by do |service| - # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 + # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 service_url = service.data_fields&.url || (service.properties && service.properties['url']) service_url&.include?('.atlassian.net') ? :cloud : :server end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index c66ce0434a4..7fbfc4c45c4 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -13,14 +13,6 @@ module Gitlab path end - # Run system command without outputting to stdout. - # - # @param cmd [Array<String>] - # @return [Boolean] - def system_silent(cmd) - Popen.popen(cmd).last.zero? - end - def force_utf8(str) str.dup.force_encoding(Encoding::UTF_8) end diff --git a/lib/gitlab/utils/inline_hash.rb b/lib/gitlab/utils/inline_hash.rb new file mode 100644 index 00000000000..41e5f3ee4c3 --- /dev/null +++ b/lib/gitlab/utils/inline_hash.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + module InlineHash + extend self + + # Transforms a Hash into an inline Hash by merging its nested keys. + # + # Input + # + # { + # 'root_param' => 'Root', + # 12 => 'number', + # symbol: 'symbol', + # nested_param: { + # key: 'Value' + # }, + # 'very' => { + # 'deep' => { + # 'nested' => { + # 12 => 'Deep nested value' + # } + # } + # } + # } + # + # + # Result + # + # { + # 'root_param' => 'Root', + # 12 => 'number', + # symbol: symbol, + # 'nested_param.key' => 'Value', + # 'very.deep.nested.12' => 'Deep nested value' + # } + # + def merge_keys(hash, prefix: nil, connector: '.') + result = {} + pairs = + if prefix + base_prefix = "#{prefix}#{connector}" + hash.map { |key, value| ["#{base_prefix}#{key}", value] } + else + hash.to_a + end + + until pairs.empty? + key, value = pairs.shift + + if value.is_a?(Hash) + value.each { |k, v| pairs.unshift ["#{key}#{connector}#{k}", v] } + else + result[key] = value + end + end + + result + end + end + end +end diff --git a/lib/gitlab/utils/safe_inline_hash.rb b/lib/gitlab/utils/safe_inline_hash.rb new file mode 100644 index 00000000000..644d87c6876 --- /dev/null +++ b/lib/gitlab/utils/safe_inline_hash.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + class SafeInlineHash + # Validates the hash size using `Gitlab::Utils::DeepSize` before merging keys using `Gitlab::Utils::InlineHash` + def initialize(hash, prefix: nil, connector: '.') + @hash = hash + end + + def self.merge_keys!(hash, prefix: nil, connector: '.') + new(hash).merge_keys!(prefix: prefix, connector: connector) + end + + def merge_keys!(prefix:, connector:) + raise ArgumentError, 'The Hash is too big' unless valid? + + Gitlab::Utils::InlineHash.merge_keys(hash, prefix: prefix, connector: connector) + end + + private + + attr_reader :hash + + def valid? + Gitlab::Utils::DeepSize.new(hash).valid? + end + end + end +end diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb index 875e8a120e9..afcdbd087d2 100644 --- a/lib/gitlab/verify/uploads.rb +++ b/lib/gitlab/verify/uploads.rb @@ -32,7 +32,7 @@ module Gitlab end def remote_object_exists?(upload) - upload.build_uploader.file.exists? + upload.retrieve_uploader.file.exists? end end end |