diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-19 08:27:35 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-19 08:27:35 +0000 |
| commit | 7e9c479f7de77702622631cff2628a9c8dcbc627 (patch) | |
| tree | c8f718a08e110ad7e1894510980d2155a6549197 /lib/gitlab | |
| parent | e852b0ae16db4052c1c567d9efa4facc81146e88 (diff) | |
| download | gitlab-ce-7e9c479f7de77702622631cff2628a9c8dcbc627.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-6-stable-eev13.6.0-rc42
Diffstat (limited to 'lib/gitlab')
183 files changed, 4215 insertions, 1528 deletions
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb index c5f843d5f1a..4bb225b63f1 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb @@ -18,24 +18,46 @@ module Gitlab end def timestamp_projection - issue_metrics_table[:first_mentioned_in_commit_at] + Arel::Nodes::NamedFunction.new('COALESCE', column_list) end override :column_list def column_list - [timestamp_projection] + [ + issue_metrics_table[:first_mentioned_in_commit_at], + mr_metrics_table[:first_commit_at] + ] end # rubocop: disable CodeReuse/ActiveRecord def apply_query_customization(query) - issue_metrics_join = mr_closing_issues_table - .join(issue_metrics_table) + query + .joins(merge_requests_closing_issues_join) + .joins(issue_metrics_join) + .joins(mr_metrics_join) + end + # rubocop: enable CodeReuse/ActiveRecord + + def issue_metrics_join + mr_closing_issues_table + .join(issue_metrics_table, Arel::Nodes::OuterJoin) .on(mr_closing_issues_table[:issue_id].eq(issue_metrics_table[:issue_id])) .join_sources + end - query.joins(:merge_requests_closing_issues).joins(issue_metrics_join) + def merge_requests_closing_issues_join + mr_table + .join(mr_closing_issues_table, Arel::Nodes::OuterJoin) + .on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])) + .join_sources + end + + def mr_metrics_join + mr_metrics_table + .join(mr_metrics_table, Arel::Nodes::OuterJoin) + .on(mr_metrics_table[:merge_request_id].eq(mr_table[:id])) + .join_sources end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb b/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb index 636bba22c23..54b3bbb3ce6 100644 --- a/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb +++ b/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb @@ -11,21 +11,36 @@ module Gitlab def execute measurement_identifiers.map do |measurement_identifier| - query_scope = ::Analytics::InstanceStatistics::Measurement::IDENTIFIER_QUERY_MAPPING[measurement_identifier]&.call + query_scope = query_mappings[measurement_identifier]&.call next if query_scope.nil? - # Determining the query range (id range) as early as possible in order to get more accurate counts. - start = query_scope.minimum(:id) - finish = query_scope.maximum(:id) - - [measurement_identifier, start, finish, recorded_at] + [measurement_identifier, *determine_start_and_finish(measurement_identifier, query_scope), recorded_at] end.compact end private attr_reader :measurement_identifiers, :recorded_at + + # Determining the query range (id range) as early as possible in order to get more accurate counts. + def determine_start_and_finish(measurement_identifier, query_scope) + queries = custom_min_max_queries[measurement_identifier] + + if queries + [queries[:minimum_query].call, queries[:maximum_query].call] + else + [query_scope.minimum(:id), query_scope.maximum(:id)] + end + end + + def custom_min_max_queries + ::Analytics::InstanceStatistics::Measurement.identifier_min_max_queries + end + + def query_mappings + ::Analytics::InstanceStatistics::Measurement.identifier_query_mapping + end end end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 6173918b453..e92bbe4f529 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -33,7 +33,8 @@ module Gitlab group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, group_testing_hook: { threshold: 5, interval: 1.minute }, profile_add_new_email: { threshold: 5, interval: 1.minute }, - profile_resend_email_confirmation: { threshold: 5, interval: 1.minute } + profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, + update_environment_canary_ingress: { threshold: 1, interval: 1.minute } }.freeze end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 001c083c778..fadd6eb848d 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -158,7 +158,7 @@ module Gitlab if Service.available_services_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included - # in the Service.available_services_names whitelist. + # in the Service.available_services_names allowlist. service = project.public_send("#{underscored_service}_service") # rubocop:disable GitlabSecurity/PublicSend if service && service.activated? && service.valid_token?(password) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 3d3f7212053..f3975fe219a 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -83,6 +83,8 @@ module Gitlab return unless ::Gitlab::Auth::CI_JOB_USER == login job = find_valid_running_job_by_token!(password) + @current_authenticated_job = job # rubocop:disable Gitlab/ModuleWithInstanceVariables + job.user end diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb new file mode 100644 index 00000000000..553571d5d00 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill design.iid for a range of projects + class BackfillDesignInternalIds + # See app/models/internal_id + # This is a direct copy of the application code with the following changes: + # - usage enum is hard-coded to the value for design_management_designs + # - init is not passed around, but ignored + class InternalId < ActiveRecord::Base + def self.track_greatest(subject, scope, new_value) + InternalIdGenerator.new(subject, scope).track_greatest(new_value) + end + + # Increments #last_value with new_value if it is greater than the current, + # and saves the record + # + # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL). + # As such, the increment is atomic and safe to be called concurrently. + def track_greatest_and_save!(new_value) + update_and_save { self.last_value = [last_value || 0, new_value].max } + end + + private + + def update_and_save(&block) + lock! + yield + # update_and_save_counter.increment(usage: usage, changed: last_value_changed?) + save! + last_value + end + end + + # See app/models/internal_id + class InternalIdGenerator + attr_reader :subject, :scope, :scope_attrs + + def initialize(subject, scope) + @subject = subject + @scope = scope + + raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty? + end + + # Create a record in internal_ids if one does not yet exist + # and set its new_value if it is higher than the current last_value + # + # Note this will acquire a ROW SHARE lock on the InternalId record + def track_greatest(new_value) + subject.transaction do + record.track_greatest_and_save!(new_value) + end + end + + def record + @record ||= (lookup || create_record) + end + + def lookup + InternalId.find_by(**scope, usage: usage_value) + end + + def usage_value + 10 # see Enums::InternalId - this is the value for design_management_designs + end + + # Create InternalId record for (scope, usage) combination, if it doesn't exist + # + # We blindly insert without synchronization. If another process + # was faster in doing this, we'll realize once we hit the unique key constraint + # violation. We can safely roll-back the nested transaction and perform + # a lookup instead to retrieve the record. + def create_record + subject.transaction(requires_new: true) do + InternalId.create!( + **scope, + usage: usage_value, + last_value: 0 + ) + end + rescue ActiveRecord::RecordNotUnique + lookup + end + end + + attr_reader :design_class + + def initialize(design_class) + @design_class = design_class + end + + def perform(relation) + start_id, end_id = relation.pluck("min(project_id), max(project_id)").flatten + table = 'design_management_designs' + + ActiveRecord::Base.connection.execute <<~SQL + WITH + starting_iids(project_id, iid) as ( + SELECT project_id, MAX(COALESCE(iid, 0)) + FROM #{table} + WHERE project_id BETWEEN #{start_id} AND #{end_id} + GROUP BY project_id + ), + with_calculated_iid(id, iid) as ( + SELECT design.id, + init.iid + ROW_NUMBER() OVER (PARTITION BY design.project_id ORDER BY design.id ASC) + FROM #{table} as design, starting_iids as init + WHERE design.project_id BETWEEN #{start_id} AND #{end_id} + AND design.iid IS NULL + AND init.project_id = design.project_id + ) + + UPDATE #{table} + SET iid = with_calculated_iid.iid + FROM with_calculated_iid + WHERE #{table}.id = with_calculated_iid.id + SQL + + # track the new greatest IID value + relation.each do |design| + current_max = design_class.where(project_id: design.project_id).maximum(:iid) + scope = { project_id: design.project_id } + InternalId.track_greatest(design, scope, current_max) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb new file mode 100644 index 00000000000..61145f6a445 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +# Based on https://community.developer.atlassian.com/t/get-rest-api-3-filter-search/29459/2, +# it's enough at the moment to simply notice if the url is from `atlassian.net` +module Gitlab + module BackgroundMigration + # Backfill the deployment_type in jira_tracker_data table + class BackfillJiraTrackerDeploymentType2 + # Migration only version of jira_tracker_data table + class JiraTrackerDataTemp < ApplicationRecord + self.table_name = 'jira_tracker_data' + + def self.encryption_options + { + key: Settings.attr_encrypted_db_key_base_32, + encode: true, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm' + } + end + + attr_encrypted :url, encryption_options + attr_encrypted :api_url, encryption_options + + enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment + end + + # Migration only version of services table + class JiraServiceTemp < ApplicationRecord + self.table_name = 'services' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + @server_ids = [] + @cloud_ids = [] + + JiraTrackerDataTemp + .where(id: start_id..stop_id, deployment_type: 0) + .each do |jira_tracker_data| + collect_deployment_type(jira_tracker_data) + end + + unless cloud_ids.empty? + JiraTrackerDataTemp.where(id: cloud_ids) + .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:cloud]) + end + + unless server_ids.empty? + JiraTrackerDataTemp.where(id: server_ids) + .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:server]) + end + + mark_jobs_as_succeeded(start_id, stop_id) + end + + private + + attr_reader :server_ids, :cloud_ids + + def client_url(jira_tracker_data) + jira_tracker_data.api_url.presence || jira_tracker_data.url.presence + end + + def server_type(url) + url.downcase.include?('.atlassian.net') ? :cloud : :server + end + + def collect_deployment_type(jira_tracker_data) + url = client_url(jira_tracker_data) + return unless url + + case server_type(url) + when :cloud + cloud_ids << jira_tracker_data.id + else + server_ids << jira_tracker_data.id + end + end + + def mark_jobs_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb b/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb new file mode 100644 index 00000000000..8a58cf9b302 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill merge request cleanup schedules of closed/merged merge requests + # without any corresponding records. + class BackfillMergeRequestCleanupSchedules + # Model used for migration added in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46782. + class MergeRequest < ActiveRecord::Base + include EachBatch + + self.table_name = 'merge_requests' + + def self.eligible + where('merge_requests.state_id IN (2, 3)') + end + end + + def perform(start_id, end_id) + eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id) + scheduled_at_column = "COALESCE(metrics.merged_at, COALESCE(metrics.latest_closed_at, merge_requests.updated_at)) + interval '14 days'" + query = + eligible_mrs + .select("merge_requests.id, #{scheduled_at_column}, NOW(), NOW()") + .joins('LEFT JOIN merge_request_metrics metrics ON metrics.merge_request_id = merge_requests.id') + + result = ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO merge_request_cleanup_schedules (merge_request_id, scheduled_at, created_at, updated_at) + #{query.to_sql} + ON CONFLICT (merge_request_id) DO NOTHING; + SQL + + ::Gitlab::BackgroundMigration::Logger.info( + message: 'Backfilled merge_request_cleanup_schedules records', + count: result.cmd_tuples + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb new file mode 100644 index 00000000000..78140b768fc --- /dev/null +++ b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class populates missing dismissal information for + # vulnerability entries. + class PopulateHasVulnerabilities + class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation + self.table_name = 'project_settings' + + UPSERT_SQL = <<~SQL + WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS ( + SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) + ) + INSERT INTO project_settings + (project_id, has_vulnerabilities, created_at, updated_at) + (SELECT * FROM upsert_data) + ON CONFLICT (project_id) + DO UPDATE SET + has_vulnerabilities = true, + updated_at = EXCLUDED.updated_at + SQL + + def self.upsert_for(project_ids) + connection.execute(UPSERT_SQL % { project_ids: project_ids.join(', ') }) + end + end + + class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'vulnerabilities' + end + + def perform(*project_ids) + ProjectSetting.upsert_for(project_ids) + rescue StandardError => e + log_error(e, project_ids) + ensure + log_info(project_ids) + end + + private + + def log_error(error, project_ids) + ::Gitlab::BackgroundMigration::Logger.error( + migrator: self.class.name, + message: error.message, + project_ids: project_ids + ) + end + + def log_info(project_ids) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: self.class.name, + message: 'Projects has been processed to populate `has_vulnerabilities` information', + count: project_ids.length + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb new file mode 100644 index 00000000000..bc0a181a06c --- /dev/null +++ b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class populates missing dismissal information for + # vulnerability entries. + class PopulateMissingVulnerabilityDismissalInformation + class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'vulnerabilities' + + has_one :finding, class_name: '::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation::Finding' + + scope :broken, -> { where('state = 2 AND (dismissed_at IS NULL OR dismissed_by_id IS NULL)') } + + def copy_dismissal_information + return unless finding&.dismissal_feedback + + update_columns( + dismissed_at: finding.dismissal_feedback.created_at, + dismissed_by_id: finding.dismissal_feedback.author_id + ) + end + end + + class Finding < ActiveRecord::Base # rubocop:disable Style/Documentation + include ShaAttribute + + self.table_name = 'vulnerability_occurrences' + + sha_attribute :project_fingerprint + + def dismissal_feedback + Feedback.dismissal.where(category: report_type, project_fingerprint: project_fingerprint, project_id: project_id).first + end + end + + class Feedback < ActiveRecord::Base # rubocop:disable Style/Documentation + DISMISSAL_TYPE = 0 + + self.table_name = 'vulnerability_feedback' + + scope :dismissal, -> { where(feedback_type: DISMISSAL_TYPE) } + end + + def perform(*vulnerability_ids) + Vulnerability.includes(:finding).where(id: vulnerability_ids).each { |vulnerability| populate_for(vulnerability) } + + log_info(vulnerability_ids) + end + + private + + def populate_for(vulnerability) + log_warning(vulnerability) unless vulnerability.copy_dismissal_information + rescue StandardError => error + log_error(error, vulnerability) + end + + def log_info(vulnerability_ids) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: self.class.name, + message: 'Dismissal information has been copied', + count: vulnerability_ids.length + ) + end + + def log_warning(vulnerability) + ::Gitlab::BackgroundMigration::Logger.warn( + migrator: self.class.name, + message: 'Could not update vulnerability!', + vulnerability_id: vulnerability.id + ) + end + + def log_error(error, vulnerability) + ::Gitlab::BackgroundMigration::Logger.error( + migrator: self.class.name, + message: error.message, + vulnerability_id: vulnerability.id + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb new file mode 100644 index 00000000000..fc79f7125e3 --- /dev/null +++ b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class updates vulnerability feedback entities with no pipeline id assigned. + class PopulateVulnerabilityFeedbackPipelineId + def perform(project_ids) + end + end + end +end + +Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId') diff --git a/lib/gitlab/background_migration/replace_blocked_by_links.rb b/lib/gitlab/background_migration/replace_blocked_by_links.rb index 26626aaef79..0c29887bb00 100644 --- a/lib/gitlab/background_migration/replace_blocked_by_links.rb +++ b/lib/gitlab/background_migration/replace_blocked_by_links.rb @@ -12,14 +12,19 @@ module Gitlab blocked_by_links = IssueLink.where(id: start_id..stop_id).where(link_type: 2) ActiveRecord::Base.transaction do - # if there is duplicit bi-directional relation (issue2 is blocked by issue1 - # and issue1 already links issue2), then we can just delete 'blocked by'. - # This should be rare as we have a pre-create check which checks if issues are - # already linked - blocked_by_links + # There could be two edge cases: + # 1) issue1 is blocked by issue2 AND issue2 blocks issue1 (type 1) + # 2) issue1 is blocked by issue2 AND issue2 is related to issue1 (type 0) + # In both cases cases we couldn't convert blocked by relation to + # `issue2 blocks issue` because there is already a link with the same + # source/target id. To avoid these conflicts, we first delete any + # "opposite" links before we update `blocked by` relation. This + # should be rare as we have a pre-create check which checks if issues + # are already linked + opposite_ids = blocked_by_links + .select('opposite_links.id') .joins('INNER JOIN issue_links as opposite_links ON issue_links.source_id = opposite_links.target_id AND issue_links.target_id = opposite_links.source_id') - .where('opposite_links.link_type': 1) - .delete_all + IssueLink.where(id: opposite_ids).delete_all blocked_by_links.update_all('source_id=target_id,target_id=source_id,link_type=1') end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 0df6e858bf4..390da014a5a 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -17,8 +17,6 @@ module Gitlab key_width: opts[:key_width].to_i, key_text: opts[:key_text] } - - @pipeline = @project.ci_pipelines.latest_successful_for_ref(@ref) end def entity @@ -42,19 +40,35 @@ module Gitlab private - # rubocop: disable CodeReuse/ActiveRecord + def successful_pipeline + @successful_pipeline ||= @project.ci_pipelines.latest_successful_for_ref(@ref) + end + + def failed_pipeline + @failed_pipeline ||= @project.ci_pipelines.latest_failed_for_ref(@ref) + end + + def running_pipeline + @running_pipeline ||= @project.ci_pipelines.latest_running_for_ref(@ref) + end + def raw_coverage - return unless @pipeline + latest = + if @job.present? + builds = ::Ci::Build + .in_pipelines([successful_pipeline, running_pipeline, failed_pipeline]) + .latest + .success + .for_ref(@ref) + .by_name(@job) + + builds.max_by(&:created_at) + else + successful_pipeline + end - if @job.blank? - @pipeline.coverage - else - @pipeline.builds - .find_by(name: @job) - .try(:coverage) - end + latest&.coverage end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index aca5a63a424..d29799f1029 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -4,11 +4,14 @@ module Gitlab module BitbucketServerImport class Importer attr_reader :recover_missing_commits - attr_reader :project, :project_key, :repository_slug, :client, :errors, :users + attr_reader :project, :project_key, :repository_slug, :client, :errors, :users, :already_imported_cache_key attr_accessor :logger REMOTE_NAME = 'bitbucket_server' BATCH_SIZE = 100 + # The base cache key to use for tracking already imported objects. + ALREADY_IMPORTED_CACHE_KEY = + 'bitbucket_server-importer/already-imported/%{project}/%{collection}' TempBranch = Struct.new(:name, :sha) @@ -36,17 +39,25 @@ module Gitlab @users = {} @temp_branches = [] @logger = Gitlab::Import::Logger.build + @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % + { project: project.id, collection: collection_method } + end + + def collection_method + :pull_requests end def execute import_repository import_pull_requests + download_lfs_objects delete_temp_branches handle_errors metrics.track_finished_import log_info(stage: "complete") + Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, 15.minutes.to_i) true end @@ -148,6 +159,14 @@ module Gitlab raise end + def download_lfs_objects + result = Projects::LfsPointers::LfsImportService.new(project).execute + + if result[:status] == :error + errors << { type: :lfs_objects, errors: "The Lfs import process failed. #{result[:message]}" } + end + end + # Bitbucket Server keeps tracks of references for open pull requests in # refs/heads/pull-requests, but closed and merged requests get moved # into hidden internal refs under stash-refs/pull-requests. Unless the @@ -158,16 +177,29 @@ module Gitlab # on the remote server. Then we have to issue a `git fetch` to download these # branches. def import_pull_requests - pull_requests = client.pull_requests(project_key, repository_slug).to_a + page = 0 + + log_info(stage: 'import_pull_requests', message: "starting") + + loop do + log_debug(stage: 'import_pull_requests', message: "importing page #{page} and batch-size #{BATCH_SIZE} from #{page * BATCH_SIZE} to #{(page + 1) * BATCH_SIZE}") + + pull_requests = client.pull_requests(project_key, repository_slug, page_offset: page, limit: BATCH_SIZE).to_a - # Creating branches on the server and fetching the newly-created branches - # may take a number of network round-trips. Do this in batches so that we can - # avoid doing a git fetch for every new branch. - pull_requests.each_slice(BATCH_SIZE) do |batch| - restore_branches(batch) if recover_missing_commits + break if pull_requests.empty? - batch.each do |pull_request| - import_bitbucket_pull_request(pull_request) + # Creating branches on the server and fetching the newly-created branches + # may take a number of network round-trips. This used to be done in batches to + # avoid doing a git fetch for every new branch, as the whole process is now + # batched, we do not need to separately do this in batches. + restore_branches(pull_requests) if recover_missing_commits + + pull_requests.each do |pull_request| + if already_imported?(pull_request) + log_info(stage: 'import_pull_requests', message: 'already imported', iid: pull_request.iid) + else + import_bitbucket_pull_request(pull_request) + end rescue StandardError => e Gitlab::ErrorTracking.log_exception( e, @@ -177,9 +209,25 @@ module Gitlab backtrace = Gitlab::BacktraceCleaner.clean_backtrace(e.backtrace) errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, backtrace: backtrace.join("\n"), raw_response: pull_request.raw } end + + log_debug(stage: 'import_pull_requests', message: "finished page #{page} and batch-size #{BATCH_SIZE}") + page += 1 end end + # Returns true if the given object has already been imported, false + # otherwise. + # + # object - The object to check. + def already_imported?(pull_request) + Gitlab::Cache::Import::Caching.set_includes?(already_imported_cache_key, pull_request.iid) + end + + # Marks the given object as "already imported". + def mark_as_imported(pull_request) + Gitlab::Cache::Import::Caching.set_add(already_imported_cache_key, pull_request.iid) + end + def delete_temp_branches @temp_branches.each do |branch| client.delete_branch(project_key, repository_slug, branch.name, branch.sha) @@ -227,6 +275,7 @@ module Gitlab end log_info(stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid) + mark_as_imported(pull_request) end def import_pull_request_comments(pull_request, merge_request) @@ -378,6 +427,10 @@ module Gitlab } end + def log_debug(details) + logger.debug(log_base_data.merge(details)) + end + def log_info(details) logger.info(log_base_data.merge(details)) end diff --git a/lib/gitlab/bulk_import/client.rb b/lib/gitlab/bulk_import/client.rb deleted file mode 100644 index c6e77a158cd..00000000000 --- a/lib/gitlab/bulk_import/client.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BulkImport - class Client - API_VERSION = 'v4'.freeze - DEFAULT_PAGE = 1.freeze - DEFAULT_PER_PAGE = 30.freeze - - ConnectionError = Class.new(StandardError) - - def initialize(uri:, token:, page: DEFAULT_PAGE, per_page: DEFAULT_PER_PAGE, api_version: API_VERSION) - @uri = URI.parse(uri) - @token = token&.strip - @page = page - @per_page = per_page - @api_version = api_version - end - - def get(resource, query = {}) - response = with_error_handling do - Gitlab::HTTP.get( - resource_url(resource), - headers: request_headers, - follow_redirects: false, - query: query.merge(request_query) - ) - end - - response.parsed_response - end - - private - - def request_query - { - page: @page, - per_page: @per_page - } - end - - def request_headers - { - 'Content-Type' => 'application/json', - 'Authorization' => "Bearer #{@token}" - } - end - - def with_error_handling - response = yield - - raise ConnectionError.new("Error #{response.code}") unless response.success? - - response - rescue *Gitlab::HTTP::HTTP_ERRORS => e - raise ConnectionError, e - end - - def base_uri - @base_uri ||= "#{@uri.scheme}://#{@uri.host}:#{@uri.port}" - end - - def api_url - Gitlab::Utils.append_path(base_uri, "/api/#{@api_version}") - end - - def resource_url(resource) - Gitlab::Utils.append_path(api_url, resource) - end - end - end -end diff --git a/lib/gitlab/chat/output.rb b/lib/gitlab/chat/output.rb index 411b1555a7d..4a55b81a9eb 100644 --- a/lib/gitlab/chat/output.rb +++ b/lib/gitlab/chat/output.rb @@ -12,7 +12,10 @@ module Gitlab PRIMARY_SECTION = 'chat_reply' # The backup trace section in case the primary one could not be found. - FALLBACK_SECTION = 'build_script' + FALLBACK_SECTION = 'step_script' + + # `step_script` used to be `build_script` before runner 13.1 + LEGACY_SECTION = 'build_script' # build - The `Ci::Build` to obtain the output from. def initialize(build) @@ -37,24 +40,6 @@ module Gitlab end end - # Returns the offset to seek to and the number of bytes to read relative - # to the offset. - def read_offset_and_length - section = find_build_trace_section(PRIMARY_SECTION) || - find_build_trace_section(FALLBACK_SECTION) - - unless section - raise( - MissingBuildSectionError, - "The build_script trace section could not be found for build #{build.id}" - ) - end - - length = section[:byte_end] - section[:byte_start] - - [section[:byte_start], length] - end - # Removes the line containing the executed command from the build output. # # output - A `String` containing the output of a trace section. @@ -88,6 +73,27 @@ module Gitlab def trace @trace ||= build.trace end + + private + + # Returns the offset to seek to and the number of bytes to read relative + # to the offset. + def read_offset_and_length + section = find_build_trace_section(PRIMARY_SECTION) || + find_build_trace_section(FALLBACK_SECTION) || + find_build_trace_section(LEGACY_SECTION) + + unless section + raise( + MissingBuildSectionError, + "The build_script trace section could not be found for build #{build.id}" + ) + end + + length = section[:byte_end] - section[:byte_start] + + [section[:byte_start], length] + end end end end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 728a66ca87f..cbecce57163 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -11,12 +11,22 @@ module Gitlab def satisfied_by?(pipeline, context) return true if pipeline.modified_paths.nil? + expanded_globs = expand_globs(pipeline, context) pipeline.modified_paths.any? do |path| - @globs.any? do |glob| + expanded_globs.any? do |glob| File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB) end end end + + def expand_globs(pipeline, context) + return @globs unless ::Feature.enabled?(:ci_variable_expansion_in_rules_changes, pipeline.project, default_enabled: true) + return @globs unless context + + @globs.map do |glob| + ExpandVariables.expand_existing(glob, context.variables) + end + end end end end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 3fbfdffe277..25fb9c0ca97 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -3,38 +3,8 @@ module Gitlab module Ci module Charts - module DailyInterval - # rubocop: disable CodeReuse/ActiveRecord - def grouped_count(query) - query - .group("DATE(#{::Ci::Pipeline.table_name}.created_at)") - .count(:created_at) - .transform_keys { |date| date.strftime(@format) } # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - # rubocop: enable CodeReuse/ActiveRecord - - def interval_step - @interval_step ||= 1.day - end - end - - module MonthlyInterval - # rubocop: disable CodeReuse/ActiveRecord - def grouped_count(query) - query - .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") - .count(:created_at) - .transform_keys(&:squish) - end - # rubocop: enable CodeReuse/ActiveRecord - - def interval_step - @interval_step ||= 1.month - end - end - class Chart - attr_reader :labels, :total, :success, :project, :pipeline_times + attr_reader :from, :to, :labels, :total, :success, :project, :pipeline_times def initialize(project) @labels = [] @@ -46,48 +16,59 @@ module Gitlab collect end + private + + attr_reader :interval + # rubocop: disable CodeReuse/ActiveRecord def collect query = project.all_pipelines - .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection + .where(::Ci::Pipeline.arel_table['created_at'].gteq(@from)) + .where(::Ci::Pipeline.arel_table['created_at'].lteq(@to)) totals_count = grouped_count(query) success_count = grouped_count(query.success) current = @from - while current < @to - label = current.strftime(@format) - - @labels << label - @total << (totals_count[label] || 0) - @success << (success_count[label] || 0) + while current <= @to + @labels << current.strftime(@format) + @total << (totals_count[current] || 0) + @success << (success_count[current] || 0) current += interval_step end end # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def grouped_count(query) + query + .group("date_trunc('#{interval}', #{::Ci::Pipeline.table_name}.created_at)") + .count(:created_at) + end + # rubocop: enable CodeReuse/ActiveRecord + + def interval_step + @interval_step ||= 1.public_send(interval) # rubocop: disable GitlabSecurity/PublicSend + end end class YearChart < Chart - include MonthlyInterval - attr_reader :to, :from - def initialize(*) @to = Date.today.end_of_month.end_of_day - @from = @to.years_ago(1).beginning_of_month.beginning_of_day - @format = '%d %B %Y' + @from = (@to - 1.year).beginning_of_month.beginning_of_day + @interval = :month + @format = '%B %Y' super end end class MonthChart < Chart - include DailyInterval - attr_reader :to, :from - def initialize(*) @to = Date.today.end_of_day - @from = 1.month.ago.beginning_of_day + @from = (@to - 1.month).beginning_of_day + @interval = :day @format = '%d %B' super @@ -95,12 +76,10 @@ module Gitlab end class WeekChart < Chart - include DailyInterval - attr_reader :to, :from - def initialize(*) @to = Date.today.end_of_day - @from = 1.week.ago.beginning_of_day + @from = (@to - 1.week).beginning_of_day + @interval = :day @format = '%d %B' super diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index 1740032e5c7..70fcc1d586a 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -18,7 +18,6 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS with_options allow_nil: true do - validates :allow_failure, boolean: true validates :when, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" @@ -48,7 +47,7 @@ module Gitlab inherit: false, metadata: { allowed_needs: %i[job bridge] } - attributes :when, :allow_failure + attributes :when def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -60,14 +59,6 @@ module Gitlab true end - def manual_action? - self.when == 'manual' - end - - def ignored? - allow_failure.nil? ? manual_action? : allow_failure - end - def value super.merge( trigger: (trigger_value if trigger_defined?), diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index ecc2c5cb729..1ce7060df22 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -24,7 +24,6 @@ module Gitlab validates :script, presence: true with_options allow_nil: true do - validates :allow_failure, boolean: true validates :when, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" @@ -118,7 +117,7 @@ module Gitlab description: 'Parallel configuration for this job.', inherit: false - attributes :script, :tags, :allow_failure, :when, :dependencies, + attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, :interruptible, :timeout, :resource_group, :release @@ -141,18 +140,10 @@ module Gitlab end end - def manual_action? - self.when == 'manual' - end - def delayed? self.when == 'delayed' end - def ignored? - allow_failure.nil? ? manual_action? : allow_failure - end - def value super.merge( before_script: before_script_value, diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index f10c509d0cc..c0315e5f901 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -32,6 +32,7 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true + validates :allow_failure, boolean: true end end @@ -64,7 +65,7 @@ module Gitlab inherit: false, default: {} - attributes :extends, :rules + attributes :extends, :rules, :allow_failure end def compose!(deps = nil) @@ -136,6 +137,14 @@ module Gitlab root_variables.merge(variables_value.to_h) end + + def manual_action? + self.when == 'manual' + end + + def ignored? + allow_failure.nil? ? manual_action? : allow_failure + end end end end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb index 2481989060e..aa34cfb3acc 100644 --- a/lib/gitlab/ci/config/entry/product/variables.rb +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -14,7 +14,7 @@ module Gitlab validations do validates :config, variables: { array_values: true } validates :config, length: { - minimum: :minimum, + minimum: 1, too_short: 'requires at least %{count} items' } end @@ -28,10 +28,6 @@ module Gitlab .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } .to_h end - - def minimum - ::Gitlab::Ci::Features.one_dimensional_matrix_enabled? ? 1 : 2 - end end end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 97ae6c4ceba..90692eafc3f 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -33,6 +33,7 @@ module Gitlab locations .compact .map(&method(:normalize_location)) + .flat_map(&method(:expand_project_files)) .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) end @@ -52,6 +53,15 @@ module Gitlab end end + def expand_project_files(location) + return location unless ::Feature.enabled?(:ci_include_multiple_files_from_project, context.project, default_enabled: true) + return location unless location[:project] + + Array.wrap(location[:file]).map do |file| + location.merge(file: file) + end + end + def normalize_location_string(location) if ::Gitlab::UrlSanitizer.valid?(location) { remote: location } diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 1b58e3ec71a..661189eea50 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -32,16 +32,12 @@ module Gitlab end # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` - # is a safe switch to disable the feature for a parituclar project when something went wrong, + # is a safe switch to disable the feature for a particular project when something went wrong, # therefore it's not supposed to be enabled by default. def self.disallow_to_create_merge_request_pipelines_in_target_project?(target_project) ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) end - def self.lint_creates_pipeline_with_dry_run?(project) - ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project, default_enabled: true) - end - def self.project_transactionless_destroy?(project) Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) end @@ -59,13 +55,21 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.one_dimensional_matrix_enabled? - ::Feature.enabled?(:one_dimensional_matrix, default_enabled: true) - end - def self.manual_bridges_enabled?(project) ::Feature.enabled?(:ci_manual_bridges, project, default_enabled: true) end + + def self.auto_rollback_available?(project) + ::Feature.enabled?(:cd_auto_rollback, project) && project&.feature_available?(:auto_rollback) + end + + def self.seed_block_run_before_workflow_rules_enabled?(project) + ::Feature.enabled?(:ci_seed_block_run_before_workflow_rules, project, default_enabled: true) + end + + def self.ci_pipeline_editor_page_enabled?(project) + ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: false) + end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 491facd0a43..a8943eadf4f 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -6,6 +6,8 @@ module Gitlab NOT_BEFORE_TIME = 5 DEFAULT_EXPIRE_TIME = 60 * 5 + NoSigningKeyError = Class.new(StandardError) + def self.for_build(build) self.new(build, ttl: build.metadata_timeout).encoded end @@ -27,7 +29,7 @@ module Gitlab private - attr_reader :build, :ttl, :key_data + attr_reader :build, :ttl def reserved_claims now = Time.now.to_i @@ -60,7 +62,17 @@ module Gitlab end def key - @key ||= OpenSSL::PKey::RSA.new(Rails.application.secrets.openid_connect_signing_key) + @key ||= begin + key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true) + Gitlab::CurrentSettings.ci_jwt_signing_key + else + Rails.application.secrets.openid_connect_signing_key + end + + raise NoSigningKeyError unless key_data + + OpenSSL::PKey::RSA.new(key_data) + end end def public_key diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 44f2ac23ce3..fb795152abe 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -24,7 +24,7 @@ module Gitlab end def validate(content, dry_run: false) - if dry_run && Gitlab::Ci::Features.lint_creates_pipeline_with_dry_run?(@project) + if dry_run simulate_pipeline_creation(content) else static_validation(content) diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 468f3bc4689..a864c843dd8 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -25,7 +25,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def auto_cancelable_pipelines - project.ci_pipelines + pipelines .where(ref: pipeline.ref) .where.not(id: pipeline.same_family_pipeline_ids) .where.not(sha: project.commit(pipeline.ref).try(:id)) @@ -33,6 +33,14 @@ module Gitlab .with_only_interruptible_builds end # rubocop: enable CodeReuse/ActiveRecord + + def pipelines + if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: false) + project.all_pipelines.ci_and_parent_sources + else + project.ci_pipelines + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index e10a0bc3718..ba86b08d209 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -19,10 +19,12 @@ module Gitlab # Build to prevent erroring out on ambiguous refs. pipeline.protected = @command.protected_ref? - ## - # Populate pipeline with block argument of CreatePipelineService#execute. - # - @command.seeds_block&.call(pipeline) + unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) + end ## # Gather all runtime build/stage errors diff --git a/lib/gitlab/ci/pipeline/chain/seed_block.rb b/lib/gitlab/ci/pipeline/chain/seed_block.rb new file mode 100644 index 00000000000..f8e62949bea --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/seed_block.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class SeedBlock < Chain::Base + include Chain::Helpers + include Gitlab::Utils::StrongMemoize + + def perform! + return unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) + + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) + + raise "Pipeline cannot be persisted by `seeds_block`" if pipeline.persisted? + end + + def break? + return false unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) + + pipeline.errors.any? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb index 42e8c365824..b20dc383419 100644 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ b/lib/gitlab/ci/pipeline/seed/environment.rb @@ -12,12 +12,23 @@ module Gitlab end def to_resource - job.project.environments - .safe_find_or_create_by(name: expanded_environment_name) + environments.safe_find_or_create_by(name: expanded_environment_name) do |environment| + environment.auto_stop_in = auto_stop_in + end end private + def environments + job.project.environments + end + + def auto_stop_in + if Feature.enabled?(:environment_auto_stop_start_on_create) + job.environment_auto_stop_in + end + end + def expanded_environment_name job.expanded_environment_name end diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb index 8c70dbb6931..09121191047 100644 --- a/lib/gitlab/ci/reports/test_case.rb +++ b/lib/gitlab/ci/reports/test_case.rb @@ -10,7 +10,7 @@ module Gitlab STATUS_ERROR = 'error' STATUS_TYPES = [STATUS_ERROR, STATUS_FAILED, STATUS_SUCCESS, STATUS_SKIPPED].freeze - attr_reader :suite_name, :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job + attr_reader :suite_name, :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job, :recent_failures def initialize(params) @suite_name = params.fetch(:suite_name) @@ -24,9 +24,15 @@ module Gitlab @attachment = params.fetch(:attachment, nil) @job = params.fetch(:job, nil) + @recent_failures = nil + @key = hash_key("#{suite_name}_#{classname}_#{name}") end + def set_recent_failures(count, base_branch) + @recent_failures = { count: count, base_branch: base_branch } + end + def has_attachment? attachment.present? end diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb new file mode 100644 index 00000000000..beceac5423a --- /dev/null +++ b/lib/gitlab/ci/reports/test_failure_history.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class TestFailureHistory + include Gitlab::Utils::StrongMemoize + + def initialize(failed_test_cases, project) + @failed_test_cases = build_map(failed_test_cases) + @project = project + end + + def load! + return unless Feature.enabled?(:test_failure_history, project) + + recent_failures_count.each do |key_hash, count| + failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master) + end + end + + private + + attr_reader :report, :project, :failed_test_cases + + def recent_failures_count + ::Ci::TestCaseFailure.recent_failures_count( + project: project, + test_case_keys: failed_test_cases.keys + ) + end + + def build_map(test_cases) + {}.tap do |hash| + test_cases.each do |test_case| + hash[test_case.key] = test_case + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb index a58de43e55e..239fc3b15e7 100644 --- a/lib/gitlab/ci/reports/test_suite_comparer.rb +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -6,6 +6,9 @@ module Gitlab class TestSuiteComparer include Gitlab::Utils::StrongMemoize + DEFAULT_MAX_TESTS = 100 + DEFAULT_MIN_TESTS = 10 + attr_reader :name, :base_suite, :head_suite def initialize(name, base_suite, head_suite) @@ -81,6 +84,29 @@ module Gitlab def error_count new_errors.count + existing_errors.count end + + # This is used to limit the presented test cases but does not affect + # total count of tests in the summary + def limited_tests + strong_memoize(:limited_tests) do + # rubocop: disable CodeReuse/ActiveRecord + OpenStruct.new( + new_failures: new_failures.take(max_tests), + existing_failures: existing_failures.take(max_tests(new_failures)), + resolved_failures: resolved_failures.take(max_tests(new_failures, existing_failures)), + new_errors: new_errors.take(max_tests), + existing_errors: existing_errors.take(max_tests(new_errors)), + resolved_errors: resolved_errors.take(max_tests(new_errors, existing_errors)) + ) + # rubocop: enable CodeReuse/ActiveRecord + end + end + + private + + def max_tests(*used) + [DEFAULT_MAX_TESTS - used.map(&:count).sum, DEFAULT_MIN_TESTS].max + end end end end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index 2171637687f..dd0bfa768a8 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -106,7 +106,7 @@ module Gitlab end def get_file(path) - File.read(path) + File.read(Rails.root.join(path).to_s) end def registration_token diff --git a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml index 82b2f5c035e..453803a6f7e 100644 --- a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml @@ -4,6 +4,7 @@ stages: - review - deploy - production + - cleanup variables: AUTO_DEVOPS_PLATFORM_TARGET: ECS diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index cba13f374f4..a13f2046291 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -160,9 +160,10 @@ include: - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml - - template: Jobs/Code-Intelligence.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml + - template: Jobs/Code-Intelligence.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml - - template: Jobs/Deploy/ECS.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml + - template: Jobs/Deploy/ECS.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml + - template: Jobs/Deploy/EC2.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 0c3598a61a7..1c25d9d583b 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -16,4 +16,14 @@ build: fi - /build/build.sh rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' + when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + +build_artifact: + stage: build + script: + - printf "To build your project, please create a build_artifact job into your .gitlab-ci.yml file.\nMore information at https://docs.gitlab.com/ee/ci/cloud_deployment\n" + - exit 1 + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index ec33020205b..fe23641802b 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10-gitlab.1" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.18" needs: [] script: - export SOURCE_CODE=$PWD @@ -34,6 +34,7 @@ code_quality: CODECLIMATE_DEBUG \ CODECLIMATE_DEV \ REPORT_STDOUT \ + REPORT_FORMAT \ ENGINE_MEMORY_LIMIT_BYTES \ ) \ --volume "$PWD":/code \ diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 33d77e39bc9..c4e194bd658 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -186,7 +186,7 @@ production_manual: when: never - if: '$CI_COMMIT_BRANCH != "master"' when: never - # $INCREMENTAL_ROLLOUT_ENABLED is for compamtibilty with pre-GitLab 11.4 syntax + # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' when: manual diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 8b921305c11..385959389de 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -183,7 +183,7 @@ production_manual: when: never - if: '$CI_COMMIT_BRANCH != "master"' when: never - # $INCREMENTAL_ROLLOUT_ENABLED is for compamtibilty with pre-GitLab 11.4 syntax + # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' when: manual diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index 317e8bfab0e..0289ba1c473 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -8,8 +8,11 @@ # # More about including CI templates: https://docs.gitlab.com/ee/ci/yaml/#includetemplate -.deploy_to_ecs: +.ecs_image: image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + +.deploy_to_ecs: + extends: .ecs_image dependencies: [] script: - ecs update-task-definition @@ -17,8 +20,6 @@ .review_ecs_base: stage: review extends: .deploy_to_ecs - environment: - name: review/$CI_COMMIT_REF_NAME .production_ecs_base: stage: production @@ -26,8 +27,18 @@ environment: name: production +.stop_review_ecs_base: + extends: .ecs_image + stage: cleanup + allow_failure: true + script: + - ecs stop-task + review_ecs: extends: .review_ecs_base + environment: + name: review/$CI_COMMIT_REF_NAME + on_stop: stop_review_ecs rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"' when: never @@ -39,8 +50,46 @@ review_ecs: when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' +stop_review_ecs: + extends: .stop_review_ecs_base + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + when: manual + review_fargate: extends: .review_ecs_base + environment: + name: review/$CI_COMMIT_REF_NAME + on_stop: stop_review_fargate + script: + - ecs update-task-definition + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + +stop_review_fargate: + extends: .stop_review_ecs_base + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"' when: never @@ -51,6 +100,7 @@ review_fargate: - if: '$CI_COMMIT_BRANCH == "master"' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + when: manual production_ecs: extends: .production_ecs_base diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index a9638f564f3..3f62d92ad13 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.33.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.34.1" environment: name: production variables: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index c3a92b67a8b..0ae8fd833c4 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -1,3 +1,9 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ + +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + stages: - build - test @@ -7,7 +13,7 @@ stages: variables: FUZZAPI_PROFILE: Quick FUZZAPI_VERSION: latest - FUZZAPI_CONFIG: "/app/.gitlab-api-fuzzing.yml" + FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml FUZZAPI_TIMEOUT: 30 FUZZAPI_REPORT: gl-api-fuzzing-report.xml # @@ -17,9 +23,70 @@ variables: # available (non 500 response to HTTP(s)) FUZZAPI_SERVICE_START_TIMEOUT: "300" # + FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine + # + +apifuzzer_fuzz_unlicensed: + stage: fuzz + allow_failure: true + rules: + - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' + - when: never + script: + - | + echo "Error: Your GitLab project is not licensed for API Fuzzing." + - exit 1 apifuzzer_fuzz: stage: fuzz + image: + name: $FUZZAPI_IMAGE + entrypoint: ["/bin/bash", "-l", "-c"] + variables: + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://apifuzzer:80 + TZ: America/Los_Angeles + services: + - name: $FUZZAPI_IMAGE + alias: apifuzzer + entrypoint: ["dotnet", "/peach/Peach.Web.dll"] + allow_failure: true + rules: + - if: $FUZZAPI_D_TARGET_IMAGE + when: never + - if: $FUZZAPI_D_WORKER_IMAGE + when: never + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + script: + # + # Validate options + - | + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Start scanning + - worker-entry + # + # Run user provided post-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + artifacts: + reports: + junit: $FUZZAPI_REPORT + +apifuzzer_fuzz_dnd: + stage: fuzz image: docker:19.03.12 variables: DOCKER_DRIVER: overlay2 @@ -28,20 +95,19 @@ apifuzzer_fuzz: FUZZAPI_API: http://apifuzzer:80 allow_failure: true rules: + - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null + when: never - if: $API_FUZZING_DISABLED when: never - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $FUZZAPI_HAR == null && - $FUZZAPI_OPENAPI == null && - $FUZZAPI_D_WORKER_IMAGE == null + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ services: - docker:19.03.12-dind script: # + # - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY # - docker network create --driver bridge $FUZZAPI_D_NETWORK @@ -56,30 +122,13 @@ apifuzzer_fuzz: --network $FUZZAPI_D_NETWORK \ -e Proxy:Port=8000 \ -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://127.0.0.1:80 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ -e GITLAB_FEATURES \ - -v $CI_PROJECT_DIR:/app \ -p 80:80 \ -p 8000:8000 \ -p 514:514 \ --restart=no \ - registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine + $FUZZAPI_IMAGE \ + dotnet /peach/Peach.Web.dll # # Start target container - | @@ -94,19 +143,31 @@ apifuzzer_fuzz: $FUZZAPI_D_TARGET_IMAGE \ ; fi # - # Start worker container + # Start worker container if provided - | if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ - echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE" \ + echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ docker run \ --name worker \ --network $FUZZAPI_D_NETWORK \ -e FUZZAPI_API=http://apifuzzer:80 \ -e FUZZAPI_PROJECT \ -e FUZZAPI_PROFILE \ - -e FUZZAPI_AUTOMATION_CMD \ -e FUZZAPI_CONFIG \ -e FUZZAPI_REPORT \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ $FUZZAPI_D_WORKER_ENV \ $FUZZAPI_D_WORKER_PORTS \ @@ -115,13 +176,49 @@ apifuzzer_fuzz: $FUZZAPI_D_WORKER_IMAGE \ ; fi # - # Wait for testing to complete if api fuzzer is scanning - - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI" != "" ]; then echo "Waiting for API Fuzzer to exit"; docker wait apifuzzer; fi + # Start API Fuzzing provided worker if no other worker present + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi; \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e TZ=America/Los_Angeles \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ + -v $CI_PROJECT_DIR:/app \ + -p 81:80 \ + -p 8001:8000 \ + -p 515:514 \ + --restart=no \ + $FUZZAPI_IMAGE \ + worker-entry \ + ; fi # - # Propagate exit code from api fuzzer (if any) - - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing exited with an error. Logs are available as job artifacts."; docker logs apifuzzer; exit 1; fi + # Propagate exit code from api fuzzing scanner (if any) + - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi # - # Run user provided pre-script + # Run user provided post-script - sh -c "$FUZZAPI_POST_SCRIPT" # after_script: @@ -129,13 +226,13 @@ apifuzzer_fuzz: # Shutdown all containers - echo "Stopping all containers" - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi - - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker stop worker; fi + - docker stop worker - docker stop apifuzzer # # Save docker logs - docker logs apifuzzer &> gl-api_fuzzing-logs.log - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi - - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker logs worker &> gl-api_fuzzing-worker-logs.log; fi + - docker logs worker &> gl-api_fuzzing-worker-logs.log # artifacts: when: always diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 21bcdd8d9b5..3cbde9d30c8 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -4,8 +4,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - - CS_MAJOR_VERSION: 2 + CS_MAJOR_VERSION: 3 container_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index e268b48d133..a1b6dc2cc1b 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -11,6 +11,14 @@ variables: COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" +coverage_fuzzing_unlicensed: + stage: test + allow_failure: true + rules: + - if: $GITLAB_FEATURES !~ /\bcoverage_fuzzing\b/ && $COVFUZZ_DISABLED == null + script: + - echo "ERROR Your GitLab project is missing licensing for Coverage Fuzzing" && exit 1 + .fuzz_base: stage: fuzz allow_failure: true diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 4418ff18d73..a51cb61da6d 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -134,6 +134,7 @@ mobsf-android-sast: name: "$SAST_ANALYZER_IMAGE" variables: SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + MOBSF_API_KEY: key rules: - if: $SAST_DISABLED when: never @@ -152,6 +153,7 @@ mobsf-ios-sast: name: "$SAST_ANALYZER_IMAGE" variables: SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + MOBSF_API_KEY: key rules: - if: $SAST_DISABLED when: never diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index 2d2e0859373..232c320562b 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -131,6 +131,8 @@ secrets: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/ + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" sobelow: extends: .download_images @@ -162,6 +164,8 @@ klar: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bklar\b/ + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" clair-vulnerabilities-db: extends: .download_images diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index b08ccf18b58..5963d7138c5 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -1,11 +1,12 @@ include: - - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml + - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml stages: - init - validate - build - deploy + - cleanup init: extends: .init diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 000a1a7f580..e455bfac9de 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -12,9 +12,6 @@ image: name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest -before_script: - - cd ${TF_ROOT} - variables: TF_ROOT: ${CI_PROJECT_DIR} @@ -26,16 +23,19 @@ cache: .init: &init stage: init script: + - cd ${TF_ROOT} - gitlab-terraform init .validate: &validate stage: validate script: + - cd ${TF_ROOT} - gitlab-terraform validate .build: &build stage: build script: + - cd ${TF_ROOT} - gitlab-terraform plan - gitlab-terraform plan-json artifacts: @@ -47,7 +47,14 @@ cache: .deploy: &deploy stage: deploy script: + - cd ${TF_ROOT} - gitlab-terraform apply when: manual only: - master + +.destroy: &destroy + stage: cleanup + script: + - gitlab-terraform destroy + when: manual diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index e99889f4a25..6f3e4ccf48d 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -75,7 +75,7 @@ module Gitlab until length <= 0 || eof? data = chunk_slice_from_offset - raise FailedToGetChunkError if data.empty? + raise FailedToGetChunkError if data.to_s.empty? chunk_bytes = [CHUNK_SIZE - chunk_offset, length].min chunk_data_slice = data.byteslice(0, chunk_bytes) @@ -100,7 +100,7 @@ module Gitlab until eof? data = chunk_slice_from_offset - raise FailedToGetChunkError if data.empty? + raise FailedToGetChunkError if data.to_s.empty? new_line = data.index("\n") diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 0ca99506311..4d7590a8e38 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -9,6 +9,11 @@ module Gitlab CONTEXT_LINES = 3 + CONFLICT_TYPES = { + "old" => "conflict_marker_their", + "new" => "conflict_marker_our" + }.freeze + attr_reader :merge_request # 'raw' holds the Gitlab::Git::Conflict::File that this instance wraps @@ -46,6 +51,34 @@ module Gitlab end end + def diff_lines_for_serializer + # calculate sections and highlight lines before changing types + sections && highlight_lines! + + sections.flat_map do |section| + if section[:conflict] + lines = [] + + initial_type = nil + section[:lines].each do |line| + if line.type != initial_type + lines << create_separator_line(line) + initial_type = line.type + end + + line.type = CONFLICT_TYPES[line.type] + lines << line + end + + lines << create_separator_line(lines.last) + + lines + else + section[:lines] + end + end + end + def sections return @sections if @sections @@ -93,9 +126,15 @@ module Gitlab lines = tail_lines elsif conflict_before - # We're at the end of the file (no conflicts after), so just remove extra - # trailing lines. + # We're at the end of the file (no conflicts after) + number_of_trailing_lines = lines.size + + # Remove extra trailing lines lines = lines.first(CONTEXT_LINES) + + if number_of_trailing_lines > CONTEXT_LINES + lines << create_match_line(lines.last) + end end end @@ -117,6 +156,10 @@ module Gitlab Gitlab::Diff::Line.new('', 'match', line.index, line.old_pos, line.new_pos) end + def create_separator_line(line) + Gitlab::Diff::Line.new('', 'conflict_marker', line.index, nil, nil) + end + # Any line beginning with a letter, an underscore, or a dollar can be used in a # match line header. Only context sections can contain match lines, as match lines # have to exist in both versions of the file. diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 2b08d3c63bb..d0579a44219 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -16,8 +16,8 @@ module Gitlab @in_memory_application_settings = nil end - def method_missing(name, *args, &block) - current_application_settings.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + def method_missing(name, *args, **kwargs, &block) + current_application_settings.send(name, *args, **kwargs, &block) # rubocop:disable GitlabSecurity/PublicSend end def respond_to_missing?(name, include_private = false) diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb index 7b01db125a9..2e469359bdc 100644 --- a/lib/gitlab/danger/commit_linter.rb +++ b/lib/gitlab/danger/commit_linter.rb @@ -191,7 +191,7 @@ module Gitlab end def subject_starts_with_lowercase? - first_char = subject.sub(/\A\[.+\]\s/, '')[0] + first_char = subject.sub(/\A(\[.+\]|\w+:)\s/, '')[0] first_char_downcased = first_char.downcase return true unless ('a'..'z').cover?(first_char_downcased) diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 783a5f1715c..89f21e8bd23 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -168,7 +168,7 @@ module Gitlab %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity, %r{\A\.codeclimate\.yml\z} => :engineering_productivity, - %r{\A\.overcommit\.yml\.example\z} => :engineering_productivity, + %r{\Alefthook.yml\z} => :engineering_productivity, %r{\A\.editorconfig\z} => :engineering_productivity, %r{Dangerfile\z} => :engineering_productivity, %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity, @@ -190,7 +190,7 @@ module Gitlab %r{\A(ee/)?vendor/} => :backend, %r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend, %r{\A[A-Z_]+_VERSION\z} => :backend, - %r{\A\.rubocop(_todo)?\.yml\z} => :backend, + %r{\A\.rubocop((_manual)?_todo)?\.yml\z} => :backend, %r{\Afile_hooks/} => :backend, %r{\A(ee/)?qa/} => :qa, @@ -200,6 +200,9 @@ module Gitlab %r{\Alocale/gitlab\.pot\z} => :none, %r{\Adata/whats_new/} => :none, + # GraphQL auto generated doc files and schema + %r{\Adoc/api/graphql/reference/} => :backend, + # Fallbacks in case the above patterns miss anything %r{\.rb\z} => :backend, %r{( diff --git a/lib/gitlab/data_builder/feature_flag.rb b/lib/gitlab/data_builder/feature_flag.rb new file mode 100644 index 00000000000..2f675ace7e1 --- /dev/null +++ b/lib/gitlab/data_builder/feature_flag.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module DataBuilder + module FeatureFlag + extend self + + def build(feature_flag, user) + { + object_kind: 'feature_flag', + project: feature_flag.project.hook_attrs, + user: user.hook_attrs, + user_url: Gitlab::UrlBuilder.build(user), + object_attributes: feature_flag.hook_attrs + } + end + end + end +end diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 11d9881aac2..6f79e965cd5 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -128,9 +128,9 @@ module Gitlab end def between_condition(start, finish) - return @column.between(start..(finish - 1)) if @column.is_a?(Arel::Attributes::Attribute) + return @column.between(start...finish) if @column.is_a?(Arel::Attributes::Attribute) - { @column => start..(finish - 1) } + { @column => start...finish } end def actual_start(start) diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index ecc05d9654a..82ea1ce26fb 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -17,23 +17,8 @@ module Gitlab end def current_partitions - result = connection.select_all(<<~SQL) - select - pg_class.relname, - parent_class.relname as base_table, - pg_get_expr(pg_class.relpartbound, inhrelid) as condition - from pg_class - inner join pg_inherits i on pg_class.oid = inhrelid - inner join pg_class parent_class on parent_class.oid = inhparent - inner join pg_namespace ON pg_namespace.oid = pg_class.relnamespace - where pg_namespace.nspname = #{connection.quote(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)} - and parent_class.relname = #{connection.quote(table_name)} - and pg_class.relispartition - order by pg_class.relname - SQL - - result.map do |record| - TimePartition.from_sql(table_name, record['relname'], record['condition']) + Gitlab::Database::PostgresPartition.for_parent_table(table_name).map do |partition| + TimePartition.from_sql(table_name, partition.name, partition.condition) end end diff --git a/lib/gitlab/database/partitioning/replace_table.rb b/lib/gitlab/database/partitioning/replace_table.rb new file mode 100644 index 00000000000..6f6af223fa2 --- /dev/null +++ b/lib/gitlab/database/partitioning/replace_table.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class ReplaceTable + DELIMITER = ";\n\n" + + attr_reader :original_table, :replacement_table, :replaced_table, :primary_key_column, + :sequence, :original_primary_key, :replacement_primary_key, :replaced_primary_key + + def initialize(original_table, replacement_table, replaced_table, primary_key_column) + @original_table = original_table + @replacement_table = replacement_table + @replaced_table = replaced_table + @primary_key_column = primary_key_column + + @sequence = default_sequence(original_table, primary_key_column) + @original_primary_key = default_primary_key(original_table) + @replacement_primary_key = default_primary_key(replacement_table) + @replaced_primary_key = default_primary_key(replaced_table) + end + + def perform + yield sql_to_replace_table if block_given? + + execute(sql_to_replace_table) + end + + private + + delegate :execute, :quote_table_name, :quote_column_name, to: :connection + def connection + @connection ||= ActiveRecord::Base.connection + end + + def default_sequence(table, column) + "#{table}_#{column}_seq" + end + + def default_primary_key(table) + "#{table}_pkey" + end + + def sql_to_replace_table + @sql_to_replace_table ||= combined_sql_statements.map(&:chomp).join(DELIMITER) + end + + def combined_sql_statements + statements = [] + + statements << alter_column_default(original_table, primary_key_column, expression: nil) + statements << alter_column_default(replacement_table, primary_key_column, + expression: "nextval('#{quote_table_name(sequence)}'::regclass)") + + statements << alter_sequence_owned_by(sequence, replacement_table, primary_key_column) + + rename_table_objects(statements, original_table, replaced_table, original_primary_key, replaced_primary_key) + rename_table_objects(statements, replacement_table, original_table, replacement_primary_key, original_primary_key) + + statements + end + + def rename_table_objects(statements, old_table, new_table, old_primary_key, new_primary_key) + statements << rename_table(old_table, new_table) + statements << rename_constraint(new_table, old_primary_key, new_primary_key) + + rename_partitions(statements, old_table, new_table) + end + + def rename_partitions(statements, old_table_name, new_table_name) + Gitlab::Database::PostgresPartition.for_parent_table(old_table_name).each do |partition| + new_partition_name = partition.name.sub(/#{old_table_name}/, new_table_name) + old_primary_key = default_primary_key(partition.name) + new_primary_key = default_primary_key(new_partition_name) + + statements << rename_constraint(partition.identifier, old_primary_key, new_primary_key) + statements << rename_table(partition.identifier, new_partition_name) + end + end + + def alter_column_default(table_name, column_name, expression:) + default_clause = expression.nil? ? 'DROP DEFAULT' : "SET DEFAULT #{expression}" + + <<~SQL + ALTER TABLE #{quote_table_name(table_name)} + ALTER COLUMN #{quote_column_name(column_name)} #{default_clause} + SQL + end + + def alter_sequence_owned_by(sequence_name, table_name, column_name) + <<~SQL + ALTER SEQUENCE #{quote_table_name(sequence_name)} + OWNED BY #{quote_table_name(table_name)}.#{quote_column_name(column_name)} + SQL + end + + def rename_table(old_name, new_name) + <<~SQL + ALTER TABLE #{quote_table_name(old_name)} + RENAME TO #{quote_table_name(new_name)} + SQL + end + + def rename_constraint(table_name, old_name, new_name) + <<~SQL + ALTER TABLE #{quote_table_name(table_name)} + RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} + SQL + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers.rb index 881177a195e..3196dd20356 100644 --- a/lib/gitlab/database/partitioning_migration_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers.rb @@ -5,6 +5,7 @@ module Gitlab module PartitioningMigrationHelpers include ForeignKeyHelpers include TableManagementHelpers + include IndexHelpers end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb new file mode 100644 index 00000000000..f367292f4b0 --- /dev/null +++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PartitioningMigrationHelpers + module IndexHelpers + include Gitlab::Database::MigrationHelpers + include Gitlab::Database::SchemaHelpers + + # Concurrently creates a new index on a partitioned table. In concept this works similarly to + # `add_concurrent_index`, and won't block reads or writes on the table while the index is being built. + # + # A special helper is required for partitioning because Postgres does not support concurrently building indexes + # on partitioned tables. This helper concurrently adds the same index to each partition, and creates the final + # index on the parent table once all of the partitions are indexed. This is the recommended safe way to add + # indexes to partitioned tables. + # + # Example: + # + # add_concurrent_partitioned_index :users, :some_column + # + # See Rails' `add_index` for more info on the available arguments. + def add_concurrent_partitioned_index(table_name, column_names, options = {}) + raise ArgumentError, 'A name is required for indexes added to partitioned tables' unless options[:name] + + partitioned_table = find_partitioned_table(table_name) + + if index_name_exists?(table_name, options[:name]) + Gitlab::AppLogger.warn "Index not created because it already exists (this may be due to an aborted" \ + " migration or similar): table_name: #{table_name}, index_name: #{options[:name]}" + + return + end + + partitioned_table.postgres_partitions.each do |partition| + partition_index_name = generated_index_name(partition.identifier, options[:name]) + partition_options = options.merge(name: partition_index_name) + + add_concurrent_index(partition.identifier, column_names, partition_options) + end + + with_lock_retries do + add_index(table_name, column_names, options) + end + end + + # Safely removes an existing index from a partitioned table. The method name is a bit inaccurate as it does not + # drop the index concurrently, but it's named as such to maintain consistency with other similar helpers, and + # indicate that this should be safe to use in a production environment. + # + # In current versions of Postgres it's impossible to drop an index concurrently, or drop an index from an + # individual partition that exists across the entire partitioned table. As a result this helper drops the index + # from the parent table, which automatically cascades to all partitions. While this does require an exclusive + # lock, dropping an index is a fast operation that won't block the table for a significant period of time. + # + # Example: + # + # remove_concurrent_partitioned_index_by_name :users, 'index_name_goes_here' + def remove_concurrent_partitioned_index_by_name(table_name, index_name) + find_partitioned_table(table_name) + + unless index_name_exists?(table_name, index_name) + Gitlab::AppLogger.warn "Index not removed because it does not exist (this may be due to an aborted " \ + "migration or similar): table_name: #{table_name}, index_name: #{index_name}" + + return + end + + with_lock_retries do + remove_index(table_name, name: index_name) + end + end + + private + + def find_partitioned_table(table_name) + partitioned_table = Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(table_name) + + raise ArgumentError, "#{table_name} is not a partitioned table" unless partitioned_table + + partitioned_table + end + + def generated_index_name(partition_name, index_name) + object_name("#{partition_name}_#{index_name}", 'index') + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index f7b0306b769..686dda80207 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -66,7 +66,10 @@ module Gitlab create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key) create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date) end - create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key) + + with_lock_retries do + create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key) + end end # Clean up a partitioned copy of an existing table. First, deletes the database function and trigger that were @@ -81,13 +84,9 @@ module Gitlab assert_not_in_transaction_block(scope: ERROR_SCOPE) with_lock_retries do - trigger_name = make_sync_trigger_name(table_name) - drop_trigger(table_name, trigger_name) + drop_sync_trigger(table_name) end - function_name = make_sync_function_name(table_name) - drop_function(function_name) - partitioned_table_name = make_partitioned_table_name(table_name) drop_table(partitioned_table_name) end @@ -177,6 +176,53 @@ module Gitlab end end + # Replaces a non-partitioned table with its partitioned copy. This is the final step in a partitioning + # migration, which makes the partitioned table ready for use by the application. The partitioned copy should be + # replaced with the original table in such a way that it appears seamless to any database clients. The replaced + # table will be renamed to "#{replaced_table}_archived". Partitions and primary key constraints will also be + # renamed to match the naming scheme of the parent table. + # + # **NOTE** This method should only be used after all other migration steps have completed successfully. + # There are several limitations to this method that MUST be handled before, or during, the swap migration: + # + # - Secondary indexes and foreign keys are not automatically recreated on the partitioned table. + # - Some types of constraints (UNIQUE and EXCLUDE) which rely on indexes, will not automatically be recreated + # on the partitioned table, since the underlying index will not be present. + # - Foreign keys referencing the original non-partitioned table, would also need to be updated to reference the + # partitioned table, but unfortunately this is not supported in PG11. + # - Views referencing the original table will not be automatically updated to reference the partitioned table. + # + # Example: + # + # replace_with_partitioned_table :audit_events + # + def replace_with_partitioned_table(table_name) + assert_table_is_allowed(table_name) + + partitioned_table_name = make_partitioned_table_name(table_name) + archived_table_name = make_archived_table_name(table_name) + primary_key_name = connection.primary_key(table_name) + + replace_table(table_name, partitioned_table_name, archived_table_name, primary_key_name) + end + + # Rolls back a migration that replaced a non-partitioned table with its partitioned copy. This can be used to + # restore the original non-partitioned table in the event of an unexpected issue. + # + # Example: + # + # rollback_replace_with_partitioned_table :audit_events + # + def rollback_replace_with_partitioned_table(table_name) + assert_table_is_allowed(table_name) + + partitioned_table_name = make_partitioned_table_name(table_name) + archived_table_name = make_archived_table_name(table_name) + primary_key_name = connection.primary_key(archived_table_name) + + replace_table(table_name, archived_table_name, partitioned_table_name, primary_key_name) + end + private def assert_table_is_allowed(table_name) @@ -190,6 +236,10 @@ module Gitlab tmp_table_name("#{table}_part") end + def make_archived_table_name(table) + "#{table}_archived" + end + def make_sync_function_name(table) object_name(table, 'table_sync_function') end @@ -270,12 +320,18 @@ module Gitlab function_name = make_sync_function_name(source_table_name) trigger_name = make_sync_trigger_name(source_table_name) - with_lock_retries do - create_sync_function(function_name, partitioned_table_name, unique_key) - create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table") + create_sync_function(function_name, partitioned_table_name, unique_key) + create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table") - create_sync_trigger(source_table_name, trigger_name, function_name) - end + create_sync_trigger(source_table_name, trigger_name, function_name) + end + + def drop_sync_trigger(source_table_name) + trigger_name = make_sync_trigger_name(source_table_name) + drop_trigger(source_table_name, trigger_name) + + function_name = make_sync_function_name(source_table_name) + drop_function(function_name) end def create_sync_function(name, partitioned_table_name, unique_key) @@ -358,6 +414,21 @@ module Gitlab end end end + + def replace_table(original_table_name, replacement_table_name, replaced_table_name, primary_key_name) + replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(original_table_name.to_s, + replacement_table_name, replaced_table_name, primary_key_name) + + with_lock_retries do + drop_sync_trigger(original_table_name) + + replace_table.perform do |sql| + say("replace_table(\"#{sql}\")") + end + + create_trigger_to_sync_tables(original_table_name, replaced_table_name, primary_key_name) + end + end end end end diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb new file mode 100644 index 00000000000..0986372586b --- /dev/null +++ b/lib/gitlab/database/postgres_partition.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresPartition < ActiveRecord::Base + self.primary_key = :identifier + + belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier' + + scope :by_identifier, ->(identifier) do + raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + + find(identifier) + end + + scope :for_parent_table, ->(name) { where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) } + + def to_s + name + end + end + end +end diff --git a/lib/gitlab/database/postgres_partitioned_table.rb b/lib/gitlab/database/postgres_partitioned_table.rb new file mode 100644 index 00000000000..5d2eaa22ee4 --- /dev/null +++ b/lib/gitlab/database/postgres_partitioned_table.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresPartitionedTable < ActiveRecord::Base + DYNAMIC_PARTITION_STRATEGIES = %w[range list].freeze + + self.primary_key = :identifier + + has_many :postgres_partitions, foreign_key: 'parent_identifier', primary_key: 'identifier' + + scope :by_identifier, ->(identifier) do + raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + + find(identifier) + end + + def self.find_by_name_in_current_schema(name) + find_by("identifier = concat(current_schema(), '.', ?)", name) + end + + def dynamic? + DYNAMIC_PARTITION_STRATEGIES.include?(strategy) + end + + def static? + !dynamic? + end + + def to_s + name + end + end + end +end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 074752fe75b..c77e000254f 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -10,6 +10,7 @@ module Gitlab def self.candidate_indexes Gitlab::Database::PostgresIndex .regular + .where('NOT expression') .not_match("^#{ConcurrentReindex::TEMPORARY_INDEX_PREFIX}") .not_match("^#{ConcurrentReindex::REPLACED_INDEX_PREFIX}") end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index a4e265eba88..d735fb55652 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -6,6 +6,7 @@ module Gitlab URL_REGEX = %r{https?://[^'" ]+}.freeze GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/.freeze REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze + VALID_LINK_ATTRIBUTES = %w[href rel target].freeze include ActionView::Helpers::SanitizeHelper @@ -66,7 +67,7 @@ module Gitlab def link_tag(name, url) sanitize( %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}, - attributes: %w[href rel target] + attributes: VALID_LINK_ATTRIBUTES ) end @@ -77,7 +78,7 @@ module Gitlab # # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"` def link_regex(regex, &url_proc) highlighted_lines.map!.with_index do |rich_line, i| - marker = StringRegexMarker.new(plain_lines[i].chomp, rich_line.html_safe) + marker = StringRegexMarker.new((plain_lines[i].chomp! || plain_lines[i]), rich_line.html_safe) marker.mark(regex, group: :name) do |text, left:, right:| url = yield(text) diff --git a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml index 1d341e6520e..95f15bd6dee 100644 --- a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml +++ b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml @@ -29,6 +29,7 @@ ignore_design_attributes: - id - issue_id - project_id + - iid ignore_version_attributes: - id diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb index c6d1e0b93a7..9af66318b89 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb @@ -18,10 +18,8 @@ module Gitlab def initialize(merge_request_diff, batch_page, batch_size, diff_options:) super(merge_request_diff, diff_options: diff_options) - batch_page ||= DEFAULT_BATCH_PAGE - batch_size ||= DEFAULT_BATCH_SIZE + @paginated_collection = load_paginated_collection(batch_page, batch_size, diff_options) - @paginated_collection = relation.page(batch_page).per(batch_size) @pagination_data = { current_page: @paginated_collection.current_page, next_page: @paginated_collection.next_page, @@ -63,6 +61,18 @@ module Gitlab def relation @merge_request_diff.merge_request_diff_files end + + def load_paginated_collection(batch_page, batch_size, diff_options) + batch_page ||= DEFAULT_BATCH_PAGE + batch_size ||= DEFAULT_BATCH_SIZE + + paths = diff_options&.fetch(:paths, nil) + + paginated_collection = relation.page(batch_page).per(batch_size) + paginated_collection = paginated_collection.by_paths(paths) if paths + + paginated_collection + end end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 379fc6af875..af9140215f0 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -8,9 +8,9 @@ module Gitlab # SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze - attr_reader :line_code, :type, :old_pos, :new_pos + attr_reader :line_code, :old_pos, :new_pos attr_writer :rich_text - attr_accessor :text, :index + attr_accessor :text, :index, :type def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text, @type, @index = text, type, index diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 803acef9a40..a5ace2be773 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -123,6 +123,7 @@ module Gitlab end extra = sanitize_request_parameters(extra) + inject_sql_query_into_extra(exception, extra) if sentry && Raven.configuration.server Raven.capture_exception(exception, tags: default_tags, extra: extra) @@ -149,6 +150,12 @@ module Gitlab filter.filter(parameters) end + def inject_sql_query_into_extra(exception, extra) + return unless exception.is_a?(ActiveRecord::StatementInvalid) + + extra[:sql] = PgQuery.normalize(exception.sql.to_s) + end + def sentry_dsn return unless Rails.env.production? || Rails.env.development? return unless Gitlab.config.sentry.enabled diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 303e1a23e6b..fc3c05c57b2 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -3,6 +3,14 @@ module Gitlab module EtagCaching class Middleware + SKIP_HEADER_KEY = 'X-Gitlab-Skip-Etag' + + class << self + def skip!(response) + response.set_header(SKIP_HEADER_KEY, '1') + end + end + def initialize(app) @app = app end @@ -22,9 +30,7 @@ module Gitlab else track_cache_miss(if_none_match, cached_value_present, route) - status, headers, body = @app.call(env) - headers['ETag'] = etag - [status, headers, body] + maybe_apply_etag(etag, *@app.call(env)) end end @@ -43,6 +49,13 @@ module Gitlab [weak_etag_format(current_value), cached_value_present] end + def maybe_apply_etag(etag, status, headers, body) + headers['ETag'] = etag unless + Gitlab::Utils.to_boolean(headers.delete(SKIP_HEADER_KEY)) + + [status, headers, body] + end + def weak_etag_format(value) %Q{W/"#{value}"} end @@ -54,7 +67,13 @@ module Gitlab add_instrument_for_cache_hit(status_code, route, request) - [status_code, { 'ETag' => etag, 'X-Gitlab-From-Cache' => 'true' }, []] + new_headers = { + 'ETag' => etag, + 'X-Gitlab-From-Cache' => 'true', + ::Gitlab::Metrics::RequestsRackMiddleware::FEATURE_CATEGORY_HEADER => route.feature_category + } + + [status_code, new_headers, []] end def track_cache_miss(if_none_match, cached_value_present, route) diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 17d9cf08367..769ac2784d1 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -3,7 +3,7 @@ module Gitlab module EtagCaching class Router - Route = Struct.new(:regexp, :name) + Route = Struct.new(:regexp, :name, :feature_category) # We enable an ETag for every request matching the regex. # To match a regex the path needs to match the following: # - Don't contain a reserved word (expect for the words used in the @@ -20,59 +20,73 @@ module Gitlab ROUTES = [ Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z), - 'issue_notes' + 'issue_notes', + 'issue_tracking' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z), - 'merge_request_notes' + 'merge_request_notes', + 'code_review' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z), - 'issue_title' + 'issue_title', + 'issue_tracking' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z), - 'commit_pipelines' + 'commit_pipelines', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z), - 'new_merge_request_pipelines' + 'new_merge_request_pipelines', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z), - 'merge_request_pipelines' + 'merge_request_pipelines', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z), - 'project_pipelines' + 'project_pipelines', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z), - 'project_pipeline' + 'project_pipeline', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z), - 'project_build' + 'project_build', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z), - 'cluster_environments' + 'cluster_environments', + 'continuous_delivery' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/environments\.json\z), - 'environments' + 'environments', + 'continuous_delivery' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z), - 'realtime_changes_import_github' + 'realtime_changes_import_github', + 'importers' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), - 'realtime_changes_import_gitea' + 'realtime_changes_import_gitea', + 'importers' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), - 'merge_request_widget' + 'merge_request_widget', + 'code_review' ) ].freeze diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 1ce3ffe4c86..6e39776bbd4 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -6,6 +6,7 @@ # Experiment options: # - environment (optional, defaults to enabled for development and GitLab.com) # - tracking_category (optional, used to set the category when tracking an experiment event) +# - use_backwards_compatible_subject_index (optional, set this to true if you need backwards compatibility) # # The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html), # which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes. @@ -30,168 +31,60 @@ module Gitlab module Experimentation EXPERIMENTS = { - signup_flow: { - tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow' - }, onboarding_issues: { - tracking_category: 'Growth::Conversion::Experiment::OnboardingIssues' - }, - suggest_pipeline: { - tracking_category: 'Growth::Expansion::Experiment::SuggestPipeline' + tracking_category: 'Growth::Conversion::Experiment::OnboardingIssues', + use_backwards_compatible_subject_index: true }, ci_notification_dot: { - tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot' + tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot', + use_backwards_compatible_subject_index: true }, upgrade_link_in_user_menu_a: { - tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA' + tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA', + use_backwards_compatible_subject_index: true }, invite_members_version_a: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA' + tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA', + use_backwards_compatible_subject_index: true }, invite_members_version_b: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB' + tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB', + use_backwards_compatible_subject_index: true + }, + invite_members_empty_group_version_a: { + tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyGroupVersionA', + use_backwards_compatible_subject_index: true }, new_create_project_ui: { - tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi' + tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi', + use_backwards_compatible_subject_index: true }, contact_sales_btn_in_app: { - tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp' + tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp', + use_backwards_compatible_subject_index: true }, customize_homepage: { - tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage' + tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage', + use_backwards_compatible_subject_index: true }, invite_email: { - tracking_category: 'Growth::Acquisition::Experiment::InviteEmail' + tracking_category: 'Growth::Acquisition::Experiment::InviteEmail', + use_backwards_compatible_subject_index: true }, invitation_reminders: { - tracking_category: 'Growth::Acquisition::Experiment::InvitationReminders' + tracking_category: 'Growth::Acquisition::Experiment::InvitationReminders', + use_backwards_compatible_subject_index: true }, group_only_trials: { - tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials' + tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials', + use_backwards_compatible_subject_index: true }, default_to_issues_board: { - tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard' + tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard', + use_backwards_compatible_subject_index: true } }.freeze - GROUP_CONTROL = :control - GROUP_EXPERIMENTAL = :experimental - - # Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent. - # Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method - # to controllers and views. It returns true when the experiment is enabled and the user is selected as part - # of the experimental group. - # - module ControllerConcern - extend ActiveSupport::Concern - - included do - before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled?, :experiment_tracking_category_and_group - end - - def set_experimentation_subject_id_cookie - return if cookies[:experimentation_subject_id].present? - - cookies.permanent.signed[:experimentation_subject_id] = { - value: SecureRandom.uuid, - secure: ::Gitlab.config.gitlab.https, - httponly: true - } - end - - def push_frontend_experiment(experiment_key) - var_name = experiment_key.to_s.camelize(:lower) - enabled = experiment_enabled?(experiment_key) - - gon.push({ experiments: { var_name => enabled } }, true) - end - - def experiment_enabled?(experiment_key) - return false if dnt_enabled? - - return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index) - return true if forced_enabled?(experiment_key) - - false - end - - def track_experiment_event(experiment_key, action, value = nil) - return if dnt_enabled? - - track_experiment_event_for(experiment_key, action, value) do |tracking_data| - ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data) - end - end - - def frontend_experimentation_tracking_data(experiment_key, action, value = nil) - return if dnt_enabled? - - track_experiment_event_for(experiment_key, action, value) do |tracking_data| - gon.push(tracking_data: tracking_data) - end - end - - def record_experiment_user(experiment_key) - return if dnt_enabled? - return unless Experimentation.enabled?(experiment_key) && current_user - - ::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user) - end - - def experiment_tracking_category_and_group(experiment_key) - "#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group')}" - end - - private - - def dnt_enabled? - Gitlab::Utils.to_boolean(request.headers['DNT']) - end - - def experimentation_subject_id - cookies.signed[:experimentation_subject_id] - end - - def experimentation_subject_index - return if experimentation_subject_id.blank? - - experimentation_subject_id.delete('-').hex % 100 - end - - def track_experiment_event_for(experiment_key, action, value) - return unless Experimentation.enabled?(experiment_key) - - yield experimentation_tracking_data(experiment_key, action, value) - end - - def experimentation_tracking_data(experiment_key, action, value) - { - category: tracking_category(experiment_key), - action: action, - property: tracking_group(experiment_key, "_group"), - label: experimentation_subject_id, - value: value - }.compact - end - - def tracking_category(experiment_key) - Experimentation.experiment(experiment_key).tracking_category - end - - def tracking_group(experiment_key, suffix = nil) - return unless Experimentation.enabled?(experiment_key) - - group = experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL - - suffix ? "#{group}#{suffix}" : group - end - - def forced_enabled?(experiment_key) - params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s - end - end - class << self def experiment(key) Experiment.new(EXPERIMENTS[key].merge(key: key)) @@ -201,7 +94,7 @@ module Gitlab return false unless EXPERIMENTS.key?(experiment_key) experiment = experiment(experiment_key) - experiment.enabled? && experiment.enabled_for_environment? + experiment.enabled_for_environment? && experiment.enabled? end def enabled_for_attribute?(experiment_key, attribute) @@ -209,13 +102,18 @@ module Gitlab enabled_for_value?(experiment_key, index) end - def enabled_for_value?(experiment_key, experimentation_subject_index) - enabled?(experiment_key) && - experiment(experiment_key).enabled_for_index?(experimentation_subject_index) + def enabled_for_value?(experiment_key, value) + enabled?(experiment_key) && experiment(experiment_key).enabled_for_index?(value) end end - Experiment = Struct.new(:key, :environment, :tracking_category, keyword_init: true) do + Experiment = Struct.new( + :key, + :environment, + :tracking_category, + :use_backwards_compatible_subject_index, + keyword_init: true + ) do def enabled? experiment_percentage > 0 end diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb new file mode 100644 index 00000000000..c6d15d7d82d --- /dev/null +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +require 'zlib' + +# Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent. +# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method +# to controllers and views. It returns true when the experiment is enabled and the user is selected as part +# of the experimental group. +# +module Gitlab + module Experimentation + module ControllerConcern + include ::Gitlab::Experimentation::GroupTypes + extend ActiveSupport::Concern + + included do + before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? + helper_method :experiment_enabled?, :experiment_tracking_category_and_group + end + + def set_experimentation_subject_id_cookie + return if cookies[:experimentation_subject_id].present? + + cookies.permanent.signed[:experimentation_subject_id] = { + value: SecureRandom.uuid, + secure: ::Gitlab.config.gitlab.https, + httponly: true + } + end + + def push_frontend_experiment(experiment_key) + var_name = experiment_key.to_s.camelize(:lower) + enabled = experiment_enabled?(experiment_key) + + gon.push({ experiments: { var_name => enabled } }, true) + end + + def experiment_enabled?(experiment_key) + return false if dnt_enabled? + + return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index(experiment_key)) + return true if forced_enabled?(experiment_key) + + false + end + + def track_experiment_event(experiment_key, action, value = nil) + return if dnt_enabled? + + track_experiment_event_for(experiment_key, action, value) do |tracking_data| + ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data) + end + end + + def frontend_experimentation_tracking_data(experiment_key, action, value = nil) + return if dnt_enabled? + + track_experiment_event_for(experiment_key, action, value) do |tracking_data| + gon.push(tracking_data: tracking_data) + end + end + + def record_experiment_user(experiment_key) + return if dnt_enabled? + return unless Experimentation.enabled?(experiment_key) && current_user + + ::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user) + end + + def experiment_tracking_category_and_group(experiment_key) + "#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group')}" + end + + private + + def dnt_enabled? + Gitlab::Utils.to_boolean(request.headers['DNT']) + end + + def experimentation_subject_id + cookies.signed[:experimentation_subject_id] + end + + def experimentation_subject_index(experiment_key) + return if experimentation_subject_id.blank? + + if Experimentation.experiment(experiment_key).use_backwards_compatible_subject_index + experimentation_subject_id.delete('-').hex % 100 + else + Zlib.crc32("#{experiment_key}#{experimentation_subject_id}") % 100 + end + end + + def track_experiment_event_for(experiment_key, action, value) + return unless Experimentation.enabled?(experiment_key) + + yield experimentation_tracking_data(experiment_key, action, value) + end + + def experimentation_tracking_data(experiment_key, action, value) + { + category: tracking_category(experiment_key), + action: action, + property: tracking_group(experiment_key, "_group"), + label: experimentation_subject_id, + value: value + }.compact + end + + def tracking_category(experiment_key) + Experimentation.experiment(experiment_key).tracking_category + end + + def tracking_group(experiment_key, suffix = nil) + return unless Experimentation.enabled?(experiment_key) + + group = experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL + + suffix ? "#{group}#{suffix}" : group + end + + def forced_enabled?(experiment_key) + params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s + end + end + end +end diff --git a/lib/gitlab/experimentation/group_types.rb b/lib/gitlab/experimentation/group_types.rb new file mode 100644 index 00000000000..8e8f7284b99 --- /dev/null +++ b/lib/gitlab/experimentation/group_types.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module Experimentation + module GroupTypes + GROUP_CONTROL = :control + GROUP_EXPERIMENTAL = :experimental + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 78c47023c08..209917073c7 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -25,7 +25,7 @@ module Gitlab # # If this value ever changes, make sure to create a migration to update # current records, and default of `ApplicationSettings#diff_max_patch_bytes`. - DEFAULT_MAX_PATCH_BYTES = 100.kilobytes + DEFAULT_MAX_PATCH_BYTES = 200.kilobytes # This is a limitation applied on the source (Gitaly), therefore we don't allow # persisting limits over that. diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 1a3409c1f84..bc712e87e99 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -302,7 +302,7 @@ module Gitlab private :archive_file_path def archive_version_path - return '' unless Feature.enabled?(:include_lfs_blobs_in_archive) + return '' unless Feature.enabled?(:include_lfs_blobs_in_archive, default_enabled: true) '@v2' end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 20ad6d0184b..e41a406ebd3 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -26,8 +26,8 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :cleanup, request, timeout: GitalyClient.fast_timeout) end - def garbage_collect(create_bitmap) - request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) + def garbage_collect(create_bitmap, prune:) + request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap, prune: prune) GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) end diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 9a7c406d981..c3cc15e10f7 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -6,10 +6,13 @@ module Gitlab [:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head'] end - def self.new_client_for(project, token: nil, parallel: true) + def self.new_client_for(project, token: nil, host: nil, parallel: true) token_to_use = token || project.import_data&.credentials&.fetch(:user) - - Client.new(token_to_use, parallel: parallel) + Client.new( + token_to_use, + host: host.presence || self.formatted_import_url(project), + parallel: parallel + ) end # Returns the ID of the ghost user. @@ -18,5 +21,17 @@ module Gitlab Gitlab::Cache::Import::Caching.read_integer(key) || Gitlab::Cache::Import::Caching.write(key, User.select(:id).ghost.id) end + + # Get formatted GitHub import URL. If github.com is in the import URL, this will return nil and octokit will use the default github.com API URL + def self.formatted_import_url(project) + url = URI.parse(project.import_url) + + unless url.host == 'github.com' + url.user = nil + url.password = nil + url.path = "/api/v3" + url.to_s + end + end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 22803c5cd71..dfe60fb5a03 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -18,6 +18,8 @@ module Gitlab attr_reader :octokit + SEARCH_MAX_REQUESTS_PER_MINUTE = 30 + # A single page of data and the corresponding page number. Page = Struct.new(:objects, :number) @@ -28,9 +30,12 @@ module Gitlab # rate limit at once. The threshold is put in place to not hit the limit # in most cases. RATE_LIMIT_THRESHOLD = 50 + SEARCH_RATE_LIMIT_THRESHOLD = 3 # token - The GitHub API token to use. # + # host - The GitHub hostname. If nil, github.com will be used. + # # per_page - The number of objects that should be displayed per page. # # parallel - When set to true hitting the rate limit will result in a @@ -39,11 +44,13 @@ module Gitlab # this value to `true` for parallel importing is crucial as # otherwise hitting the rate limit will result in a thread # being blocked in a `sleep()` call for up to an hour. - def initialize(token, per_page: 100, parallel: true) + def initialize(token, host: nil, per_page: 100, parallel: true) + @host = host @octokit = ::Octokit::Client.new( access_token: token, per_page: per_page, - api_endpoint: api_endpoint + api_endpoint: api_endpoint, + web_endpoint: web_endpoint ) @octokit.connection_options[:ssl] = { verify: verify_ssl } @@ -148,8 +155,26 @@ module Gitlab end end + def search_repos_by_name(name) + each_page(:search_repositories, search_query(str: name, type: :name)) + end + + def search_query(str:, type:, include_collaborations: true, include_orgs: true) + query = "#{str} in:#{type} is:public,private user:#{octokit.user.login}" + + query = [query, collaborations_subquery].join(' ') if include_collaborations + query = [query, organizations_subquery].join(' ') if include_orgs + + query + end + # Returns `true` if we're still allowed to perform API calls. + # Search API has rate limit of 30, use lowered threshold when search is used. def requests_remaining? + if requests_limit == SEARCH_MAX_REQUESTS_PER_MINUTE + return remaining_requests > SEARCH_RATE_LIMIT_THRESHOLD + end + remaining_requests > RATE_LIMIT_THRESHOLD end @@ -157,6 +182,10 @@ module Gitlab octokit.rate_limit.remaining end + def requests_limit + octokit.rate_limit.limit + end + def raise_or_wait_for_rate_limit rate_limit_counter.increment @@ -181,7 +210,11 @@ module Gitlab end def api_endpoint - custom_api_endpoint || default_api_endpoint + @host || custom_api_endpoint || default_api_endpoint + end + + def web_endpoint + @host || custom_api_endpoint || ::Octokit::Default.web_endpoint end def custom_api_endpoint @@ -213,6 +246,20 @@ module Gitlab 'The number of GitHub API calls performed when importing projects' ) end + + private + + def collaborations_subquery + each_object(:repos, nil, { affiliation: 'collaborator' }) + .map { |repo| "repo:#{repo.full_name}" } + .join(' ') + end + + def organizations_subquery + each_object(:organizations) + .map { |org| "org:#{org.login}" } + .join(' ') + end end end end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb index 6a181caf65d..cb6b2017208 100644 --- a/lib/gitlab/github_import/sequential_importer.rb +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -25,10 +25,11 @@ module Gitlab # project - The project to import the data into. # token - The token to use for the GitHub API. - def initialize(project, token: nil) + # host - The GitHub hostname. If nil, github.com will be used. + def initialize(project, token: nil, host: nil) @project = project @client = GithubImport - .new_client_for(project, token: token, parallel: false) + .new_client_for(project, token: token, host: host, parallel: false) end def execute diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 10660649623..2d41ad76618 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -28,6 +28,7 @@ module Gitlab gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites') + gon.select2_css_path = ActionController::Base.helpers.stylesheet_path('lazy_bundles/select2.css') gon.test_env = Rails.env.test? gon.disable_animations = Gitlab.config.gitlab['disable_animations'] gon.suggested_label_colors = LabelsHelper.suggested_colors @@ -58,9 +59,13 @@ module Gitlab # args - Any additional arguments to pass to `Feature.enabled?`. This allows # you to check if a flag is enabled for a particular user. def push_frontend_feature_flag(name, *args, **kwargs) - var_name = name.to_s.camelize(:lower) enabled = Feature.enabled?(name, *args, **kwargs) + push_to_gon_features(name, enabled) + end + + def push_to_gon_features(name, enabled) + var_name = name.to_s.camelize(:lower) # Here the `true` argument signals gon that the value should be merged # into any existing ones, instead of overwriting them. This allows you to # use this method to push multiple feature flags. diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb index ac149cadb5b..a0dccbcdab3 100644 --- a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb +++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb @@ -41,6 +41,8 @@ module Gitlab data.map! { |v| utf8_encode_values(v) } when String encode_utf8(data) + when Integer + data end end end diff --git a/lib/gitlab/grape_logging/loggers/content_logger.rb b/lib/gitlab/grape_logging/loggers/content_logger.rb new file mode 100644 index 00000000000..658953adc80 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/content_logger.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module GrapeLogging + module Loggers + class ContentLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + { + content_length: request.env['CONTENT_LENGTH'], + content_range: request.env['HTTP_CONTENT_RANGE'] + }.compact + end + end + end + end +end diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb index cbf3e7b8429..e8db619f88a 100644 --- a/lib/gitlab/graphql/authorize/authorize_field_service.rb +++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb @@ -46,6 +46,8 @@ module Gitlab # Returns any authorize metadata from @field def field_authorizations + return [] if @field.metadata[:authorize] == true + Array.wrap(@field.metadata[:authorize]) end @@ -54,7 +56,7 @@ module Gitlab # The field is a built-in/scalar type, or a list of scalars # authorize using the parent's object parent_typed_object.object - elsif @field.connection? || resolved_type.is_a?(Array) + elsif @field.connection? || @field.type.list? || resolved_type.is_a?(Array) # The field is a connection or a list of non-built-in types, we'll # authorize each element when rendering nil @@ -75,16 +77,25 @@ module Gitlab # no need to do anything elsif authorizing_object # Authorizing fields representing scalars, or a simple field with an object - resolved_type if allowed_access?(current_user, authorizing_object) + ::Gitlab::Graphql::Lazy.with_value(authorizing_object) do |object| + resolved_type if allowed_access?(current_user, object) + end elsif @field.connection? - # A connection with pagination, modify the visible nodes on the - # connection type in place - resolved_type.object.edge_nodes.to_a.keep_if { |node| allowed_access?(current_user, node) } - resolved_type - elsif resolved_type.is_a? Array + ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |type| + # A connection with pagination, modify the visible nodes on the + # connection type in place + nodes = to_nodes(type) + nodes.keep_if { |node| allowed_access?(current_user, node) } if nodes + type + end + elsif @field.type.list? || resolved_type.is_a?(Array) # A simple list of rendered types each object being an object to authorize - resolved_type.select do |single_object_type| - allowed_access?(current_user, realized(single_object_type).object) + ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |items| + items.select do |single_object_type| + object_type = realized(single_object_type) + object = object_type.try(:object) || object_type + allowed_access?(current_user, object) + end end else raise "Can't authorize #{@field}" @@ -93,18 +104,23 @@ module Gitlab # Ensure that we are dealing with realized objects, not delayed promises def realized(thing) - case thing - when BatchLoader::GraphQL - thing.sync - when GraphQL::Execution::Lazy - thing.value # part of the private api, but we need to unwrap it here. + ::Gitlab::Graphql::Lazy.force(thing) + end + + # Try to get the connection + # can be at type.object or at type + def to_nodes(type) + if type.respond_to?(:nodes) + type.nodes + elsif type.respond_to?(:object) + to_nodes(type.object) else - thing + nil end end def allowed_access?(current_user, object) - object = object.sync if object.respond_to?(:sync) + object = realized(object) authorizations.all? do |ability| Ability.allowed?(current_user, ability, object) diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb index dcd0e12cbfc..503b1064b11 100644 --- a/lib/gitlab/graphql/docs/helper.rb +++ b/lib/gitlab/graphql/docs/helper.rb @@ -81,11 +81,15 @@ module Gitlab # We are ignoring connections and built in types for now, # they should be added when queries are generated. def objects - graphql_object_types.select do |object_type| + object_types = graphql_object_types.select do |object_type| !object_type[:name]["Connection"] && !object_type[:name]["Edge"] && !object_type[:name]["__"] end + + object_types.each do |type| + type[:fields] += type[:connections] + end end # We ignore the built-in enum types. diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index ec052943589..97df4233905 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -14,6 +14,8 @@ CAUTION: **Caution:** Fields that are deprecated are marked with **{warning-solid}**. + Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-process) can be found + in [Removed Items](../removed_items.md). \ :plain diff --git a/lib/gitlab/graphql/lazy.rb b/lib/gitlab/graphql/lazy.rb index a7f7610a041..3cc11047387 100644 --- a/lib/gitlab/graphql/lazy.rb +++ b/lib/gitlab/graphql/lazy.rb @@ -3,17 +3,41 @@ module Gitlab module Graphql class Lazy + include Gitlab::Utils::StrongMemoize + + def initialize(&block) + @proc = block + end + + def force + strong_memoize(:force) { self.class.force(@proc.call) } + end + + def then(&block) + self.class.new { yield force } + end + # Force evaluation of a (possibly) lazy value def self.force(value) case value + when ::Gitlab::Graphql::Lazy + value.force when ::BatchLoader::GraphQL value.sync + when ::GraphQL::Execution::Lazy + value.value # part of the private api, but we can force this as well when ::Concurrent::Promise - value.execute.value + value.execute if value.state == :unscheduled + + value.value # value.value(10.seconds) else value end end + + def self.with_value(unforced, &block) + self.new { unforced }.then(&block) + end end end end diff --git a/lib/gitlab/graphql/loaders/batch_model_loader.rb b/lib/gitlab/graphql/loaders/batch_model_loader.rb index 164fe74148c..9b85ba164d4 100644 --- a/lib/gitlab/graphql/loaders/batch_model_loader.rb +++ b/lib/gitlab/graphql/loaders/batch_model_loader.rb @@ -12,14 +12,11 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def find - BatchLoader::GraphQL.for({ model: model_class, id: model_id.to_i }).batch do |loader_info, loader| - per_model = loader_info.group_by { |info| info[:model] } - per_model.each do |model, info| - ids = info.map { |i| i[:id] } - results = model.where(id: ids) + BatchLoader::GraphQL.for(model_id.to_i).batch(key: model_class) do |ids, loader, args| + model = args[:key] + results = model.where(id: ids) - results.each { |record| loader.call({ model: model, id: record.id }, record) } - end + results.each { |record| loader.call(record.id, record) } end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb index 941a4f434a1..b8535575da5 100644 --- a/lib/gitlab/graphql/present/instrumentation.rb +++ b/lib/gitlab/graphql/present/instrumentation.rb @@ -4,6 +4,8 @@ module Gitlab module Graphql module Present class Instrumentation + SAFE_CONTEXT_KEYS = %i[current_user].freeze + def instrument(type, field) return field unless field.metadata[:type_class] @@ -22,7 +24,8 @@ module Gitlab next old_resolver.call(presented_type, args, context) end - presenter = presented_in.presenter_class.new(object, **context.to_h) + attrs = safe_context_values(context) + presenter = presented_in.presenter_class.new(object, **attrs) # we have to use the new `authorized_new` method, as `new` is protected wrapped = presented_type.class.authorized_new(presenter, context) @@ -34,6 +37,12 @@ module Gitlab resolve(resolve_with_presenter) end end + + private + + def safe_context_values(context) + context.to_h.slice(*SAFE_CONTEXT_KEYS) + end end end end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 5fec50eecd2..dd872caee0e 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -4,10 +4,10 @@ module Gitlab class GroupSearchResults < SearchResults attr_reader :group - def initialize(current_user, query, limit_projects = nil, group:, default_project_filter: false, sort: nil, filters: {}) + def initialize(current_user, query, limit_projects = nil, group:, default_project_filter: false, order_by: nil, sort: nil, filters: {}) @group = group - super(current_user, query, limit_projects, default_project_filter: default_project_filter, sort: sort, filters: filters) + super(current_user, query, limit_projects, default_project_filter: default_project_filter, order_by: order_by, sort: sort, filters: filters) end # rubocop:disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/hook_data/release_builder.rb b/lib/gitlab/hook_data/release_builder.rb new file mode 100644 index 00000000000..b15c260f4a8 --- /dev/null +++ b/lib/gitlab/hook_data/release_builder.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class ReleaseBuilder < BaseBuilder + def self.safe_hook_attributes + %i[ + id + created_at + description + name + released_at + tag + ].freeze + end + + alias_method :release, :object + + def build(action) + attrs = { + object_kind: object_kind, + project: release.project.hook_attrs, + description: absolute_image_urls(release.description), + url: Gitlab::UrlBuilder.build(release), + action: action, + assets: { + count: release.assets_count, + links: release.links.map(&:hook_attrs), + sources: release.sources.map(&:hook_attrs) + }, + commit: release.commit.hook_attrs + } + + release.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) + .merge!(attrs) + end + + private + + def object_kind + release.class.name.underscore + end + end + end +end diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index e56b88dfce0..33054a5b9bf 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -24,7 +24,9 @@ module Gitlab return 'PO-syntax errors' => [parse_error] end - validate_entries + Gitlab::I18n.with_locale(locale) do + validate_entries + end end def parse_po @@ -156,12 +158,10 @@ module Gitlab end def validate_translation(errors, entry) - Gitlab::I18n.with_locale(locale) do - if entry.has_plural? - translate_plural(entry) - else - translate_singular(entry) - end + if entry.has_plural? + translate_plural(entry) + else + translate_singular(entry) end # `sprintf` could raise an `ArgumentError` when invalid passing something @@ -230,9 +230,7 @@ module Gitlab # This calls the C function that defines the pluralization rule, it can # return a boolean (`false` represents 0, `true` represents 1) or an integer # that specifies the plural form to be used for the given number - pluralization_result = Gitlab::I18n.with_locale(locale) do - FastGettext.pluralisation_rule.call(counter) - end + pluralization_result = FastGettext.pluralisation_rule.call(counter) case pluralization_result when false diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 7b8689069d8..8e78f6e274a 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -55,9 +55,17 @@ module Gitlab end def project_tree - @project_tree ||= Gitlab::ImportExport::Project::TreeRestorer.new(user: current_user, - shared: shared, - project: project) + @project_tree ||= project_tree_class.new(user: current_user, + shared: shared, + project: project) + end + + def project_tree_class + sample_data_template? ? Gitlab::ImportExport::Project::Sample::TreeRestorer : Gitlab::ImportExport::Project::TreeRestorer + end + + def sample_data_template? + project&.import_data&.data&.dig('sample_data') end def avatar_restorer diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb index 0d9839b86cf..5c8edd485e5 100644 --- a/lib/gitlab/import_export/json/ndjson_reader.rb +++ b/lib/gitlab/import_export/json/ndjson_reader.rb @@ -29,9 +29,9 @@ module Gitlab json_decode(data) end - def consume_relation(importable_path, key) + def consume_relation(importable_path, key, mark_as_consumed: true) Enumerator.new do |documents| - next unless @consumed_relations.add?("#{importable_path}/#{key}") + next if mark_as_consumed && !@consumed_relations.add?("#{importable_path}/#{key}") # This reads from `tree/project/merge_requests.ndjson` path = file_path(importable_path, "#{key}.ndjson") @@ -44,11 +44,6 @@ module Gitlab end end - # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330) - def clear_consumed_relations - @consumed_relations.clear - end - private def json_decode(string) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index a0526ba0414..ae7ddbc5eba 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -185,6 +185,7 @@ excluded_attributes: - :secret - :encrypted_secret_token - :encrypted_secret_token_iv + - :repository_read_only merge_request_diff: - :external_diff - :stored_externally @@ -410,8 +411,25 @@ ee: - :deploy_access_levels - :service_desk_setting - :security_setting + - :push_rule included_attributes: issuable_sla: - :issue - :due_at + push_rule: + - :force_push_regex + - :delete_branch_regex + - :commit_message_regex + - :author_email_regex + - :file_name_regex + - :branch_name_regex + - :commit_message_negative_regex + - :max_file_size + - :deny_delete_tag + - :member_check + - :is_sample + - :prevent_secrets + - :reject_unsigned_commits + - :commit_committer_check + - :regexp_uses_re2 diff --git a/lib/gitlab/import_export/project/sample/date_calculator.rb b/lib/gitlab/import_export/project/sample/date_calculator.rb index 2d989d21166..543fd25d883 100644 --- a/lib/gitlab/import_export/project/sample/date_calculator.rb +++ b/lib/gitlab/import_export/project/sample/date_calculator.rb @@ -9,7 +9,6 @@ module Gitlab def initialize(dates) @dates = dates.dup - @dates.flatten! @dates.compact! @dates.sort! @dates.map! { |date| date.to_time.to_f } diff --git a/lib/gitlab/import_export/project/sample/relation_factory.rb b/lib/gitlab/import_export/project/sample/relation_factory.rb new file mode 100644 index 00000000000..6e59174f9a3 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/relation_factory.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class RelationFactory < Project::RelationFactory + DATE_MODELS = %i[issues milestones].freeze + + def initialize(date_calculator:, **args) + super(**args) + + @date_calculator = date_calculator + end + + private + + def setup_models + super + + # Override due date attributes in data hash for Sample Data templates + # Dates are moved by taking the closest one to average and moving that (and rest around it) to the date of import + override_date_attributes + end + + def override_date_attributes + return unless DATE_MODELS.include?(@relation_name) + + @relation_hash['start_date'] = calculate_by_closest_date(@relation_hash['start_date']&.to_time) + @relation_hash['due_date'] = calculate_by_closest_date(@relation_hash['due_date']&.to_time) + end + + def calculate_by_closest_date(date) + return unless date + + @date_calculator.calculate_by_closest_date_to_average(date) + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb new file mode 100644 index 00000000000..44ccb67a531 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class RelationTreeRestorer < ImportExport::RelationTreeRestorer + def initialize(*args) + super + + @date_calculator = Gitlab::ImportExport::Project::Sample::DateCalculator.new(dates) + end + + private + + def relation_factory_params(*args) + super.merge(date_calculator: @date_calculator) + end + + def dates + return [] if relation_reader.legacy? + + RelationFactory::DATE_MODELS.flat_map do |tag| + relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model| + model.first['due_date'] + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb deleted file mode 100644 index b0c3940b5f9..00000000000 --- a/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Project - module Sample - class SampleDataRelationTreeRestorer < RelationTreeRestorer - DATE_MODELS = %i[issues milestones].freeze - - def initialize(*args) - super - - date_calculator - end - - private - - def build_relation(relation_key, relation_definition, data_hash) - # Override due date attributes in data hash for Sample Data templates - # Dates are moved by taking the closest one to average and moving that (and rest around it) to the date of import - # TODO: To move this logic to RelationFactory (see: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465333) - override_date_attributes!(relation_key, data_hash) - super - end - - def override_date_attributes!(relation_key, data_hash) - return unless DATE_MODELS.include?(relation_key.to_sym) - - data_hash['start_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['start_date'].to_time) unless data_hash['start_date'].nil? - data_hash['due_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['due_date'].to_time) unless data_hash['due_date'].nil? - end - - # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330) - def dates - unless relation_reader.legacy? - DATE_MODELS.map do |tag| - relation_reader.consume_relation(@importable_path, tag).map { |model| model.first['due_date'] }.tap do - relation_reader.clear_consumed_relations - end - end - end - end - - def date_calculator - @date_calculator ||= Gitlab::ImportExport::Project::Sample::DateCalculator.new(dates) - end - end - end - end - end -end diff --git a/lib/gitlab/import_export/project/sample/tree_restorer.rb b/lib/gitlab/import_export/project/sample/tree_restorer.rb new file mode 100644 index 00000000000..1d4b5328cb9 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/tree_restorer.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class TreeRestorer < Project::TreeRestorer + def relation_tree_restorer_class + RelationTreeRestorer + end + + def relation_factory + RelationFactory + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index b1d647281ab..fb9e5be1877 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -85,11 +85,7 @@ module Gitlab end def relation_tree_restorer_class - sample_data_template? ? Sample::SampleDataRelationTreeRestorer : RelationTreeRestorer - end - - def sample_data_template? - @project&.import_data&.data&.dig('sample_data') + RelationTreeRestorer end def members_mapper diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 26e7d2cf765..428bcbe8dc5 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -86,6 +86,10 @@ module Gitlab mkdir_p(File.join(uploads_export_path, secret)) download_or_copy_upload(upload, upload_path) + rescue Errno::ENAMETOOLONG => e + # Do not fail entire project export if downloaded file has filename that exceeds 255 characters. + # Ignore raised exception, skip such upload, log the error and keep going with the export instead. + Gitlab::ErrorTracking.log_exception(e, project_id: @project.id) end end end diff --git a/lib/gitlab/instrumentation/throttle.rb b/lib/gitlab/instrumentation/throttle.rb new file mode 100644 index 00000000000..0b7e990fb2e --- /dev/null +++ b/lib/gitlab/instrumentation/throttle.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + class Throttle + KEY = :instrumentation_throttle_safelist + + def self.safelist + Gitlab::SafeRequestStore[KEY] + end + + def self.safelist=(name) + Gitlab::SafeRequestStore[KEY] = name + end + end + end +end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 3a29d2e7efa..d7228099eaf 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -21,6 +21,7 @@ module Gitlab instrument_rugged(payload) instrument_redis(payload) instrument_elasticsearch(payload) + instrument_throttle(payload) end def instrument_gitaly(payload) @@ -56,6 +57,11 @@ module Gitlab payload[:elasticsearch_duration_s] = Gitlab::Instrumentation::ElasticsearchTransport.query_time end + def instrument_throttle(payload) + safelist = Gitlab::Instrumentation::Throttle.safelist + payload[:throttle_safelist] = safelist if safelist.present? + end + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the # `enqueued_at` field or `created_at` field is available. # diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 29cfec443e8..8565f664cd4 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -67,15 +67,6 @@ module Gitlab ::JSON.pretty_generate(object, opts) end - # Feature detection for using Oj instead of the `json` gem. - # - # @return [Boolean] - def enable_oj? - return false unless feature_table_exists? - - Feature.enabled?(:oj_json, default_enabled: true) - end - private # Convert JSON string into Ruby through toggleable adapters. @@ -91,11 +82,7 @@ module Gitlab def adapter_load(string, *args, **opts) opts = standardize_opts(opts) - if enable_oj? - Oj.load(string, opts) - else - ::JSON.parse(string, opts) - end + Oj.load(string, opts) rescue Oj::ParseError, Encoding::UndefinedConversionError => ex raise parser_error.new(ex) end @@ -120,11 +107,7 @@ module Gitlab # # @return [String] def adapter_dump(object, *args, **opts) - if enable_oj? - Oj.dump(object, opts) - else - ::JSON.dump(object, *args) - end + Oj.dump(object, opts) end # Generates JSON for an object but with fewer options, using toggleable adapters. @@ -135,11 +118,7 @@ module Gitlab def adapter_generate(object, opts = {}) opts = standardize_opts(opts) - if enable_oj? - Oj.generate(object, opts) - else - ::JSON.generate(object, opts) - end + Oj.generate(object, opts) end # Take a JSON standard options hash and standardize it to work across adapters @@ -149,11 +128,8 @@ module Gitlab # @return [Hash] def standardize_opts(opts) opts ||= {} - - if enable_oj? - opts[:mode] = :rails - opts[:symbol_keys] = opts[:symbolize_keys] || opts[:symbolize_names] - end + opts[:mode] = :rails + opts[:symbol_keys] = opts[:symbolize_keys] || opts[:symbolize_names] opts end @@ -213,7 +189,7 @@ module Gitlab # @param object [Object] # @return [String] def self.call(object, env = nil) - if Gitlab::Json.enable_oj? && Feature.enabled?(:grape_gitlab_json, default_enabled: true) + if Feature.enabled?(:grape_gitlab_json, default_enabled: true) Gitlab::Json.dump(object) else Grape::Formatter::Json.call(object, env) diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb deleted file mode 100644 index 49d2969f7f3..00000000000 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class BaseCommand - attr_reader :name, :files - - def initialize(rbac:, name:, files:) - @rbac = rbac - @name = name - @files = files - end - - def rbac? - @rbac - end - - def pod_resource - pod_service_account_name = rbac? ? service_account_name : nil - - Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate - end - - def generate_script - <<~HEREDOC - set -xeo pipefail - HEREDOC - end - - def pod_name - "install-#{name}" - end - - def config_map_resource - Gitlab::Kubernetes::ConfigMap.new(name, files).generate - end - - def service_account_resource - return unless rbac? - - Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate - end - - def cluster_role_binding_resource - return unless rbac? - - subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] - - Gitlab::Kubernetes::ClusterRoleBinding.new( - cluster_role_binding_name, - cluster_role_name, - subjects - ).generate - end - - def file_names - files.keys - end - - private - - def files_dir - "/data/helm/#{name}/config" - end - - def namespace - Gitlab::Kubernetes::Helm::NAMESPACE - end - - def service_account_name - Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT - end - - def cluster_role_binding_name - Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING - end - - def cluster_role_name - Gitlab::Kubernetes::Helm::CLUSTER_ROLE - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/certificate.rb b/lib/gitlab/kubernetes/helm/certificate.rb deleted file mode 100644 index 598714e0874..00000000000 --- a/lib/gitlab/kubernetes/helm/certificate.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module Kubernetes - module Helm - class Certificate - INFINITE_EXPIRY = 1000.years - SHORT_EXPIRY = 30.minutes - - attr_reader :key, :cert - - def key_string - @key.to_s - end - - def cert_string - @cert.to_pem - end - - def self.from_strings(key_string, cert_string) - key = OpenSSL::PKey::RSA.new(key_string) - cert = OpenSSL::X509::Certificate.new(cert_string) - new(key, cert) - end - - def self.generate_root - _issue(signed_by: nil, expires_in: INFINITE_EXPIRY, certificate_authority: true) - end - - def issue(expires_in: SHORT_EXPIRY) - self.class._issue(signed_by: self, expires_in: expires_in, certificate_authority: false) - end - - private - - def self._issue(signed_by:, expires_in:, certificate_authority:) - key = OpenSSL::PKey::RSA.new(4096) - public_key = key.public_key - - subject = OpenSSL::X509::Name.parse("/C=US") - - cert = OpenSSL::X509::Certificate.new - cert.subject = subject - - cert.issuer = signed_by&.cert&.subject || subject - - cert.not_before = Time.now - cert.not_after = expires_in.from_now - cert.public_key = public_key - cert.serial = 0x0 - cert.version = 2 - - if certificate_authority - extension_factory = OpenSSL::X509::ExtensionFactory.new - extension_factory.subject_certificate = cert - extension_factory.issuer_certificate = cert - cert.add_extension(extension_factory.create_extension('subjectKeyIdentifier', 'hash')) - cert.add_extension(extension_factory.create_extension('basicConstraints', 'CA:TRUE', true)) - cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true)) - end - - cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new) - - new(key, cert) - end - - def initialize(key, cert) - @key = key - @cert = cert - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb deleted file mode 100644 index a9e93c0c90e..00000000000 --- a/lib/gitlab/kubernetes/helm/client_command.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module ClientCommand - def init_command - <<~SHELL.chomp - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - SHELL - end - - def repository_command - ['helm', 'repo', 'add', name, repository].shelljoin if repository - end - - private - - def repository_update_command - 'helm repo update' - end - - def optional_tls_flags - return [] unless files.key?(:'ca.pem') - - [ - '--tls', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tls-cert', "#{files_dir}/cert.pem", - '--tls-key', "#{files_dir}/key.pem" - ] - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb deleted file mode 100644 index f8b9601bc98..00000000000 --- a/lib/gitlab/kubernetes/helm/delete_command.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class DeleteCommand < BaseCommand - include ClientCommand - - attr_reader :predelete, :postdelete - - def initialize(predelete: nil, postdelete: nil, **args) - super(**args) - @predelete = predelete - @postdelete = postdelete - end - - def generate_script - super + [ - init_command, - predelete, - delete_command, - postdelete - ].compact.join("\n") - end - - def pod_name - "uninstall-#{name}" - end - - def delete_command - ['helm', 'delete', '--purge', name].shelljoin - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb deleted file mode 100644 index e4844e255c5..00000000000 --- a/lib/gitlab/kubernetes/helm/init_command.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class InitCommand < BaseCommand - def generate_script - super + [ - init_helm_command - ].join("\n") - end - - private - - def init_helm_command - command = %w[helm init] + init_command_flags - - command.shelljoin - end - - def init_command_flags - tls_flags + optional_service_account_flag - end - - def tls_flags - [ - '--tiller-tls', - '--tiller-tls-verify', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tiller-tls-cert', "#{files_dir}/cert.pem", - '--tiller-tls-key', "#{files_dir}/key.pem" - ] - end - - def optional_service_account_flag - return [] unless rbac? - - ['--service-account', service_account_name] - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb deleted file mode 100644 index d166842fce6..00000000000 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class InstallCommand < BaseCommand - include ClientCommand - - attr_reader :chart, :repository, :preinstall, :postinstall - attr_accessor :version - - def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args) - super(**args) - @chart = chart - @version = version - @repository = repository - @preinstall = preinstall - @postinstall = postinstall - end - - def generate_script - super + [ - init_command, - repository_command, - repository_update_command, - preinstall, - install_command, - postinstall - ].compact.join("\n") - end - - private - - # Uses `helm upgrade --install` which means we can use this for both - # installation and uprade of applications - def install_command - command = ['helm', 'upgrade', name, chart] + - install_flag + - rollback_support_flag + - reset_values_flag + - optional_version_flag + - rbac_create_flag + - namespace_flag + - value_flag - - command.shelljoin - end - - def install_flag - ['--install'] - end - - def reset_values_flag - ['--reset-values'] - end - - def value_flag - ['-f', "/data/helm/#{name}/config/values.yaml"] - end - - def namespace_flag - ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] - end - - def rbac_create_flag - if rbac? - %w[--set rbac.create=true,rbac.enabled=true] - else - %w[--set rbac.create=false,rbac.enabled=false] - end - end - - def optional_version_flag - return [] unless version - - ['--version', version] - end - - def rollback_support_flag - ['--atomic', '--cleanup-on-fail'] - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/patch_command.rb b/lib/gitlab/kubernetes/helm/patch_command.rb deleted file mode 100644 index a33dbdac134..00000000000 --- a/lib/gitlab/kubernetes/helm/patch_command.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -# PatchCommand is for updating values in installed charts without overwriting -# existing values. -module Gitlab - module Kubernetes - module Helm - class PatchCommand < BaseCommand - include ClientCommand - - attr_reader :chart, :repository - attr_accessor :version - - def initialize(chart:, version:, repository: nil, **args) - super(**args) - - # version is mandatory to prevent chart mismatches - # we do not want our values interpreted in the context of the wrong version - raise ArgumentError, 'version is required' if version.blank? - - @chart = chart - @version = version - @repository = repository - end - - def generate_script - super + [ - init_command, - repository_command, - repository_update_command, - upgrade_command - ].compact.join("\n") - end - - private - - def upgrade_command - command = ['helm', 'upgrade', name, chart] + - reuse_values_flag + - version_flag + - namespace_flag + - value_flag - - command.shelljoin - end - - def reuse_values_flag - ['--reuse-values'] - end - - def value_flag - ['-f', "/data/helm/#{name}/config/values.yaml"] - end - - def namespace_flag - ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] - end - - def version_flag - ['--version', version] - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index 75484f80070..9d0207e6b1f 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -27,7 +27,7 @@ module Gitlab def container_specification { name: 'helm', - image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{Gitlab::Kubernetes::Helm::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}", + image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{command.class::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}-alpine-3.12", env: generate_pod_env(command), command: %w(/bin/sh), args: %w(-c $(COMMAND_SCRIPT)) @@ -50,11 +50,10 @@ module Gitlab end def generate_pod_env(command) - { - HELM_VERSION: Gitlab::Kubernetes::Helm::HELM_VERSION, - TILLER_NAMESPACE: namespace_name, + command.env.merge( + HELM_VERSION: command.class::HELM_VERSION, COMMAND_SCRIPT: command.generate_script - }.map { |key, value| { name: key, value: value } } + ).map { |key, value| { name: key, value: value } } end def volumes_specification diff --git a/lib/gitlab/kubernetes/helm/reset_command.rb b/lib/gitlab/kubernetes/helm/reset_command.rb deleted file mode 100644 index f1f7938039c..00000000000 --- a/lib/gitlab/kubernetes/helm/reset_command.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class ResetCommand < BaseCommand - include ClientCommand - - def generate_script - super + [ - reset_helm_command, - delete_tiller_replicaset, - delete_tiller_clusterrolebinding - ].join("\n") - end - - def pod_name - "uninstall-#{name}" - end - - private - - # This method can be delete once we upgrade Helm to > 12.13.0 - # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/27096#note_159695900 - # - # Tracking this method to be removed here: - # https://gitlab.com/gitlab-org/gitlab-foss/issues/52791#note_199374155 - def delete_tiller_replicaset - delete_args = %w[replicaset -n gitlab-managed-apps -l name=tiller] - - Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) - end - - def delete_tiller_clusterrolebinding - delete_args = %w[clusterrolebinding tiller-admin] - - Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) - end - - def reset_helm_command - command = %w[helm reset] + optional_tls_flags - - command.shelljoin - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v2/base_command.rb b/lib/gitlab/kubernetes/helm/v2/base_command.rb new file mode 100644 index 00000000000..931c2248310 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/base_command.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + class BaseCommand + attr_reader :name, :files + + HELM_VERSION = '2.16.9' + + def initialize(rbac:, name:, files:) + @rbac = rbac + @name = name + @files = files + end + + def env + { TILLER_NAMESPACE: namespace } + end + + def rbac? + @rbac + end + + def pod_resource + pod_service_account_name = rbac? ? service_account_name : nil + + Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate + end + + def generate_script + <<~HEREDOC + set -xeo pipefail + HEREDOC + end + + def pod_name + "install-#{name}" + end + + def config_map_resource + Gitlab::Kubernetes::ConfigMap.new(name, files).generate + end + + def service_account_resource + return unless rbac? + + Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate + end + + def cluster_role_binding_resource + return unless rbac? + + subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] + + Gitlab::Kubernetes::ClusterRoleBinding.new( + cluster_role_binding_name, + cluster_role_name, + subjects + ).generate + end + + def file_names + files.keys + end + + private + + def files_dir + "/data/helm/#{name}/config" + end + + def namespace + Gitlab::Kubernetes::Helm::NAMESPACE + end + + def service_account_name + Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT + end + + def cluster_role_binding_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING + end + + def cluster_role_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/certificate.rb b/lib/gitlab/kubernetes/helm/v2/certificate.rb new file mode 100644 index 00000000000..f603ff44ef3 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/certificate.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true +module Gitlab + module Kubernetes + module Helm + module V2 + class Certificate + INFINITE_EXPIRY = 1000.years + SHORT_EXPIRY = 30.minutes + + attr_reader :key, :cert + + def key_string + @key.to_s + end + + def cert_string + @cert.to_pem + end + + def self.from_strings(key_string, cert_string) + key = OpenSSL::PKey::RSA.new(key_string) + cert = OpenSSL::X509::Certificate.new(cert_string) + new(key, cert) + end + + def self.generate_root + _issue(signed_by: nil, expires_in: INFINITE_EXPIRY, certificate_authority: true) + end + + def issue(expires_in: SHORT_EXPIRY) + self.class._issue(signed_by: self, expires_in: expires_in, certificate_authority: false) + end + + private + + def self._issue(signed_by:, expires_in:, certificate_authority:) + key = OpenSSL::PKey::RSA.new(4096) + public_key = key.public_key + + subject = OpenSSL::X509::Name.parse("/C=US") + + cert = OpenSSL::X509::Certificate.new + cert.subject = subject + + cert.issuer = signed_by&.cert&.subject || subject + + cert.not_before = Time.now.utc + cert.not_after = expires_in.from_now.utc + cert.public_key = public_key + cert.serial = 0x0 + cert.version = 2 + + if certificate_authority + extension_factory = OpenSSL::X509::ExtensionFactory.new + extension_factory.subject_certificate = cert + extension_factory.issuer_certificate = cert + cert.add_extension(extension_factory.create_extension('subjectKeyIdentifier', 'hash')) + cert.add_extension(extension_factory.create_extension('basicConstraints', 'CA:TRUE', true)) + cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true)) + end + + cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new) + + new(key, cert) + end + + def initialize(key, cert) + @key = key + @cert = cert + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/client_command.rb b/lib/gitlab/kubernetes/helm/v2/client_command.rb new file mode 100644 index 00000000000..88693a28d6c --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/client_command.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + module ClientCommand + def init_command + <<~SHELL.chomp + export HELM_HOST="localhost:44134" + tiller -listen ${HELM_HOST} -alsologtostderr & + helm init --client-only + SHELL + end + + def repository_command + ['helm', 'repo', 'add', name, repository].shelljoin if repository + end + + private + + def repository_update_command + 'helm repo update' + end + + def optional_tls_flags + return [] unless files.key?(:'ca.pem') + + [ + '--tls', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tls-cert', "#{files_dir}/cert.pem", + '--tls-key', "#{files_dir}/key.pem" + ] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/delete_command.rb b/lib/gitlab/kubernetes/helm/v2/delete_command.rb new file mode 100644 index 00000000000..4d52fc1398f --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/delete_command.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + class DeleteCommand < BaseCommand + include ClientCommand + + attr_reader :predelete, :postdelete + + def initialize(predelete: nil, postdelete: nil, **args) + super(**args) + @predelete = predelete + @postdelete = postdelete + end + + def generate_script + super + [ + init_command, + predelete, + delete_command, + postdelete + ].compact.join("\n") + end + + def pod_name + "uninstall-#{name}" + end + + def delete_command + ['helm', 'delete', '--purge', name].shelljoin + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/init_command.rb b/lib/gitlab/kubernetes/helm/v2/init_command.rb new file mode 100644 index 00000000000..f8b52feb5b6 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/init_command.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + class InitCommand < BaseCommand + def generate_script + super + [ + init_helm_command + ].join("\n") + end + + private + + def init_helm_command + command = %w[helm init] + init_command_flags + + command.shelljoin + end + + def init_command_flags + tls_flags + optional_service_account_flag + end + + def tls_flags + [ + '--tiller-tls', + '--tiller-tls-verify', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tiller-tls-cert', "#{files_dir}/cert.pem", + '--tiller-tls-key', "#{files_dir}/key.pem" + ] + end + + def optional_service_account_flag + return [] unless rbac? + + ['--service-account', service_account_name] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/install_command.rb b/lib/gitlab/kubernetes/helm/v2/install_command.rb new file mode 100644 index 00000000000..10e16723e45 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/install_command.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + class InstallCommand < BaseCommand + include ClientCommand + + attr_reader :chart, :repository, :preinstall, :postinstall + attr_accessor :version + + def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args) + super(**args) + @chart = chart + @version = version + @repository = repository + @preinstall = preinstall + @postinstall = postinstall + end + + def generate_script + super + [ + init_command, + repository_command, + repository_update_command, + preinstall, + install_command, + postinstall + ].compact.join("\n") + end + + private + + # Uses `helm upgrade --install` which means we can use this for both + # installation and uprade of applications + def install_command + command = ['helm', 'upgrade', name, chart] + + install_flag + + rollback_support_flag + + reset_values_flag + + optional_version_flag + + rbac_create_flag + + namespace_flag + + value_flag + + command.shelljoin + end + + def install_flag + ['--install'] + end + + def reset_values_flag + ['--reset-values'] + end + + def value_flag + ['-f', "/data/helm/#{name}/config/values.yaml"] + end + + def namespace_flag + ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] + end + + def rbac_create_flag + if rbac? + %w[--set rbac.create=true,rbac.enabled=true] + else + %w[--set rbac.create=false,rbac.enabled=false] + end + end + + def optional_version_flag + return [] unless version + + ['--version', version] + end + + def rollback_support_flag + ['--atomic', '--cleanup-on-fail'] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/patch_command.rb b/lib/gitlab/kubernetes/helm/v2/patch_command.rb new file mode 100644 index 00000000000..2855e6444b1 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/patch_command.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# PatchCommand is for updating values in installed charts without overwriting +# existing values. +module Gitlab + module Kubernetes + module Helm + module V2 + class PatchCommand < BaseCommand + include ClientCommand + + attr_reader :chart, :repository + attr_accessor :version + + def initialize(chart:, version:, repository: nil, **args) + super(**args) + + # version is mandatory to prevent chart mismatches + # we do not want our values interpreted in the context of the wrong version + raise ArgumentError, 'version is required' if version.blank? + + @chart = chart + @version = version + @repository = repository + end + + def generate_script + super + [ + init_command, + repository_command, + repository_update_command, + upgrade_command + ].compact.join("\n") + end + + private + + def upgrade_command + command = ['helm', 'upgrade', name, chart] + + reuse_values_flag + + version_flag + + namespace_flag + + value_flag + + command.shelljoin + end + + def reuse_values_flag + ['--reuse-values'] + end + + def value_flag + ['-f', "/data/helm/#{name}/config/values.yaml"] + end + + def namespace_flag + ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] + end + + def version_flag + ['--version', version] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/reset_command.rb b/lib/gitlab/kubernetes/helm/v2/reset_command.rb new file mode 100644 index 00000000000..172a0884c49 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/reset_command.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + class ResetCommand < BaseCommand + include ClientCommand + + def generate_script + super + [ + reset_helm_command, + delete_tiller_replicaset, + delete_tiller_clusterrolebinding + ].join("\n") + end + + def pod_name + "uninstall-#{name}" + end + + private + + # This method can be delete once we upgrade Helm to > 12.13.0 + # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/27096#note_159695900 + # + # Tracking this method to be removed here: + # https://gitlab.com/gitlab-org/gitlab-foss/issues/52791#note_199374155 + def delete_tiller_replicaset + delete_args = %w[replicaset -n gitlab-managed-apps -l name=tiller] + + Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) + end + + def delete_tiller_clusterrolebinding + delete_args = %w[clusterrolebinding tiller-admin] + + Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) + end + + def reset_helm_command + command = %w[helm reset] + optional_tls_flags + + command.shelljoin + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v3/base_command.rb b/lib/gitlab/kubernetes/helm/v3/base_command.rb new file mode 100644 index 00000000000..ca1bf5462f0 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v3/base_command.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V3 + class BaseCommand + attr_reader :name, :files + + HELM_VERSION = '3.2.4' + + def initialize(rbac:, name:, files:) + @rbac = rbac + @name = name + @files = files + end + + def env + {} + end + + def rbac? + @rbac + end + + def pod_resource + pod_service_account_name = rbac? ? service_account_name : nil + + Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate + end + + def generate_script + <<~HEREDOC + set -xeo pipefail + HEREDOC + end + + def pod_name + "install-#{name}" + end + + def config_map_resource + Gitlab::Kubernetes::ConfigMap.new(name, files).generate + end + + def service_account_resource + return unless rbac? + + Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate + end + + def cluster_role_binding_resource + return unless rbac? + + subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] + + Gitlab::Kubernetes::ClusterRoleBinding.new( + cluster_role_binding_name, + cluster_role_name, + subjects + ).generate + end + + def file_names + files.keys + end + + def repository_command + ['helm', 'repo', 'add', name, repository].shelljoin if repository + end + + private + + def repository_update_command + 'helm repo update' + end + + def namespace_flag + ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] + end + + def namespace + Gitlab::Kubernetes::Helm::NAMESPACE + end + + def service_account_name + Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT + end + + def cluster_role_binding_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING + end + + def cluster_role_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v3/delete_command.rb b/lib/gitlab/kubernetes/helm/v3/delete_command.rb new file mode 100644 index 00000000000..f628e852f54 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v3/delete_command.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V3 + class DeleteCommand < BaseCommand + attr_reader :predelete, :postdelete + + def initialize(predelete: nil, postdelete: nil, **args) + super(**args) + @predelete = predelete + @postdelete = postdelete + end + + def generate_script + super + [ + predelete, + delete_command, + postdelete + ].compact.join("\n") + end + + def pod_name + "uninstall-#{name}" + end + + def delete_command + ['helm', 'uninstall', name, *namespace_flag].shelljoin + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v3/install_command.rb b/lib/gitlab/kubernetes/helm/v3/install_command.rb new file mode 100644 index 00000000000..20d17f49115 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v3/install_command.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V3 + class InstallCommand < BaseCommand + attr_reader :chart, :repository, :preinstall, :postinstall + attr_accessor :version + + def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args) + super(**args) + @chart = chart + @version = version + @repository = repository + @preinstall = preinstall + @postinstall = postinstall + end + + def generate_script + super + [ + repository_command, + repository_update_command, + preinstall, + install_command, + postinstall + ].compact.join("\n") + end + + private + + # Uses `helm upgrade --install` which means we can use this for both + # installation and uprade of applications + def install_command + command = ['helm', 'upgrade', name, chart] + + install_flag + + rollback_support_flag + + reset_values_flag + + optional_version_flag + + rbac_create_flag + + namespace_flag + + value_flag + + command.shelljoin + end + + def install_flag + ['--install'] + end + + def reset_values_flag + ['--reset-values'] + end + + def value_flag + ['-f', "/data/helm/#{name}/config/values.yaml"] + end + + def rbac_create_flag + if rbac? + %w[--set rbac.create=true,rbac.enabled=true] + else + %w[--set rbac.create=false,rbac.enabled=false] + end + end + + def optional_version_flag + return [] unless version + + ['--version', version] + end + + def rollback_support_flag + ['--atomic', '--cleanup-on-fail'] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v3/patch_command.rb b/lib/gitlab/kubernetes/helm/v3/patch_command.rb new file mode 100644 index 00000000000..00f340591e7 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v3/patch_command.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# PatchCommand is for updating values in installed charts without overwriting +# existing values. +module Gitlab + module Kubernetes + module Helm + module V3 + class PatchCommand < BaseCommand + attr_reader :chart, :repository + attr_accessor :version + + def initialize(chart:, version:, repository: nil, **args) + super(**args) + + # version is mandatory to prevent chart mismatches + # we do not want our values interpreted in the context of the wrong version + raise ArgumentError, 'version is required' if version.blank? + + @chart = chart + @version = version + @repository = repository + end + + def generate_script + super + [ + repository_command, + repository_update_command, + upgrade_command + ].compact.join("\n") + end + + private + + def upgrade_command + command = ['helm', 'upgrade', name, chart] + + reuse_values_flag + + version_flag + + namespace_flag + + value_flag + + command.shelljoin + end + + def reuse_values_flag + ['--reuse-values'] + end + + def value_flag + ['-f', "/data/helm/#{name}/config/values.yaml"] + end + + def version_flag + ['--version', version] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 13cd6dcad3f..a25f005d81e 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -61,18 +61,11 @@ module Gitlab # RBAC methods delegates to the apis/rbac.authorization.k8s.io api # group client delegate :update_cluster_role_binding, - to: :rbac_client - - # RBAC methods delegates to the apis/rbac.authorization.k8s.io api - # group client - delegate :create_role, - :get_role, - :update_role, - to: :rbac_client - - # RBAC methods delegates to the apis/rbac.authorization.k8s.io api - # group client - delegate :update_role_binding, + :create_role, + :get_role, + :update_role, + :delete_role_binding, + :update_role_binding, to: :rbac_client # non-entity methods that can only work with the core client @@ -182,10 +175,21 @@ module Gitlab end end + def patch_ingress(*args) + extensions_client.discover unless extensions_client.discovered + + if extensions_client.respond_to?(:patch_ingress) + extensions_client.patch_ingress(*args) + else + networking_client.patch_ingress(*args) + end + end + def create_or_update_cluster_role_binding(resource) update_cluster_role_binding(resource) end + # Note that we cannot update roleRef as that is immutable def create_or_update_role_binding(resource) update_role_binding(resource) end diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index f7eaafeb446..4482610523e 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -24,6 +24,7 @@ module Gitlab @api ||= ::Octokit::Client.new( access_token: access_token, api_endpoint: api_endpoint, + web_endpoint: web_endpoint, # If there is no config, we're connecting to github.com and we # should verify ssl. connection_options: { @@ -85,6 +86,10 @@ module Gitlab end end + def web_endpoint + host.presence || ::Octokit::Default.web_endpoint + end + def config Gitlab::Auth::OAuth::Provider.config_for('github') end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 3f9fd1b1a19..a17e3b1ad5c 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -36,7 +36,7 @@ module Gitlab } end - @client = Client.new(credentials[:user], opts) + @client = Client.new(credentials[:user], **opts) end def execute @@ -303,6 +303,8 @@ module Gitlab end imported!(resource_type) + rescue ::Octokit::NotFound => e + errors << { type: resource_type, errors: e.message } end def imported?(resource_type) diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index f6bda0dbea4..23d7eb67312 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -3,43 +3,70 @@ module Gitlab module Metrics class RequestsRackMiddleware - HTTP_METHODS = %w(delete get head options patch post put).to_set.freeze + HTTP_METHODS = { + "delete" => %w(200 202 204 303 400 401 403 404 500 503), + "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 422 429 500 503), + "head" => %w(200 204 301 302 303 401 403 404 410 500), + "options" => %w(200 404), + "patch" => %w(200 202 204 400 403 404 409 416 500), + "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 422 429 500 503), + "put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500) + }.freeze HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze FEATURE_CATEGORY_HEADER = 'X-Gitlab-Feature-Category' FEATURE_CATEGORY_DEFAULT = 'unknown' + # These were the top 5 categories at a point in time, chosen as a + # reasonable default. If we initialize every category we'll end up + # with an explosion in unused metric combinations, but we want the + # most common ones to be always present. + FEATURE_CATEGORIES_TO_INITIALIZE = ['authentication_and_authorization', + 'code_review', 'continuous_integration', + 'not_owned', 'source_code_management', + FEATURE_CATEGORY_DEFAULT].freeze + def initialize(app) @app = app end - def self.http_request_total - @http_request_total ||= ::Gitlab::Metrics.counter(:http_requests_total, 'Request count') + def self.http_requests_total + ::Gitlab::Metrics.counter(:http_requests_total, 'Request count') end def self.rack_uncaught_errors_count - @rack_uncaught_errors_count ||= ::Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') + ::Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') end def self.http_request_duration_seconds - @http_request_duration_seconds ||= ::Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', - {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) + ::Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', + {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) end def self.http_health_requests_total - @http_health_requests_total ||= ::Gitlab::Metrics.counter(:http_health_requests_total, 'Health endpoint request count') + ::Gitlab::Metrics.counter(:http_health_requests_total, 'Health endpoint request count') end - def self.initialize_http_request_duration_seconds - HTTP_METHODS.each do |method| + def self.initialize_metrics + # This initialization is done to avoid gaps in scraped metrics after + # restarts. It makes sure all counters/histograms are available at + # process start. + # + # For example `rate(http_requests_total{status="500"}[1m])` would return + # no data until the first 500 error would occur. + HTTP_METHODS.each do |method, statuses| http_request_duration_seconds.get({ method: method }) + + statuses.product(FEATURE_CATEGORIES_TO_INITIALIZE) do |status, feature_category| + http_requests_total.get({ method: method, status: status, feature_category: feature_category }) + end end end def call(env) method = env['REQUEST_METHOD'].downcase - method = 'INVALID' unless HTTP_METHODS.include?(method) + method = 'INVALID' unless HTTP_METHODS.key?(method) started = Time.now.to_f health_endpoint = health_endpoint?(env['PATH_INFO']) status = 'undefined' @@ -61,9 +88,13 @@ module Gitlab raise ensure if health_endpoint - RequestsRackMiddleware.http_health_requests_total.increment(status: status, method: method) + RequestsRackMiddleware.http_health_requests_total.increment(status: status.to_s, method: method) else - RequestsRackMiddleware.http_request_total.increment(status: status, method: method, feature_category: feature_category || FEATURE_CATEGORY_DEFAULT) + RequestsRackMiddleware.http_requests_total.increment( + status: status.to_s, + method: method, + feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT + ) end end end diff --git a/lib/gitlab/middleware/handle_malformed_strings.rb b/lib/gitlab/middleware/handle_malformed_strings.rb new file mode 100644 index 00000000000..84f7e2e1b14 --- /dev/null +++ b/lib/gitlab/middleware/handle_malformed_strings.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + # There is no valid reason for a request to contain a malformed string + # so just return HTTP 400 (Bad Request) if we receive one + class HandleMalformedStrings + include ActionController::HttpAuthentication::Basic + + NULL_BYTE_REGEX = Regexp.new(Regexp.escape("\u0000")).freeze + + attr_reader :app + + def initialize(app) + @app = app + end + + def call(env) + return [400, { 'Content-Type' => 'text/plain' }, ['Bad Request']] if request_contains_malformed_string?(env) + + app.call(env) + end + + private + + def request_contains_malformed_string?(env) + return false if ENV['DISABLE_REQUEST_VALIDATION'] == '1' + + # Duplicate the env, so it is not modified when accessing the parameters + # https://github.com/rails/rails/blob/34991a6ae2fc68347c01ea7382fa89004159e019/actionpack/lib/action_dispatch/http/parameters.rb#L59 + # The modification causes problems with our multipart middleware + request = ActionDispatch::Request.new(env.dup) + + return true if malformed_path?(request.path) + return true if credentials_malformed?(request) + + request.params.values.any? do |value| + param_has_null_byte?(value) + end + rescue ActionController::BadRequest + # If we can't build an ActionDispatch::Request something's wrong + # This would also happen if `#params` contains invalid UTF-8 + # in this case we'll return a 400 + # + true + end + + def malformed_path?(path) + string_malformed?(Rack::Utils.unescape(path)) + rescue ArgumentError + # Rack::Utils.unescape raised this, path is malformed. + true + end + + def credentials_malformed?(request) + credentials = if has_basic_credentials?(request) + decode_credentials(request).presence + else + request.authorization.presence + end + + return false unless credentials + + string_malformed?(credentials) + end + + def param_has_null_byte?(value, depth = 0) + # Guard against possible attack sending large amounts of nested params + # Should be safe as deeply nested params are highly uncommon. + return false if depth > 2 + + depth += 1 + + if value.respond_to?(:match) + string_malformed?(value) + elsif value.respond_to?(:values) + value.values.any? do |hash_value| + param_has_null_byte?(hash_value, depth) + end + elsif value.is_a?(Array) + value.any? do |array_value| + param_has_null_byte?(array_value, depth) + end + else + false + end + end + + def string_malformed?(string) + # We're using match rather than include, because that will raise an ArgumentError + # when the string contains invalid UTF8 + # + # We try to encode the string from ASCII-8BIT to UTF8. If we failed to do + # so for certain characters in the string, those chars are probably incomplete + # multibyte characters. + string.encode(Encoding::UTF_8).match?(NULL_BYTE_REGEX) + rescue ArgumentError, Encoding::UndefinedConversionError + # If we're here, we caught a malformed string. Return true + true + end + end + end +end diff --git a/lib/gitlab/middleware/handle_null_bytes.rb b/lib/gitlab/middleware/handle_null_bytes.rb deleted file mode 100644 index c88dfb6ee0b..00000000000 --- a/lib/gitlab/middleware/handle_null_bytes.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Middleware - # There is no valid reason for a request to contain a null byte (U+0000) - # so just return HTTP 400 (Bad Request) if we receive one - class HandleNullBytes - NULL_BYTE_REGEX = Regexp.new(Regexp.escape("\u0000")).freeze - - attr_reader :app - - def initialize(app) - @app = app - end - - def call(env) - return [400, {}, ["Bad Request"]] if request_has_null_byte?(env) - - app.call(env) - end - - private - - def request_has_null_byte?(request) - return false if ENV['REJECT_NULL_BYTES'] == "1" - - request = Rack::Request.new(request) - - request.params.values.any? do |value| - param_has_null_byte?(value) - end - end - - def param_has_null_byte?(value, depth = 0) - # Guard against possible attack sending large amounts of nested params - # Should be safe as deeply nested params are highly uncommon. - return false if depth > 2 - - depth += 1 - - if value.respond_to?(:match) - string_contains_null_byte?(value) - elsif value.respond_to?(:values) - value.values.any? do |hash_value| - param_has_null_byte?(hash_value, depth) - end - elsif value.is_a?(Array) - value.any? do |array_value| - param_has_null_byte?(array_value, depth) - end - else - false - end - end - - def string_contains_null_byte?(string) - string.match?(NULL_BYTE_REGEX) - end - end - end -end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index cfea4aaddf3..101172cdfcc 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -9,20 +9,19 @@ module Gitlab APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance' - WHITELISTED_GIT_ROUTES = { - 'repositories/git_http' => %w{git_upload_pack git_receive_pack} + ALLOWLISTED_GIT_ROUTES = { + 'repositories/git_http' => %w{git_upload_pack} }.freeze - WHITELISTED_GIT_LFS_ROUTES = { - 'repositories/lfs_api' => %w{batch}, - 'repositories/lfs_locks_api' => %w{verify create unlock} + ALLOWLISTED_GIT_LFS_BATCH_ROUTES = { + 'repositories/lfs_api' => %w{batch} }.freeze - WHITELISTED_GIT_REVISION_ROUTES = { + ALLOWLISTED_GIT_REVISION_ROUTES = { 'projects/compare' => %w{create} }.freeze - WHITELISTED_SESSION_ROUTES = { + ALLOWLISTED_SESSION_ROUTES = { 'sessions' => %w{destroy}, 'admin/sessions' => %w{create destroy} }.freeze @@ -55,7 +54,7 @@ module Gitlab def disallowed_request? DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && - !whitelisted_routes + !allowlisted_routes end def json_request? @@ -87,8 +86,8 @@ module Gitlab end # Overridden in EE module - def whitelisted_routes - workhorse_passthrough_route? || internal_route? || lfs_route? || compare_git_revisions_route? || sidekiq_route? || session_route? || graphql_query? + def allowlisted_routes + workhorse_passthrough_route? || internal_route? || lfs_batch_route? || compare_git_revisions_route? || sidekiq_route? || session_route? || graphql_query? end # URL for requests passed through gitlab-workhorse to rails-web @@ -96,9 +95,9 @@ module Gitlab def workhorse_passthrough_route? # Calling route_hash may be expensive. Only do it if we think there's a possible match return false unless request.post? && - request.path.end_with?('.git/git-upload-pack', '.git/git-receive-pack') + request.path.end_with?('.git/git-upload-pack') - WHITELISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + ALLOWLISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end def internal_route? @@ -109,18 +108,16 @@ module Gitlab # Calling route_hash may be expensive. Only do it if we think there's a possible match return false unless request.post? && request.path.end_with?('compare') - WHITELISTED_GIT_REVISION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + ALLOWLISTED_GIT_REVISION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end - def lfs_route? + # Batch upload requests are blocked in: + # https://gitlab.com/gitlab-org/gitlab/blob/master/app/controllers/repositories/lfs_api_controller.rb#L106 + def lfs_batch_route? # Calling route_hash may be expensive. Only do it if we think there's a possible match - unless request.path.end_with?('/info/lfs/objects/batch', - '/info/lfs/locks', '/info/lfs/locks/verify') || - %r{/info/lfs/locks/\d+/unlock\z}.match?(request.path) - return false - end + return unless request.path.end_with?('/info/lfs/objects/batch') - WHITELISTED_GIT_LFS_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + ALLOWLISTED_GIT_LFS_BATCH_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end def session_route? @@ -128,7 +125,7 @@ module Gitlab return false unless request.post? && request.path.end_with?('/users/sign_out', '/admin/session', '/admin/session/destroy') - WHITELISTED_SESSION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + ALLOWLISTED_SESSION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end def sidekiq_route? diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb index 2dd7d08a58b..a3c0fdcf467 100644 --- a/lib/gitlab/octokit/middleware.rb +++ b/lib/gitlab/octokit/middleware.rb @@ -8,7 +8,7 @@ module Gitlab end def call(env) - Gitlab::UrlBlocker.validate!(env[:url], { allow_localhost: allow_local_requests?, allow_local_network: allow_local_requests? }) + Gitlab::UrlBlocker.validate!(env[:url], allow_localhost: allow_local_requests?, allow_local_network: allow_local_requests?) @app.call(env) end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index b60ecb6631b..541f9b06842 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -96,16 +96,6 @@ module Gitlab args[:strategy_class] = args[:strategy_class].constantize end - # Providers that are known to depend on rack-oauth2, like those using - # Omniauth::Strategies::OpenIDConnect, need to be quirked so the - # client_auth_method argument value is passed as a symbol. - if (args[:strategy_class] == OmniAuth::Strategies::OpenIDConnect || - args[:name] == 'openid_connect') && - args[:client_auth_method].is_a?(String) - - args[:client_auth_method] = args[:client_auth_method].to_sym - end - args end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 706c16f6149..ad0a5c80604 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -49,6 +49,9 @@ module Gitlab s search sent_notifications + sitemap + sitemap.xml + sitemap.xml.gz slash-command-logo.png snippets unsubscribes @@ -251,6 +254,14 @@ module Gitlab %r{#{personal_snippet_path_regex}|#{project_snippet_path_regex}} end + def container_image_regex + @container_image_regex ||= %r{([\w\.-]+\/){0,1}[\w\.-]+}.freeze + end + + def container_image_blob_sha_regex + @container_image_blob_sha_regex ||= %r{[\w+.-]+:?\w+}.freeze + end + private def personal_snippet_path_regex diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 333564bee01..6719dc8362b 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -4,11 +4,11 @@ module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref - def initialize(current_user, query, project:, repository_ref: nil, sort: nil, filters: {}) + def initialize(current_user, query, project:, repository_ref: nil, order_by: nil, sort: nil, filters: {}) @project = project @repository_ref = repository_ref.presence - super(current_user, query, [project], sort: sort, filters: filters) + super(current_user, query, [project], order_by: order_by, sort: sort, filters: filters) end def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil) @@ -75,15 +75,6 @@ module Gitlab @commits_count ||= commits(limit: count_limit).count end - def single_commit_result? - return false if commits_count != 1 - - counts = %i(limited_milestones_count limited_notes_count - limited_merge_requests_count limited_issues_count - limited_blobs_count wiki_blobs_count) - counts.all? { |count_method| public_send(count_method) == 0 } # rubocop:disable GitlabSecurity/PublicSend - end - private def paginated_commits(page, per_page) diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index dd7a27ead01..1294e475145 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -29,9 +29,9 @@ module Gitlab # Anything, including `/cmd arg` which are ignored by this filter # ` - ^.*`\n* + `\n* .+? - \n*`$ + \n*` ) }mix.freeze diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index c8c949a9363..1986b7a1789 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -56,21 +56,21 @@ module Gitlab @updates[:merge] = params[:merge_request_diff_head_sha] end - desc 'Toggle the Work In Progress status' + desc 'Toggle the Draft status' explanation do noun = quick_action_target.to_ability_name.humanize(capitalize: false) if quick_action_target.work_in_progress? - _("Unmarks this %{noun} as Work In Progress.") + _("Unmarks this %{noun} as a draft.") else - _("Marks this %{noun} as Work In Progress.") + _("Marks this %{noun} as a draft.") end % { noun: noun } end execution_message do noun = quick_action_target.to_ability_name.humanize(capitalize: false) if quick_action_target.work_in_progress? - _("Unmarked this %{noun} as Work In Progress.") + _("Unmarked this %{noun} as a draft.") else - _("Marked this %{noun} as Work In Progress.") + _("Marked this %{noun} as a draft.") end % { noun: noun } end @@ -80,7 +80,7 @@ module Gitlab # Allow it to mark as WIP on MR creation page _or_ through MR notes. (quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)) end - command :wip do + command :draft, :wip do @updates[:wip_event] = quick_action_target.work_in_progress? ? 'unwip' : 'wip' end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 5584323789b..6f80c7d439f 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -18,6 +18,10 @@ module Gitlab pool.with { |redis| yield redis } end + def version + with { |redis| redis.info['redis_version'] } + end + def pool @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 01aff48b08b..d7501fc7068 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -4,7 +4,7 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project - merge_request snippet commit commit_range directly_addressed_user epic iteration).freeze + merge_request snippet commit commit_range directly_addressed_user epic iteration vulnerability).freeze attr_accessor :project, :current_user, :author # This counter is increased by a number of references filtered out by # banzai reference exctractor. Note that this counter is stateful and @@ -38,7 +38,7 @@ module Gitlab end REFERABLES.each do |type| - define_method("#{type}s") do + define_method(type.to_s.pluralize) do @references[type] ||= references(type) end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 1b169b6186b..4ae6297f6f5 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -22,6 +22,10 @@ module Gitlab @composer_package_version_regex ||= %r{^v?(\d+(\.(\d+|x))*(-.+)?)}.freeze end + def composer_dev_version_regex + @composer_dev_version_regex ||= %r{(^dev-)|(-dev$)}.freeze + end + def package_name_regex @package_name_regex ||= %r{\A\@?(([\w\-\.\+]*)\/)*([\w\-\.]+)@?(([\w\-\.\+]*)\/)*([\w\-\.]*)\z}.freeze end diff --git a/lib/gitlab/repository_size_checker.rb b/lib/gitlab/repository_size_checker.rb index 03d9f961dd9..0ed31176dd8 100644 --- a/lib/gitlab/repository_size_checker.rb +++ b/lib/gitlab/repository_size_checker.rb @@ -32,18 +32,24 @@ module Gitlab def changes_will_exceed_size_limit?(change_size) return false unless enabled? - change_size > limit || exceeded_size(change_size) > 0 + above_size_limit? || exceeded_size(change_size) > 0 end # @param change_size [int] in bytes def exceeded_size(change_size = 0) - current_size + change_size - limit + size = current_size + change_size - limit + + [size, 0].max end def error_message @error_message_object ||= ::Gitlab::RepositorySizeErrorMessage.new(self) end + def additional_repo_storage_available? + false + end + private attr_reader :namespace diff --git a/lib/gitlab/repository_size_error_message.rb b/lib/gitlab/repository_size_error_message.rb index 556190453de..8da840779c9 100644 --- a/lib/gitlab/repository_size_error_message.rb +++ b/lib/gitlab/repository_size_error_message.rb @@ -4,7 +4,7 @@ module Gitlab class RepositorySizeErrorMessage include ActiveSupport::NumberHelper - delegate :current_size, :limit, :exceeded_size, to: :@checker + delegate :current_size, :limit, :exceeded_size, :additional_repo_storage_available?, to: :@checker # @param checher [RepositorySizeChecker] def initialize(checker) @@ -24,7 +24,11 @@ module Gitlab end def new_changes_error - "Your push to this repository would cause it to exceed the size limit of #{formatted(limit)} so it has been rejected. #{more_info_message}" + if additional_repo_storage_available? + "Your push to this repository has been rejected because it would exceed storage limits. Please contact your GitLab administrator for more information." + else + "Your push to this repository would cause it to exceed the size limit of #{formatted(limit)} so it has been rejected. #{more_info_message}" + end end def more_info_message diff --git a/lib/gitlab/repository_url_builder.rb b/lib/gitlab/repository_url_builder.rb index 2b88af1f77c..a2d0d50d20b 100644 --- a/lib/gitlab/repository_url_builder.rb +++ b/lib/gitlab/repository_url_builder.rb @@ -4,9 +4,6 @@ module Gitlab module RepositoryUrlBuilder class << self def build(path, protocol: :ssh) - # TODO: See https://gitlab.com/gitlab-org/gitlab/-/issues/213021 - path = path.sub('@snippets', 'snippets') - case protocol when :ssh ssh_url(path) diff --git a/lib/gitlab/robots_txt/parser.rb b/lib/gitlab/robots_txt/parser.rb index b9a3837e468..604d2f9b35b 100644 --- a/lib/gitlab/robots_txt/parser.rb +++ b/lib/gitlab/robots_txt/parser.rb @@ -3,34 +3,68 @@ module Gitlab module RobotsTxt class Parser - attr_reader :disallow_rules + DISALLOW_REGEX = /^disallow: /i.freeze + ALLOW_REGEX = /^allow: /i.freeze + + attr_reader :disallow_rules, :allow_rules def initialize(content) @raw_content = content - @disallow_rules = parse_raw_content! + @disallow_rules, @allow_rules = parse_raw_content! end def disallowed?(path) + return false if allow_rules.any? { |rule| path =~ rule } + disallow_rules.any? { |rule| path =~ rule } end private - # This parser is very basic as it only knows about `Disallow:` lines, - # and simply ignores all other lines. + # This parser is very basic as it only knows about `Disallow:` + # and `Allow:` lines, and simply ignores all other lines. # - # Order of predecence, 'Allow:`, etc are ignored for now. + # Patterns ending in `$`, and `*` for 0 or more characters are recognized. + # + # It is case insensitive and `Allow` rules takes precedence + # over `Disallow`. def parse_raw_content! - @raw_content.each_line.map do |line| - if line.start_with?('Disallow:') - value = line.sub('Disallow:', '').strip - value = Regexp.escape(value).gsub('\*', '.*') - Regexp.new("^#{value}") - else - nil + disallowed = [] + allowed = [] + + @raw_content.each_line.each do |line| + if disallow_rule?(line) + disallowed << get_disallow_pattern(line) + elsif allow_rule?(line) + allowed << get_allow_pattern(line) end - end.compact + end + + [disallowed, allowed] + end + + def disallow_rule?(line) + line =~ DISALLOW_REGEX + end + + def get_disallow_pattern(line) + get_pattern(line, DISALLOW_REGEX) + end + + def allow_rule?(line) + line =~ ALLOW_REGEX + end + + def get_allow_pattern(line) + get_pattern(line, ALLOW_REGEX) + end + + def get_pattern(line, rule_regex) + value = line.sub(rule_regex, '').strip + value = Regexp.escape(value).gsub('\*', '.*') + value = value.sub(/\\\$$/, '$') + Regexp.new("^#{value}") end end end diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb index fc1abc064c7..183e582925d 100644 --- a/lib/gitlab/search/found_blob.rb +++ b/lib/gitlab/search/found_blob.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize include BlobActiveModel - attr_reader :project, :content_match, :blob_path + attr_reader :project, :content_match, :blob_path, :highlight_line PATH_REGEXP = /\A(?<ref>[^:]*):(?<path>[^\x00]*)\x00/.freeze CONTENT_REGEXP = /^(?<ref>[^:]*):(?<path>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze @@ -26,6 +26,7 @@ module Gitlab @binary_basename = opts.fetch(:basename, nil) @ref = opts.fetch(:ref, nil) @startline = opts.fetch(:startline, nil) + @highlight_line = opts.fetch(:highlight_line, nil) @binary_data = opts.fetch(:data, nil) @per_page = opts.fetch(:per_page, 20) @project = opts.fetch(:project, nil) diff --git a/lib/gitlab/search/sort_options.rb b/lib/gitlab/search/sort_options.rb new file mode 100644 index 00000000000..3395c34d171 --- /dev/null +++ b/lib/gitlab/search/sort_options.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Search + module SortOptions + def sort_and_direction(order_by, sort) + # Due to different uses of sort param in web vs. API requests we prefer + # order_by when present + case [order_by, sort] + when %w[created_at asc], [nil, 'created_asc'] + :created_at_asc + when %w[created_at desc], [nil, 'created_desc'] + :created_at_desc + else + :unknown + end + end + module_function :sort_and_direction # rubocop: disable Style/AccessModifierDeclarations + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index b81264c5d0c..0091ae1e8ce 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -7,7 +7,7 @@ module Gitlab DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 20 - attr_reader :current_user, :query, :sort, :filters + attr_reader :current_user, :query, :order_by, :sort, :filters # Limit search results by passed projects # It allows us to search only for projects user has access to @@ -19,11 +19,12 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, query, limit_projects = nil, sort: nil, default_project_filter: false, filters: {}) + def initialize(current_user, query, limit_projects = nil, order_by: nil, sort: nil, default_project_filter: false, filters: {}) @current_user = current_user @query = query @limit_projects = limit_projects || Project.all @default_project_filter = default_project_filter + @order_by = order_by @sort = sort @filters = filters end @@ -94,10 +95,6 @@ module Gitlab @limited_users_count ||= limited_count(users) end - def single_commit_result? - false - end - def count_limit COUNT_LIMIT end @@ -132,13 +129,15 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def apply_sort(scope) - case sort - when 'oldest' + # Due to different uses of sort param we prefer order_by when + # present + case ::Gitlab::Search::SortOptions.sort_and_direction(order_by, sort) + when :created_at_asc scope.reorder('created_at ASC') - when 'newest' + when :created_at_desc scope.reorder('created_at DESC') else - scope + scope.reorder('created_at DESC') end end # rubocop: enable CodeReuse/ActiveRecord @@ -219,7 +218,7 @@ module Gitlab params[:state] = filters[:state] if filters.key?(:state) - if [true, false].include?(filters[:confidential]) && Feature.enabled?(:search_filter_by_confidential) + if [true, false].include?(filters[:confidential]) params[:confidential] = filters[:confidential] end end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 4df6a50c8dd..259d3e300b6 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -99,6 +99,7 @@ module Gitlab config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } config[:bin_dir] = Gitlab.config.gitaly.client_path config[:gitlab] = { url: Gitlab.config.gitlab.url } + config[:logging] = { dir: Rails.root.join('log').to_s } TomlRB.dump(config) end diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb index 1e5d23a8405..e471517c50a 100644 --- a/lib/gitlab/sidekiq_cluster/cli.rb +++ b/lib/gitlab/sidekiq_cluster/cli.rb @@ -47,16 +47,24 @@ module Gitlab option_parser.parse!(argv) + # Remove with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 + if @queue_selector && @experimental_queue_selector + raise CommandError, + 'You cannot specify --queue-selector and --experimental-queue-selector together' + end + all_queues = SidekiqConfig::CliMethods.all_queues(@rails_path) queue_names = SidekiqConfig::CliMethods.worker_queues(@rails_path) queue_groups = argv.map do |queues| next queue_names if queues == '*' - # When using the experimental queue query syntax, we treat - # each queue group as a worker attribute query, and resolve - # the queues for the queue group using this query. - if @experimental_queue_selector + # When using the queue query syntax, we treat each queue group + # as a worker attribute query, and resolve the queues for the + # queue group using this query. + + # Simplify with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 + if @queue_selector || @experimental_queue_selector SidekiqConfig::CliMethods.query_workers(queues, all_queues) else SidekiqConfig::CliMethods.expand_queues(queues.split(','), queue_names) @@ -182,7 +190,12 @@ module Gitlab @rails_path = path end - opt.on('--experimental-queue-selector', 'EXPERIMENTAL: Run workers based on the provided selector') do |experimental_queue_selector| + opt.on('--queue-selector', 'Run workers based on the provided selector') do |queue_selector| + @queue_selector = queue_selector + end + + # Remove with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 + opt.on('--experimental-queue-selector', 'DEPRECATED: use --queue-selector-instead') do |experimental_queue_selector| @experimental_queue_selector = experimental_queue_selector end diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb index 326dfdae661..dc81c34c4d0 100644 --- a/lib/gitlab/sidekiq_logging/logs_jobs.rb +++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb @@ -16,7 +16,7 @@ module Gitlab # Add process id params job['pid'] = ::Process.pid - job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] + job.delete('args') unless SidekiqLogArguments.enabled? job end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb index 6fdef4c354e..63e8bee4443 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb @@ -8,6 +8,7 @@ module Gitlab STRATEGIES = { until_executing: UntilExecuting, + until_executed: UntilExecuted, none: None }.freeze diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb new file mode 100644 index 00000000000..df5df590281 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + module Strategies + class Base + def initialize(duplicate_job) + @duplicate_job = duplicate_job + end + + def schedule(job) + raise NotImplementedError + end + + def perform(_job) + raise NotImplementedError + end + + private + + attr_reader :duplicate_job + + def strategy_name + self.class.name.to_s.demodulize.underscore.humanize.downcase + end + + def check! + # The default expiry time is the DuplicateJob::DUPLICATE_KEY_TTL already + # Only the strategies de-duplicating when scheduling + duplicate_job.check! + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb new file mode 100644 index 00000000000..59b0e7e29da --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + module Strategies + module DeduplicatesWhenScheduling + def initialize(duplicate_job) + @duplicate_job = duplicate_job + end + + def schedule(job) + if deduplicatable_job? && check! && duplicate_job.duplicate? + job['duplicate-of'] = duplicate_job.existing_jid + + if duplicate_job.droppable? + Gitlab::SidekiqLogging::DeduplicationLogger.instance.log( + job, "dropped #{strategy_name}", duplicate_job.options) + return false + end + end + + yield + end + + private + + def deduplicatable_job? + !duplicate_job.scheduled? || duplicate_job.options[:including_scheduled] + end + + def check! + duplicate_job.check!(expiry) + end + + def expiry + return DuplicateJob::DUPLICATE_KEY_TTL unless duplicate_job.scheduled? + + time_diff = duplicate_job.scheduled_at.to_i - Time.now.to_i + + time_diff > 0 ? time_diff : DuplicateJob::DUPLICATE_KEY_TTL + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb index cd101cd16b6..acbe0efaafa 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb @@ -5,10 +5,7 @@ module Gitlab module DuplicateJobs module Strategies # This strategy will never deduplicate a job - class None - def initialize(_duplicate_job) - end - + class None < Base def schedule(_job) yield end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb new file mode 100644 index 00000000000..738efa36fc8 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + module Strategies + # This strategy takes a lock before scheduling the job in a queue and + # removes the lock after the job has executed preventing a new job to be queued + # while a job is still executing. + class UntilExecuted < Base + include DeduplicatesWhenScheduling + + def perform(_job) + yield + + duplicate_job.delete! + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb index 46ce0eb4a91..68d66383b2b 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb @@ -7,50 +7,14 @@ module Gitlab # This strategy takes a lock before scheduling the job in a queue and # removes the lock before the job starts allowing a new job to be queued # while a job is still executing. - class UntilExecuting - def initialize(duplicate_job) - @duplicate_job = duplicate_job - end - - def schedule(job) - if deduplicatable_job? && check! && duplicate_job.duplicate? - job['duplicate-of'] = duplicate_job.existing_jid - - if duplicate_job.droppable? - Gitlab::SidekiqLogging::DeduplicationLogger.instance.log( - job, "dropped until executing", duplicate_job.options) - return false - end - end - - yield - end + class UntilExecuting < Base + include DeduplicatesWhenScheduling def perform(_job) duplicate_job.delete! yield end - - private - - attr_reader :duplicate_job - - def deduplicatable_job? - !duplicate_job.scheduled? || duplicate_job.options[:including_scheduled] - end - - def check! - duplicate_job.check!(expiry) - end - - def expiry - return DuplicateJob::DUPLICATE_KEY_TTL unless duplicate_job.scheduled? - - time_diff = duplicate_job.scheduled_at.to_i - Time.now.to_i - - time_diff > 0 ? time_diff : DuplicateJob::DUPLICATE_KEY_TTL - end end end end diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb index ff24ec69ab0..0a2cee75af7 100644 --- a/lib/gitlab/static_site_editor/config/generated_config.rb +++ b/lib/gitlab/static_site_editor/config/generated_config.rb @@ -34,7 +34,7 @@ module Gitlab delegate :project, to: :repository def supported_extensions - %w[.md].freeze + %w[.md .md.erb].freeze end def commit_id @@ -50,8 +50,6 @@ module Gitlab end def extension_supported? - return true if path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project) - supported_extensions.any? { |ext| path.end_with?(ext) } end diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 50e09bdcdd6..e84937ec4ad 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -105,6 +105,20 @@ module Gitlab files.map { |t| { name: t.name } } end end + + def template_subsets(project = nil) + return [] if project && !project.repository.exists? + + if categories.any? + categories.keys.map do |category| + files = self.by_category(category, project) + [category, files.map { |t| { key: t.key, name: t.name, content: t.content } }] + end.to_h + else + files = self.all(project) + files.map { |t| { key: t.key, name: t.name, content: t.content } } + end + end end end end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 02d354ec43a..19be468e3d5 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'snowplow-tracker' - module Gitlab module Tracking SNOWPLOW_NAMESPACE = 'gl' @@ -27,16 +25,11 @@ module Gitlab end def event(category, action, label: nil, property: nil, value: nil, context: nil) - return unless enabled? - - snowplow.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) + snowplow.event(category, action, label: label, property: property, value: value, context: context) end def self_describing_event(schema_url, event_data_json, context: nil) - return unless enabled? - - event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, event_data_json) - snowplow.track_self_describing_event(event_json, context, (Time.now.to_f * 1000).to_i) + snowplow.self_describing_event(schema_url, event_data_json, context: context) end def snowplow_options(group) @@ -54,19 +47,7 @@ module Gitlab private def snowplow - @snowplow ||= SnowplowTracker::Tracker.new( - emitter, - SnowplowTracker::Subject.new, - SNOWPLOW_NAMESPACE, - Gitlab::CurrentSettings.snowplow_app_id - ) - end - - def emitter - SnowplowTracker::AsyncEmitter.new( - Gitlab::CurrentSettings.snowplow_collector_hostname, - protocol: 'https' - ) + @snowplow ||= Gitlab::Tracking::Destinations::Snowplow.new end end end diff --git a/lib/gitlab/tracking/destinations/base.rb b/lib/gitlab/tracking/destinations/base.rb new file mode 100644 index 00000000000..00e92e0bd57 --- /dev/null +++ b/lib/gitlab/tracking/destinations/base.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Destinations + class Base + def event(category, action, label: nil, property: nil, value: nil, context: nil) + raise NotImplementedError, "#{self} does not implement #{__method__}" + end + end + end + end +end diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb new file mode 100644 index 00000000000..9cebcfe5ee1 --- /dev/null +++ b/lib/gitlab/tracking/destinations/snowplow.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'snowplow-tracker' + +module Gitlab + module Tracking + module Destinations + class Snowplow < Base + extend ::Gitlab::Utils::Override + + override :event + def event(category, action, label: nil, property: nil, value: nil, context: nil) + return unless enabled? + + tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) + end + + def self_describing_event(schema_url, event_data_json, context: nil) + return unless enabled? + + event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, event_data_json) + tracker.track_self_describing_event(event_json, context, (Time.now.to_f * 1000).to_i) + end + + private + + def enabled? + Gitlab::CurrentSettings.snowplow_enabled? + end + + def tracker + @tracker ||= SnowplowTracker::Tracker.new( + emitter, + SnowplowTracker::Subject.new, + Gitlab::Tracking::SNOWPLOW_NAMESPACE, + Gitlab::CurrentSettings.snowplow_app_id + ) + end + + def emitter + SnowplowTracker::AsyncEmitter.new( + Gitlab::CurrentSettings.snowplow_collector_hostname, + protocol: 'https' + ) + end + end + end + end +end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 9213b5ebab2..eece2c343d2 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -49,7 +49,7 @@ module Gitlab return [uri, nil] unless address_info ip_address = ip_address(address_info) - return [uri, nil] if domain_whitelisted?(uri) || ip_whitelisted?(ip_address, port: get_port(uri)) + return [uri, nil] if domain_allowed?(uri) || ip_allowed?(ip_address, port: get_port(uri)) protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, dns_rebind_protection) @@ -65,8 +65,8 @@ module Gitlab protected_uri_with_hostname end - def blocked_url?(*args) - validate!(*args) + def blocked_url?(url, **kwargs) + validate!(url, **kwargs) false rescue BlockedUrlError @@ -113,8 +113,8 @@ module Gitlab end rescue SocketError # If the dns rebinding protection is not enabled or the domain - # is whitelisted we avoid the dns rebinding checks - return if domain_whitelisted?(uri) || !dns_rebind_protection + # is allowed we avoid the dns rebinding checks + return if domain_allowed?(uri) || !dns_rebind_protection # In the test suite we use a lot of mocked urls that are either invalid or # don't exist. In order to avoid modifying a ton of tests and factories @@ -253,12 +253,12 @@ module Gitlab (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end - def domain_whitelisted?(uri) - Gitlab::UrlBlockers::UrlWhitelist.domain_whitelisted?(uri.normalized_host, port: get_port(uri)) + def domain_allowed?(uri) + Gitlab::UrlBlockers::UrlAllowlist.domain_allowed?(uri.normalized_host, port: get_port(uri)) end - def ip_whitelisted?(ip_address, port: nil) - Gitlab::UrlBlockers::UrlWhitelist.ip_whitelisted?(ip_address, port: port) + def ip_allowed?(ip_address, port: nil) + Gitlab::UrlBlockers::UrlAllowlist.ip_allowed?(ip_address, port: port) end def config diff --git a/lib/gitlab/url_blockers/domain_whitelist_entry.rb b/lib/gitlab/url_blockers/domain_allowlist_entry.rb index b94e8ee3f69..b65bd9e1a92 100644 --- a/lib/gitlab/url_blockers/domain_whitelist_entry.rb +++ b/lib/gitlab/url_blockers/domain_allowlist_entry.rb @@ -2,7 +2,7 @@ module Gitlab module UrlBlockers - class DomainWhitelistEntry + class DomainAllowlistEntry attr_reader :domain, :port def initialize(domain, port: nil) diff --git a/lib/gitlab/url_blockers/ip_whitelist_entry.rb b/lib/gitlab/url_blockers/ip_allowlist_entry.rb index 88c76574d3d..b293afe166c 100644 --- a/lib/gitlab/url_blockers/ip_whitelist_entry.rb +++ b/lib/gitlab/url_blockers/ip_allowlist_entry.rb @@ -2,7 +2,7 @@ module Gitlab module UrlBlockers - class IpWhitelistEntry + class IpAllowlistEntry attr_reader :ip, :port # Argument ip should be an IPAddr object diff --git a/lib/gitlab/url_blockers/url_whitelist.rb b/lib/gitlab/url_blockers/url_allowlist.rb index 59f74dde7fc..60238bea75a 100644 --- a/lib/gitlab/url_blockers/url_whitelist.rb +++ b/lib/gitlab/url_blockers/url_allowlist.rb @@ -2,43 +2,41 @@ module Gitlab module UrlBlockers - class UrlWhitelist + class UrlAllowlist class << self - def ip_whitelisted?(ip_string, port: nil) + def ip_allowed?(ip_string, port: nil) return false if ip_string.blank? - ip_whitelist, _ = outbound_local_requests_whitelist_arrays + ip_allowlist, _ = outbound_local_requests_allowlist_arrays ip_obj = Gitlab::Utils.string_to_ip_object(ip_string) - ip_whitelist.any? do |ip_whitelist_entry| - ip_whitelist_entry.match?(ip_obj, port) + ip_allowlist.any? do |ip_allowlist_entry| + ip_allowlist_entry.match?(ip_obj, port) end end - def domain_whitelisted?(domain_string, port: nil) + def domain_allowed?(domain_string, port: nil) return false if domain_string.blank? - _, domain_whitelist = outbound_local_requests_whitelist_arrays + _, domain_allowlist = outbound_local_requests_allowlist_arrays - domain_whitelist.any? do |domain_whitelist_entry| - domain_whitelist_entry.match?(domain_string, port) + domain_allowlist.any? do |domain_allowlist_entry| + domain_allowlist_entry.match?(domain_string, port) end end private - attr_reader :ip_whitelist, :domain_whitelist - # We cannot use Gitlab::CurrentSettings as ApplicationSetting itself # calls this class. This ends up in a cycle where # Gitlab::CurrentSettings creates an ApplicationSetting which then # calls this method. # # See https://gitlab.com/gitlab-org/gitlab/issues/9833 - def outbound_local_requests_whitelist_arrays + def outbound_local_requests_allowlist_arrays return [[], []] unless ApplicationSetting.current - ApplicationSetting.current.outbound_local_requests_whitelist_arrays + ApplicationSetting.current.outbound_local_requests_allowlist_arrays end end end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 1e522ae63b6..ce59e10241e 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -32,6 +32,8 @@ module Gitlab instance.milestone_url(object, **options) when Note note_url(object, **options) + when Release + instance.release_url(object, **options) when Project instance.project_url(object, **options) when Snippet diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 68f24559b1f..4b0dd54683b 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -40,8 +40,11 @@ module Gitlab with_finished_at(:recording_ce_finished_at) do license_usage_data + .merge(system_usage_data_license) + .merge(system_usage_data_settings) .merge(system_usage_data) .merge(system_usage_data_monthly) + .merge(system_usage_data_weekly) .merge(features_usage_data) .merge(components_usage_data) .merge(cycle_analytics_usage_data) @@ -157,6 +160,8 @@ module Gitlab projects_with_tracing_enabled: count(ProjectTracingSetting), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_service_enabled: count(AlertsService.active), + projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), + projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id), projects_with_terraform_states: distinct_count(::Terraform::State, :project_id), @@ -212,9 +217,11 @@ module Gitlab # rubocop: enable UsageData/LargeTable: packages: count(::Packages::Package.where(last_28_days_time_period)), personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)), - project_snippets: count(ProjectSnippet.where(last_28_days_time_period)) + project_snippets: count(ProjectSnippet.where(last_28_days_time_period)), + projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(last_28_days_time_period), :project_id) }.merge( - snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)) + snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)), + aggregated_metrics_monthly ).tap do |data| data[:snippets] = data[:personal_snippets] + data[:project_snippets] end @@ -222,6 +229,27 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def system_usage_data_license + { + license: {} + } + end + + def system_usage_data_settings + { + settings: {} + } + end + + def system_usage_data_weekly + { + counts_weekly: { + }.merge( + aggregated_metrics_weekly + ) + } + end + def cycle_analytics_usage_data Gitlab::CycleAnalytics::UsageData.new.to_json rescue ActiveRecord::StatementInvalid @@ -500,6 +528,7 @@ module Gitlab key => { configure: usage_activity_by_stage_configure(time_period), create: usage_activity_by_stage_create(time_period), + enablement: usage_activity_by_stage_enablement(time_period), manage: usage_activity_by_stage_manage(time_period), monitor: usage_activity_by_stage_monitor(time_period), package: usage_activity_by_stage_package(time_period), @@ -555,6 +584,11 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # Empty placeholder allows this to match the pattern used by other sections + def usage_activity_by_stage_enablement(time_period) + {} + end + # Omitted because no user, creator or author associated: `campaigns_imported_from_github`, `ldap_group_links` # rubocop: disable CodeReuse/ActiveRecord def usage_activity_by_stage_manage(time_period) @@ -564,7 +598,11 @@ module Gitlab users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), + bulk_imports: { + gitlab: distinct_count(::BulkImport.where(time_period, source_type: :gitlab), :user_id) + }, projects_imported: { + total: count(Project.where(time_period).where.not(import_type: nil)), gitlab_project: projects_imported_count('gitlab_project', time_period), gitlab: projects_imported_count('gitlab', time_period), github: projects_imported_count('github', time_period), @@ -577,7 +615,8 @@ module Gitlab issues_imported: { jira: distinct_count(::JiraImportState.where(time_period), :user_id), fogbugz: projects_imported_count('fogbugz', time_period), - phabricator: projects_imported_count('phabricator', time_period) + phabricator: projects_imported_count('phabricator', time_period), + csv: distinct_count(Issues::CsvImport.where(time_period), :user_id) }, groups_imported: distinct_count(::GroupImportState.where(time_period), :user_id) } @@ -592,7 +631,10 @@ module Gitlab operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period), start: user_minimum_id, finish: user_maximum_id), - projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id) + projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id), + projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id), + projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id), + projects_with_alert_incidents: distinct_count(::Issue.incident.with_alert_management_alerts.where(time_period), :project_id) } end # rubocop: enable CodeReuse/ActiveRecord @@ -664,6 +706,22 @@ module Gitlab { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } end + def aggregated_metrics_monthly + return {} unless Feature.enabled?(:product_analytics_aggregated_metrics) + + { + aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_monthly_data + } + end + + def aggregated_metrics_weekly + return {} unless Feature.enabled?(:product_analytics_aggregated_metrics) + + { + aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_weekly_data + } + end + def analytics_unique_visits_data results = ::Gitlab::Analytics::UniqueVisits.analytics_events.each_with_object({}) do |target, hash| hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } @@ -709,7 +767,8 @@ module Gitlab data = { action_monthly_active_users_project_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::PUSH_ACTION, action_monthly_active_users_design_management: Gitlab::UsageDataCounters::TrackUniqueEvents::DESIGN_ACTION, - action_monthly_active_users_wiki_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::WIKI_ACTION + action_monthly_active_users_wiki_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::WIKI_ACTION, + action_monthly_active_users_git_write: Gitlab::UsageDataCounters::TrackUniqueEvents::GIT_WRITE_ACTION } data.each do |key, event| diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml new file mode 100644 index 00000000000..97ec8423b95 --- /dev/null +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml @@ -0,0 +1,17 @@ +#- name: unique name of aggregated metric +# operator: aggregation operator. Valid values are: +# - "OR": counts unique elements that were observed triggering any of following events +# - "AND": counts unique elements that were observed triggering all of following events +# events: list of events names to aggregate into metric. All events in this list must have the same 'redis_slot' and 'aggregation' attributes +# see from lib/gitlab/usage_data_counters/known_events/ for the list of valid events. +# feature_flag: name of development feature flag that will be checked before metrics aggregation is performed. +# Corresponding feature flag should have `default_enabled` attribute set to `false`. +# This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked. +--- +- name: product_analytics_test_metrics_union + operator: OR + events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] + feature_flag: product_analytics_aggregated_metrics +- name: product_analytics_test_metrics_intersection + operator: AND + events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] diff --git a/lib/gitlab/usage_data_counters/designs_counter.rb b/lib/gitlab/usage_data_counters/designs_counter.rb index 22188b555d2..07e1963f9fb 100644 --- a/lib/gitlab/usage_data_counters/designs_counter.rb +++ b/lib/gitlab/usage_data_counters/designs_counter.rb @@ -1,42 +1,8 @@ # frozen_string_literal: true module Gitlab::UsageDataCounters - class DesignsCounter - extend Gitlab::UsageDataCounters::RedisCounter - + class DesignsCounter < BaseCounter KNOWN_EVENTS = %w[create update delete].freeze - - UnknownEvent = Class.new(StandardError) - - class << self - # Each event gets a unique Redis key - def redis_key(event) - raise UnknownEvent, event unless KNOWN_EVENTS.include?(event.to_s) - - "USAGE_DESIGN_MANAGEMENT_DESIGNS_#{event}".upcase - end - - def count(event) - increment(redis_key(event)) - end - - def read(event) - total_count(redis_key(event)) - end - - def totals - KNOWN_EVENTS.map { |event| [counter_key(event), read(event)] }.to_h - end - - def fallback_totals - KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h - end - - private - - def counter_key(event) - "design_management_designs_#{event}".to_sym - end - end + PREFIX = 'design_management_designs' end end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index eb132ef0967..573ad1dce35 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -5,18 +5,28 @@ module Gitlab module HLLRedisCounter DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days - DEFAULT_REDIS_SLOT = ''.freeze - - UnknownEvent = Class.new(StandardError) - UnknownAggregation = Class.new(StandardError) - - KNOWN_EVENTS_PATH = 'lib/gitlab/usage_data_counters/known_events.yml'.freeze + DEFAULT_REDIS_SLOT = '' + + EventError = Class.new(StandardError) + UnknownEvent = Class.new(EventError) + UnknownAggregation = Class.new(EventError) + AggregationMismatch = Class.new(EventError) + SlotMismatch = Class.new(EventError) + CategoryMismatch = Class.new(EventError) + UnknownAggregationOperator = Class.new(EventError) + InvalidContext = Class.new(EventError) + + KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) ALLOWED_AGGREGATIONS = %i(daily weekly).freeze + UNION_OF_AGGREGATED_METRICS = 'OR' + INTERSECTION_OF_AGGREGATED_METRICS = 'AND' + ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze + AGGREGATED_METRICS_PATH = File.expand_path('aggregated_metrics/*.yml', __dir__) # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # - # All events should be added to know_events file lib/gitlab/usage_data_counters/known_events.yml + # All events should be added to known_events yml files lib/gitlab/usage_data_counters/known_events/ # # Event example: # @@ -25,6 +35,7 @@ module Gitlab # category: compliance # Group events in categories # expiry: 29 # Optional expiration time in days, default value 29 days for daily and 6.weeks for weekly # aggregation: daily # Aggregation level, keys are stored daily or weekly + # feature_flag: # The event feature flag # # Usage: # @@ -33,28 +44,24 @@ module Gitlab class << self include Gitlab::Utils::UsageData - def track_event(entity_id, event_name, time = Time.zone.now) - return unless Gitlab::CurrentSettings.usage_ping_enabled? - - event = event_for(event_name) - - raise UnknownEvent.new("Unknown event #{event_name}") unless event.present? - - Gitlab::Redis::HLL.add(key: redis_key(event, time), value: entity_id, expiry: expiry(event)) + def track_event(value, event_name, time = Time.zone.now) + track(value, event_name, time: time) end - def unique_events(event_names:, start_date:, end_date:) - events = events_for(Array(event_names)) - - raise 'Events should be in same slot' unless events_in_same_slot?(events) - raise 'Events should be in same category' unless events_in_same_category?(events) - raise 'Events should have same aggregation level' unless events_same_aggregation?(events) - - aggregation = events.first[:aggregation] + def track_event_in_context(value, event_name, context, time = Time.zone.now) + return if context.blank? + return unless context.in?(valid_context_list) - keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date) + track(value, event_name, context: context, time: time) + end - redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } + def unique_events(event_names:, start_date:, end_date:, context: '') + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events| + raise SlotMismatch, events unless events_in_same_slot?(events) + raise CategoryMismatch, events unless events_in_same_category?(events) + raise AggregationMismatch, events unless events_same_aggregation?(events) + raise InvalidContext if context.present? && !context.in?(valid_context_list) + end end def categories @@ -72,8 +79,8 @@ module Gitlab events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| - hash["#{event}_weekly"] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) - hash["#{event}_monthly"] = unique_events(event_names: event, start_date: 4.weeks.ago.to_date, end_date: Date.current) + hash["#{event}_weekly"] = unique_events(event_names: [event], start_date: 7.days.ago.to_date, end_date: Date.current) + hash["#{event}_monthly"] = unique_events(event_names: [event], start_date: 4.weeks.ago.to_date, end_date: Date.current) end if eligible_for_totals?(events_names) @@ -89,8 +96,136 @@ module Gitlab event_for(event_name).present? end + def aggregated_metrics_monthly_data + aggregated_metrics_data(4.weeks.ago.to_date) + end + + def aggregated_metrics_weekly_data + aggregated_metrics_data(7.days.ago.to_date) + end + + def known_events + @known_events ||= load_events(KNOWN_EVENTS_PATH) + end + + def aggregated_metrics + @aggregated_metrics ||= load_events(AGGREGATED_METRICS_PATH) + end + private + def track(value, event_name, context: '', time: Time.zone.now) + return unless Gitlab::CurrentSettings.usage_ping_enabled? + + event = event_for(event_name) + raise UnknownEvent, "Unknown event #{event_name}" unless event.present? + + Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: value, expiry: expiry(event)) + end + + # The aray of valid context on which we allow tracking + def valid_context_list + Plan.all_plans + end + + def aggregated_metrics_data(start_date) + aggregated_metrics.each_with_object({}) do |aggregation, weekly_data| + next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: false, type: :development) + + weekly_data[aggregation[:name]] = calculate_count_for_aggregation(aggregation, start_date: start_date, end_date: Date.current) + end + end + + def calculate_count_for_aggregation(aggregation, start_date:, end_date:) + case aggregation[:operator] + when UNION_OF_AGGREGATED_METRICS + calculate_events_union(event_names: aggregation[:events], start_date: start_date, end_date: end_date) + when INTERSECTION_OF_AGGREGATED_METRICS + calculate_events_intersections(event_names: aggregation[:events], start_date: start_date, end_date: end_date) + else + raise UnknownAggregationOperator, "Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}" + end + end + + # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle + # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391 + def calculate_events_intersections(event_names:, start_date:, end_date:, subset_powers_cache: Hash.new({})) + # calculate power of intersection of all given metrics from inclusion exclusion principle + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + + # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... + subset_powers_data = subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) + + # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| + power_of_union_of_all_events = begin + subset_powers_cache[event_names.size][event_names.join('_+_')] ||= \ + calculate_events_union(event_names: event_names, start_date: start_date, end_date: end_date) + end + + # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, + # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + subset_powers_size_even = subset_powers_data.size.even? + + # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => + sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) + + # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| + sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_events : -power_of_union_of_all_events) + end + + def sum_subset_powers(subset_powers_data, subset_powers_size_even) + sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| + (index + 1).odd? ? value : -value + end + + (subset_powers_size_even ? -1 : 1) * sum_without_sign + end + + def subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) + subset_sizes = (1..(event_names.size - 1)) + + subset_sizes.map do |subset_size| + if subset_size > 1 + # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) + event_names.combination(subset_size).sum do |events_subset| + subset_powers_cache[subset_size][events_subset.join('_&_')] ||= \ + calculate_events_intersections(event_names: events_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache) + end + else + # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... + event_names.sum do |event| + subset_powers_cache[subset_size][event] ||= \ + unique_events(event_names: event, start_date: start_date, end_date: end_date) + end + end + end + end + + def calculate_events_union(event_names:, start_date:, end_date:) + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| + raise SlotMismatch, events unless events_in_same_slot?(events) + raise AggregationMismatch, events unless events_same_aggregation?(events) + end + end + + def count_unique_events(event_names:, start_date:, end_date:, context: '') + events = events_for(Array(event_names).map(&:to_s)) + + yield events if block_given? + + aggregation = events.first[:aggregation] + + keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context) + redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } + end + # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level # and if there are more than 1 event def eligible_for_totals?(events_names) @@ -100,16 +235,22 @@ module Gitlab events_in_same_slot?(events) && events_in_same_category?(events) && events_same_aggregation?(events) end - def keys_for_aggregation(aggregation, events:, start_date:, end_date:) + def keys_for_aggregation(aggregation, events:, start_date:, end_date:, context: '') if aggregation.to_sym == :daily - daily_redis_keys(events: events, start_date: start_date, end_date: end_date) + daily_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) else - weekly_redis_keys(events: events, start_date: start_date, end_date: end_date) + weekly_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) end end - def known_events - @known_events ||= YAML.load_file(Rails.root.join(KNOWN_EVENTS_PATH)).map(&:with_indifferent_access) + def load_events(wildcard) + Dir[wildcard].each_with_object([]) do |path, events| + events.push(*load_yaml_from_path(path)) + end + end + + def load_yaml_from_path(path) + YAML.safe_load(File.read(path))&.map(&:with_indifferent_access) end def known_events_names @@ -141,7 +282,7 @@ module Gitlab end def event_for(event_name) - known_events.find { |event| event[:name] == event_name } + known_events.find { |event| event[:name] == event_name.to_s } end def events_for(event_names) @@ -153,17 +294,26 @@ module Gitlab end # Compose the key in order to store events daily or weekly - def redis_key(event, time) + def redis_key(event, time, context = '') raise UnknownEvent.new("Unknown event #{event[:name]}") unless known_events_names.include?(event[:name].to_s) raise UnknownAggregation.new("Use :daily or :weekly aggregation") unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) + key = apply_slot(event) + key = apply_time_aggregation(key, time, event) + key = "#{context}_#{key}" if context.present? + key + end + + def apply_slot(event) slot = redis_slot(event) - key = if slot.present? - event[:name].to_s.gsub(slot, "{#{slot}}") - else - "{#{event[:name]}}" - end + if slot.present? + event[:name].to_s.gsub(slot, "{#{slot}}") + else + "{#{event[:name]}}" + end + end + def apply_time_aggregation(key, time, event) if event[:aggregation].to_sym == :daily year_day = time.strftime('%G-%j') "#{year_day}-#{key}" @@ -173,21 +323,29 @@ module Gitlab end end - def daily_redis_keys(events:, start_date:, end_date:) + def daily_redis_keys(events:, start_date:, end_date:, context: '') (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date) } + events.map { |event| redis_key(event, date, context) } end.flatten end - def weekly_redis_keys(events:, start_date:, end_date:) + def validate_aggregation_operator!(operator) + return true if ALLOWED_METRICS_AGGREGATIONS.include?(operator) + + raise UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}") + end + + def weekly_redis_keys(events:, start_date:, end_date:, context: '') weeks = end_date.to_date.cweek - start_date.to_date.cweek weeks = 1 if weeks == 0 (0..(weeks - 1)).map do |week_increment| - events.map { |event| redis_key(event, start_date + week_increment * 7.days) } + events.map { |event| redis_key(event, start_date + week_increment * 7.days, context) } end.flatten end end end end end + +Gitlab::UsageDataCounters::HLLRedisCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::HLLRedisCounter') diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index e8839875109..da013a06777 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -9,14 +9,12 @@ module Gitlab ISSUE_CREATED = 'g_project_management_issue_created' ISSUE_CLOSED = 'g_project_management_issue_closed' ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed' - ISSUE_ITERATION_CHANGED = 'g_project_management_issue_iteration_changed' ISSUE_LABEL_CHANGED = 'g_project_management_issue_label_changed' ISSUE_MADE_CONFIDENTIAL = 'g_project_management_issue_made_confidential' ISSUE_MADE_VISIBLE = 'g_project_management_issue_made_visible' ISSUE_MILESTONE_CHANGED = 'g_project_management_issue_milestone_changed' ISSUE_REOPENED = 'g_project_management_issue_reopened' ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed' - ISSUE_WEIGHT_CHANGED = 'g_project_management_issue_weight_changed' ISSUE_CROSS_REFERENCED = 'g_project_management_issue_cross_referenced' ISSUE_MOVED = 'g_project_management_issue_moved' ISSUE_RELATED = 'g_project_management_issue_related' @@ -24,15 +22,15 @@ module Gitlab ISSUE_MARKED_AS_DUPLICATE = 'g_project_management_issue_marked_as_duplicate' ISSUE_LOCKED = 'g_project_management_issue_locked' ISSUE_UNLOCKED = 'g_project_management_issue_unlocked' - ISSUE_ADDED_TO_EPIC = 'g_project_management_issue_added_to_epic' - ISSUE_REMOVED_FROM_EPIC = 'g_project_management_issue_removed_from_epic' - ISSUE_CHANGED_EPIC = 'g_project_management_issue_changed_epic' ISSUE_DESIGNS_ADDED = 'g_project_management_issue_designs_added' ISSUE_DESIGNS_MODIFIED = 'g_project_management_issue_designs_modified' ISSUE_DESIGNS_REMOVED = 'g_project_management_issue_designs_removed' ISSUE_DUE_DATE_CHANGED = 'g_project_management_issue_due_date_changed' ISSUE_TIME_ESTIMATE_CHANGED = 'g_project_management_issue_time_estimate_changed' ISSUE_TIME_SPENT_CHANGED = 'g_project_management_issue_time_spent_changed' + ISSUE_COMMENT_ADDED = 'g_project_management_issue_comment_added' + ISSUE_COMMENT_EDITED = 'g_project_management_issue_comment_edited' + ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed' class << self def track_issue_created_action(author:, time: Time.zone.now) @@ -75,14 +73,6 @@ module Gitlab track_unique_action(ISSUE_MILESTONE_CHANGED, author, time) end - def track_issue_iteration_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_ITERATION_CHANGED, author, time) - end - - def track_issue_weight_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_WEIGHT_CHANGED, author, time) - end - def track_issue_cross_referenced_action(author:, time: Time.zone.now) track_unique_action(ISSUE_CROSS_REFERENCED, author, time) end @@ -111,18 +101,6 @@ module Gitlab track_unique_action(ISSUE_UNLOCKED, author, time) end - def track_issue_added_to_epic_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_ADDED_TO_EPIC, author, time) - end - - def track_issue_removed_from_epic_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_REMOVED_FROM_EPIC, author, time) - end - - def track_issue_changed_epic_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_CHANGED_EPIC, author, time) - end - def track_issue_designs_added_action(author:, time: Time.zone.now) track_unique_action(ISSUE_DESIGNS_ADDED, author, time) end @@ -147,6 +125,18 @@ module Gitlab track_unique_action(ISSUE_TIME_SPENT_CHANGED, author, time) end + def track_issue_comment_added_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_ADDED, author, time) + end + + def track_issue_comment_edited_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_EDITED, author, time) + end + + def track_issue_comment_removed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_REMOVED, author, time) + end + private def track_unique_action(action, author, time) @@ -159,3 +149,5 @@ module Gitlab end end end + +Gitlab::UsageDataCounters::IssueActivityUniqueCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::IssueActivityUniqueCounter') diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index bc56c5d6d9b..85f16ea807b 100644 --- a/lib/gitlab/usage_data_counters/known_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -4,114 +4,141 @@ redis_slot: compliance category: compliance aggregation: weekly + feature_flag: track_unique_visits - name: g_compliance_audit_events category: compliance redis_slot: compliance aggregation: weekly + feature_flag: track_unique_visits - name: i_compliance_audit_events category: compliance redis_slot: compliance aggregation: weekly + feature_flag: track_unique_visits - name: i_compliance_credential_inventory category: compliance redis_slot: compliance aggregation: weekly + feature_flag: track_unique_visits - name: a_compliance_audit_events_api category: compliance redis_slot: compliance aggregation: weekly + feature_flag: usage_data_a_compliance_audit_events_api # Analytics category - name: g_analytics_contribution category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_insights category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_issues category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_productivity category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_valuestream category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_pipelines category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_code_reviews category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_valuestream category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_insights category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_issues category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_repo category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: i_analytics_cohorts category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: i_analytics_dev_ops_score category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_merge_request category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_merge_request category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: i_analytics_instance_statistics category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_edit_by_web_ide category: ide_edit redis_slot: edit expiry: 29 aggregation: daily + feature_flag: track_editor_edit_actions - name: g_edit_by_sfe category: ide_edit redis_slot: edit expiry: 29 aggregation: daily + feature_flag: track_editor_edit_actions - name: g_edit_by_snippet_ide category: ide_edit redis_slot: edit expiry: 29 aggregation: daily + feature_flag: track_editor_edit_actions - name: i_search_total category: search redis_slot: search aggregation: weekly + feature_flag: search_track_unique_users - name: i_search_advanced category: search redis_slot: search aggregation: weekly + feature_flag: search_track_unique_users - name: i_search_paid category: search redis_slot: search aggregation: weekly + feature_flag: search_track_unique_users - name: wiki_action category: source_code aggregation: daily @@ -121,6 +148,9 @@ - name: project_action category: source_code aggregation: daily +- name: git_write_action + category: source_code + aggregation: daily - name: merge_request_action category: source_code aggregation: daily @@ -133,173 +163,242 @@ redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_alert_status_changed - name: incident_management_alert_assigned redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_alert_assigned - name: incident_management_alert_todo redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_alert_todo - name: incident_management_incident_created redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_created - name: incident_management_incident_reopened redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_reopened - name: incident_management_incident_closed redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_closed - name: incident_management_incident_assigned redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_assigned - name: incident_management_incident_todo redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_todo - name: incident_management_incident_comment redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_comment - name: incident_management_incident_zoom_meeting redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_zoom_meeting - name: incident_management_incident_published redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_published - name: incident_management_incident_relate redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_relate - name: incident_management_incident_unrelate redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_unrelate - name: incident_management_incident_change_confidential redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_change_confidential # Testing category - name: i_testing_test_case_parsed category: testing redis_slot: testing aggregation: weekly + feature_flag: usage_data_i_testing_test_case_parsed # Project Management group - name: g_project_management_issue_title_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_description_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_assignee_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_made_confidential category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_made_visible category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_created category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_closed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_reopened category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_label_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_milestone_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_iteration_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_weight_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_cross_referenced category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_moved category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_related category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_unrelated category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_marked_as_duplicate category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_locked category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_unlocked category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_added_to_epic category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_removed_from_epic category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_changed_epic category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_added category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_modified category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_removed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_due_date_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_time_estimate_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_time_spent_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_comment_added + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_comment_edited + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_comment_removed + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_health_status_changed + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +# Secrets Management +- name: i_ci_secrets_management_vault_build_created + category: ci_secrets_management + redis_slot: ci_secrets_management + aggregation: weekly + feature_flag: usage_data_i_ci_secrets_management_vault_build_created diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml new file mode 100644 index 00000000000..7ed02aa2a85 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -0,0 +1,265 @@ +--- +- name: i_package_maven_user_push + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_deploy_token_push + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_user_delete + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_deploy_token_delete + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_user_pull + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_deploy_token_pull + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_user_push + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_deploy_token_push + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_user_delete + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_deploy_token_delete + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_user_pull + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_deploy_token_pull + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_user_push + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_deploy_token_push + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_user_delete + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_deploy_token_delete + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_user_pull + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_deploy_token_pull + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_user_push + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_deploy_token_push + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_user_delete + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_deploy_token_delete + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_user_pull + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_deploy_token_pull + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_user_push + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_deploy_token_push + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_user_delete + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_deploy_token_delete + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_user_pull + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_deploy_token_pull + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_user_push + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_deploy_token_push + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_user_delete + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_deploy_token_delete + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_user_pull + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_deploy_token_pull + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_user_push + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_deploy_token_push + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_user_delete + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_deploy_token_delete + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_user_pull + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_deploy_token_pull + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_user_push + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_deploy_token_push + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_user_delete + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_deploy_token_delete + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_user_pull + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_deploy_token_pull + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_user_push + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_deploy_token_push + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_user_delete + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_deploy_token_delete + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_user_pull + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_deploy_token_pull + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_user_push + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_deploy_token_push + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_user_delete + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_deploy_token_delete + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_user_pull + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_deploy_token_pull + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_user_push + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_deploy_token_push + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_user_delete + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_deploy_token_delete + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_user_pull + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_deploy_token_pull + category: tag_packages + aggregation: weekly + redis_slot: package diff --git a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb index 8886a106da8..3c5989d1e11 100644 --- a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb +++ b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb @@ -3,7 +3,7 @@ module Gitlab module UsageDataCounters class StaticSiteEditorCounter < BaseCounter - KNOWN_EVENTS = %w[views].freeze + KNOWN_EVENTS = %w[views commits merge_requests].freeze PREFIX = 'static_site_editor' class << self diff --git a/lib/gitlab/usage_data_counters/track_unique_events.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb index 7053744b665..95380ae0b1d 100644 --- a/lib/gitlab/usage_data_counters/track_unique_events.rb +++ b/lib/gitlab/usage_data_counters/track_unique_events.rb @@ -8,6 +8,9 @@ module Gitlab PUSH_ACTION = :project_action MERGE_REQUEST_ACTION = :merge_request_action + GIT_WRITE_ACTIONS = [WIKI_ACTION, DESIGN_ACTION, PUSH_ACTION].freeze + GIT_WRITE_ACTION = :git_write_action + ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({ wiki: { created: WIKI_ACTION, @@ -41,6 +44,8 @@ module Gitlab return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, transformed_action.to_s, time) + + track_git_write_action(author_id, transformed_action, time) end def count_unique_events(event_action:, date_from:, date_to:) @@ -64,6 +69,12 @@ module Gitlab def valid_action?(action) Event.actions.key?(action) end + + def track_git_write_action(author_id, transformed_action, time) + return unless GIT_WRITE_ACTIONS.include?(transformed_action) + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, GIT_WRITE_ACTION, time) + end end end end diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb index 00fcd42a9af..9f2f4ac3971 100644 --- a/lib/gitlab/usage_data_counters/web_ide_counter.rb +++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb @@ -2,54 +2,43 @@ module Gitlab module UsageDataCounters - class WebIdeCounter - extend RedisCounter - KNOWN_EVENTS = %i[commits views merge_requests previews terminals pipelines].freeze + class WebIdeCounter < BaseCounter + KNOWN_EVENTS = %w[commits views merge_requests previews terminals pipelines].freeze PREFIX = 'web_ide' class << self def increment_commits_count - increment(redis_key('commits')) + count('commits') end def increment_merge_requests_count - increment(redis_key('merge_requests')) + count('merge_requests') end def increment_views_count - increment(redis_key('views')) + count('views') end def increment_terminals_count - increment(redis_key('terminals')) + count('terminals') end def increment_pipelines_count - increment(redis_key('pipelines')) + count('pipelines') end def increment_previews_count return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? - increment(redis_key('previews')) - end - - def totals - KNOWN_EVENTS.map { |event| [counter_key(event), total_count(redis_key(event))] }.to_h - end - - def fallback_totals - KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h + count('previews') end private def redis_key(event) - "#{PREFIX}_#{event}_count".upcase - end + require_known_event(event) - def counter_key(event) - "#{PREFIX}_#{event}".to_sym + "#{prefix}_#{event}_count".upcase end end end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 1c6ddc2e70f..eec89e1ab72 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -90,9 +90,13 @@ module Gitlab def can_collaborate?(ref) assert_project! + can_push? || branch_allows_collaboration_for?(ref) + end + + def branch_allows_collaboration_for?(ref) # Checking for an internal project or group to prevent an infinite loop: # https://gitlab.com/gitlab-org/gitlab/issues/36805 - can_push? || (!project.internal? && project.branch_allows_collaboration?(user, ref)) + (!project.internal? && project.branch_allows_collaboration?(user, ref)) end def permission_cache diff --git a/lib/gitlab/webpack/dev_server_middleware.rb b/lib/gitlab/webpack/dev_server_middleware.rb index 069e68e8d29..88f2a4455c6 100644 --- a/lib/gitlab/webpack/dev_server_middleware.rb +++ b/lib/gitlab/webpack/dev_server_middleware.rb @@ -16,14 +16,14 @@ module Gitlab super(app, backend: "#{@proxy_scheme}://#{@proxy_host}:#{@proxy_port}", **opts) end - # disable SSL check since any cert used here will likely be self-signed - def rewrite_env(env) - env["rack.ssl_verify_none"] = true - env - end - def perform_request(env) if @proxy_path && env['PATH_INFO'].start_with?("/#{@proxy_path}") + # disable SSL check since any cert used here will likely be self-signed + env['rack.ssl_verify_none'] = true + + # ensure we pass the expected Host header so webpack-dev-server doesn't complain + env['HTTP_HOST'] = "#{@proxy_host}:#{@proxy_port}" + if relative_url_root = Rails.application.config.relative_url_root env['SCRIPT_NAME'] = "" env['REQUEST_PATH'].sub!(/\A#{Regexp.escape(relative_url_root)}/, '') diff --git a/lib/gitlab/whats_new.rb b/lib/gitlab/whats_new.rb index e95ace2c475..69ccb48c544 100644 --- a/lib/gitlab/whats_new.rb +++ b/lib/gitlab/whats_new.rb @@ -2,27 +2,39 @@ module Gitlab module WhatsNew - CACHE_DURATION = 1.day + CACHE_DURATION = 1.hour WHATS_NEW_FILES_PATH = Rails.root.join('data', 'whats_new', '*.yml') private - def whats_new_most_recent_release_items - Rails.cache.fetch('whats_new:release_items', expires_in: CACHE_DURATION) do - file = File.read(most_recent_release_file_path) + def whats_new_release_items(page: 1) + Rails.cache.fetch(whats_new_items_cache_key(page), expires_in: CACHE_DURATION) do + index = page - 1 + file_path = whats_new_file_paths[index] + + next if file_path.nil? + + file = File.read(file_path) items = YAML.safe_load(file, permitted_classes: [Date]) items if items.is_a?(Array) end rescue => e - Gitlab::ErrorTracking.track_exception(e, yaml_file_path: most_recent_release_file_path) + Gitlab::ErrorTracking.track_exception(e, page: page) nil end - def most_recent_release_file_path - @most_recent_release_file_path ||= Dir.glob(WHATS_NEW_FILES_PATH).max + def whats_new_file_paths + @whats_new_file_paths ||= Rails.cache.fetch('whats_new:file_paths', expires_in: CACHE_DURATION) do + Dir.glob(WHATS_NEW_FILES_PATH).sort.reverse + end + end + + def whats_new_items_cache_key(page) + filename = /\d*\_\d*\_\d*/.match(whats_new_file_paths&.first) + "whats_new:release_items:file-#{filename}:page-#{page}" end end end diff --git a/lib/gitlab/with_feature_category.rb b/lib/gitlab/with_feature_category.rb new file mode 100644 index 00000000000..65d21daf78a --- /dev/null +++ b/lib/gitlab/with_feature_category.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module WithFeatureCategory + extend ActiveSupport::Concern + include Gitlab::ClassAttributes + + class_methods do + def feature_category(category, actions = []) + feature_category_configuration[category] ||= [] + feature_category_configuration[category] += actions.map(&:to_s) + + validate_config!(feature_category_configuration) + end + + def feature_category_for_action(action) + category_config = feature_category_configuration.find do |_, actions| + actions.empty? || actions.include?(action) + end + + category_config&.first || superclass_feature_category_for_action(action) + end + + private + + def validate_config!(config) + empty = config.find { |_, actions| actions.empty? } + duplicate_actions = config.values.map(&:uniq).flatten.group_by(&:itself).select { |_, v| v.count > 1 }.keys + + if config.length > 1 && empty + raise ArgumentError, "#{empty.first} is defined for all actions, but other categories are set" + end + + if duplicate_actions.any? + raise ArgumentError, "Actions have multiple feature categories: #{duplicate_actions.join(', ')}" + end + end + + def feature_category_configuration + class_attributes[:feature_category_config] ||= {} + end + + def superclass_feature_category_for_action(action) + return unless superclass.respond_to?(:feature_category_for_action) + + superclass.feature_category_for_action(action) + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index eb780a2f7f6..8e7af8876a4 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -270,7 +270,7 @@ module Gitlab prefix: metadata['ArchivePrefix'], format: format, path: path.presence || "", - include_lfs_blobs: Feature.enabled?(:include_lfs_blobs_in_archive) + include_lfs_blobs: Feature.enabled?(:include_lfs_blobs_in_archive, default_enabled: true) ).to_proto ) } |