diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 09:55:51 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 09:55:51 +0000 |
commit | e8d2c2579383897a1dd7f9debd359abe8ae8373d (patch) | |
tree | c42be41678c2586d49a75cabce89322082698334 /lib/gitlab | |
parent | fc845b37ec3a90aaa719975f607740c22ba6a113 (diff) | |
download | gitlab-ce-e8d2c2579383897a1dd7f9debd359abe8ae8373d.tar.gz |
Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42
Diffstat (limited to 'lib/gitlab')
295 files changed, 3246 insertions, 1362 deletions
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index 530e53f9d10..8eb067ed0ec 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -31,6 +31,10 @@ module Gitlab raise NotImplementedError end + def hash_code + Digest::SHA256.hexdigest(self.class.identifier.to_s) + end + # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query. # Example: get me all the Issue records between start event end end event def timestamp_projection diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb index 723486231b1..3546a7e3ddb 100644 --- a/lib/gitlab/analytics/unique_visits.rb +++ b/lib/gitlab/analytics/unique_visits.rb @@ -3,10 +3,6 @@ module Gitlab module Analytics class UniqueVisits - def track_visit(*args, **kwargs) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(*args, **kwargs) - end - # Returns number of unique visitors for given targets in given time frame # # @param [String, Array[<String>]] targets ids of targets to count visits on. Special case for :any diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 580c7042f1e..8cab2f65726 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -156,10 +156,10 @@ module Gitlab underscored_service = matched_login['service'].underscore - return unless Integration.available_services_names.include?(underscored_service) + return unless Integration.available_integration_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included - # in the Integration.available_services_names allowlist. + # in the Integration.available_integration_names allowlist. accessor = Project.integration_association_name(underscored_service) service = project.public_send(accessor) # rubocop:disable GitlabSecurity/PublicSend diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 416e36c7ccb..0796f23fbfe 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -89,9 +89,11 @@ module Gitlab job.user end - # We only allow Private Access Tokens with `api` scope to be used by web + # We allow Private Access Tokens with `api` scope to be used by web # requests on RSS feeds or ICS files for backwards compatibility. # It is also used by GraphQL/API requests. + # And to allow accessing /archive programatically as it was a big pain point + # for users https://gitlab.com/gitlab-org/gitlab/-/issues/28978. def find_user_from_web_access_token(request_format, scopes: [:api]) return unless access_token && valid_web_access_format?(request_format) @@ -269,6 +271,8 @@ module Gitlab ics_request? when :api api_request? + when :archive + archive_request? if Feature.enabled?(:allow_archive_as_web_access_format, default_enabled: :yaml) end end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 3853709698b..47eca74aa5b 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -53,11 +53,7 @@ module Gitlab if results.nil? response = ldap.get_operation_result - - unless response.code == 0 - Gitlab::AppLogger.warn("LDAP search error: #{response.message}") - end - + check_empty_response_code(response) [] else results @@ -136,6 +132,16 @@ module Gitlab def renew_connection_adapter @ldap = Net::LDAP.new(config.adapter_options) end + + def check_empty_response_code(response) + if config.retry_empty_result_with_codes.include?(response.code) + raise Net::LDAP::Error, "Got empty results with response code: #{response.code}, message: #{response.message}" + end + + unless response.code == 0 + Gitlab::AppLogger.warn("LDAP search error: #{response.message}") + end + end end end end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 441f0d14b39..7bfe776fed0 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -163,6 +163,10 @@ module Gitlab options['timeout'].to_i end + def retry_empty_result_with_codes + options.fetch('retry_empty_result_with_codes', []) + end + def external_groups options['external_groups'] || [] end diff --git a/lib/gitlab/auth/u2f_webauthn_converter.rb b/lib/gitlab/auth/u2f_webauthn_converter.rb index f85b2248aeb..20b5d2ddc88 100644 --- a/lib/gitlab/auth/u2f_webauthn_converter.rb +++ b/lib/gitlab/auth/u2f_webauthn_converter.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'webauthn/u2f_migrator' + module Gitlab module Auth class U2fWebauthnConverter diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb new file mode 100644 index 00000000000..a0d0791b6af --- /dev/null +++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill draft column on open merge requests based on regex parsing of + # their titles. + # + class BackfillDraftStatusOnMergeRequests + # Migration only version of MergeRequest table + class MergeRequest < ActiveRecord::Base + include EachBatch + + self.table_name = 'merge_requests' + + def self.eligible + where(state_id: 1) + .where(draft: false) + .where("title ~* ?", '^\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP') + end + end + + def perform(start_id, end_id) + eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id) + + return if eligible_mrs.empty? + + eligible_mrs.each_slice(10) do |slice| + MergeRequest.where(id: slice).update_all(draft: true) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb b/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb new file mode 100644 index 00000000000..170af90805a --- /dev/null +++ b/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will populate the upvotes_count field + # for each issue + class BackfillUpvotesCountOnIssues + BATCH_SIZE = 1_000 + + def perform(start_id, stop_id) + (start_id..stop_id).step(BATCH_SIZE).each do |offset| + update_issue_upvotes_count(offset, offset + BATCH_SIZE) + end + end + + private + + def execute(sql) + @connection ||= ::ActiveRecord::Base.connection + @connection.execute(sql) + end + + def update_issue_upvotes_count(batch_start, batch_stop) + execute(<<~SQL) + UPDATE issues + SET upvotes_count = sub_q.count_all + FROM ( + SELECT COUNT(*) AS count_all, e.awardable_id AS issue_id + FROM award_emoji AS e + WHERE e.name = 'thumbsup' AND + e.awardable_type = 'Issue' AND + e.awardable_id BETWEEN #{batch_start} AND #{batch_stop} + GROUP BY issue_id + ) AS sub_q + WHERE sub_q.issue_id = issues.id; + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_orphaned_deployments.rb b/lib/gitlab/background_migration/delete_orphaned_deployments.rb new file mode 100644 index 00000000000..9ac4111ff0f --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_deployments.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for deleting orphaned deployments. + class DeleteOrphanedDeployments + include Database::MigrationHelpers + + def perform(start_id, end_id) + orphaned_deployments + .where(id: start_id..end_id) + .delete_all + + mark_job_as_succeeded(start_id, end_id) + end + + def orphaned_deployments + define_batchable_model('deployments') + .where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb new file mode 100644 index 00000000000..e694e5359cd --- /dev/null +++ b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb @@ -0,0 +1,287 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates author and committer names and emails from + # merge_request_diff_commits to two columns that point to + # merge_request_diff_commit_users. + # + # rubocop: disable Metrics/ClassLength + class MigrateMergeRequestDiffCommitUsers + # The number of user rows in merge_request_diff_commit_users to get in a + # single query. + USER_ROWS_PER_QUERY = 1_000 + + # The number of rows in merge_request_diff_commits to get in a single + # query. + COMMIT_ROWS_PER_QUERY = 10_000 + + # The number of rows in merge_request_diff_commits to update in a single + # query. + # + # Tests in staging revealed that increasing the number of updates per + # query translates to a longer total runtime for a migration. For example, + # given the same range of rows to migrate, 1000 updates per query required + # a total of roughly 15 seconds. On the other hand, 5000 updates per query + # required a total of roughly 25 seconds. For this reason, we use a value + # of 1000 rows per update. + UPDATES_PER_QUERY = 1_000 + + # rubocop: disable Style/Documentation + class MergeRequestDiffCommit < ActiveRecord::Base + include FromUnion + extend ::SuppressCompositePrimaryKeyWarning + + self.table_name = 'merge_request_diff_commits' + + # Yields each row to migrate in the given range. + # + # This method uses keyset pagination to ensure we don't retrieve + # potentially tens of thousands (or even hundreds of thousands) of rows + # in a single query. Such queries could time out, or increase the amount + # of memory needed to process the data. + # + # We can't use `EachBatch` and similar approaches, as + # merge_request_diff_commits doesn't have a single monotonically + # increasing primary key. + def self.each_row_to_migrate(start_id, stop_id, &block) + order = Pagination::Keyset::Order.build( + %w[merge_request_diff_id relative_order].map do |col| + Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: col, + order_expression: self.arel_table[col.to_sym].asc, + nullable: :not_nullable, + distinct: false + ) + end + ) + + scope = MergeRequestDiffCommit + .where(merge_request_diff_id: start_id...stop_id) + .order(order) + + Pagination::Keyset::Iterator + .new(scope: scope, use_union_optimization: true) + .each_batch(of: COMMIT_ROWS_PER_QUERY) { |rows| rows.each(&block) } + end + end + # rubocop: enable Style/Documentation + + # rubocop: disable Style/Documentation + class MergeRequestDiffCommitUser < ActiveRecord::Base + self.table_name = 'merge_request_diff_commit_users' + + def self.union(queries) + from("(#{queries.join("\nUNION ALL\n")}) #{table_name}") + end + end + # rubocop: enable Style/Documentation + + def perform(start_id, stop_id) + # This Hash maps user names + emails to their corresponding rows in + # merge_request_diff_commit_users. + user_mapping = {} + + user_details, diff_rows_to_update = get_data_to_update(start_id, stop_id) + + get_user_rows_in_batches(user_details, user_mapping) + create_missing_users(user_details, user_mapping) + update_commit_rows(diff_rows_to_update, user_mapping) + + Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'MigrateMergeRequestDiffCommitUsers', + [start_id, stop_id] + ) + end + + # Returns the data we'll use to determine what merge_request_diff_commits + # rows to update, and what data to use for populating their + # commit_author_id and committer_id columns. + def get_data_to_update(start_id, stop_id) + # This Set is used to retrieve users that already exist in + # merge_request_diff_commit_users. + users = Set.new + + # This Hash maps the primary key of every row in + # merge_request_diff_commits to the (trimmed) author and committer + # details to use for updating the row. + to_update = {} + + MergeRequestDiffCommit.each_row_to_migrate(start_id, stop_id) do |row| + author = [prepare(row.author_name), prepare(row.author_email)] + committer = [prepare(row.committer_name), prepare(row.committer_email)] + + to_update[[row.merge_request_diff_id, row.relative_order]] = + [author, committer] + + users << author if author[0] || author[1] + users << committer if committer[0] || committer[1] + end + + [users, to_update] + end + + # Gets any existing rows in merge_request_diff_commit_users in batches. + # + # This method may end up having to retrieve lots of rows. To reduce the + # overhead, we batch queries into a UNION query. We limit the number of + # queries per UNION so we don't end up sending a single query containing + # too many SELECT statements. + def get_user_rows_in_batches(users, user_mapping) + users.each_slice(USER_ROWS_PER_QUERY) do |pairs| + queries = pairs.map do |(name, email)| + MergeRequestDiffCommitUser.where(name: name, email: email).to_sql + end + + MergeRequestDiffCommitUser.union(queries).each do |row| + user_mapping[[row.name.to_s, row.email.to_s]] = row + end + end + end + + # Creates any users for which no row exists in + # merge_request_diff_commit_users. + # + # Not all users queried may exist yet, so we need to create any missing + # ones; making sure we handle concurrent creations of the same user + def create_missing_users(users, mapping) + create = [] + + users.each do |(name, email)| + create << { name: name, email: email } unless mapping[[name, email]] + end + + return if create.empty? + + MergeRequestDiffCommitUser + .insert_all(create, returning: %w[id name email]) + .each do |row| + mapping[[row['name'], row['email']]] = MergeRequestDiffCommitUser + .new(id: row['id'], name: row['name'], email: row['email']) + end + + # It's possible for (name, email) pairs to be inserted concurrently, + # resulting in the above insert not returning anything. Here we get any + # remaining users that were created concurrently. + get_user_rows_in_batches( + users.reject { |pair| mapping.key?(pair) }, + mapping + ) + end + + # Updates rows in merge_request_diff_commits with their new + # commit_author_id and committer_id values. + def update_commit_rows(to_update, user_mapping) + to_update.each_slice(UPDATES_PER_QUERY) do |slice| + updates = {} + + slice.each do |(diff_id, order), (author, committer)| + author_id = user_mapping[author]&.id + committer_id = user_mapping[committer]&.id + + updates[[diff_id, order]] = [author_id, committer_id] + end + + bulk_update_commit_rows(updates) + end + end + + # Bulk updates rows in the merge_request_diff_commits table with their new + # author and/or committer ID values. + # + # Updates are batched together to reduce the overhead of having to produce + # a single UPDATE for every row, as we may end up having to update + # thousands of rows at once. + # + # The query produced by this method is along the lines of the following: + # + # UPDATE merge_request_diff_commits + # SET commit_author_id = + # CASE + # WHEN (merge_request_diff_id, relative_order) = (x, y) THEN X + # WHEN ... + # END, + # committer_id = + # CASE + # WHEN (merge_request_diff_id, relative_order) = (x, y) THEN Y + # WHEN ... + # END + # WHERE (merge_request_diff_id, relative_order) IN ( (x, y), ... ) + # + # The `mapping` argument is a Hash in the following format: + # + # { [merge_request_diff_id, relative_order] => [author_id, committer_id] } + # + # rubocop: disable Metrics/AbcSize + def bulk_update_commit_rows(mapping) + author_case = Arel::Nodes::Case.new + committer_case = Arel::Nodes::Case.new + primary_values = [] + + mapping.each do |diff_id_and_order, (author_id, committer_id)| + primary_value = Arel::Nodes::Grouping.new(diff_id_and_order) + + primary_values << primary_value + + if author_id + author_case.when(primary_key.eq(primary_value)).then(author_id) + end + + if committer_id + committer_case.when(primary_key.eq(primary_value)).then(committer_id) + end + end + + if author_case.conditions.empty? && committer_case.conditions.empty? + return + end + + fields = [] + + # Statements such as `SET x = CASE END` are not valid SQL statements, so + # we omit setting an ID field if there are no values to populate it + # with. + if author_case.conditions.any? + fields << [arel_table[:commit_author_id], author_case] + end + + if committer_case.conditions.any? + fields << [arel_table[:committer_id], committer_case] + end + + query = Arel::UpdateManager.new + .table(arel_table) + .where(primary_key.in(primary_values)) + .set(fields) + .to_sql + + MergeRequestDiffCommit.connection.execute(query) + end + # rubocop: enable Metrics/AbcSize + + def primary_key + Arel::Nodes::Grouping.new( + [arel_table[:merge_request_diff_id], arel_table[:relative_order]] + ) + end + + def arel_table + MergeRequestDiffCommit.arel_table + end + + # Prepares a value to be inserted into a column in the table + # `merge_request_diff_commit_users`. Values in this table are limited to + # 512 characters. + # + # We treat empty strings as NULL values, as there's no point in (for + # example) storing a row where both the name and Email are an empty + # string. In addition, if we treated them differently we could end up with + # two rows: one where field X is NULL, and one where field X is an empty + # string. This is redundant, so we avoid storing such data. + def prepare(value) + value.present? ? value[0..511] : nil + end + end + # rubocop: enable Metrics/ClassLength + end +end diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb index 091e6660bac..83aa36a11e6 100644 --- a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb +++ b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true # rubocop:disable Style/Documentation -require "webauthn/u2f_migrator" module Gitlab module BackgroundMigration diff --git a/lib/gitlab/background_migration/populate_latest_pipeline_ids.rb b/lib/gitlab/background_migration/populate_latest_pipeline_ids.rb new file mode 100644 index 00000000000..a2b25a293fe --- /dev/null +++ b/lib/gitlab/background_migration/populate_latest_pipeline_ids.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +module Gitlab + module BackgroundMigration + class PopulateLatestPipelineIds + class ProjectSetting < ActiveRecord::Base + include EachBatch + + self.table_name = 'project_settings' + + scope :in_range, -> (start_id, end_id) { where(id: start_id..end_id) } + scope :has_vulnerabilities_without_latest_pipeline_set, -> do + joins('LEFT OUTER JOIN vulnerability_statistics vs ON vs.project_id = project_settings.project_id') + .where(vs: { latest_pipeline_id: nil }) + .where('has_vulnerabilities IS TRUE') + end + end + + def perform(start_id, end_id) + # no-op + end + end + end +end + +Gitlab::BackgroundMigration::PopulateLatestPipelineIds.prepend_mod diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb index 7da933c7b11..4026a91903f 100644 --- a/lib/gitlab/background_migration/user_mentions/models/note.rb +++ b/lib/gitlab/background_migration/user_mentions/models/note.rb @@ -21,7 +21,7 @@ module Gitlab belongs_to :project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" def for_personal_snippet? - noteable && noteable.class.name == 'PersonalSnippet' + noteable && noteable.instance_of?(PersonalSnippet) end def for_project_noteable? diff --git a/lib/gitlab/backup_logger.rb b/lib/gitlab/backup_logger.rb new file mode 100644 index 00000000000..fad36b860ae --- /dev/null +++ b/lib/gitlab/backup_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class BackupLogger < Gitlab::JsonLogger + def self.file_name_noext + 'backup_json' + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index e59494c9d9c..f79baadb8ea 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -63,10 +63,7 @@ module Gitlab return users[username] if users.key?(username) - users[username] = User.select(:id) - .joins(:identities) - .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) - .try(:id) + users[username] = User.by_provider_and_extern_uid(:bitbucket, username).select(:id).first&.id end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 9e958eb52fb..137f76bc96d 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -50,8 +50,6 @@ module Gitlab def load_status return if loaded? - return unless Gitlab::Ci::Features.pipeline_status_omit_commit_sha_in_cache_key?(project) || commit - if has_cache? load_from_cache else @@ -119,11 +117,7 @@ module Gitlab end def cache_key - if Gitlab::Ci::Features.pipeline_status_omit_commit_sha_in_cache_key?(project) - "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:project:#{project.id}:pipeline_status" - else - "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:project:#{project.id}:pipeline_status:#{commit&.sha}" - end + "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:project:#{project.id}:pipeline_status" end def commit diff --git a/lib/gitlab/cache/helpers.rb b/lib/gitlab/cache/helpers.rb new file mode 100644 index 00000000000..7b11d6bc9ff --- /dev/null +++ b/lib/gitlab/cache/helpers.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + module Helpers + # @return [ActiveSupport::Duration] + DEFAULT_EXPIRY = 1.day + + # @return [ActiveSupport::Cache::Store] + def cache + Rails.cache + end + + def render_cached(obj_or_collection, with:, cache_context: -> (_) { current_user&.cache_key }, expires_in: Gitlab::Cache::Helpers::DEFAULT_EXPIRY, **presenter_args) + json = + if obj_or_collection.is_a?(Enumerable) + cached_collection( + obj_or_collection, + presenter: with, + presenter_args: presenter_args, + context: cache_context, + expires_in: expires_in + ) + else + cached_object( + obj_or_collection, + presenter: with, + presenter_args: presenter_args, + context: cache_context, + expires_in: expires_in + ) + end + + render Gitlab::Json::PrecompiledJson.new(json) + end + + private + + # Optionally uses a `Proc` to add context to a cache key + # + # @param object [Object] must respond to #cache_key + # @param context [Proc] a proc that will be called with the object as an argument, and which should return a + # string or array of strings to be combined into the cache key + # @return [String] + def contextual_cache_key(presenter, object, context) + return object.cache_key if context.nil? + + [presenter.class.name, object.cache_key, context.call(object)].flatten.join(":") + end + + # Used for fetching or rendering a single object + # + # @param object [Object] the object to render + # @param presenter [Grape::Entity] + # @param presenter_args [Hash] keyword arguments to be passed to the entity + # @param context [Proc] + # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry + # @return [String] + def cached_object(object, presenter:, presenter_args:, context:, expires_in:) + cache.fetch(contextual_cache_key(presenter, object, context), expires_in: expires_in) do + Gitlab::Json.dump(presenter.represent(object, **presenter_args).as_json) + end + end + + # Used for fetching or rendering multiple objects + # + # @param objects [Enumerable<Object>] the objects to render + # @param presenter [Grape::Entity] + # @param presenter_args [Hash] keyword arguments to be passed to the entity + # @param context [Proc] + # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry + # @return [Array<String>] + def cached_collection(collection, presenter:, presenter_args:, context:, expires_in:) + json = fetch_multi(presenter, collection, context: context, expires_in: expires_in) do |obj| + Gitlab::Json.dump(presenter.represent(obj, **presenter_args).as_json) + end + + json.values + end + + # An adapted version of ActiveSupport::Cache::Store#fetch_multi. + # + # The original method only provides the missing key to the block, + # not the missing object, so we have to create a map of cache keys + # to the objects to allow us to pass the object to the missing value + # block. + # + # The result is that this is functionally identical to `#fetch`. + def fetch_multi(presenter, *objs, context:, **kwargs) + objs.flatten! + map = multi_key_map(presenter, objs, context: context) + + # TODO: `contextual_cache_key` should be constructed based on the guideline https://docs.gitlab.com/ee/development/redis.html#multi-key-commands. + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + cache.fetch_multi(*map.keys, **kwargs) do |key| + yield map[key] + end + end + end + + # @param objects [Enumerable<Object>] objects which _must_ respond to `#cache_key` + # @param context [Proc] a proc that can be called to help generate each cache key + # @return [Hash] + def multi_key_map(presenter, objects, context:) + objects.index_by do |object| + contextual_cache_key(presenter, object, context) + end + end + end + end +end diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 86441973941..4cbc0231bce 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -173,6 +173,34 @@ module Gitlab val ? true : false end + # Adds a value to a hash. + # + # raw_key - The key of the hash to add to. + # field - The field to add to the hash. + # value - The field value to add to the hash. + # timeout - The new timeout of the key. + def self.hash_add(raw_key, field, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.multi do |m| + m.hset(key, field, value) + m.expire(key, timeout) + end + end + end + + # Returns the values of the given hash. + # + # raw_key - The key of the set to check. + def self.values_from_hash(raw_key) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.hgetall(key) + end + end + def self.cache_key_for(raw_key) "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}" end diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb index be8009750da..0538fe68474 100644 --- a/lib/gitlab/changelog/config.rb +++ b/lib/gitlab/changelog/config.rb @@ -52,7 +52,12 @@ module Gitlab end if (template = hash['template']) - config.template = Parser.new.parse_and_transform(template) + config.template = + begin + TemplateParser::Parser.new.parse_and_transform(template) + rescue TemplateParser::Error => e + raise Error, e.message + end end if (categories = hash['categories']) @@ -73,7 +78,12 @@ module Gitlab def initialize(project) @project = project @date_format = DEFAULT_DATE_FORMAT - @template = Parser.new.parse_and_transform(DEFAULT_TEMPLATE) + @template = + begin + TemplateParser::Parser.new.parse_and_transform(DEFAULT_TEMPLATE) + rescue TemplateParser::Error => e + raise Error, e.message + end @categories = {} @tag_regex = DEFAULT_TAG_REGEX end diff --git a/lib/gitlab/changelog/release.rb b/lib/gitlab/changelog/release.rb index f2a01c2b0dc..c0b6a5c5679 100644 --- a/lib/gitlab/changelog/release.rb +++ b/lib/gitlab/changelog/release.rb @@ -54,7 +54,7 @@ module Gitlab end def to_markdown - state = EvalState.new + state = TemplateParser::EvalState.new data = { 'categories' => entries_for_template } # While not critical, we would like release sections to be separated by @@ -63,7 +63,12 @@ module Gitlab # # Since it can be a bit tricky to get this right in a template, we # enforce an empty line separator ourselves. - markdown = @config.template.evaluate(state, data).strip + markdown = + begin + @config.template.evaluate(state, data).strip + rescue TemplateParser::ParseError => e + raise Error, e.message + end # The release header can't be changed using the Liquid template, as we # need this to be in a known format. Without this restriction, we won't diff --git a/lib/gitlab/checks/container_moved.rb b/lib/gitlab/checks/container_moved.rb new file mode 100644 index 00000000000..41180bbf246 --- /dev/null +++ b/lib/gitlab/checks/container_moved.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class ContainerMoved < PostPushMessage + REDIRECT_NAMESPACE = "redirect_namespace" + + def initialize(repository, user, protocol, redirected_path) + @redirected_path = redirected_path + + super(repository, user, protocol) + end + + def message + <<~MESSAGE + #{container.class.model_name.human} '#{redirected_path}' was moved to '#{container.full_path}'. + + Please update your Git remote: + + git remote set-url origin #{url_to_repo} + MESSAGE + end + + private + + attr_reader :redirected_path + + def self.message_key(user, repository) + "#{REDIRECT_NAMESPACE}:#{user.id}:#{repository.gl_repository}" + end + + # TODO: Remove in the next release + # https://gitlab.com/gitlab-org/gitlab/-/issues/292030 + def self.legacy_message_key(user, repository) + return unless repository.project + + "#{REDIRECT_NAMESPACE}:#{user.id}:#{repository.project.id}" + end + end + end +end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index 51013b69755..84069a1249b 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -7,9 +7,9 @@ module Gitlab ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' def validate! - # This feature flag is used for disabling integrify check on some envs + # This feature flag is used for disabling integrity check on some envs # because these costy calculations may cause performance issues - return unless Feature.enabled?(:lfs_check, default_enabled: true) + return unless Feature.enabled?(:lfs_check, project, default_enabled: :yaml) return unless project.lfs_enabled? diff --git a/lib/gitlab/checks/post_push_message.rb b/lib/gitlab/checks/post_push_message.rb index 0d93e7ac8a1..09407f45e54 100644 --- a/lib/gitlab/checks/post_push_message.rb +++ b/lib/gitlab/checks/post_push_message.rb @@ -9,21 +9,32 @@ module Gitlab @protocol = protocol end - def self.fetch_message(user_id, project_id) - key = message_key(user_id, project_id) + def self.fetch_message(user, repository) + key = message_key(user, repository) + + # Also check for messages in the legacy key + # TODO: Remove in the next release + # https://gitlab.com/gitlab-org/gitlab/-/issues/292030 + legacy_key = legacy_message_key(user, repository) if respond_to?(:legacy_message_key) Gitlab::Redis::SharedState.with do |redis| message = redis.get(key) redis.del(key) - message + + if legacy_key + legacy_message = redis.get(legacy_key) + redis.del(legacy_key) + end + + legacy_message || message end end def add_message - return unless user.present? && project.present? + return unless user && repository Gitlab::Redis::SharedState.with do |redis| - key = self.class.message_key(user.id, project.id) + key = self.class.message_key(user, repository) redis.setex(key, 5.minutes, message) end end @@ -39,7 +50,7 @@ module Gitlab delegate :project, to: :repository, allow_nil: true delegate :container, to: :repository, allow_nil: false - def self.message_key(user_id, project_id) + def self.message_key(user, repository) raise NotImplementedError end diff --git a/lib/gitlab/checks/project_created.rb b/lib/gitlab/checks/project_created.rb index 362562068e9..ea3be883be6 100644 --- a/lib/gitlab/checks/project_created.rb +++ b/lib/gitlab/checks/project_created.rb @@ -21,8 +21,16 @@ module Gitlab private - def self.message_key(user_id, project_id) - "#{PROJECT_CREATED}:#{user_id}:#{project_id}" + def self.message_key(user, repository) + "#{PROJECT_CREATED}:#{user.id}:#{repository.gl_repository}" + end + + # TODO: Remove in the next release + # https://gitlab.com/gitlab-org/gitlab/-/issues/292030 + def self.legacy_message_key(user, repository) + return unless repository.project + + "#{PROJECT_CREATED}:#{user.id}:#{repository.project.id}" end def project_url diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb deleted file mode 100644 index 4cc35de9c2d..00000000000 --- a/lib/gitlab/checks/project_moved.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Checks - class ProjectMoved < PostPushMessage - REDIRECT_NAMESPACE = "redirect_namespace" - - def initialize(repository, user, protocol, redirected_path) - @redirected_path = redirected_path - - super(repository, user, protocol) - end - - def message - <<~MESSAGE - Project '#{redirected_path}' was moved to '#{project.full_path}'. - - Please update your Git remote: - - git remote set-url origin #{url_to_repo} - MESSAGE - end - - private - - attr_reader :redirected_path - - def self.message_key(user_id, project_id) - "#{REDIRECT_NAMESPACE}:#{user_id}:#{project_id}" - end - end - end -end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 466706384c0..8f2d47e7ccc 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -76,8 +76,14 @@ module Gitlab @section_header = true end - def set_section_duration(duration) - @section_duration = Time.at(duration.to_i).utc.strftime('%M:%S') + def set_section_duration(duration_in_seconds) + duration = ActiveSupport::Duration.build(duration_in_seconds.to_i) + hours = duration.in_hours.floor + hours = hours > 0 ? "%02d" % hours : nil + minutes = "%02d" % duration.parts[:minutes].to_i + seconds = "%02d" % duration.parts[:seconds].to_i + + @section_duration = [hours, minutes, seconds].compact.join(':') end def flush_current_segment! diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 6118ff49928..56eeb5eeb06 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -36,8 +36,7 @@ module Gitlab }, if: :expose_as_present? validates :expose_as, type: String, length: { maximum: 100 }, if: :expose_as_present? validates :expose_as, format: { with: EXPOSE_AS_REGEX, message: EXPOSE_AS_ERROR_MESSAGE }, if: :expose_as_present? - validates :exclude, array_of_strings: true, if: :exclude_enabled? - validates :exclude, absence: { message: 'feature is disabled' }, unless: :exclude_enabled? + validates :exclude, array_of_strings: true validates :reports, type: Hash validates :when, inclusion: { in: %w[on_success on_failure always], @@ -60,10 +59,6 @@ module Gitlab !@config[:expose_as].nil? end - - def exclude_enabled? - ::Gitlab::Ci::Features.artifacts_exclude_enabled? - end end end end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 4db25fb0930..e45dbfa243f 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -15,7 +15,7 @@ module Gitlab %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance browser_performance load_performance license_scanning metrics lsif dotenv cobertura terraform accessibility cluster_applications - requirements coverage_fuzzing api_fuzzing].freeze + requirements coverage_fuzzing api_fuzzing cluster_image_scanning].freeze attributes ALLOWED_KEYS @@ -32,6 +32,7 @@ module Gitlab validates :secret_detection, array_of_strings_or_string: true validates :dependency_scanning, array_of_strings_or_string: true validates :container_scanning, array_of_strings_or_string: true + validates :cluster_image_scanning, array_of_strings_or_string: true validates :dast, array_of_strings_or_string: true validates :performance, array_of_strings_or_string: true validates :browser_performance, array_of_strings_or_string: true diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index fe69a170404..d26a903c1f8 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -6,18 +6,6 @@ module Gitlab # Ci::Features is a class that aggregates all CI/CD feature flags in one place. # module Features - def self.artifacts_exclude_enabled? - ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) - end - - def self.pipeline_latest? - ::Feature.enabled?(:ci_pipeline_latest, default_enabled: true) - end - - def self.pipeline_status_omit_commit_sha_in_cache_key?(project) - Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project, default_enabled: true) - end - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` # is a safe switch to disable the feature for a particular project when something went wrong, # therefore it's not supposed to be enabled by default. @@ -34,10 +22,6 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.display_quality_on_mr_diff?(project) - ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: :yaml) - end - def self.gldropdown_tags_enabled? ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) end diff --git a/lib/gitlab/ci/matching/runner_matcher.rb b/lib/gitlab/ci/matching/runner_matcher.rb index 63642674936..a729ca8a821 100644 --- a/lib/gitlab/ci/matching/runner_matcher.rb +++ b/lib/gitlab/ci/matching/runner_matcher.rb @@ -18,6 +18,7 @@ module Gitlab # class RunnerMatcher ATTRIBUTES = %i[ + runner_ids runner_type public_projects_minutes_cost_factor private_projects_minutes_cost_factor diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 66fc6741252..ef7447fa83d 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -10,10 +10,7 @@ module Gitlab def perform! raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result - - if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml) - raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result - end + raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result # Allocate next IID. This operation must be outside of transactions of pipeline creations. pipeline.ensure_project_iid! @@ -51,13 +48,9 @@ module Gitlab end def root_variables - if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml) - ::Gitlab::Ci::Variables::Helpers.merge_variables( - @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables - ) - else - @command.yaml_processor_result.root_variables - end + ::Gitlab::Ci::Variables::Helpers.merge_variables( + @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + ) end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 299b27a5f13..54d92745992 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,11 +11,16 @@ module Gitlab delegate :dig, to: :@seed_attributes - def initialize(context, attributes, previous_stages) + def initialize(context, attributes, previous_stages, current_stage) @context = context @pipeline = context.pipeline @seed_attributes = attributes - @previous_stages = previous_stages + @stages_for_needs_lookup = if Feature.enabled?(:ci_same_stage_job_needs, @pipeline.project, default_enabled: :yaml) + (previous_stages + [current_stage]).compact + else + previous_stages + end + @needs_attributes = dig(:needs_attributes) @resource_group_key = attributes.delete(:resource_group_key) @job_variables = @seed_attributes.delete(:job_variables) @@ -67,6 +72,7 @@ module Gitlab .deep_merge(rules_attributes) .deep_merge(allow_failure_criteria_attributes) .deep_merge(@cache.cache_attributes) + .deep_merge(runner_tags) end def bridge? @@ -148,14 +154,18 @@ module Gitlab @needs_attributes.flat_map do |need| next if need[:optional] - result = @previous_stages.any? do |stage| - stage.seeds_names.include?(need[:name]) - end + result = need_present?(need) - "'#{name}' job needs '#{need[:name]}' job, but it was not added to the pipeline" unless result + "'#{name}' job needs '#{need[:name]}' job, but '#{need[:name]}' is not in any previous stage" unless result end.compact end + def need_present?(need) + @stages_for_needs_lookup.any? do |stage| + stage.seeds_names.include?(need[:name]) + end + end + def max_needs_allowed @pipeline.project.actual_limits.ci_needs_size_limit end @@ -202,6 +212,16 @@ module Gitlab end end + def runner_tags + { tag_list: evaluate_runner_tags }.compact + end + + def evaluate_runner_tags + @seed_attributes[:tag_list]&.map do |tag| + ExpandVariables.expand_existing(tag, evaluate_context.variables) + end + end + # If a job uses `allow_failure:exit_codes` and `rules:allow_failure` # we need to prevent the exit codes from being persisted because they # would break the behavior defined by `rules:allow_failure`. @@ -213,8 +233,6 @@ module Gitlab end def recalculate_yaml_variables! - return unless ::Feature.enabled?(:ci_workflow_rules_variables, @pipeline.project, default_enabled: :yaml) - @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance ) @@ -224,3 +242,5 @@ module Gitlab end end end + +Gitlab::Ci::Pipeline::Seed::Build.prepend_mod_with('Gitlab::Ci::Pipeline::Seed::Build') diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index c988ea10e41..018fb260986 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -17,7 +17,7 @@ module Gitlab @previous_stages = previous_stages @builds = attributes.fetch(:builds).map do |attributes| - Seed::Build.new(context, attributes, previous_stages) + Seed::Build.new(context, attributes, previous_stages, self) end end diff --git a/lib/gitlab/ci/pipeline_object_hierarchy.rb b/lib/gitlab/ci/pipeline_object_hierarchy.rb index de3262b10e0..e05a617f4fc 100644 --- a/lib/gitlab/ci/pipeline_object_hierarchy.rb +++ b/lib/gitlab/ci/pipeline_object_hierarchy.rb @@ -21,7 +21,7 @@ module Gitlab middle_table[:source_pipeline_id].eq(objects_table[:id]).and( middle_table[:pipeline_id].eq(cte.table[:id]) ).and( - same_project_condition + project_condition ) end @@ -29,15 +29,15 @@ module Gitlab middle_table[:pipeline_id].eq(objects_table[:id]).and( middle_table[:source_pipeline_id].eq(cte.table[:id]) ).and( - same_project_condition + project_condition ) end - def same_project_condition - if options[:same_project] - middle_table[:source_project_id].eq(middle_table[:project_id]) - else - Arel.sql('TRUE') + def project_condition + case options[:project_condition] + when :same then middle_table[:source_project_id].eq(middle_table[:project_id]) + when :different then middle_table[:source_project_id].not_eq(middle_table[:project_id]) + else Arel.sql('TRUE') end end end diff --git a/lib/gitlab/ci/reports/security/analyzer.rb b/lib/gitlab/ci/reports/security/analyzer.rb new file mode 100644 index 00000000000..b88eaf87cef --- /dev/null +++ b/lib/gitlab/ci/reports/security/analyzer.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Analyzer + attr_reader :id, :name, :version, :vendor + + def initialize(id:, name:, version:, vendor:) + @id = id + @name = name + @version = version + @vendor = vendor + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/concerns/fingerprint_path_from_file.rb b/lib/gitlab/ci/reports/security/concerns/fingerprint_path_from_file.rb new file mode 100644 index 00000000000..ec1d80e11c8 --- /dev/null +++ b/lib/gitlab/ci/reports/security/concerns/fingerprint_path_from_file.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + module Concerns + module FingerprintPathFromFile + extend ActiveSupport::Concern + + def fingerprint_path + File.basename(file_path.to_s) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/identifier.rb b/lib/gitlab/ci/reports/security/identifier.rb new file mode 100644 index 00000000000..4ba943cdcbc --- /dev/null +++ b/lib/gitlab/ci/reports/security/identifier.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Identifier + attr_reader :external_id + attr_reader :external_type + attr_reader :fingerprint + attr_reader :name + attr_reader :url + + def initialize(external_id:, external_type:, name:, url: nil) + @external_id = external_id + @external_type = external_type + @name = name + @url = url + + @fingerprint = generate_fingerprint + end + + def key + fingerprint + end + + def to_hash + %i[ + external_id + external_type + fingerprint + name + url + ].each_with_object({}) do |key, hash| + hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def ==(other) + other.external_type == external_type && + other.external_id == external_id + end + + def type_identifier? + cwe? || wasc? + end + + def cve? + external_type.to_s.casecmp?('cve') + end + + def cwe? + external_type.to_s.casecmp?('cwe') + end + + def wasc? + external_type.to_s.casecmp?('wasc') + end + + private + + def generate_fingerprint + Digest::SHA1.hexdigest("#{external_type}:#{external_id}") + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/link.rb b/lib/gitlab/ci/reports/security/link.rb new file mode 100644 index 00000000000..1c4c05cd9ac --- /dev/null +++ b/lib/gitlab/ci/reports/security/link.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Link + attr_accessor :name, :url + + def initialize(name: nil, url: nil) + @name = name + @url = url + end + + def to_hash + { + name: name, + url: url + }.compact + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/scan.rb b/lib/gitlab/ci/reports/security/scan.rb new file mode 100644 index 00000000000..7dd0acc868b --- /dev/null +++ b/lib/gitlab/ci/reports/security/scan.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Scan + attr_accessor :type, :status, :start_time, :end_time + + def initialize(params = {}) + @type = params.dig('type') + @status = params.dig('status') + @start_time = params.dig('start_time') + @end_time = params.dig('end_time') + end + + def to_hash + { + type: type, + status: status, + start_time: start_time, + end_time: end_time + }.compact + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/scanned_resource.rb b/lib/gitlab/ci/reports/security/scanned_resource.rb new file mode 100644 index 00000000000..605577eafcd --- /dev/null +++ b/lib/gitlab/ci/reports/security/scanned_resource.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class ScannedResource + include Gitlab::Utils::StrongMemoize + + attr_reader :request_method + attr_reader :request_uri + + delegate :scheme, :host, :port, :path, :query, to: :request_uri, prefix: :url + + def initialize(uri, request_method) + raise ArgumentError unless uri.is_a?(URI) + + @request_method = request_method + @request_uri = uri + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/scanner.rb b/lib/gitlab/ci/reports/security/scanner.rb new file mode 100644 index 00000000000..c1de03cea44 --- /dev/null +++ b/lib/gitlab/ci/reports/security/scanner.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Scanner + ANALYZER_ORDER = { + "bundler_audit" => 1, + "retire.js" => 2, + "gemnasium" => 3, + "gemnasium-maven" => 3, + "gemnasium-python" => 3, + "bandit" => 1, + "semgrep" => 2 + }.freeze + + attr_accessor :external_id, :name, :vendor, :version + + alias_method :key, :external_id + + def initialize(external_id:, name:, vendor:, version:) + @external_id = external_id + @name = name + @vendor = vendor + @version = version + end + + def to_hash + { + external_id: external_id.to_s, + name: name.to_s, + vendor: vendor.presence + }.compact + end + + def ==(other) + other.external_id == external_id + end + + def <=>(other) + sort_keys.compact <=> other.sort_keys.compact + end + + protected + + def sort_keys + @sort_keys ||= [order, external_id, name, vendor] + end + + private + + def order + ANALYZER_ORDER.fetch(external_id, Float::INFINITY) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 66f51f63585..dbbb9a01dab 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -31,7 +31,8 @@ module Gitlab project_deleted: 'pipeline project was deleted', user_blocked: 'pipeline user was blocked', ci_quota_exceeded: 'no more CI minutes available', - no_matching_runner: 'no matching runner available' + no_matching_runner: 'no matching runner available', + trace_size_exceeded: 'log size limit exceeded' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 5368e020a50..3b2da773102 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -95,11 +95,7 @@ module Gitlab end def any_skipped_or_ignored? - if ::Feature.enabled?(:ci_fix_pipeline_status_for_dag_needs_manual, @project, default_enabled: :yaml) - any_of?(:skipped) || any_of?(:ignored) - else - any_of?(:skipped) - end + any_of?(:skipped) || any_of?(:ignored) end def consume_all_statuses(all_statuses) diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index e7ed2081f6a..f60f5243666 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -18,6 +18,10 @@ module Gitlab @user = user end + def id + "#{group}-#{subject.id}" + end + def icon raise NotImplementedError end diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml index c06ef83c180..ebb0b5948f1 100644 --- a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml + # This template is on early stage of development. # Use it with caution. For usage instruction please read # https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v2.3.0/README.md diff --git a/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml index 267027a1b8a..60173cab54a 100644 --- a/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml + stages: - provision - review diff --git a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml index 453803a6f7e..17e49440784 100644 --- a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml + stages: - build - test diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 2ff36bcc657..64e3b695e27 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml + # Read more about how to use this script on this blog post https://about.gitlab.com/2019/01/28/android-publishing-with-gitlab-and-fastlane/ # You will also need to configure your build.gradle, Dockerfile, and fastlane configuration to make this work. # If you are looking for a simpler template that does not publish, see the Android template. diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml index d20dabc0b00..b8a4c59c233 100644 --- a/lib/gitlab/ci/templates/Android.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android.gitlab-ci.yml + # Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny # If you are interested in using Android with FastLane for publishing take a look at the Android-Fastlane template. diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 207e2cf074a..adb5d430d46 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -1,4 +1,10 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml + # Auto DevOps +# # This CI/CD configuration provides a standard pipeline for # * building a Docker image (using a buildpack if necessary), # * storing the image in the container registry, diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index 67e58d9ee99..1910913f2bd 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -1,4 +1,9 @@ -# see https://docs.gitlab.com/ee/ci/yaml/README.html for all available options +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml + +# See https://docs.gitlab.com/ee/ci/yaml/README.html for all available options # you can delete this line if you're not using Docker image: busybox:latest diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index 33a2a534508..bdcd3240380 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -1,6 +1,12 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/C++.gitlab-ci.yml + # use the official gcc image, based on debian # can use verions as well, like gcc:5.2 # see https://hub.docker.com/_/gcc/ + image: gcc build: diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index d879e27dfcb..f166da9bdd6 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -1,4 +1,9 @@ -# This file uses Test Kitchen with the kitchen-dokken driver to +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Chef.gitlab-ci.yml + +# This template uses Test Kitchen with the kitchen-dokken driver to # perform functional testing. Doing so requires that your runner be a # Docker runner configured for privileged mode. Please see # https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index 0c5850bdb8e..0f9e28c9a8e 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml + # Based on openjdk:8, already includes lein image: clojure:lein-2.7.0 # If you need to configure a database, add a `services` section here diff --git a/lib/gitlab/ci/templates/Composer.gitlab-ci.yml b/lib/gitlab/ci/templates/Composer.gitlab-ci.yml index 5d9c68d3031..911acf8aff2 100644 --- a/lib/gitlab/ci/templates/Composer.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Composer.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Composer.gitlab-ci.yml + # Publishes a tag/branch to Composer Packages of the current project publish: image: curlimages/curl:latest diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml index 538f96c4084..856a097e6e0 100644 --- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml + # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/crystallang/crystal/ image: "crystallang/crystal:latest" diff --git a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml index cc383f89b0c..a50e722f18a 100644 --- a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Dart.gitlab-ci.yml + # https://hub.docker.com/r/google/dart image: google/dart:2.8.4 diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index c657c7e8eb1..d2d3b3ed61e 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Django.gitlab-ci.yml + # Official framework image. Look for the different tagged releases at: # https://hub.docker.com/r/library/python image: python:latest diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml index d0c63ab6edf..8f5f0e2c451 100644 --- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -1,8 +1,14 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Docker.gitlab-ci.yml + # Build a Docker image with CI/CD and push to the GitLab registry. # Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html # # This template uses one generic job with conditional builds # for the default branch and all other (MR) branches. + docker-build: # Use the official docker image. image: docker:latest diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml index 7271526ab1b..1ceaf9fc86b 100644 --- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml + image: elixir:latest # Pick zero or more services to be used on all builds. diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index 504ece611ca..d176ce19299 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml + code_quality: stage: test image: "cirrusci/flutter:1.22.5" diff --git a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml index 07d0de5f9e5..38036c1f964 100644 --- a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml + # This is a sample GitLab CI/CD configuration file that should run without any modifications. # It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, # it uses echo commands to simulate the pipeline execution. diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index 1b686bc6cc0..b5dd0005013 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Go.gitlab-ci.yml + image: golang:latest variables: diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml index cbf4d58bdad..76f0c9f8427 100644 --- a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -1,6 +1,12 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml + # This is the Gradle build system for JVM applications # https://gradle.org/ # https://github.com/gradle/gradle + image: gradle:alpine # Disable the Gradle daemon for Continuous Integration servers as correctness diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index efcd1d3ddc0..3c514d7b0c6 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Grails.gitlab-ci.yml + # This template uses the java:8 docker image because there isn't any # official Grails image at this moment # diff --git a/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml deleted file mode 100644 index 90812083917..00000000000 --- a/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml +++ /dev/null @@ -1,9 +0,0 @@ -# This file is a template demonstrating the `script` keyword. -# Learn more about this keyword here: https://docs.gitlab.com/ee/ci/yaml/README.html#script - -# After committing this template, visit CI/CD > Jobs to see the script output. - -job: - script: - # provide a shell script as argument for this keyword. - - echo "Hello World" diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 6af79728dc8..80125a9bc01 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -1,6 +1,6 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ # -# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-variables variables: @@ -38,9 +38,6 @@ bandit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -57,9 +54,6 @@ brakeman-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -77,9 +71,6 @@ eslint-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -100,9 +91,6 @@ flawfinder-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -120,9 +108,6 @@ kubesec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -138,9 +123,6 @@ gosec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -157,9 +139,6 @@ gosec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" @@ -194,9 +173,6 @@ nodejs-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -213,9 +189,6 @@ phpcs-security-audit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -232,9 +205,6 @@ pmd-apex-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -251,9 +221,6 @@ security-code-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -271,9 +238,6 @@ semgrep-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -294,9 +258,6 @@ sobelow-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -313,9 +274,6 @@ spotbugs-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index be0efc9180b..4687a07d05b 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Julia.gitlab-ci.yml + # This is an example .gitlab-ci.yml file to test (and optionally report the coverage # results of) your [Julia][1] packages. Please refer to the [documentation][2] # for more information about package development in Julia. diff --git a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml index e4ed7fadfaa..1bc258d30c4 100644 --- a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml + --- variables: # Feel free to choose the image that suits you best. diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index 5d2c8024524..43e4ac02d41 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml + # Official framework image. Look for the different tagged releases at: # https://hub.docker.com/r/library/php image: php:latest diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 97d0f611f47..dfa46d7af61 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Maven.gitlab-ci.yml + # Build JAVA applications using Apache Maven (http://maven.apache.org) # For docker image tags see https://hub.docker.com/_/maven/ # diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 36fe27f54c2..2f214347ec3 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Mono.gitlab-ci.yml + # This is a simple gitlab continuous integration template (compatible with the shared runner provided on gitlab.com) # using the official mono docker image to build a visual studio project. # diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index 92379ded77c..e48801b7970 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml + # Official framework image. Look for the different tagged releases at: # https://hub.docker.com/r/library/node/tags/ image: node:latest diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 45bddb1bc6a..7c8bbe464af 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml + image: openshift/origin-cli stages: diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index 84e8223e69b..281bf7e3dd9 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/PHP.gitlab-ci.yml + # Select image from https://hub.docker.com/_/php/ image: php:latest @@ -8,9 +13,9 @@ cache: before_script: - apt-get update -yqq - - apt-get install -yqq git libmcrypt-dev libpq-dev libcurl4-gnutls-dev libicu-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev + - apt-get install -yqq git libpq-dev libcurl4-gnutls-dev libicu-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev libonig-dev libzip-dev # Install PHP extensions - - docker-php-ext-install mbstring mcrypt pdo_pgsql curl json intl gd xml zip bz2 opcache + - docker-php-ext-install mbstring pdo_pgsql curl intl gd xml zip bz2 opcache # Install & enable Xdebug for code coverage reports - pecl install xdebug - docker-php-ext-enable xdebug diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml index 0b03ba6c3d8..3db712c6dc5 100644 --- a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Packer.gitlab-ci.yml + image: name: hashicorp/packer:latest entrypoint: diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index 90cd8472916..55cf22b6601 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml + # Full project: https://gitlab.com/pages/brunch image: node:4.2.2 diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml index 7435afef572..2f518d667a5 100644 --- a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml + # Full project: https://gitlab.com/pages/doxygen image: alpine diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index 708c5063cc6..d3726fe34c5 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml + image: node:latest # This folder is cached between builds diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 694446dd6c9..17ed1d2e87f 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml + # Full project: https://gitlab.com/pages/plain-html pages: stage: deploy diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index a2fd6620909..9e48ac9fcdc 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml + # Full project: https://gitlab.com/pages/harp image: node:4.2.2 diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml index fd75e47e899..a6f94a4d80e 100644 --- a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml + # Full project: https://gitlab.com/pages/hexo image: node:10.15.3 diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index a6a605e35f0..cfc4a1d904a 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml + --- # All available Hugo versions are listed here: # https://gitlab.com/pages/hugo/container_registry diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml index 1be2f4bad76..59e55efaee0 100644 --- a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml + # Full project: https://gitlab.com/pages/hyde image: python:2.7 diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml index 886b6c36249..8e15570fd1a 100644 --- a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml + # This template uses the java:8 docker image because there isn't any # official JBake image at this moment # diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index 01e063c50ad..e0ad2e55f7d 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml + # Template project: https://gitlab.com/pages/jekyll # Docs: https://docs.gitlab.com/ee/pages/ image: ruby:2.6 diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index e39aa8a2063..26fac92d0dc 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml + # Jigsaw is a simple static sites generator with Laravel's Blade. # # Full project: https://github.com/tightenco/jigsaw diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml index 13d3089f4fa..9b5c1198c6c 100644 --- a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml + # Full project: https://gitlab.com/pages/hyde image: python:2.7 diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index e65cf3928f2..d97f0b7beb7 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml + # Full project: https://gitlab.com/pages/metalsmith image: node:4.2.2 diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml index 377fd8c396e..17ce0ef3659 100644 --- a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml + # Full project: https://gitlab.com/pages/middleman image: ruby:2.6 diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml index 89281b41b66..a3ce96da244 100644 --- a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml + # Full project: https://gitlab.com/pages/nanoc image: ruby:2.6 diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml index 8fd4702b90d..4abdf66a21c 100644 --- a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml + # Full project: https://gitlab.com/pages/octopress image: ruby:2.6 diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml index 09c6649fc13..7d52a407848 100644 --- a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml + # Full project: https://gitlab.com/pages/pelican image: python:2.7-alpine diff --git a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml index 9fa8b07f7cb..961941ac4d0 100644 --- a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml + image: node:10-alpine # specify the location of the Open API Specification files within your project diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index abce887d45b..aec41c137a4 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Python.gitlab-ci.yml + # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/library/python/tags/ image: python:latest diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 1bdaaeede43..490fc779e17 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml + # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/library/ruby/tags/ image: ruby:latest diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index 94117a79d1c..869c1782352 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Rust.gitlab-ci.yml + # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/library/rust/tags/ image: "rust:latest" diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index e081e20564a..ff8f9601189 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Scala.gitlab-ci.yml + # Official OpenJDK Java image. Look for the different tagged releases at # https://hub.docker.com/_/openjdk/ . A Java image is not required # but an image with a JVM speeds up the build a bit. diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 0c4c39cbcd6..009061ce844 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -1,6 +1,11 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml -# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ +# +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 0c4c39cbcd6..ceeefa8aea6 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -1,6 +1,11 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.lastest.gitlab-ci.yml -# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ +# +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: diff --git a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml new file mode 100644 index 00000000000..f4f066cc7c2 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml @@ -0,0 +1,34 @@ +# Use this template to enable cluster image scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#customize-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the `CIS_KUBECONFIG` variable to allow analyzer to connect to your Kubernetes cluster and fetch found vulnerabilities. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#available-variables + +variables: + CIS_ANALYZER_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/cluster-image-scanning:0 + +cluster_image_scanning: + image: "$CIS_ANALYZER_IMAGE" + stage: test + allow_failure: true + artifacts: + reports: + cluster_image_scanning: gl-cluster-image-scanning-report.json + paths: [gl-cluster-image-scanning-report.json] + dependencies: [] + script: + - /analyzer run + rules: + - if: $CLUSTER_IMAGE_SCANNING_DISABLED + when: never + - if: '($KUBECONFIG == null || $KUBECONFIG == "") && ($CIS_KUBECONFIG == null || $CIS_KUBECONFIG == "")' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bcluster_image_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index bd163f9db94..89e6743b0e4 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + # Use this template to enable container scanning in your project. # You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` # keyword. @@ -13,7 +18,7 @@ # - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the # DOCKERFILE_PATH variable. # -# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 2dbfb80b419..7243f240eed 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -1,6 +1,11 @@ -# Read more about this feature https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml -# Configure coverage fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Read more about this feature https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing +# +# Configure coverage fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/#available-cicd-variables variables: diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index 9170e943e9d..a2933085d4e 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dast-API.gitlab-ci.yml + # To use this template, add the following to your .gitlab-ci.yml file: # # include: @@ -13,7 +18,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html -# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables variables: diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index a0564a16c07..3e7ab9b5c3b 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml + stages: - build - test @@ -5,7 +10,7 @@ stages: - dast variables: - DAST_VERSION: 1 + DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 5521a4a781b..0802868d67f 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml + # To use this template, add the following to your .gitlab-ci.yml file: # # include: @@ -10,10 +15,10 @@ # - test # - deploy # - dast - +# # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ - -# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: @@ -43,15 +48,10 @@ dast: $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && - $REVIEW_DISABLED && $DAST_WEBSITE == null && - $DAST_API_SPECIFICATION == null + $REVIEW_DISABLED when: never - if: $CI_COMMIT_BRANCH && $CI_KUBERNETES_ACTIVE && $GITLAB_FEATURES =~ /\bdast\b/ - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdast\b/ && - $DAST_WEBSITE - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdast\b/ && - $DAST_API_SPECIFICATION + $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index e936364c86c..ac7d87a4cda 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.lastest.gitlab-ci.yml + # To use this template, add the following to your .gitlab-ci.yml file: # # include: @@ -13,7 +18,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ -# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 8df5ce79fe8..aa7b394a13c 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -1,6 +1,11 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml + # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/ # -# Configure dependency scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure dependency scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/index.html#available-variables variables: diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index 870684c9f1d..1249b8d6fdc 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,6 +1,11 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml + # Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html # -# Configure license scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure license scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/compliance/license_compliance/#available-variables variables: diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index d410c49b9a4..e30777d8401 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -1,16 +1,18 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml + # This template should be used when Security Products (https://about.gitlab.com/handbook/engineering/development/secure/#security-products) # have to be downloaded and stored locally. # # Usage: # -# ``` -# include: -# - template: Secure-Binaries.gitlab-ci.yml -# ``` +# include: +# - template: Secure-Binaries.gitlab-ci.yml # # Docs: https://docs.gitlab.com/ee/topics/airgap/ - variables: SECURE_BINARIES_ANALYZERS: >- bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, @@ -222,7 +224,7 @@ license-finder: dast: extends: .download_images variables: - SECURE_BINARIES_ANALYZER_VERSION: "1" + SECURE_BINARIES_ANALYZER_VERSION: "2" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml index 280e75d46f5..55648437191 100644 --- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml + # GitLab Serverless template image: alpine:latest diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index cca0ba5d38e..eedb3b7a310 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Swift.gitlab-ci.yml + # Lifted from: https://about.gitlab.com/2016/03/10/setting-up-gitlab-ci-for-ios-projects/ # This file assumes an own GitLab CI runner, setup on a macOS system. stages: diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 62b32d7c2db..272b980b4b2 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml + include: - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index 5963d7138c5..d34a847f2d5 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml + include: - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index e8a99a6ea06..22c40d8a8b8 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml + # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/accessibility_testing.html stages: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index f0621165f8a..e0df9799917 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml + # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html stages: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml index f0621165f8a..ad24ebae8d4 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml + # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html stages: diff --git a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml index 584e6966180..4d0f8c10a20 100644 --- a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml + rspec-rails-modified-path-specs: image: ruby:2.6 stage: .pre diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml index cd23af562e5..53fabcfc721 100644 --- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml + # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/load_performance_testing.html stages: diff --git a/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml b/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml index 05635cf71be..26b2c8694cd 100644 --- a/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml + # Read more on when to use this template at # https://docs.gitlab.com/ee/ci/yaml/#workflowrules diff --git a/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml b/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml index 50ff4c1f60b..28c25f48972 100644 --- a/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml + # Read more on when to use this template at # https://docs.gitlab.com/ee/ci/yaml/#workflowrules diff --git a/lib/gitlab/ci/templates/dotNET-Core.yml b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml index 40ca296d7bd..edd0fb0ba07 100644 --- a/lib/gitlab/ci/templates/dotNET-Core.yml +++ b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml @@ -1,7 +1,11 @@ ---- +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.yml + # This is a simple example illustrating how to build and test .NET Core project # with GitLab Continuous Integration / Continuous Delivery. - +# # ### Specify the Docker image # # Instead of installing .NET Core SDK manually, a docker image is used @@ -26,14 +30,6 @@ variables: # NOTE: Please edit this path so it matches the structure of your project! SOURCE_CODE_PATH: '*/*/' -# ### Define stage list -# -# In this example there are only two stages. -# Initially, the project will be built and then tested. -stages: - - build - - test - # ### Define global cache rule # # Before building the project, all dependencies (e.g. third-party NuGet packages) diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index b29f45323f5..dd88953b9a4 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml + # The following script will work for any project that can be built from command line by msbuild # It uses powershell shell executor, so you need to add the following line to your config.toml file # (located in gitlab-runner.exe directory): @@ -19,7 +24,6 @@ # The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473), # running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes - # place project specific paths in variables to make the rest of the script more generic variables: EXE_RELEASE_FOLDER: 'YourApp\bin\Release' diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml index 87aea8527d1..0b75c298167 100644 --- a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml + # This is a very simple template that mainly relies on FastLane to build and distribute your app. # Read more about how to use this template on the blog post https://about.gitlab.com/2019/03/06/ios-publishing-with-gitlab-and-fastlane/ # You will also need fastlane and signing configuration for this to work, along with a MacOS runner. diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index 536cf9bd8d8..bfea437b8f1 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/npm.gitlab-ci.yml + publish: image: node:latest stage: deploy diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 84eb860a168..f9798023838 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -189,7 +189,7 @@ module Gitlab raise ArchiveError, 'Job is not finished yet' unless job.complete? if trace_artifact - unsafe_trace_cleanup! if Feature.enabled?(:erase_traces_from_already_archived_jobs_when_archiving_again, job.project, default_enabled: :yaml) + unsafe_trace_cleanup! raise AlreadyArchivedError, 'Could not archive again' end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index a8c1002f2b9..c94fa84f608 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -46,6 +46,10 @@ module Gitlab @jobs.each do |name, job| validate_job!(name, job) end + + if ::Feature.enabled?(:ci_same_stage_job_needs, @opts[:project], default_enabled: :yaml) + YamlProcessor::Dag.check_circular_dependencies!(@jobs) + end end def validate_job!(name, job) @@ -99,10 +103,16 @@ module Gitlab job_stage_index = stage_index(name) dependency_stage_index = stage_index(dependency) - # A dependency might be defined later in the configuration - # with a stage that does not exist - unless dependency_stage_index.present? && dependency_stage_index < job_stage_index - error!("#{name} job: #{dependency_type} #{dependency} is not defined in prior stages") + if ::Feature.enabled?(:ci_same_stage_job_needs, @opts[:project], default_enabled: :yaml) + unless dependency_stage_index.present? && dependency_stage_index <= job_stage_index + error!("#{name} job: #{dependency_type} #{dependency} is not defined in current or prior stages") + end + else + # A dependency might be defined later in the configuration + # with a stage that does not exist + unless dependency_stage_index.present? && dependency_stage_index < job_stage_index + error!("#{name} job: #{dependency_type} #{dependency} is not defined in prior stages") + end end end diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb new file mode 100644 index 00000000000..0140218d9bc --- /dev/null +++ b/lib/gitlab/ci/yaml_processor/dag.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# Represents Dag pipeline +module Gitlab + module Ci + class YamlProcessor + class Dag + include TSort + + MissingNodeError = Class.new(StandardError) + + def initialize(nodes) + @nodes = nodes + end + + def self.check_circular_dependencies!(jobs) + nodes = jobs.values.to_h do |job| + name = job[:name].to_s + needs = job.dig(:needs, :job).to_a + + [name, needs.map { |need| need[:name].to_s }] + end + + new(nodes).tsort + rescue TSort::Cyclic + raise ValidationError, 'The pipeline has circular dependencies.' + rescue MissingNodeError + end + + def tsort_each_child(node, &block) + raise MissingNodeError, "node #{node} is missing" unless @nodes[node] + + @nodes[node].each(&block) + end + + def tsort_each_node(&block) + @nodes.each_key(&block) + end + end + end + end +end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index d7b31946ab0..842920ba02e 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -37,6 +37,7 @@ module Gitlab allow_webpack_dev_server(settings_hash) if Rails.env.development? allow_cdn(settings_hash) if ENV['GITLAB_CDN_HOST'].present? + allow_customersdot(settings_hash) if Rails.env.development? && ENV['CUSTOMER_PORTAL_URL'].present? settings_hash end @@ -85,6 +86,12 @@ module Gitlab def self.append_to_directive(settings_hash, directive, text) settings_hash['directives'][directive] = "#{settings_hash['directives'][directive]} #{text}".strip end + + def self.allow_customersdot(settings_hash) + customersdot_host = ENV['CUSTOMER_PORTAL_URL'] + + append_to_directive(settings_hash, 'frame_src', customersdot_host) + end end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index aa419d75df2..a269b8d0366 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,15 +2,15 @@ module Gitlab module Database + CI_DATABASE_NAME = 'ci' + # This constant is used when renaming tables concurrently. # If you plan to rename a table using the `rename_table_safely` method, add your table here one milestone before the rename. # Example: # TABLES_TO_BE_RENAMED = { # 'old_name' => 'new_name' # }.freeze - TABLES_TO_BE_RENAMED = { - 'services' => 'integrations' - }.freeze + TABLES_TO_BE_RENAMED = {}.freeze # Minimum PostgreSQL version requirement per documentation: # https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements @@ -68,6 +68,25 @@ module Gitlab end end + def self.has_config?(database_name) + Gitlab::Application.config.database_configuration[Rails.env].include?(database_name.to_s) + end + + def self.main_database?(name) + # The database is `main` if it is a first entry in `database.yml` + # Rails internally names them `primary` to avoid confusion + # with broad `primary` usage we use `main` instead + # + # TODO: The explicit `== 'main'` is needed in a transition period till + # the `database.yml` is not migrated into `main:` syntax + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65243 + ActiveRecord::Base.configurations.primary?(name.to_s) || name.to_s == 'main' + end + + def self.ci_database?(name) + name.to_s == CI_DATABASE_NAME + end + def self.username config['username'] || ENV['USER'] end @@ -333,6 +352,16 @@ module Gitlab end end + def self.dbname(ar_connection) + if ar_connection.respond_to?(:pool) && + ar_connection.pool.respond_to?(:db_config) && + ar_connection.pool.db_config.respond_to?(:database) + return ar_connection.pool.db_config.database + end + + 'unknown' + end + # inside_transaction? will return true if the caller is running within a transaction. Handles special cases # when running inside a test environment, where tests may be wrapped in transactions def self.inside_transaction? diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb index e7e3c1766a9..eda991efbd5 100644 --- a/lib/gitlab/database/as_with_materialized.rb +++ b/lib/gitlab/database/as_with_materialized.rb @@ -24,6 +24,8 @@ module Gitlab end # Note: to be deleted after the minimum PG version is set to 12.0 + # Update the documentation together when deleting the method + # https://docs.gitlab.com/ee/development/merge_request_performance_guidelines.html#use-ctes-wisely def self.materialized_if_supported materialized_supported? ? 'MATERIALIZED' : '' end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 9a1dc4ee17d..03bd02d7554 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -44,6 +44,51 @@ module Gitlab # TODO: Switch to individual job interval (prereq: https://gitlab.com/gitlab-org/gitlab/-/issues/328801) duration.to_f / batched_migration.interval end + + def split_and_retry! + with_lock do + raise 'Only failed jobs can be split' unless failed? + + new_batch_size = batch_size / 2 + + raise 'Job cannot be split further' if new_batch_size < 1 + + batching_strategy = batched_migration.batch_class.new + next_batch_bounds = batching_strategy.next_batch( + batched_migration.table_name, + batched_migration.column_name, + batch_min_value: min_value, + batch_size: new_batch_size + ) + midpoint = next_batch_bounds.last + + # We don't want the midpoint to go over the existing max_value because + # those IDs would already be in the next batched migration job. + # This could happen when a lot of records in the current batch are deleted. + # + # In this case, we just lower the batch size so that future calls to this + # method could eventually split the job if it continues to fail. + if midpoint >= max_value + update!(batch_size: new_batch_size, attempts: 0) + else + old_max_value = max_value + + update!( + batch_size: new_batch_size, + max_value: midpoint, + attempts: 0, + started_at: nil, + finished_at: nil, + metrics: {} + ) + + new_record = dup + new_record.min_value = midpoint.next + new_record.max_value = old_max_value + new_record.save! + end + end + end end end end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 36e89023c86..9d66824da51 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -10,7 +10,7 @@ module Gitlab self.table_name = :batched_background_migrations has_many :batched_jobs, foreign_key: :batched_background_migration_id - has_one :last_job, -> { order(id: :desc) }, + has_one :last_job, -> { order(max_value: :desc) }, class_name: 'Gitlab::Database::BackgroundMigration::BatchedJob', foreign_key: :batched_background_migration_id @@ -29,11 +29,16 @@ module Gitlab paused: 0, active: 1, finished: 3, - failed: 4 + failed: 4, + finalizing: 5 } attribute :pause_ms, :integer, default: 100 + def self.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + for_configuration(job_class_name, table_name, column_name, job_arguments).first + end + def self.active_migration active.queue_order.first end diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 67fe6c536e6..14e3919986e 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -4,6 +4,12 @@ module Gitlab module Database module BackgroundMigration class BatchedMigrationRunner + FailedToFinalize = Class.new(RuntimeError) + + def self.finalize(job_class_name, table_name, column_name, job_arguments) + new.finalize(job_class_name, table_name, column_name, job_arguments) + end + def initialize(migration_wrapper = BatchedMigrationWrapper.new) @migration_wrapper = migration_wrapper end @@ -37,10 +43,35 @@ module Gitlab raise 'this method is not intended for use in real environments' end - while migration.active? - run_migration_job(migration) + run_migration_while(migration, :active) + end - migration.reload_last_job + # Finalize migration for given configuration. + # + # If the migration is already finished, do nothing. Otherwise change its status to `finalizing` + # in order to prevent it being picked up by the background worker. Perform all pending jobs, + # then keep running until migration is finished. + def finalize(job_class_name, table_name, column_name, job_arguments) + migration = BatchedMigration.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + + configuration = { + job_class_name: job_class_name, + table_name: table_name, + column_name: column_name, + job_arguments: job_arguments + } + + if migration.nil? + Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" + elsif migration.finished? + Gitlab::AppLogger.warn "Batched background migration for the given configuration is already finished: #{configuration}" + else + migration.finalizing! + migration.batched_jobs.pending.each { |job| migration_wrapper.perform(job) } + + run_migration_while(migration, :finalizing) + + raise FailedToFinalize unless migration.finished? end end @@ -90,6 +121,14 @@ module Gitlab active_migration.finished! end end + + def run_migration_while(migration, status) + while migration.status == status.to_s + run_migration_job(migration) + + migration.reload_last_job + end + end end end end diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 9002d39e1ee..49f56b5be97 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -18,7 +18,7 @@ # batch_count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) # batch_count(Namespace.group(:type)) # batch_distinct_count(::Project, :creator_id) -# batch_distinct_count(::Project.with_active_services.service_desk_enabled.where(time_period), start: ::User.minimum(:id), finish: ::User.maximum(:id)) +# batch_distinct_count(::Project.aimed_for_deletion.service_desk_enabled.where(time_period), start: ::User.minimum(:id), finish: ::User.maximum(:id)) # batch_distinct_count(Project.group(:visibility_level), :creator_id) # batch_sum(User, :sign_in_count) # batch_sum(Issue.group(:state_id), :weight)) @@ -41,159 +41,5 @@ module Gitlab include BatchCount end end - - class BatchCounter - FALLBACK = -1 - MIN_REQUIRED_BATCH_SIZE = 1_250 - DEFAULT_SUM_BATCH_SIZE = 1_000 - MAX_ALLOWED_LOOPS = 10_000 - SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep - ALLOWED_MODES = [:itself, :distinct].freeze - FALLBACK_FINISH = 0 - OFFSET_BY_ONE = 1 - - # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 - DEFAULT_DISTINCT_BATCH_SIZE = 10_000 - DEFAULT_BATCH_SIZE = 100_000 - - def initialize(relation, column: nil, operation: :count, operation_args: nil) - @relation = relation - @column = column || relation.primary_key - @operation = operation - @operation_args = operation_args - end - - def unwanted_configuration?(finish, batch_size, start) - (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || - (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || - (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || - start >= finish - end - - def count(batch_size: nil, mode: :itself, start: nil, finish: nil) - raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? - - check_mode!(mode) - - # non-distinct have better performance - batch_size ||= batch_size_for_mode_and_operation(mode, @operation) - - start = actual_start(start) - finish = actual_finish(finish) - - raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 - return FALLBACK if unwanted_configuration?(finish, batch_size, start) - - results = nil - batch_start = start - - while batch_start < finish - begin - batch_end = [batch_start + batch_size, finish].min - batch_relation = build_relation_batch(batch_start, batch_end, mode) - - op_args = @operation_args - if @operation == :count && @operation_args.blank? && use_loose_index_scan_for_distinct_values?(mode) - op_args = [Gitlab::Database::LooseIndexScanDistinctCount::COLUMN_ALIAS] - end - - results = merge_results(results, batch_relation.send(@operation, *op_args)) # rubocop:disable GitlabSecurity/PublicSend - batch_start = batch_end - rescue ActiveRecord::QueryCanceled => error - # retry with a safe batch size & warmer cache - if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE - batch_size /= 2 - else - log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) - return FALLBACK - end - rescue Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError => error - Gitlab::AppJsonLogger - .error( - event: 'batch_count', - relation: @relation.table_name, - operation: @operation, - operation_args: @operation_args, - mode: mode, - message: "LooseIndexScanDistinctCount column error: #{error.message}" - ) - - return FALLBACK - end - - sleep(SLEEP_TIME_IN_SECONDS) - end - - results - end - - def merge_results(results, object) - return object unless results - - if object.is_a?(Hash) - results.merge!(object) { |_, a, b| a + b } - else - results + object - end - end - - private - - def build_relation_batch(start, finish, mode) - if use_loose_index_scan_for_distinct_values?(mode) - Gitlab::Database::LooseIndexScanDistinctCount.new(@relation, @column).build_query(from: start, to: finish) - else - @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def batch_size_for_mode_and_operation(mode, operation) - return DEFAULT_SUM_BATCH_SIZE if operation == :sum - - mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE - end - - def between_condition(start, finish) - return @column.between(start...finish) if @column.is_a?(Arel::Attributes::Attribute) - - { @column => start...finish } - end - - def actual_start(start) - start || @relation.unscope(:group, :having).minimum(@column) || 0 - end - - def actual_finish(finish) - (finish || @relation.unscope(:group, :having).maximum(@column) || FALLBACK_FINISH) + OFFSET_BY_ONE - end - - def check_mode!(mode) - raise "The mode #{mode.inspect} is not supported" unless ALLOWED_MODES.include?(mode) - raise 'Use distinct count for optimized distinct counting' if @relation.limit(1).distinct_value.present? && mode != :distinct - raise 'Use distinct count only with non id fields' if @column == :id && mode == :distinct - end - - def log_canceled_batch_fetch(batch_start, mode, query, error) - Gitlab::AppJsonLogger - .error( - event: 'batch_count', - relation: @relation.table_name, - operation: @operation, - operation_args: @operation_args, - start: batch_start, - mode: mode, - query: query, - message: "Query has been canceled with message: #{error.message}" - ) - end - - def use_loose_index_scan_for_distinct_values?(mode) - Feature.enabled?(:loose_index_scan_for_distinct_values) && not_group_by_query? && mode == :distinct - end - - def not_group_by_query? - !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank? - end - end end end diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb new file mode 100644 index 00000000000..5f2e404c9da --- /dev/null +++ b/lib/gitlab/database/batch_counter.rb @@ -0,0 +1,159 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class BatchCounter + FALLBACK = -1 + MIN_REQUIRED_BATCH_SIZE = 1_250 + DEFAULT_SUM_BATCH_SIZE = 1_000 + MAX_ALLOWED_LOOPS = 10_000 + SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep + ALLOWED_MODES = [:itself, :distinct].freeze + FALLBACK_FINISH = 0 + OFFSET_BY_ONE = 1 + + # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 + DEFAULT_DISTINCT_BATCH_SIZE = 10_000 + DEFAULT_BATCH_SIZE = 100_000 + + def initialize(relation, column: nil, operation: :count, operation_args: nil) + @relation = relation + @column = column || relation.primary_key + @operation = operation + @operation_args = operation_args + end + + def unwanted_configuration?(finish, batch_size, start) + (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || + (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || + (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || + start >= finish + end + + def count(batch_size: nil, mode: :itself, start: nil, finish: nil) + raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? + + check_mode!(mode) + + # non-distinct have better performance + batch_size ||= batch_size_for_mode_and_operation(mode, @operation) + + start = actual_start(start) + finish = actual_finish(finish) + + raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 + return FALLBACK if unwanted_configuration?(finish, batch_size, start) + + results = nil + batch_start = start + + while batch_start < finish + begin + batch_end = [batch_start + batch_size, finish].min + batch_relation = build_relation_batch(batch_start, batch_end, mode) + + op_args = @operation_args + if @operation == :count && @operation_args.blank? && use_loose_index_scan_for_distinct_values?(mode) + op_args = [Gitlab::Database::LooseIndexScanDistinctCount::COLUMN_ALIAS] + end + + results = merge_results(results, batch_relation.send(@operation, *op_args)) # rubocop:disable GitlabSecurity/PublicSend + batch_start = batch_end + rescue ActiveRecord::QueryCanceled => error + # retry with a safe batch size & warmer cache + if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE + batch_size /= 2 + else + log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) + return FALLBACK + end + rescue Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError => error + Gitlab::AppJsonLogger + .error( + event: 'batch_count', + relation: @relation.table_name, + operation: @operation, + operation_args: @operation_args, + mode: mode, + message: "LooseIndexScanDistinctCount column error: #{error.message}" + ) + + return FALLBACK + end + + sleep(SLEEP_TIME_IN_SECONDS) + end + + results + end + + def merge_results(results, object) + return object unless results + + if object.is_a?(Hash) + results.merge!(object) { |_, a, b| a + b } + else + results + object + end + end + + private + + def build_relation_batch(start, finish, mode) + if use_loose_index_scan_for_distinct_values?(mode) + Gitlab::Database::LooseIndexScanDistinctCount.new(@relation, @column).build_query(from: start, to: finish) + else + @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def batch_size_for_mode_and_operation(mode, operation) + return DEFAULT_SUM_BATCH_SIZE if operation == :sum + + mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE + end + + def between_condition(start, finish) + return @column.between(start...finish) if @column.is_a?(Arel::Attributes::Attribute) + + { @column => start...finish } + end + + def actual_start(start) + start || @relation.unscope(:group, :having).minimum(@column) || 0 + end + + def actual_finish(finish) + (finish || @relation.unscope(:group, :having).maximum(@column) || FALLBACK_FINISH) + OFFSET_BY_ONE + end + + def check_mode!(mode) + raise "The mode #{mode.inspect} is not supported" unless ALLOWED_MODES.include?(mode) + raise 'Use distinct count for optimized distinct counting' if @relation.limit(1).distinct_value.present? && mode != :distinct + raise 'Use distinct count only with non id fields' if @column == :id && mode == :distinct + end + + def log_canceled_batch_fetch(batch_start, mode, query, error) + Gitlab::AppJsonLogger + .error( + event: 'batch_count', + relation: @relation.table_name, + operation: @operation, + operation_args: @operation_args, + start: batch_start, + mode: mode, + query: query, + message: "Query has been canceled with message: #{error.message}" + ) + end + + def use_loose_index_scan_for_distinct_values?(mode) + Feature.enabled?(:loose_index_scan_for_distinct_values) && not_group_by_query? && mode == :distinct + end + + def not_group_by_query? + !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank? + end + end + end +end diff --git a/lib/gitlab/database/custom_structure.rb b/lib/gitlab/database/custom_structure.rb deleted file mode 100644 index e4404e73a63..00000000000 --- a/lib/gitlab/database/custom_structure.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - class CustomStructure - CUSTOM_DUMP_FILE = 'db/gitlab_structure.sql' - - def dump - File.open(self.class.custom_dump_filepath, 'wb') do |io| - io << "-- this file tracks custom GitLab data, such as foreign keys referencing partitioned tables\n" - io << "-- more details can be found in the issue: https://gitlab.com/gitlab-org/gitlab/-/issues/201872\n\n" - - dump_partitioned_foreign_keys(io) if partitioned_foreign_keys_exist? - end - end - - def self.custom_dump_filepath - Rails.root.join(CUSTOM_DUMP_FILE) - end - - private - - def dump_partitioned_foreign_keys(io) - io << "COPY partitioned_foreign_keys (#{partitioned_fk_columns.join(", ")}) FROM STDIN;\n" - - PartitioningMigrationHelpers::PartitionedForeignKey.find_each do |fk| - io << fk.attributes.values_at(*partitioned_fk_columns).join("\t") << "\n" - end - io << "\\.\n" - end - - def partitioned_foreign_keys_exist? - return false unless PartitioningMigrationHelpers::PartitionedForeignKey.table_exists? - - PartitioningMigrationHelpers::PartitionedForeignKey.exists? - end - - def partitioned_fk_columns - @partitioned_fk_columns ||= PartitioningMigrationHelpers::PartitionedForeignKey.column_names - end - end - end -end diff --git a/lib/gitlab/database/dynamic_model_helpers.rb b/lib/gitlab/database/dynamic_model_helpers.rb index 7439591be99..220062f1bc6 100644 --- a/lib/gitlab/database/dynamic_model_helpers.rb +++ b/lib/gitlab/database/dynamic_model_helpers.rb @@ -3,6 +3,8 @@ module Gitlab module Database module DynamicModelHelpers + BATCH_SIZE = 1_000 + def define_batchable_model(table_name) Class.new(ActiveRecord::Base) do include EachBatch @@ -12,7 +14,7 @@ module Gitlab end end - def each_batch(table_name, scope: ->(table) { table.all }, of: 1000) + def each_batch(table_name, scope: ->(table) { table.all }, of: BATCH_SIZE) if transaction_open? raise <<~MSG.squish each_batch should not run inside a transaction, you can disable @@ -25,7 +27,7 @@ module Gitlab .each_batch(of: of) { |batch| yield batch } end - def each_batch_range(table_name, scope: ->(table) { table.all }, of: 1000) + def each_batch_range(table_name, scope: ->(table) { table.all }, of: BATCH_SIZE) each_batch(table_name, scope: scope, of: of) do |batch| yield batch.pluck('MIN(id), MAX(id)').first end diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index 88743cd2e75..31d41a6d6c0 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -85,7 +85,6 @@ module Gitlab # Returns true if load balancing is to be enabled. def self.enable? return false if Gitlab::Runtime.rake? - return false if Gitlab::Runtime.sidekiq? && !Gitlab::Utils.to_boolean(ENV['ENABLE_LOAD_BALANCING_FOR_SIDEKIQ'], default: false) return false unless self.configured? true diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index a833bb8491f..a5d67ebc050 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -147,15 +147,15 @@ module Gitlab raise 'Failed to determine the write location of the primary database' end - # Returns true if all hosts have caught up to the given transaction - # write location. - def all_caught_up?(location) - @host_list.hosts.all? { |host| host.caught_up?(location) } - end - # Returns true if there was at least one host that has caught up with the given transaction. # # In case of a retry, this method also stores the set of hosts that have caught up. + # + # UPD: `select_caught_up_hosts` seems to have redundant logic managing host list (`:gitlab_load_balancer_valid_hosts`), + # while we only need a single host: https://gitlab.com/gitlab-org/gitlab/-/issues/326125#note_615271604 + # Also, shuffling the list afterwards doesn't seem to be necessary. + # This may be improved by merging this method with `select_up_to_date_host`. + # Could be removed when `:load_balancing_refine_load_balancer_methods` FF is rolled out def select_caught_up_hosts(location) all_hosts = @host_list.hosts valid_hosts = all_hosts.select { |host| host.caught_up?(location) } @@ -179,6 +179,8 @@ module Gitlab # Returns true if there was at least one host that has caught up with the given transaction. # Similar to `#select_caught_up_hosts`, picks a random host, to rotate replicas we use. # Unlike `#select_caught_up_hosts`, does not iterate over all hosts if finds any. + # + # It is going to be merged with `select_caught_up_hosts`, because they intend to do the same. def select_up_to_date_host(location) all_hosts = @host_list.hosts.shuffle host = all_hosts.find { |host| host.caught_up?(location) } @@ -190,6 +192,7 @@ module Gitlab true end + # Could be removed when `:load_balancing_refine_load_balancer_methods` FF is rolled out def set_consistent_hosts_for_request(hosts) RequestStore[VALID_HOSTS_CACHE_KEY] = hosts end diff --git a/lib/gitlab/database/load_balancing/rack_middleware.rb b/lib/gitlab/database/load_balancing/rack_middleware.rb index 4734ff99bd3..8e7e6865402 100644 --- a/lib/gitlab/database/load_balancing/rack_middleware.rb +++ b/lib/gitlab/database/load_balancing/rack_middleware.rb @@ -39,6 +39,8 @@ module Gitlab result = @app.call(env) + ActiveSupport::Notifications.instrument('web_transaction_completed.load_balancing') + stick_if_necessary(env) result diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb index 524d69c00c0..0e36ebbc3ee 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -5,27 +5,29 @@ module Gitlab module LoadBalancing class SidekiqClientMiddleware def call(worker_class, job, _queue, _redis_pool) + # Mailers can't be constantized worker_class = worker_class.to_s.safe_constantize - mark_data_consistency_location(worker_class, job) + if load_balancing_enabled?(worker_class) + job['worker_data_consistency'] = worker_class.get_data_consistency + set_data_consistency_location!(job) unless location_already_provided?(job) + else + job['worker_data_consistency'] = ::WorkerAttributes::DEFAULT_DATA_CONSISTENCY + end yield end private - def mark_data_consistency_location(worker_class, job) - # Mailers can't be constantized - return unless worker_class - return unless worker_class.include?(::ApplicationWorker) - return unless worker_class.get_data_consistency_feature_flag_enabled? - - return if location_already_provided?(job) - - job['worker_data_consistency'] = worker_class.get_data_consistency - - return unless worker_class.utilizes_load_balancing_capabilities? + def load_balancing_enabled?(worker_class) + worker_class && + worker_class.include?(::ApplicationWorker) && + worker_class.utilizes_load_balancing_capabilities? && + worker_class.get_data_consistency_feature_flag_enabled? + end + def set_data_consistency_location!(job) if Session.current.use_primary? job['database_write_location'] = load_balancer.primary_write_location else diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 9bd0adf8dbd..0551750568a 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -7,8 +7,18 @@ module Gitlab JobReplicaNotUpToDate = Class.new(StandardError) def call(worker, job, _queue) - if requires_primary?(worker.class, job) + worker_class = worker.class + strategy = select_load_balancing_strategy(worker_class, job) + + job['load_balancing_strategy'] = strategy.to_s + + if use_primary?(strategy) Session.current.use_primary! + elsif strategy == :retry + raise JobReplicaNotUpToDate, "Sidekiq job #{worker_class} JID-#{job['jid']} couldn't use the replica."\ + " Replica was not up to date." + else + # this means we selected an up-to-date replica, but there is nothing to do in this case. end yield @@ -23,31 +33,42 @@ module Gitlab Session.clear_session end - def requires_primary?(worker_class, job) - return true unless worker_class.include?(::ApplicationWorker) - return true unless worker_class.utilizes_load_balancing_capabilities? - return true unless worker_class.get_data_consistency_feature_flag_enabled? - - location = job['database_write_location'] || job['database_replica_location'] + def use_primary?(strategy) + strategy.start_with?('primary') + end - return true unless location + def select_load_balancing_strategy(worker_class, job) + return :primary unless load_balancing_available?(worker_class) - job_data_consistency = worker_class.get_data_consistency - job[:data_consistency] = job_data_consistency.to_s + location = job['database_write_location'] || job['database_replica_location'] + return :primary_no_wal unless location if replica_caught_up?(location) - job[:database_chosen] = 'replica' - false - elsif job_data_consistency == :delayed && not_yet_retried?(job) - job[:database_chosen] = 'retry' - raise JobReplicaNotUpToDate, "Sidekiq job #{worker_class} JID-#{job['jid']} couldn't use the replica."\ - " Replica was not up to date." + # Happy case: we can read from a replica. + retried_before?(worker_class, job) ? :replica_retried : :replica + elsif can_retry?(worker_class, job) + # Optimistic case: The worker allows retries and we have retries left. + :retry else - job[:database_chosen] = 'primary' - true + # Sad case: we need to fall back to the primary. + :primary end end + def load_balancing_available?(worker_class) + worker_class.include?(::ApplicationWorker) && + worker_class.utilizes_load_balancing_capabilities? && + worker_class.get_data_consistency_feature_flag_enabled? + end + + def can_retry?(worker_class, job) + worker_class.get_data_consistency == :delayed && not_yet_retried?(job) + end + + def retried_before?(worker_class, job) + worker_class.get_data_consistency == :delayed && !not_yet_retried?(job) + end + def not_yet_retried?(job) # if `retry_count` is `nil` it indicates that this job was never retried # the `0` indicates that this is a first retry @@ -59,11 +80,7 @@ module Gitlab end def replica_caught_up?(location) - if Feature.enabled?(:sidekiq_load_balancing_rotate_up_to_date_replica) - load_balancer.select_up_to_date_host(location) - else - load_balancer.host.caught_up?(location) - end + load_balancer.select_up_to_date_host(location) end end end diff --git a/lib/gitlab/database/load_balancing/srv_resolver.rb b/lib/gitlab/database/load_balancing/srv_resolver.rb index 20da525f4d2..1f599ef4a27 100644 --- a/lib/gitlab/database/load_balancing/srv_resolver.rb +++ b/lib/gitlab/database/load_balancing/srv_resolver.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'net/dns' + module Gitlab module Database module LoadBalancing diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb index efbd7099300..8e1aa079216 100644 --- a/lib/gitlab/database/load_balancing/sticking.rb +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -33,8 +33,10 @@ module Gitlab return true unless location - load_balancer.all_caught_up?(location).tap do |caught_up| - unstick(namespace, id) if caught_up + load_balancer.select_up_to_date_host(location).tap do |found| + ActiveSupport::Notifications.instrument('caught_up_replica_pick.load_balancing', { result: found } ) + + unstick(namespace, id) if found end end @@ -51,8 +53,14 @@ module Gitlab # write location. If no such location exists, err on the side of caution. return false unless location - load_balancer.select_caught_up_hosts(location).tap do |selected| - unstick(namespace, id) if selected + if ::Feature.enabled?(:load_balancing_refine_load_balancer_methods) + load_balancer.select_up_to_date_host(location).tap do |selected| + unstick(namespace, id) if selected + end + else + load_balancer.select_caught_up_hosts(location).tap do |selected| + unstick(namespace, id) if selected + end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index d155abefdc8..842ab4f7b80 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -217,11 +217,12 @@ module Gitlab # source - The source table containing the foreign key. # target - The target table the key points to. # column - The name of the column to create the foreign key on. + # target_column - The name of the referenced column, defaults to "id". # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". # name - The name of the foreign key. # - def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, name: nil, validate: true) + def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, target_column: :id, name: nil, validate: true) # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. if transaction_open? @@ -231,7 +232,8 @@ module Gitlab options = { column: column, on_delete: on_delete, - name: name.presence || concurrent_foreign_key_name(source, column) + name: name.presence || concurrent_foreign_key_name(source, column), + primary_key: target_column } if foreign_key_exists?(source, target, **options) @@ -252,7 +254,7 @@ module Gitlab ALTER TABLE #{source} ADD CONSTRAINT #{options[:name]} FOREIGN KEY (#{options[:column]}) - REFERENCES #{target} (id) + REFERENCES #{target} (#{target_column}) #{on_delete_statement(options[:on_delete])} NOT VALID; EOF @@ -389,12 +391,14 @@ module Gitlab # * +logger+ - [Gitlab::JsonLogger] # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES` def with_lock_retries(*args, **kwargs, &block) + raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) merged_args = { klass: self.class, logger: Gitlab::BackgroundMigration::Logger }.merge(kwargs) - Gitlab::Database::WithLockRetries.new(**merged_args).run(&block) + Gitlab::Database::WithLockRetries.new(**merged_args) + .run(raise_on_exhaustion: raise_on_exhaustion, &block) end def true_value @@ -1106,7 +1110,16 @@ module Gitlab Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" elsif !migration.finished? raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ - "but it is '#{migration.status}': #{configuration}" + "but it is '#{migration.status}':" \ + "\t#{configuration}" \ + "\n\n" \ + "Finalize it manualy by running" \ + "\n\n" \ + "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.inspect.gsub(',', '\,')}']" \ + "\n\n" \ + "For more information, check the documentation" \ + "\n\n" \ + "\thttps://docs.gitlab.com/ee/user/admin_area/monitoring/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished" end end @@ -1610,6 +1623,13 @@ into similar problems in the future (e.g. when new tables are created). raise end + def rename_constraint(table_name, old_name, new_name) + execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} + RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} + SQL + end + private def validate_check_constraint_name!(constraint_name) diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index fa30ffb62f5..28491a934a0 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -107,7 +107,10 @@ module Gitlab batch_counter = 0 model_class.each_batch(of: batch_size) do |relation, index| - start_id, end_id = relation.pluck(Arel.sql("MIN(#{primary_column_name}), MAX(#{primary_column_name})")).first + max = relation.arel_table[primary_column_name].maximum + min = relation.arel_table[primary_column_name].minimum + + start_id, end_id = relation.pluck(min, max).first # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for # the same time, which is not helpful in most cases where we wish to diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index 82ea1ce26fb..4c68399cb68 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -4,16 +4,17 @@ module Gitlab module Database module Partitioning class MonthlyStrategy - attr_reader :model, :partitioning_key + attr_reader :model, :partitioning_key, :retain_for # We create this many partitions in the future HEADROOM = 6.months delegate :table_name, to: :model - def initialize(model, partitioning_key) + def initialize(model, partitioning_key, retain_for: nil) @model = model @partitioning_key = partitioning_key + @retain_for = retain_for end def current_partitions @@ -27,13 +28,21 @@ module Gitlab desired_partitions - current_partitions end + def extra_partitions + current_partitions - desired_partitions + end + private def desired_partitions [].tap do |parts| min_date, max_date = relevant_range - parts << partition_for(upper_bound: min_date) + if pruning_old_partitions? && min_date <= oldest_active_date + min_date = oldest_active_date.beginning_of_month + else + parts << partition_for(upper_bound: min_date) + end while min_date < max_date next_date = min_date.next_month @@ -52,13 +61,17 @@ module Gitlab # to start from MINVALUE to a specific date `x`. The range returned # does not include the range of the first, half-unbounded partition. def relevant_range - if first_partition = current_partitions.min + if (first_partition = current_partitions.min) # Case 1: First partition starts with MINVALUE, i.e. from is nil -> start with first real partition # Case 2: Rather unexpectedly, first partition does not start with MINVALUE, i.e. from is not nil # In this case, use first partition beginning as a start min_date = first_partition.from || first_partition.to end + if pruning_old_partitions? + min_date ||= oldest_active_date + end + # In case we don't have a partition yet min_date ||= Date.today min_date = min_date.beginning_of_month @@ -72,6 +85,14 @@ module Gitlab TimePartition.new(table_name, lower_bound, upper_bound) end + def pruning_old_partitions? + Feature.enabled?(:partition_pruning_dry_run) && retain_for.present? + end + + def oldest_active_date + (Date.today - retain_for).beginning_of_month + end + def connection ActiveRecord::Base.connection end diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_manager.rb index d4b2b8d50e2..c2a9422a42a 100644 --- a/lib/gitlab/database/partitioning/partition_creator.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -3,7 +3,7 @@ module Gitlab module Database module Partitioning - class PartitionCreator + class PartitionManager def self.register(model) raise ArgumentError, "Only models with a #partitioning_strategy can be registered." unless model.respond_to?(:partitioning_strategy) @@ -15,7 +15,7 @@ module Gitlab end LEASE_TIMEOUT = 1.minute - LEASE_KEY = 'database_partition_creation_%s' + MANAGEMENT_LEASE_KEY = 'database_partition_management_%s' attr_reader :models @@ -23,23 +23,25 @@ module Gitlab @models = models end - def create_partitions + def sync_partitions Gitlab::AppLogger.info("Checking state of dynamic postgres partitions") models.each do |model| # Double-checking before getting the lease: - # The prevailing situation is no missing partitions - next if missing_partitions(model).empty? + # The prevailing situation is no missing partitions and no extra partitions + next if missing_partitions(model).empty? && extra_partitions(model).empty? - only_with_exclusive_lease(model) do + only_with_exclusive_lease(model, lease_key: MANAGEMENT_LEASE_KEY) do partitions_to_create = missing_partitions(model) + create(partitions_to_create) unless partitions_to_create.empty? - next if partitions_to_create.empty? - - create(model, partitions_to_create) + if Feature.enabled?(:partition_pruning_dry_run) + partitions_to_detach = extra_partitions(model) + detach(partitions_to_detach) unless partitions_to_detach.empty? + end end rescue StandardError => e - Gitlab::AppLogger.error("Failed to create partition(s) for #{model.table_name}: #{e.class}: #{e.message}") + Gitlab::AppLogger.error("Failed to create / detach partition(s) for #{model.table_name}: #{e.class}: #{e.message}") end end @@ -51,15 +53,22 @@ module Gitlab model.partitioning_strategy.missing_partitions end - def only_with_exclusive_lease(model) - lease = Gitlab::ExclusiveLease.new(LEASE_KEY % model.table_name, timeout: LEASE_TIMEOUT) + def extra_partitions(model) + return [] unless Feature.enabled?(:partition_pruning_dry_run) + return [] unless connection.table_exists?(model.table_name) + + model.partitioning_strategy.extra_partitions + end + + def only_with_exclusive_lease(model, lease_key:) + lease = Gitlab::ExclusiveLease.new(lease_key % model.table_name, timeout: LEASE_TIMEOUT) yield if lease.try_obtain ensure lease&.cancel end - def create(model, partitions) + def create(partitions) connection.transaction do with_lock_retries do partitions.each do |partition| @@ -71,6 +80,18 @@ module Gitlab end end + def detach(partitions) + connection.transaction do + with_lock_retries do + partitions.each { |p| detach_one_partition(p) } + end + end + end + + def detach_one_partition(partition) + Gitlab::AppLogger.info("Planning to detach #{partition.partition_name} for table #{partition.table}") + end + def with_lock_retries(&block) Gitlab::Database::WithLockRetries.new( klass: self.class, diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb index 9ec9ae684a5..ad122fd47fe 100644 --- a/lib/gitlab/database/partitioning/partition_monitoring.rb +++ b/lib/gitlab/database/partitioning/partition_monitoring.rb @@ -6,7 +6,7 @@ module Gitlab class PartitionMonitoring attr_reader :models - def initialize(models = PartitionCreator.models) + def initialize(models = PartitionManager.models) @models = models end diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb index 4402c42b136..f1aa7871245 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb @@ -79,135 +79,6 @@ module Gitlab "#{prefix}#{hashed_identifier}" end - - # Creates a "foreign key" that references a partitioned table. Because foreign keys referencing partitioned - # tables are not supported in PG11, this does not create a true database foreign key, but instead implements the - # same functionality at the database level by using triggers. - # - # Example: - # - # add_partitioned_foreign_key :issues, :projects - # - # Available options: - # - # :column - name of the referencing column (otherwise inferred from the referenced table name) - # :primary_key - name of the primary key in the referenced table (defaults to id) - # :on_delete - supports either :cascade for ON DELETE CASCADE or :nullify for ON DELETE SET NULL - # - def add_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id, on_delete: :cascade) - cascade_delete = extract_cascade_option(on_delete) - - update_foreign_keys(from_table, to_table, column, primary_key, cascade_delete) do |current_keys, existing_key, specified_key| - if existing_key.nil? - unless specified_key.save - raise "failed to create foreign key: #{specified_key.errors.full_messages.to_sentence}" - end - - current_keys << specified_key - else - Gitlab::AppLogger.warn "foreign key not added because it already exists: #{specified_key}" - current_keys - end - end - end - - # Drops a "foreign key" that references a partitioned table. This method ONLY applies to foreign keys previously - # created through the `add_partitioned_foreign_key` method. Standard database foreign keys should be managed - # through the familiar Rails helpers. - # - # Example: - # - # remove_partitioned_foreign_key :issues, :projects - # - # Available options: - # - # :column - name of the referencing column (otherwise inferred from the referenced table name) - # :primary_key - name of the primary key in the referenced table (defaults to id) - # - def remove_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id) - update_foreign_keys(from_table, to_table, column, primary_key) do |current_keys, existing_key, specified_key| - if existing_key - existing_key.delete - current_keys.delete(existing_key) - else - Gitlab::AppLogger.warn "foreign key not removed because it doesn't exist: #{specified_key}" - end - - current_keys - end - end - - private - - def fk_function_name(table) - object_name(table, 'fk_cascade_function') - end - - def fk_trigger_name(table) - object_name(table, 'fk_cascade_trigger') - end - - def fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete) - PartitionedForeignKey.new(from_table: from_table.to_s, to_table: to_table.to_s, from_column: from_column.to_s, - to_column: to_column.to_s, cascade_delete: cascade_delete) - end - - def update_foreign_keys(from_table, to_table, from_column, to_column, cascade_delete = nil) - assert_not_in_transaction_block(scope: 'partitioned foreign key') - - from_column ||= "#{to_table.to_s.singularize}_id" - specified_key = fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete) - - current_keys = PartitionedForeignKey.by_referenced_table(to_table).to_a - existing_key = find_existing_key(current_keys, specified_key) - - final_keys = yield current_keys, existing_key, specified_key - - fn_name = fk_function_name(to_table) - trigger_name = fk_trigger_name(to_table) - - with_lock_retries do - drop_trigger(to_table, trigger_name, if_exists: true) - - if final_keys.empty? - drop_function(fn_name, if_exists: true) - else - create_or_replace_fk_function(fn_name, final_keys) - create_trigger(to_table, trigger_name, fn_name, fires: 'AFTER DELETE') - end - end - end - - def extract_cascade_option(on_delete) - case on_delete - when :cascade then true - when :nullify then false - else raise ArgumentError, "invalid option #{on_delete} for :on_delete" - end - end - - def find_existing_key(keys, key) - keys.find { |k| k.from_table == key.from_table && k.from_column == key.from_column } - end - - def create_or_replace_fk_function(fn_name, fk_specs) - create_trigger_function(fn_name, replace: true) do - cascade_statements = build_cascade_statements(fk_specs) - cascade_statements << 'RETURN OLD;' - - cascade_statements.join("\n") - end - end - - def build_cascade_statements(foreign_keys) - foreign_keys.map do |fks| - if fks.cascade_delete? - "DELETE FROM #{fks.from_table} WHERE #{fks.from_column} = OLD.#{fks.to_column};" - else - "UPDATE #{fks.from_table} SET #{fks.from_column} = NULL WHERE #{fks.from_column} = OLD.#{fks.to_column};" - end - end - end end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb deleted file mode 100644 index f9a90511f9b..00000000000 --- a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module PartitioningMigrationHelpers - class PartitionedForeignKey < ApplicationRecord - validates_with PartitionedForeignKeyValidator - - scope :by_referenced_table, ->(table) { where(to_table: table) } - end - end - end -end diff --git a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb deleted file mode 100644 index 089cf2b8931..00000000000 --- a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module PartitioningMigrationHelpers - class PartitionedForeignKeyValidator < ActiveModel::Validator - def validate(record) - validate_key_part(record, :from_table, :from_column) - validate_key_part(record, :to_table, :to_column) - end - - private - - def validate_key_part(record, table_field, column_field) - if !connection.table_exists?(record[table_field]) - record.errors.add(table_field, 'must be a valid table') - elsif !connection.column_exists?(record[table_field], record[column_field]) - record.errors.add(column_field, 'must be a valid column') - end - end - - def connection - ActiveRecord::Base.connection - end - end - end - end -end diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index aa46b98be5d..580cab5622d 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -11,17 +11,17 @@ module Gitlab # In order to not use a possible complex time consuming query when calculating min and max values, # the start and finish can be sent specifically, start and finish should contain max and min values for PRIMARY KEY of # relation (most cases `id` column) rather than counted attribute eg: - # estimate_distinct_count(start: ::Project.with_active_services.minimum(:id), finish: ::Project.with_active_services.maximum(:id)) + # estimate_distinct_count(start: ::Project.aimed_for_deletion.minimum(:id), finish: ::Project.aimed_for_deletion.maximum(:id)) # # Grouped relations are NOT supported yet. # # @example Usage # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project, :creator_id).execute - # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project.with_active_services.service_desk_enabled.where(time_period)) + # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project.aimed_for_deletion.service_desk_enabled.where(time_period)) # .execute( # batch_size: 1_000, - # start: ::Project.with_active_services.service_desk_enabled.where(time_period).minimum(:id), - # finish: ::Project.with_active_services.service_desk_enabled.where(time_period).maximum(:id) + # start: ::Project.aimed_for_deletion.service_desk_enabled.where(time_period).minimum(:id), + # finish: ::Project.aimed_for_deletion.service_desk_enabled.where(time_period).maximum(:id) # ) # # @note HyperLogLog is an PROBABILISTIC algorithm that ESTIMATES distinct count of given attribute value for supplied relation diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 6e734834841..58e4e7e7924 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -7,6 +7,7 @@ module Gitlab self.table_name = 'postgres_indexes' self.primary_key = 'identifier' + self.inheritance_column = :_type_disabled has_one :bloat_estimate, class_name: 'Gitlab::Database::PostgresIndexBloatEstimate', foreign_key: :identifier has_many :reindexing_actions, class_name: 'Gitlab::Database::Reindexing::ReindexAction', foreign_key: :index_identifier @@ -17,12 +18,12 @@ module Gitlab find(identifier) end - # A 'regular' index is a non-unique index, - # that does not serve an exclusion constraint and - # is defined on a table that is not partitioned. - scope :regular, -> { where(unique: false, partitioned: false, exclusion: false)} + # Indexes with reindexing support + scope :reindexing_support, -> { where(partitioned: false, exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES) } - scope :not_match, ->(regex) { where("name !~ ?", regex)} + scope :not_match, ->(regex) { where("name !~ ?", regex) } + + scope :match, ->(regex) { where("name ~* ?", regex) } scope :not_recently_reindexed, -> do recent_actions = Reindexing::ReindexAction.recent.where('index_identifier = identifier') @@ -30,10 +31,19 @@ module Gitlab where('NOT EXISTS (?)', recent_actions) end + def reset + reload # rubocop:disable Cop/ActiveRecordAssociationReload + clear_memoization(:bloat_size) + end + def bloat_size strong_memoize(:bloat_size) { bloat_estimate&.bloat_size || 0 } end + def relative_bloat_level + bloat_size / ondisk_size_bytes.to_f + end + def to_s name end diff --git a/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb b/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb index 59bd24d3c37..a2e7f4befab 100644 --- a/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb +++ b/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb @@ -7,8 +7,7 @@ module Gitlab extend ActiveSupport::Concern def dump_schema_information # :nodoc: - versions = schema_migration.all_versions - Gitlab::Database::SchemaVersionFiles.touch_all(versions) if versions.any? + Gitlab::Database::SchemaMigrations.touch_all(self) nil end diff --git a/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb index 9f664fa2137..71d2554844e 100644 --- a/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb +++ b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb @@ -22,7 +22,7 @@ module Gitlab end def force_disconnect_timer - @force_disconnect_timer ||= ConnectionTimer.starting_now + @force_disconnect_timer ||= ::Gitlab::Database::ConnectionTimer.starting_now end end end diff --git a/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb b/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb index cf8342941c4..30060ecb34f 100644 --- a/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb +++ b/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb @@ -6,9 +6,10 @@ module Gitlab module LoadSchemaVersionsMixin extend ActiveSupport::Concern - def structure_load(*args) - super(*args) - Gitlab::Database::SchemaVersionFiles.load_all + def structure_load(...) + super(...) + + Gitlab::Database::SchemaMigrations.load_all(connection) end end end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 0cfad690283..841e04ccbd1 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -6,6 +6,8 @@ module Gitlab # Number of indexes to reindex per invocation DEFAULT_INDEXES_PER_INVOCATION = 2 + SUPPORTED_TYPES = %w(btree gist).freeze + # candidate_indexes: Array of Gitlab::Database::PostgresIndex def self.perform(candidate_indexes, how_many: DEFAULT_INDEXES_PER_INVOCATION) IndexSelection.new(candidate_indexes).take(how_many).each do |index| @@ -15,10 +17,8 @@ module Gitlab def self.candidate_indexes Gitlab::Database::PostgresIndex - .regular - .where('NOT expression') - .not_match("^#{ConcurrentReindex::TEMPORARY_INDEX_PREFIX}") - .not_match("^#{ConcurrentReindex::REPLACED_INDEX_PREFIX}") + .not_match("#{ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$") + .reindexing_support end end end diff --git a/lib/gitlab/database/reindexing/concurrent_reindex.rb b/lib/gitlab/database/reindexing/concurrent_reindex.rb deleted file mode 100644 index 7e2dd55d21b..00000000000 --- a/lib/gitlab/database/reindexing/concurrent_reindex.rb +++ /dev/null @@ -1,154 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module Reindexing - class ConcurrentReindex - include Gitlab::Utils::StrongMemoize - - ReindexError = Class.new(StandardError) - - PG_IDENTIFIER_LENGTH = 63 - TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' - REPLACED_INDEX_PREFIX = 'old_reindex_' - STATEMENT_TIMEOUT = 9.hours - - # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock, - # which only conflicts with DDL and vacuum. We therefore execute this with a rather - # high lock timeout and a long pause in between retries. This is an alternative to - # setting a high statement timeout, which would lead to a long running query with effects - # on e.g. vacuum. - REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 - - attr_reader :index, :logger - - def initialize(index, logger: Gitlab::AppLogger) - @index = index - @logger = logger - end - - def perform - raise ReindexError, 'UNIQUE indexes are currently not supported' if index.unique? - raise ReindexError, 'partitioned indexes are currently not supported' if index.partitioned? - raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion? - raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name.start_with?(TEMPORARY_INDEX_PREFIX, REPLACED_INDEX_PREFIX) - - logger.info "Starting reindex of #{index}" - - with_rebuilt_index do |replacement_index| - swap_index(replacement_index) - end - end - - private - - def with_rebuilt_index - if Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) - logger.debug("dropping dangling index from previous run (if it exists): #{replacement_index_name}") - remove_index(index.schema, replacement_index_name) - end - - create_replacement_index_statement = index.definition - .sub(/CREATE INDEX #{index.name}/, "CREATE INDEX CONCURRENTLY #{replacement_index_name}") - - logger.info("creating replacement index #{replacement_index_name}") - logger.debug("replacement index definition: #{create_replacement_index_statement}") - - set_statement_timeout do - connection.execute(create_replacement_index_statement) - end - - replacement_index = Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) - - unless replacement_index.valid_index? - message = 'replacement index was created as INVALID' - logger.error("#{message}, cleaning up") - raise ReindexError, "failed to reindex #{index}: #{message}" - end - - # Some expression indexes (aka functional indexes) - # require additional statistics. The existing statistics - # are tightly bound to the original index. We have to - # rebuild statistics for the new index before dropping - # the original one. - rebuild_statistics if index.expression? - - yield replacement_index - ensure - begin - remove_index(index.schema, replacement_index_name) - rescue StandardError => e - logger.error(e) - end - end - - def swap_index(replacement_index) - logger.info("swapping replacement index #{replacement_index} with #{index}") - - with_lock_retries do - rename_index(index.schema, index.name, replaced_index_name) - rename_index(replacement_index.schema, replacement_index.name, index.name) - rename_index(index.schema, replaced_index_name, replacement_index.name) - end - end - - def rename_index(schema, old_index_name, new_index_name) - connection.execute(<<~SQL) - ALTER INDEX #{quote_table_name(schema)}.#{quote_table_name(old_index_name)} - RENAME TO #{quote_table_name(new_index_name)} - SQL - end - - def remove_index(schema, name) - logger.info("Removing index #{schema}.#{name}") - - retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( - timing_configuration: REMOVE_INDEX_RETRY_CONFIG, - klass: self.class, - logger: logger - ) - - retries.run(raise_on_exhaustion: false) do - connection.execute(<<~SQL) - DROP INDEX CONCURRENTLY - IF EXISTS #{quote_table_name(schema)}.#{quote_table_name(name)} - SQL - end - end - - def rebuild_statistics - logger.info("rebuilding table statistics for #{index.schema}.#{index.tablename}") - - connection.execute(<<~SQL) - ANALYZE #{quote_table_name(index.schema)}.#{quote_table_name(index.tablename)} - SQL - end - - def replacement_index_name - @replacement_index_name ||= "#{TEMPORARY_INDEX_PREFIX}#{index.indexrelid}" - end - - def replaced_index_name - @replaced_index_name ||= "#{REPLACED_INDEX_PREFIX}#{index.indexrelid}" - end - - def with_lock_retries(&block) - arguments = { klass: self.class, logger: logger } - Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block) - end - - def set_statement_timeout - execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) - yield - ensure - execute('RESET statement_timeout') - end - - delegate :execute, :quote_table_name, to: :connection - def connection - @connection ||= ActiveRecord::Base.connection - end - end - end - end -end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index d68f47b5b6c..13298f67ca9 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -41,7 +41,7 @@ module Gitlab end def perform_for(index, action) - ConcurrentReindex.new(index).perform + ReindexConcurrently.new(index).perform rescue StandardError action.state = :failed diff --git a/lib/gitlab/database/reindexing/index_selection.rb b/lib/gitlab/database/reindexing/index_selection.rb index 406e70791df..2186384e7d7 100644 --- a/lib/gitlab/database/reindexing/index_selection.rb +++ b/lib/gitlab/database/reindexing/index_selection.rb @@ -6,6 +6,12 @@ module Gitlab class IndexSelection include Enumerable + # Only reindex indexes with a relative bloat level (bloat estimate / size) higher than this + MINIMUM_RELATIVE_BLOAT = 0.2 + + # Only consider indexes with a total ondisk size in this range (before reindexing) + INDEX_SIZE_RANGE = (1.gigabyte..100.gigabyte).freeze + delegate :each, to: :indexes def initialize(candidates) @@ -24,11 +30,12 @@ module Gitlab # we force a N+1 pattern here and estimate bloat on a per-index # basis. - @indexes ||= filter_candidates.sort_by(&:bloat_size).reverse - end - - def filter_candidates - candidates.not_recently_reindexed + @indexes ||= candidates + .not_recently_reindexed + .where(ondisk_size_bytes: INDEX_SIZE_RANGE) + .sort_by(&:relative_bloat_level) # forced N+1 + .reverse + .select { |candidate| candidate.relative_bloat_level >= MINIMUM_RELATIVE_BLOAT } end end end diff --git a/lib/gitlab/database/reindexing/reindex_action.rb b/lib/gitlab/database/reindexing/reindex_action.rb index 7e58201889f..ff465fffb74 100644 --- a/lib/gitlab/database/reindexing/reindex_action.rb +++ b/lib/gitlab/database/reindexing/reindex_action.rb @@ -10,7 +10,7 @@ module Gitlab enum state: { started: 0, finished: 1, failed: 2 } # Amount of time to consider a previous reindexing *recent* - RECENT_THRESHOLD = 7.days + RECENT_THRESHOLD = 10.days scope :recent, -> { where(state: :finished).where('action_end > ?', Time.zone.now - RECENT_THRESHOLD) } diff --git a/lib/gitlab/database/reindexing/reindex_concurrently.rb b/lib/gitlab/database/reindexing/reindex_concurrently.rb new file mode 100644 index 00000000000..8d9f9f5abdd --- /dev/null +++ b/lib/gitlab/database/reindexing/reindex_concurrently.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + # This is a >= PG12 reindexing strategy based on `REINDEX CONCURRENTLY` + class ReindexConcurrently + ReindexError = Class.new(StandardError) + + TEMPORARY_INDEX_PATTERN = '\_ccnew[0-9]*' + STATEMENT_TIMEOUT = 9.hours + PG_MAX_INDEX_NAME_LENGTH = 63 + + # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock, + # which only conflicts with DDL and vacuum. We therefore execute this with a rather + # high lock timeout and a long pause in between retries. This is an alternative to + # setting a high statement timeout, which would lead to a long running query with effects + # on e.g. vacuum. + REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 + + attr_reader :index, :logger + + def initialize(index, logger: Gitlab::AppLogger) + @index = index + @logger = logger + end + + def perform + raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion? + raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name =~ /#{TEMPORARY_INDEX_PATTERN}/ + + # Expression indexes require additional statistics in `pg_statistic`: + # select * from pg_statistic where starelid = (select oid from pg_class where relname = 'some_index'); + # + # In PG12, this has been fixed in https://gitlab.com/postgres/postgres/-/commit/b17ff07aa3eb142d2cde2ea00e4a4e8f63686f96. + # Discussion happened in https://www.postgresql.org/message-id/flat/CAFcNs%2BqpFPmiHd1oTXvcPdvAHicJDA9qBUSujgAhUMJyUMb%2BSA%40mail.gmail.com + # following a GitLab.com incident that surfaced this (https://gitlab.com/gitlab-com/gl-infra/production/-/issues/2885). + # + # While this has been backpatched, we continue to disable expression indexes until further review. + raise ReindexError, 'expression indexes are currently not supported' if index.expression? + + begin + with_logging do + set_statement_timeout do + execute("REINDEX INDEX CONCURRENTLY #{quote_table_name(index.schema)}.#{quote_table_name(index.name)}") + end + end + ensure + cleanup_dangling_indexes + end + end + + private + + def with_logging + bloat_size = index.bloat_size + ondisk_size_before = index.ondisk_size_bytes + + logger.info( + message: "Starting reindex of #{index}", + index: index.identifier, + table: index.tablename, + estimated_bloat_bytes: bloat_size, + index_size_before_bytes: ondisk_size_before, + relative_bloat_level: index.relative_bloat_level + ) + + duration = Benchmark.realtime do + yield + end + + index.reset + + logger.info( + message: "Finished reindex of #{index}", + index: index.identifier, + table: index.tablename, + estimated_bloat_bytes: bloat_size, + index_size_before_bytes: ondisk_size_before, + index_size_after_bytes: index.ondisk_size_bytes, + relative_bloat_level: index.relative_bloat_level, + duration_s: duration.round(2) + ) + end + + def cleanup_dangling_indexes + Gitlab::Database::PostgresIndex.match("#{TEMPORARY_INDEX_PATTERN}$").each do |lingering_index| + # Example lingering index name: some_index_ccnew1 + + # Example prefix: 'some_index' + prefix = lingering_index.name.gsub(/#{TEMPORARY_INDEX_PATTERN}/, '') + + # Example suffix: '_ccnew1' + suffix = lingering_index.name.match(/#{TEMPORARY_INDEX_PATTERN}/)[0] + + # Only remove if the lingering index name could have been chosen + # as a result of a REINDEX operation (considering that PostgreSQL + # truncates index names to 63 chars and adds a suffix). + if index.name[0...PG_MAX_INDEX_NAME_LENGTH - suffix.length] == prefix + remove_index(lingering_index) + end + end + end + + def remove_index(index) + logger.info("Removing dangling index #{index.identifier}") + + retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( + timing_configuration: REMOVE_INDEX_RETRY_CONFIG, + klass: self.class, + logger: logger + ) + + retries.run(raise_on_exhaustion: false) do + execute("DROP INDEX CONCURRENTLY IF EXISTS #{quote_table_name(index.schema)}.#{quote_table_name(index.name)}") + end + end + + def with_lock_retries(&block) + arguments = { klass: self.class, logger: logger } + Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block) + end + + def set_statement_timeout + execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) + yield + ensure + execute('RESET statement_timeout') + end + + delegate :execute, :quote_table_name, to: :connection + def connection + @connection ||= ActiveRecord::Base.connection + end + end + end + end +end diff --git a/lib/gitlab/database/schema_cleaner.rb b/lib/gitlab/database/schema_cleaner.rb index 8f93da2b66c..c3cdcf1450d 100644 --- a/lib/gitlab/database/schema_cleaner.rb +++ b/lib/gitlab/database/schema_cleaner.rb @@ -30,11 +30,7 @@ module Gitlab structure.gsub!(/\n{3,}/, "\n\n") io << structure.strip - io << <<~MSG - -- schema_migrations.version information is no longer stored in this file, - -- but instead tracked in the db/schema_migrations directory - -- see https://gitlab.com/gitlab-org/gitlab/-/issues/218590 for details - MSG + io << "\n" nil end diff --git a/lib/gitlab/database/schema_migrations.rb b/lib/gitlab/database/schema_migrations.rb new file mode 100644 index 00000000000..1c49ed8d946 --- /dev/null +++ b/lib/gitlab/database/schema_migrations.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaMigrations + def self.touch_all(connection) + context = Gitlab::Database::SchemaMigrations::Context.new(connection) + + Gitlab::Database::SchemaMigrations::Migrations.new(context).touch_all + end + + def self.load_all(connection) + context = Gitlab::Database::SchemaMigrations::Context.new(connection) + + Gitlab::Database::SchemaMigrations::Migrations.new(context).load_all + end + end + end +end diff --git a/lib/gitlab/database/schema_migrations/context.rb b/lib/gitlab/database/schema_migrations/context.rb new file mode 100644 index 00000000000..bd8b9bed2c1 --- /dev/null +++ b/lib/gitlab/database/schema_migrations/context.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaMigrations + class Context + attr_reader :connection + + def initialize(connection) + @connection = connection + end + + def schema_directory + @schema_directory ||= + if ActiveRecord::Base.configurations.primary?(database_name) + File.join(db_dir, 'schema_migrations') + else + File.join(db_dir, "#{database_name}_schema_migrations") + end + end + + def versions_to_create + versions_from_database = @connection.schema_migration.all_versions + versions_from_migration_files = @connection.migration_context.migrations.map { |m| m.version.to_s } + + versions_from_database & versions_from_migration_files + end + + private + + def database_name + @database_name ||= @connection.pool.db_config.name + end + + def db_dir + @db_dir ||= Rails.application.config.paths["db"].first + end + end + end + end +end diff --git a/lib/gitlab/database/schema_migrations/migrations.rb b/lib/gitlab/database/schema_migrations/migrations.rb new file mode 100644 index 00000000000..3b16b7f1b81 --- /dev/null +++ b/lib/gitlab/database/schema_migrations/migrations.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaMigrations + class Migrations + MIGRATION_VERSION_GLOB = '20[0-9][0-9]*' + + def initialize(context) + @context = context + end + + def touch_all + return unless @context.versions_to_create.any? + + version_filepaths = version_filenames.map { |f| File.join(schema_directory, f) } + FileUtils.rm(version_filepaths) + + @context.versions_to_create.each do |version| + version_filepath = File.join(schema_directory, version) + + File.open(version_filepath, 'w') do |file| + file << Digest::SHA256.hexdigest(version) + end + end + end + + def load_all + return if version_filenames.empty? + + values = version_filenames.map { |vf| "('#{@context.connection.quote_string(vf)}')" } + + @context.connection.execute(<<~SQL) + INSERT INTO schema_migrations (version) + VALUES #{values.join(',')} + ON CONFLICT DO NOTHING + SQL + end + + private + + def schema_directory + @context.schema_directory + end + + def version_filenames + @version_filenames ||= Dir.glob(MIGRATION_VERSION_GLOB, base: schema_directory) + end + end + end + end +end diff --git a/lib/gitlab/database/schema_version_files.rb b/lib/gitlab/database/schema_version_files.rb deleted file mode 100644 index 27a942404ef..00000000000 --- a/lib/gitlab/database/schema_version_files.rb +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - class SchemaVersionFiles - SCHEMA_DIRECTORY = 'db/schema_migrations' - MIGRATION_DIRECTORIES = %w[db/migrate db/post_migrate].freeze - MIGRATION_VERSION_GLOB = '20[0-9][0-9]*' - - def self.touch_all(versions_from_database) - versions_from_migration_files = find_versions_from_migration_files - - version_filepaths = find_version_filenames.map { |f| schema_directory.join(f) } - FileUtils.rm(version_filepaths) - - versions_to_create = versions_from_database & versions_from_migration_files - versions_to_create.each do |version| - version_filepath = schema_directory.join(version) - - File.open(version_filepath, 'w') do |file| - file << Digest::SHA256.hexdigest(version) - end - end - end - - def self.load_all - version_filenames = find_version_filenames - return if version_filenames.empty? - - values = version_filenames.map { |vf| "('#{connection.quote_string(vf)}')" } - connection.execute(<<~SQL) - INSERT INTO schema_migrations (version) - VALUES #{values.join(',')} - ON CONFLICT DO NOTHING - SQL - end - - def self.schema_directory - @schema_directory ||= Rails.root.join(SCHEMA_DIRECTORY) - end - - def self.migration_directories - @migration_directories ||= MIGRATION_DIRECTORIES.map { |dir| Rails.root.join(dir) } - end - - def self.find_version_filenames - Dir.glob(MIGRATION_VERSION_GLOB, base: schema_directory) - end - - def self.find_versions_from_migration_files - migration_directories.each_with_object([]) do |directory, migration_versions| - directory_migrations = Dir.glob(MIGRATION_VERSION_GLOB, base: directory) - directory_versions = directory_migrations.map! { |m| m.split('_').first } - - migration_versions.concat(directory_versions) - end - end - - def self.connection - ActiveRecord::Base.connection - end - end - end -end diff --git a/lib/gitlab/database_importers/instance_administrators/create_group.rb b/lib/gitlab/database_importers/instance_administrators/create_group.rb index d9425810405..79244120776 100644 --- a/lib/gitlab/database_importers/instance_administrators/create_group.rb +++ b/lib/gitlab/database_importers/instance_administrators/create_group.rb @@ -89,7 +89,7 @@ module Gitlab end def track_event(result) - ::Gitlab::Tracking.event("instance_administrators_group", "group_created") + ::Gitlab::Tracking.event("instance_administrators_group", "group_created", namespace: result[:group]) success(result) end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index d1ada8c723e..57d354eb907 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -75,13 +75,13 @@ module Gitlab if response # In the add_prometheus_manual_configuration method, the Prometheus - # server_address config is saved as an api_url in the PrometheusService - # model. There are validates hooks in the PrometheusService model that - # check if the project associated with the PrometheusService is the + # server_address config is saved as an api_url in the Integrations::Prometheus + # model. There are validates hooks in the Integrations::Prometheus model that + # check if the project associated with the Integrations::Prometheus is the # self_monitoring project. It checks # Gitlab::CurrentSettings.self_monitoring_project_id, which is why the # Gitlab::CurrentSettings cache needs to be expired here, so that - # PrometheusService sees the latest self_monitoring_project_id. + # Integrations::Prometheus sees the latest self_monitoring_project_id. Gitlab::CurrentSettings.expire_current_application_settings success(result) else @@ -107,10 +107,10 @@ module Gitlab return success(result) unless prometheus_enabled? return success(result) unless prometheus_server_address.present? - service = result[:project].find_or_initialize_service('prometheus') + prometheus = result[:project].find_or_initialize_integration('prometheus') - unless service.update(prometheus_service_attributes) - log_error('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}' % { errors: service.errors.full_messages }) + unless prometheus.update(prometheus_integration_attributes) + log_error('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}' % { errors: prometheus.errors.full_messages }) return error(_('Could not save prometheus manual configuration')) end @@ -118,7 +118,8 @@ module Gitlab end def track_event(result) - ::Gitlab::Tracking.event("self_monitoring", "project_created") + project = result[:project] + ::Gitlab::Tracking.event("self_monitoring", "project_created", project: project, namespace: project.namespace) success(result) end @@ -156,7 +157,7 @@ module Gitlab ::Gitlab::Prometheus::Internal.uri end - def prometheus_service_attributes + def prometheus_integration_attributes { api_url: internal_prometheus_server_address_uri, manual_configuration: true, diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 35581952f4a..0ba23b8ffc7 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -449,7 +449,7 @@ module Gitlab end def alternate_viewer_class - return unless viewer.class == DiffViewer::Renamed + return unless viewer.instance_of?(DiffViewer::Renamed) find_renderable_viewer_class(RICH_VIEWERS) || (DiffViewer::Text if text?) end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index f3f0f227a8c..6d04c4874c7 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -85,14 +85,14 @@ module Gitlab # No-op end + def overflow? + raw_diff_files.overflow? + end + private def empty_pagination_data - { - current_page: nil, - next_page: nil, - total_pages: nil - } + { total_pages: nil } end def diff_stats_collection diff --git a/lib/gitlab/diff/file_collection/commit.rb b/lib/gitlab/diff/file_collection/commit.rb index 7b1d6171e82..0f8f408d326 100644 --- a/lib/gitlab/diff/file_collection/commit.rb +++ b/lib/gitlab/diff/file_collection/commit.rb @@ -10,6 +10,10 @@ module Gitlab diff_options: diff_options, diff_refs: commit.diff_refs) end + + def cache_key + ['commit', @diffable.id] + end end end end diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb index 663bad95db7..badebabb192 100644 --- a/lib/gitlab/diff/file_collection/compare.rb +++ b/lib/gitlab/diff/file_collection/compare.rb @@ -14,6 +14,10 @@ module Gitlab def unfold_diff_lines(positions) # no-op end + + def cache_key + ['compare', @diffable.head.id, @diffable.base.id] + end end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index d2ca86fdfe7..692186fc323 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -6,6 +6,8 @@ module Gitlab class MergeRequestDiffBase < Base extend ::Gitlab::Utils::Override + delegate :real_size, :overflow?, :cache_key, to: :@merge_request_diff + def initialize(merge_request_diff, diff_options:) @merge_request_diff = merge_request_diff @@ -44,10 +46,6 @@ module Gitlab diff_stats_cache.clear end - def real_size - @merge_request_diff.real_size - end - private def highlight_cache @@ -58,7 +56,7 @@ module Gitlab def diff_stats_cache strong_memoize(:diff_stats_cache) do - Gitlab::Diff::StatsCache.new(cachable_key: @merge_request_diff.cache_key) + Gitlab::Diff::StatsCache.new(cachable_key: cache_key) end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb index 5ff7c88970c..0a601bde612 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb @@ -21,9 +21,7 @@ module Gitlab @paginated_collection = load_paginated_collection(batch_page, batch_size, diff_options) @pagination_data = { - current_page: current_page, - next_page: next_page, - total_pages: total_pages + total_pages: @paginated_collection.blank? ? nil : relation.size } end @@ -62,24 +60,6 @@ module Gitlab @merge_request_diff.merge_request_diff_files end - def current_page - return if @paginated_collection.blank? - - batch_gradual_load? ? nil : @paginated_collection.current_page - end - - def next_page - return if @paginated_collection.blank? - - batch_gradual_load? ? nil : @paginated_collection.next_page - end - - def total_pages - return if @paginated_collection.blank? - - batch_gradual_load? ? relation.size : @paginated_collection.total_pages - end - # rubocop: disable CodeReuse/ActiveRecord def load_paginated_collection(batch_page, batch_size, diff_options) batch_page ||= DEFAULT_BATCH_PAGE @@ -87,21 +67,12 @@ module Gitlab paths = diff_options&.fetch(:paths, nil) - paginated_collection = if batch_gradual_load? - relation.offset(batch_page).limit([batch_size.to_i, DEFAULT_BATCH_SIZE].min) - else - relation.page(batch_page).per(batch_size) - end - + paginated_collection = relation.offset(batch_page).limit([batch_size.to_i, DEFAULT_BATCH_SIZE].min) paginated_collection = paginated_collection.by_paths(paths) if paths paginated_collection end # rubocop: enable CodeReuse/ActiveRecord - - def batch_gradual_load? - Feature.enabled?(:diffs_gradual_load, @merge_request_diff.project, default_enabled: true) - end end end end diff --git a/lib/gitlab/diff/position_tracer/line_strategy.rb b/lib/gitlab/diff/position_tracer/line_strategy.rb index e3c1e549b96..8bacc781f61 100644 --- a/lib/gitlab/diff/position_tracer/line_strategy.rb +++ b/lib/gitlab/diff/position_tracer/line_strategy.rb @@ -95,7 +95,7 @@ module Gitlab if c_line # If the line is still in D but also in C, it has turned from an # added line into an unchanged one. - new_position = new_position(cd_diff, c_line, d_line) + new_position = new_position(cd_diff, c_line, d_line, position.line_range) if valid_position?(new_position) # If the line is still in the MR, we don't treat this as outdated. { position: new_position, outdated: false } @@ -108,7 +108,7 @@ module Gitlab end else # If the line is still in D and not in C, it is still added. - { position: new_position(cd_diff, nil, d_line), outdated: false } + { position: new_position(cd_diff, nil, d_line, position.line_range), outdated: false } end else # If the line is no longer in D, it has been removed from the MR. @@ -143,7 +143,7 @@ module Gitlab { position: new_position(bd_diff, nil, d_line), outdated: true } else # If the line is still in C and not in D, it is still removed. - { position: new_position(cd_diff, c_line, nil), outdated: false } + { position: new_position(cd_diff, c_line, nil, position.line_range), outdated: false } end else # If the line is no longer in C, it has been removed outside of the MR. @@ -174,7 +174,7 @@ module Gitlab if c_line && d_line # If the line is still in C and D, it is still unchanged. - new_position = new_position(cd_diff, c_line, d_line) + new_position = new_position(cd_diff, c_line, d_line, position.line_range) if valid_position?(new_position) # If the line is still in the MR, we don't treat this as outdated. { position: new_position, outdated: false } @@ -188,7 +188,7 @@ module Gitlab # If the line is still in D but no longer in C, it has turned from # an unchanged line into an added one. # We don't treat this as outdated since the line is still in the MR. - { position: new_position(cd_diff, nil, d_line), outdated: false } + { position: new_position(cd_diff, nil, d_line, position.line_range), outdated: false } else # !d_line && (c_line || !c_line) # If the line is no longer in D, it has turned from an unchanged line # into a removed one. @@ -196,12 +196,15 @@ module Gitlab end end - def new_position(diff_file, old_line, new_line) - Position.new( + def new_position(diff_file, old_line, new_line, line_range = nil) + params = { diff_file: diff_file, old_line: old_line, - new_line: new_line - ) + new_line: new_line, + line_range: line_range + }.compact + + Position.new(**params) end def valid_position?(position) diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index e927a5641e5..b110d39818d 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -61,7 +61,8 @@ module Gitlab params: { title: mail.subject, description: message_including_reply - } + }, + spam_params: nil ).execute end diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index fd3143488b1..d508cf9360e 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -4,14 +4,6 @@ module Gitlab module Email module Handler module ReplyProcessing - private - - attr_reader :project_id, :project_slug, :project_path, :incoming_email_token - - def author - raise NotImplementedError - end - # rubocop:disable Gitlab/ModuleWithInstanceVariables def project return @project if instance_variable_defined?(:@project) @@ -27,6 +19,14 @@ module Gitlab end # rubocop:enable Gitlab/ModuleWithInstanceVariables + private + + attr_reader :project_id, :project_slug, :project_path, :incoming_email_token + + def author + raise NotImplementedError + end + def message @message ||= process_message end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 05daa08530e..84b55079cea 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -42,18 +42,10 @@ module Gitlab end end - def metrics_params - super.merge(project: project&.full_path) - end - def metrics_event :receive_email_service_desk end - private - - attr_reader :project_id, :project_path, :service_desk_key - def project strong_memoize(:project) do @project = service_desk_key ? project_from_key : super @@ -62,6 +54,10 @@ module Gitlab end end + private + + attr_reader :project_id, :project_path, :service_desk_key + def project_from_key return unless match = service_desk_key.match(PROJECT_KEY_PATTERN) @@ -83,7 +79,8 @@ module Gitlab description: message_including_template, confidential: true, external_author: from_address - } + }, + spam_params: nil ).execute raise InvalidIssueError unless @issue.persisted? @@ -95,6 +92,7 @@ module Gitlab def send_thank_you_email Notify.service_desk_thank_you_email(@issue.id).deliver_later + Gitlab::Metrics::BackgroundTransaction.current&.add_event(:service_desk_thank_you_email) end def message_including_template diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb index d563de6c77e..88140c67804 100644 --- a/lib/gitlab/email/message/in_product_marketing/verify.rb +++ b/lib/gitlab/email/message/in_product_marketing/verify.rb @@ -68,7 +68,7 @@ module Gitlab private def ci_link - link(s_('InProductMarketing|how easy it is to get started'), help_page_url('ci/README')) + link(s_('InProductMarketing|how easy it is to get started'), help_page_url('ci/index')) end def quick_start_link diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 8139a294269..242def826be 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -22,6 +22,9 @@ module Gitlab handler.execute.tap do Gitlab::Metrics::BackgroundTransaction.current&.add_event(handler.metrics_event, handler.metrics_params) end + rescue StandardError => e + Gitlab::Metrics::BackgroundTransaction.current&.add_event('email_receiver_error', error: e.class.name) + raise e end def mail_metadata @@ -33,7 +36,11 @@ module Gitlab references: Array(mail.references), delivered_to: delivered_to.map(&:value), envelope_to: envelope_to.map(&:value), - x_envelope_to: x_envelope_to.map(&:value) + x_envelope_to: x_envelope_to.map(&:value), + meta: { + client_id: "email/#{mail.from.first}", + project: handler&.project&.full_path + } } end diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb index e2a9192806f..e835deeea2c 100644 --- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb +++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb @@ -35,7 +35,12 @@ module Gitlab # Worse in new version, no setter! Have to poke at the # instance variable - exception.value = message if message + if message.present? + exceptions.each do |exception| + exception.value = message if valid_exception?(exception) + end + end + event.extra[:grpc_debug_error_string] = debug_str if debug_str end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index fe3dd4759d6..0cf3969b490 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -62,9 +62,6 @@ module Gitlab learn_gitlab_b: { tracking_category: 'Growth::Activation::Experiment::LearnGitLabB', rollout_strategy: :user - }, - in_product_marketing_emails: { - tracking_category: 'Growth::Activation::Experiment::InProductMarketingEmails' } }.freeze diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index ca9205a8f8c..2a43f0d5ca9 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -11,6 +11,7 @@ module Gitlab module Experimentation module ControllerConcern include ::Gitlab::Experimentation::GroupTypes + include Gitlab::Tracking::Helpers extend ActiveSupport::Concern included do @@ -101,10 +102,6 @@ module Gitlab private - def dnt_enabled? - Gitlab::Utils.to_boolean(request.headers['DNT']) - end - def experimentation_subject_id cookies.signed[:experimentation_subject_id] end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index a2215366bdc..d5ced2045f5 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -80,7 +80,7 @@ module Gitlab def shas_eql?(sha1, sha2) return true if sha1.nil? && sha2.nil? return false if sha1.nil? || sha2.nil? - return false unless sha1.class == sha2.class + return false unless sha1.instance_of?(sha2.class) # If either of the shas is below the minimum length, we cannot be sure # that they actually refer to the same commit because of hash collision. diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e38c7b516ee..70d072e8082 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -354,9 +354,13 @@ module Gitlab end end - def new_commits(newrev) + def new_commits(newrevs) wrapped_gitaly_errors do - gitaly_ref_client.list_new_commits(newrev) + if Feature.enabled?(:list_commits) + gitaly_commit_client.list_commits(Array.wrap(newrevs) + %w[--not --all]) + else + Array.wrap(newrevs).flat_map { |newrev| gitaly_ref_client.list_new_commits(newrev) } + end end end @@ -370,6 +374,20 @@ module Gitlab end end + # List blobs reachable via a set of revisions. Supports the + # pseudo-revisions `--not` and `--all`. Uses the minimum of + # GitalyClient.medium_timeout and dynamic timeout if the dynamic + # timeout is set, otherwise it'll always use the medium timeout. + def blobs(revisions, dynamic_timeout: nil) + revisions = revisions.reject { |rev| rev.blank? || rev == ::Gitlab::Git::BLANK_SHA } + + return [] if revisions.blank? + + wrapped_gitaly_errors do + gitaly_blob_client.list_blobs(revisions, limit: REV_LIST_COMMIT_LIMIT, dynamic_timeout: dynamic_timeout) + end + end + def count_commits(options) options = process_count_commits_options(options.dup) @@ -869,12 +887,6 @@ module Gitlab end end - def rebase_in_progress?(rebase_id) - wrapped_gitaly_errors do - gitaly_repository_client.rebase_in_progress?(rebase_id) - end - end - def squash(user, squash_id, start_sha:, end_sha:, author:, message:) wrapped_gitaly_errors do gitaly_operation_client.user_squash(user, squash_id, start_sha, end_sha, author, message) diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index 2c798844798..05ae3391040 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -3,10 +3,10 @@ module Gitlab module Git class User - attr_reader :username, :name, :email, :gl_id + attr_reader :username, :name, :email, :gl_id, :timezone def self.from_gitlab(gitlab_user) - new(gitlab_user.username, gitlab_user.name, gitlab_user.commit_email, Gitlab::GlId.gl_id(gitlab_user)) + new(gitlab_user.username, gitlab_user.name, gitlab_user.commit_email, Gitlab::GlId.gl_id(gitlab_user), gitlab_user.timezone) end def self.from_gitaly(gitaly_user) @@ -14,23 +14,30 @@ module Gitlab gitaly_user.gl_username, Gitlab::EncodingHelper.encode!(gitaly_user.name), Gitlab::EncodingHelper.encode!(gitaly_user.email), - gitaly_user.gl_id + gitaly_user.gl_id, + gitaly_user.timezone ) end - def initialize(username, name, email, gl_id) + def initialize(username, name, email, gl_id, timezone) @username = username @name = name @email = email @gl_id = gl_id + + @timezone = if Feature.enabled?(:add_timezone_to_web_operations) + timezone + else + Time.zone.tzinfo.name + end end def ==(other) - [username, name, email, gl_id] == [other.username, other.name, other.email, other.gl_id] + [username, name, email, gl_id, timezone] == [other.username, other.name, other.email, other.gl_id, other.timezone] end def to_gitaly - Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name.b, email: email.b) + Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name.b, email: email.b, timezone: timezone) end end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 5616b61de07..194f5da0a5c 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -54,8 +54,11 @@ module Gitlab attr_reader :repository - def self.default_ref - 'master' + # TODO remove argument when issue + # https://gitlab.com/gitlab-org/gitlab/-/issues/329190 + # is closed. + def self.default_ref(container = nil) + Gitlab::DefaultBranch.value(object: container) end # Initialize with a Gitlab::Git::Repository instance diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index b2a65d9f2d8..642a77ced11 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -9,7 +9,6 @@ module Gitlab ForbiddenError = Class.new(StandardError) NotFoundError = Class.new(StandardError) TimeoutError = Class.new(StandardError) - ProjectMovedError = Class.new(NotFoundError) # Use the magic string '_any' to indicate we do not know what the # changes are. This is also what gitlab-shell does. @@ -148,11 +147,11 @@ module Gitlab raise NotFoundError, not_found_message if container.nil? check_project! if project? + add_container_moved_message! end def check_project! check_project_accessibility! - add_project_moved_message! end def check_custom_action @@ -221,12 +220,12 @@ module Gitlab error_message(:project_not_found) end - def add_project_moved_message! + def add_container_moved_message! return if redirected_path.nil? - project_moved = Checks::ProjectMoved.new(repository, user, protocol, redirected_path) + container_moved = Checks::ContainerMoved.new(repository, user, protocol, redirected_path) - project_moved.add_message + container_moved.add_message end def check_command_disabled! @@ -499,13 +498,23 @@ module Gitlab end def check_changes_size - changes_size = 0 + changes_size = + if Feature.enabled?(:git_access_batched_changes_size, project, default_enabled: :yaml) + revs = ['--not', '--all', '--not'] + revs += changes_list.map { |change| change[:newrev] } - changes_list.each do |change| - changes_size += repository.new_blobs(change[:newrev]).sum(&:size) + repository.blobs(revs).sum(&:size) + else + changes_size = 0 - check_size_against_limit(changes_size) - end + changes_list.each do |change| + changes_size += repository.new_blobs(change[:newrev]).sum(&:size) + end + + changes_size + end + + check_size_against_limit(changes_size) end def check_size_against_limit(size) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index f4a89edecd1..2c26da037da 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -120,7 +120,7 @@ module Gitlab raise "storage #{storage.inspect} is missing a gitaly_address" end - unless URI(address).scheme.in?(%w(tcp unix tls)) + unless %w(tcp unix tls).include?(URI(address).scheme) raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'" end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index e4c8dc150a5..362ecbd845d 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -19,6 +19,25 @@ module Gitlab consume_blob_response(response) end + def list_blobs(revisions, limit: 0, bytes_limit: 0, dynamic_timeout: nil) + request = Gitaly::ListBlobsRequest.new( + repository: @gitaly_repo, + revisions: Array.wrap(revisions), + limit: limit, + bytes_limit: bytes_limit + ) + + timeout = + if dynamic_timeout + [dynamic_timeout, GitalyClient.medium_timeout].min + else + GitalyClient.medium_timeout + end + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :list_blobs, request, timeout: timeout) + GitalyClient::BlobsStitcher.new(GitalyClient::ListBlobsAdapter.new(response)) + end + def batch_lfs_pointers(blob_ids) return [] if blob_ids.empty? diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb index 2f6d146b5c4..6c51b4cf8c6 100644 --- a/lib/gitlab/gitaly_client/blobs_stitcher.rb +++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb @@ -35,8 +35,8 @@ module Gitlab Gitlab::Git::Blob.new( id: blob_data[:oid], - mode: blob_data[:mode].to_s(8), - name: File.basename(blob_data[:path]), + mode: blob_data[:mode]&.to_s(8), + name: blob_data[:path] && File.basename(blob_data[:path]), path: blob_data[:path], size: blob_data[:size], commit_id: blob_data[:revision], diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 3d24b4d53a4..b894207f0aa 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -248,6 +248,16 @@ module Gitlab consume_commits_response(response) end + def list_commits(revisions) + request = Gitaly::ListCommitsRequest.new( + repository: @gitaly_repo, + revisions: Array.wrap(revisions) + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) + consume_commits_response(response) + end + def list_commits_by_oid(oids) return [] if oids.empty? diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index fc40c23611a..300800189f1 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -49,7 +49,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage, timeout: GitalyClient.long_timeout) if response.resolution_error.present? raise Gitlab::Git::Conflict::Resolver::ResolutionError, response.resolution_error diff --git a/lib/gitlab/gitaly_client/list_blobs_adapter.rb b/lib/gitlab/gitaly_client/list_blobs_adapter.rb new file mode 100644 index 00000000000..bba668e4dc6 --- /dev/null +++ b/lib/gitlab/gitaly_client/list_blobs_adapter.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module GitalyClient + class ListBlobsAdapter + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + @rpc_response.each do |msg| + msg.blobs.each do |blob| + yield blob + end + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 6a75096ff80..009aeaf868a 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -152,23 +152,6 @@ module Gitlab ) end - def rebase_in_progress?(rebase_id) - request = Gitaly::IsRebaseInProgressRequest.new( - repository: @gitaly_repo, - rebase_id: rebase_id.to_s - ) - - response = GitalyClient.call( - @storage, - :repository_service, - :is_rebase_in_progress, - request, - timeout: GitalyClient.fast_timeout - ) - - response.in_progress - end - def squash_in_progress?(squash_id) request = Gitaly::IsSquashInProgressRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index f66dc3010ea..4cc0269673f 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -52,7 +52,7 @@ module Gitlab @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path'] storage['path'] = Deprecated - @hash = storage.with_indifferent_access + @hash = ActiveSupport::HashWithIndifferentAccess.new(storage) end def gitaly_address diff --git a/lib/gitlab/github_import/importer/diff_notes_importer.rb b/lib/gitlab/github_import/importer/diff_notes_importer.rb index 966f12c5c2f..49cbc8f7a42 100644 --- a/lib/gitlab/github_import/importer/diff_notes_importer.rb +++ b/lib/gitlab/github_import/importer/diff_notes_importer.rb @@ -22,6 +22,10 @@ module Gitlab :pull_requests_comments end + def object_type + :diff_note + end + def id_for_already_imported_cache(note) note.id end diff --git a/lib/gitlab/github_import/importer/issues_importer.rb b/lib/gitlab/github_import/importer/issues_importer.rb index ac6d0666b3a..6cc1a61b332 100644 --- a/lib/gitlab/github_import/importer/issues_importer.rb +++ b/lib/gitlab/github_import/importer/issues_importer.rb @@ -18,6 +18,10 @@ module Gitlab ImportIssueWorker end + def object_type + :issue + end + def collection_method :issues end diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index c74a7706117..40248ecbd31 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -18,6 +18,10 @@ module Gitlab ImportLfsObjectWorker end + def object_type + :lfs_object + end + def collection_method :lfs_objects end @@ -26,6 +30,8 @@ module Gitlab lfs_objects = Projects::LfsPointers::LfsObjectDownloadListService.new(project).execute lfs_objects.each do |object| + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + yield object end rescue StandardError => e diff --git a/lib/gitlab/github_import/importer/notes_importer.rb b/lib/gitlab/github_import/importer/notes_importer.rb index 5aec760ea5f..ca1d7d60515 100644 --- a/lib/gitlab/github_import/importer/notes_importer.rb +++ b/lib/gitlab/github_import/importer/notes_importer.rb @@ -18,6 +18,10 @@ module Gitlab ImportNoteWorker end + def object_type + :note + end + def collection_method :issues_comments end diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb index 8173fdd5e3e..640914acf4d 100644 --- a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb @@ -43,7 +43,7 @@ module Gitlab def missing_author_note s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*") % { - author: pull_request.merged_by.login, + author: pull_request.merged_by&.login || 'ghost', timestamp: pull_request.merged_at } end diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index f476ee13392..dd5b7c93ced 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -69,8 +69,8 @@ module Gitlab author_id: author_id, note: note, system: false, - created_at: review.submitted_at, - updated_at: review.submitted_at + created_at: submitted_at, + updated_at: submitted_at }.merge(extra) end @@ -80,8 +80,8 @@ module Gitlab approval_attribues = { merge_request_id: merge_request.id, user_id: user_id, - created_at: review.submitted_at, - updated_at: review.submitted_at + created_at: submitted_at, + updated_at: submitted_at } result = ::Approval.insert( @@ -105,6 +105,10 @@ module Gitlab Note.create!(attributes) end + + def submitted_at + @submitted_at ||= (review.submitted_at || merge_request.updated_at) + end end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 28cd3f802a2..b2f099761b1 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -22,6 +22,10 @@ module Gitlab pr.number end + def object_type + :pull_request + end + def each_object_to_import super do |pr| update_repository if update_repository?(pr) diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb index 94472cd341e..287e0ea7f7f 100644 --- a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb @@ -22,6 +22,10 @@ module Gitlab :pull_requests_merged_by end + def object_type + :pull_request_merged_by + end + def id_for_already_imported_cache(merge_request) merge_request.id end @@ -30,6 +34,8 @@ module Gitlab project.merge_requests.with_state(:merged).find_each do |merge_request| next if already_imported?(merge_request) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + pull_request = client.pull_request(project.import_source, merge_request.iid) yield(pull_request) diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index 809a518d13a..e389acbf877 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -29,6 +29,10 @@ module Gitlab :pull_request_reviews end + def object_type + :pull_request_review + end + def id_for_already_imported_cache(review) review.id end @@ -57,6 +61,8 @@ module Gitlab project.merge_requests.find_each do |merge_request| next if already_imported?(merge_request) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + client .pull_request_reviews(project.import_source, merge_request.iid) .each do |review| @@ -81,6 +87,8 @@ module Gitlab page.objects.each do |review| next if already_imported?(review) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + review.merge_request_id = merge_request.id yield(review) diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index e5f4dabe42d..0b1c221bbec 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -13,7 +13,7 @@ module Gitlab # author - An instance of `Gitlab::GithubImport::Representation::User` # exists - Boolean that indicates the user exists in the GitLab database. def initialize(text, author, exists = false) - @text = text + @text = text.to_s @author = author @exists = exists end diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb new file mode 100644 index 00000000000..e4835504c2d --- /dev/null +++ b/lib/gitlab/github_import/object_counter.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +# Count objects fetched or imported from Github. +module Gitlab + module GithubImport + class ObjectCounter + OPERATIONS = %w[fetched imported].freeze + PROJECT_COUNTER_LIST_KEY = 'github-importer/object-counters-list/%{project}/%{operation}' + PROJECT_COUNTER_KEY = 'github-importer/object-counter/%{project}/%{operation}/%{object_type}' + + GLOBAL_COUNTER_KEY = 'github_importer_%{operation}_%{object_type}' + GLOBAL_COUNTER_DESCRIPTION = 'The number of %{operation} Github %{object_type}' + + CACHING = Gitlab::Cache::Import::Caching + + class << self + def increment(project, object_type, operation) + validate_operation!(operation) + + increment_project_counter(project, object_type, operation) + increment_global_counter(object_type, operation) + end + + def summary(project) + OPERATIONS.each_with_object({}) do |operation, result| + result[operation] = {} + + CACHING + .values_from_set(counter_list_key(project, operation)) + .sort + .each do |counter| + object_type = counter.split('/').last + result[operation][object_type] = CACHING.read_integer(counter) + end + end + end + + private + + # Global counters are long lived, in Prometheus, + # and it's used to report the health of the Github Importer + # in the Grafana Dashboard + # https://dashboards.gitlab.net/d/2zgM_rImz/github-importer?orgId=1 + def increment_global_counter(object_type, operation) + key = GLOBAL_COUNTER_KEY % { + operation: operation, + object_type: object_type + } + description = GLOBAL_COUNTER_DESCRIPTION % { + operation: operation, + object_type: object_type.to_s.humanize + } + + Gitlab::Metrics.counter(key.to_sym, description).increment + end + + # Project counters are short lived, in Redis, + # and it's used to report how successful a project + # import was with the #summary method. + def increment_project_counter(project, object_type, operation) + counter_key = PROJECT_COUNTER_KEY % { project: project.id, operation: operation, object_type: object_type } + + add_counter_to_list(project, operation, counter_key) + + CACHING.increment(counter_key) + end + + def add_counter_to_list(project, operation, key) + CACHING.set_add(counter_list_key(project, operation), key) + end + + def counter_list_key(project, operation) + PROJECT_COUNTER_LIST_KEY % { project: project.id, operation: operation } + end + + def validate_operation!(operation) + unless operation.to_s.presence_in(OPERATIONS) + raise ArgumentError, "Operation must be #{OPERATIONS.join(' or ')}" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 92f9e8a646d..4598429d568 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -103,6 +103,8 @@ module Gitlab page.objects.each do |object| next if already_imported?(object) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + yield object # We mark the object as imported immediately so we don't end up @@ -129,6 +131,10 @@ module Gitlab Gitlab::Cache::Import::Caching.set_add(already_imported_cache_key, id) end + def object_type + raise NotImplementedError + end + # Returns the ID to use for the cache used for checking if an object has # already been imported or not. # diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb index 3205259a1ed..08b3160fc4c 100644 --- a/lib/gitlab/github_import/representation/pull_request_review.rb +++ b/lib/gitlab/github_import/representation/pull_request_review.rb @@ -29,7 +29,7 @@ module Gitlab hash = Representation.symbolize_hash(raw_hash) hash[:author] &&= Representation::User.from_json_hash(hash[:author]) - hash[:submitted_at] = Time.parse(hash[:submitted_at]).in_time_zone + hash[:submitted_at] = Time.parse(hash[:submitted_at]).in_time_zone if hash[:submitted_at].present? new(hash) end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 8d584415202..058cd1ebd57 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -138,7 +138,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def query_id_for_github_id(id) - User.for_github_id(id).pluck(:id).first + User.by_provider_and_extern_uid(:github, id).select(:id).first&.id end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e84863deba8..0cd33b9f5b7 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -56,8 +56,8 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def gitlab_user_id(project, gitlab_id) - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s) - (user && user.id) || project.creator_id + user_id = User.by_provider_and_extern_uid(:gitlab, gitlab_id).select(:id).first&.id + user_id || project.creator_id end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/global_id/deprecations.rb b/lib/gitlab/global_id/deprecations.rb index ac4a44e0e10..2753e2b8372 100644 --- a/lib/gitlab/global_id/deprecations.rb +++ b/lib/gitlab/global_id/deprecations.rb @@ -9,11 +9,12 @@ module Gitlab # Example: # # DEPRECATIONS = [ - # Deprecation.new(old_model_name: 'PrometheusService', new_model_name: 'Integrations::Prometheus', milestone: '14.0') + # Deprecation.new(old_model_name: 'PrometheusService', new_model_name: 'Integrations::Prometheus', milestone: '14.1') # ].freeze DEPRECATIONS = [ # This works around an accidentally released argument named as `"EEIterationID"` in 7000489db. - Deprecation.new(old_model_name: 'EEIteration', new_model_name: 'Iteration', milestone: '13.3') + Deprecation.new(old_model_name: 'EEIteration', new_model_name: 'Iteration', milestone: '13.3'), + Deprecation.new(old_model_name: 'PrometheusService', new_model_name: 'Integrations::Prometheus', milestone: '14.1') ].freeze # Maps of the DEPRECATIONS Hash for quick access. diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 14f9c7f2191..16a8b5f959e 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -23,6 +23,7 @@ module Gitlab gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab.revision + gon.feature_category = Gitlab::ApplicationContext.current_context_attribute(:feature_category).presence gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path @@ -32,6 +33,7 @@ module Gitlab gon.disable_animations = Gitlab.config.gitlab['disable_animations'] gon.suggested_label_colors = LabelsHelper.suggested_colors gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week + gon.time_display_relative = true gon.ee = Gitlab.ee? gon.dot_com = Gitlab.com? @@ -40,6 +42,7 @@ module Gitlab gon.current_username = current_user.username gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url + gon.time_display_relative = current_user.time_display_relative end # Initialize gon.features with any flags that should be diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb index f1b74999897..5a9d21e7469 100644 --- a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -4,11 +4,42 @@ module Gitlab module Graphql module Pagination module Keyset + # https://gitlab.com/gitlab-org/gitlab/-/issues/334973 # Use the generic keyset implementation if the given ActiveRecord scope supports it. # Note: this module is temporary, at some point it will be merged with Keyset::Connection module GenericKeysetPagination extend ActiveSupport::Concern + # rubocop: disable Naming/PredicateName + # rubocop: disable CodeReuse/ActiveRecord + def has_next_page + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(items) + + strong_memoize(:generic_keyset_pagination_has_next_page) do + if before + # If `before` is specified, that points to a specific record, + # even if it's the last one. Since we're asking for `before`, + # then the specific record we're pointing to is in the + # next page + true + elsif first + case sliced_nodes + when Array + sliced_nodes.size > limit_value + else + # If we count the number of requested items plus one (`limit_value + 1`), + # then if we get `limit_value + 1` then we know there is a next page + sliced_nodes.limit(1).offset(limit_value).exists? + # replacing relation count + # relation_count(set_limit(sliced_nodes, limit_value + 1)) == limit_value + 1 + end + else + false + end + end + end + + # rubocop: enable CodeReuse/ActiveRecord def ordered_items raise ArgumentError, 'Relation must have a primary key' unless items.primary_key.present? diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index d05ced00a6b..afe1554aec1 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -11,11 +11,9 @@ module Gitlab end def self.too_large?(size) - file_size_limit = Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] + return false unless size.to_i > self.file_size_limit - return false unless size.to_i > file_size_limit - - over_highlight_size_limit.increment(source: "file size: #{file_size_limit}") if Feature.enabled?(:track_file_size_over_highlight_limit) + over_highlight_size_limit.increment(source: "file size: #{self.file_size_limit}") if Feature.enabled?(:track_file_size_over_highlight_limit) true end @@ -51,6 +49,16 @@ module Gitlab attr_reader :context + def self.file_size_limit + if Feature.enabled?(:one_megabyte_file_size_limit) + 1024.kilobytes + else + Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] + end + end + + private_class_method :file_size_limit + def custom_language return unless @language diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index 2d1bb515058..181ce447b52 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -8,6 +8,7 @@ module Gitlab labels total_time_spent time_change + severity ].freeze def self.safe_hook_attributes @@ -51,7 +52,8 @@ module Gitlab assignee_ids: issue.assignee_ids, assignee_id: issue.assignee_ids.first, # This key is deprecated labels: issue.labels_hook_attrs, - state: issue.state + state: issue.state, + severity: issue.severity } issue.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 30e72b58e21..7ebbe9f1c14 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -40,26 +40,26 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 1, 'cs_CZ' => 1, - 'de' => 18, + 'de' => 17, 'en' => 100, 'eo' => 1, - 'es' => 40, + 'es' => 38, 'fil_PH' => 1, - 'fr' => 13, + 'fr' => 12, 'gl_ES' => 1, 'id_ID' => 0, 'it' => 2, - 'ja' => 44, + 'ja' => 42, 'ko' => 13, 'nl_NL' => 1, - 'pl_PL' => 3, + 'pl_PL' => 5, 'pt_BR' => 21, - 'ru' => 30, - 'tr_TR' => 17, - 'uk' => 42, - 'zh_CN' => 69, - 'zh_HK' => 3, - 'zh_TW' => 4 + 'ru' => 29, + 'tr_TR' => 16, + 'uk' => 41, + 'zh_CN' => 67, + 'zh_HK' => 2, + 'zh_TW' => 3 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index d000c331b6d..a84978a2a80 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -61,7 +61,9 @@ tree: - :push_event_payload - :suggestions - merge_request_diff: - - :merge_request_diff_commits + - merge_request_diff_commits: + - :commit_author + - :committer - :merge_request_diff_files - events: - :push_event_payload @@ -201,6 +203,10 @@ excluded_attributes: - :verification_failure merge_request_diff_commits: - :merge_request_diff_id + - :commit_author_id + - :committer_id + merge_request_diff_commit_user: + - :id merge_request_diff_detail: - :merge_request_diff_id - :verification_retry_at @@ -223,6 +229,7 @@ excluded_attributes: - :promoted_to_epic_id - :blocking_issues_count - :service_desk_reply_to + - :upvotes_count merge_request: - :milestone_id - :sprint_id diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index 831e38f3034..0753625b978 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -28,6 +28,7 @@ module Gitlab def find return if epic? && group.nil? + return find_diff_commit_user if diff_commit_user? super end @@ -81,6 +82,13 @@ module Gitlab end end + def find_diff_commit_user + find_with_cache do + MergeRequest::DiffCommitUser + .find_or_create(@attributes['name'], @attributes['email']) + end + end + def label? klass == Label end @@ -101,6 +109,10 @@ module Gitlab klass == DesignManagement::Design end + def diff_commit_user? + klass == MergeRequest::DiffCommitUser + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 4678396f97e..102fcedd2fc 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -31,7 +31,9 @@ module Gitlab ci_cd_settings: 'ProjectCiCdSetting', error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', links: 'Releases::Link', - metrics_setting: 'ProjectMetricsSetting' }.freeze + metrics_setting: 'ProjectMetricsSetting', + commit_author: 'MergeRequest::DiffCommitUser', + committer: 'MergeRequest::DiffCommitUser' }.freeze BUILD_MODELS = %i[Ci::Build commit_status].freeze @@ -56,6 +58,7 @@ module Gitlab external_pull_request external_pull_requests DesignManagement::Design + MergeRequest::DiffCommitUser ].freeze def create diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 46b82240ef7..d5f924ae2bd 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -37,7 +37,7 @@ module Gitlab ActiveRecord::Base.no_touching do update_params! - BulkInsertableAssociations.with_bulk_insert(enabled: @importable.class == ::Project) do + BulkInsertableAssociations.with_bulk_insert(enabled: @importable.instance_of?(::Project)) do fix_ci_pipelines_not_sorted_on_legacy_project_json! create_relations! end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index b5a5f8fd984..8a64abb9f62 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -1,14 +1,12 @@ # frozen_string_literal: true -require 'redis' - module Gitlab module Instrumentation module RedisInterceptor APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax xread xreadgroup].freeze def call(*args, &block) - start = Time.now # must come first so that 'start' is always defined + start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined instrumentation_class.instance_count_request instrumentation_class.redis_cluster_validate!(args.first) @@ -17,7 +15,7 @@ module Gitlab instrumentation_class.instance_count_exception(ex) raise ex ensure - duration = Time.now - start + duration = Gitlab::Metrics::System.monotonic_time - start unless APDEX_EXCLUDE.include?(command_from_args(args)) instrumentation_class.instance_observe_duration(duration) @@ -99,7 +97,3 @@ module Gitlab end end end - -class ::Redis::Client - prepend ::Gitlab::Instrumentation::RedisInterceptor -end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index a865a6392f0..c10c4a7bedf 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -29,6 +29,7 @@ module Gitlab instrument_rack_attack(payload) instrument_cpu(payload) instrument_thread_memory_allocations(payload) + instrument_load_balancing(payload) end def instrument_gitaly(payload) @@ -104,6 +105,12 @@ module Gitlab payload.merge!(counters) if counters end + def instrument_load_balancing(payload) + load_balancing_payload = ::Gitlab::Metrics::Subscribers::LoadBalancing.load_balancing_payload + + payload.merge!(load_balancing_payload) + end + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the # `enqueued_at` field or `created_at` field is available. # diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb index 9d7254f49f7..b87c9936570 100644 --- a/lib/gitlab/integrations/sti_type.rb +++ b/lib/gitlab/integrations/sti_type.rb @@ -5,9 +5,9 @@ module Gitlab class StiType < ActiveRecord::Type::String NAMESPACED_INTEGRATIONS = Set.new(%w( Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog - Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker - Jenkins Jira Mattermost MattermostSlashCommands MicrosoftTeams MockCi Packagist PipelinesEmail Pivotaltracker - Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit Youtrack WebexTeams + Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker Jenkins Jira Mattermost + MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker + Prometheus Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit Youtrack WebexTeams )).freeze def cast(value) diff --git a/lib/gitlab/jira_import.rb b/lib/gitlab/jira_import.rb index 75d6fdc07b6..60344e4be68 100644 --- a/lib/gitlab/jira_import.rb +++ b/lib/gitlab/jira_import.rb @@ -19,10 +19,10 @@ module Gitlab return unless configuration_check - jira_service = project.jira_service + jira_integration = project.jira_integration - raise Projects::ImportService::Error, _('Jira integration not configured.') unless jira_service&.active? - raise Projects::ImportService::Error, _('Unable to connect to the Jira instance. Please check your Jira integration configuration.') unless jira_service&.valid_connection? + raise Projects::ImportService::Error, _('Jira integration not configured.') unless jira_integration&.active? + raise Projects::ImportService::Error, _('Unable to connect to the Jira instance. Please check your Jira integration configuration.') unless jira_integration&.valid_connection? end def self.jira_item_cache_key(project_id, jira_item_id, collection_type) diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb index 688254bf91f..2b83f0492cb 100644 --- a/lib/gitlab/jira_import/base_importer.rb +++ b/lib/gitlab/jira_import/base_importer.rb @@ -14,7 +14,7 @@ module Gitlab raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key @project = project - @client = project.jira_service.client + @client = project.jira_integration.client @formatter = Gitlab::ImportFormatter.new end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 767ce310b5a..f1370a40222 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -228,6 +228,14 @@ module Gitlab raise UnsupportedFormatError end + + def render_in(_view_context) + to_s + end + + def format + :json + end end class LimitedEncoder diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index a4663314b3b..86c0aa2b48d 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -25,12 +25,6 @@ module Gitlab write_secret end - def included_in_gitlab_com_rollout?(project) - return true unless ::Gitlab.com? - - Feature.enabled?(:kubernetes_agent_on_gitlab_com, project, default_enabled: :yaml) - end - # Return GitLab KAS version # # @return [String] version diff --git a/lib/gitlab/kas/client.rb b/lib/gitlab/kas/client.rb index 6675903e692..842ee98e4da 100644 --- a/lib/gitlab/kas/client.rb +++ b/lib/gitlab/kas/client.rb @@ -49,14 +49,14 @@ module Gitlab end def kas_endpoint_url - Gitlab::Kas.internal_url.delete_prefix('grpc://') + Gitlab::Kas.internal_url.sub(%r{^grpc://|^grpcs://}, '') end def credentials - if Rails.env.test? || Rails.env.development? - :this_channel_is_insecure - else + if URI(Gitlab::Kas.internal_url).scheme == 'grpcs' GRPC::Core::ChannelCredentials.new + else + :this_channel_is_insecure end end diff --git a/lib/gitlab/kubernetes/cilium_network_policy.rb b/lib/gitlab/kubernetes/cilium_network_policy.rb index f77b3e8de99..e333d3818b9 100644 --- a/lib/gitlab/kubernetes/cilium_network_policy.rb +++ b/lib/gitlab/kubernetes/cilium_network_policy.rb @@ -12,7 +12,7 @@ module Gitlab # We are modeling existing kubernetes resource and don't have # control over amount of parameters. # rubocop:disable Metrics/ParameterLists - def initialize(name:, namespace:, selector:, ingress:, resource_version: nil, description: nil, labels: nil, creation_timestamp: nil, egress: nil, annotations: nil) + def initialize(name:, namespace:, selector:, ingress:, resource_version: nil, description: nil, labels: nil, creation_timestamp: nil, egress: nil, annotations: nil, environment_ids: []) @name = name @description = description @namespace = namespace @@ -23,6 +23,7 @@ module Gitlab @ingress = ingress @egress = egress @annotations = annotations + @environment_ids = environment_ids end # rubocop:enable Metrics/ParameterLists @@ -49,7 +50,7 @@ module Gitlab nil end - def self.from_resource(resource) + def self.from_resource(resource, environment_ids = []) return unless resource return if !resource[:metadata] || !resource[:spec] @@ -65,7 +66,8 @@ module Gitlab creation_timestamp: metadata[:creationTimestamp], selector: spec[:endpointSelector], ingress: spec[:ingress], - egress: spec[:egress] + egress: spec[:egress], + environment_ids: environment_ids ) end @@ -83,7 +85,7 @@ module Gitlab private - attr_reader :name, :description, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress, :annotations + attr_reader :name, :description, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress, :annotations, :environment_ids def selector @selector ||= {} diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb index c8e9b987443..e6111db5b17 100644 --- a/lib/gitlab/kubernetes/network_policy.rb +++ b/lib/gitlab/kubernetes/network_policy.rb @@ -8,7 +8,8 @@ module Gitlab KIND = 'NetworkPolicy' - def initialize(name:, namespace:, selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil) + # rubocop:disable Metrics/ParameterLists + def initialize(name:, namespace:, selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil, environment_ids: []) @name = name @namespace = namespace @labels = labels @@ -17,7 +18,9 @@ module Gitlab @policy_types = policy_types @ingress = ingress @egress = egress + @environment_ids = environment_ids end + # rubocop:enable Metrics/ParameterLists def self.from_yaml(manifest) return unless manifest @@ -40,7 +43,7 @@ module Gitlab nil end - def self.from_resource(resource) + def self.from_resource(resource, environment_ids = []) return unless resource return if !resource[:metadata] || !resource[:spec] @@ -54,7 +57,8 @@ module Gitlab selector: spec[:podSelector], policy_types: spec[:policyTypes], ingress: spec[:ingress], - egress: spec[:egress] + egress: spec[:egress], + environment_ids: environment_ids ) end @@ -69,7 +73,7 @@ module Gitlab private - attr_reader :name, :namespace, :labels, :creation_timestamp, :policy_types, :ingress, :egress + attr_reader :name, :namespace, :labels, :creation_timestamp, :policy_types, :ingress, :egress, :environment_ids def selector @selector ||= {} diff --git a/lib/gitlab/kubernetes/network_policy_common.rb b/lib/gitlab/kubernetes/network_policy_common.rb index 99517454508..de91833b734 100644 --- a/lib/gitlab/kubernetes/network_policy_common.rb +++ b/lib/gitlab/kubernetes/network_policy_common.rb @@ -16,7 +16,8 @@ module Gitlab creation_timestamp: creation_timestamp, manifest: manifest, is_autodevops: autodevops?, - is_enabled: enabled? + is_enabled: enabled?, + environment_ids: environment_ids } end diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index e85d1314eda..7ae1b195ec6 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -35,12 +35,7 @@ module Gitlab def find_by_external_uid return unless id - identities = ::Identity.arel_table - - User.select(:id) - .joins(:identities) - .find_by(identities[:provider].eq(:github).and(identities[:extern_uid].eq(id))) - .try(:id) + User.by_provider_and_extern_uid(:github, id).select(:id).first&.id end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index dce7cdb31a1..83fd74310d0 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -25,7 +25,6 @@ module Gitlab ::Gitlab::InstrumentationHelper.add_instrumentation_data(payload) payload[:queue_duration_s] = event.payload[:queue_duration_s] if event.payload[:queue_duration_s] - payload[:response] = event.payload[:response] if event.payload[:response] payload[:etag_route] = event.payload[:etag_route] if event.payload[:etag_route] payload[Labkit::Correlation::CorrelationId::LOG_KEY] = event.payload[Labkit::Correlation::CorrelationId::LOG_KEY] || Labkit::Correlation::CorrelationId.current_id diff --git a/lib/gitlab/memory/instrumentation.rb b/lib/gitlab/memory/instrumentation.rb index e800fe14cf1..4e93d6a9ece 100644 --- a/lib/gitlab/memory/instrumentation.rb +++ b/lib/gitlab/memory/instrumentation.rb @@ -21,24 +21,13 @@ module Gitlab Thread.current.respond_to?(:memory_allocations) end - # This method changes a global state - def self.ensure_feature_flag! + def self.start_thread_memory_allocations return unless available? - enabled = Feature.enabled?(:trace_memory_allocations, default_enabled: true) - return if enabled == Thread.trace_memory_allocations - MUTEX.synchronize do - # This enables or disables feature dynamically - # based on a feature flag - Thread.trace_memory_allocations = enabled + # This method changes a global state + Thread.trace_memory_allocations = true end - end - - def self.start_thread_memory_allocations - return unless available? - - ensure_feature_flag! # it will return `nil` if disabled Thread.current.memory_allocations diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index 2c4793eb75f..c8591a81a05 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -7,14 +7,9 @@ module Gitlab module Metrics module Dashboard class Finder - # Dashboards that should not be part of the list of all dashboards - # displayed on the metrics dashboard page. - PREDEFINED_DASHBOARD_EXCLUSION_LIST = [ - # This dashboard is only useful in the self monitoring project. - ::Metrics::Dashboard::SelfMonitoringDashboardService, - - # This dashboard is displayed on the K8s cluster settings health page. - ::Metrics::Dashboard::ClusterDashboardService + PREDEFINED_DASHBOARD_LIST = [ + ::Metrics::Dashboard::PodDashboardService, + ::Metrics::Dashboard::SystemDashboardService ].freeze class << self @@ -90,11 +85,7 @@ module Gitlab return [self_monitoring_service] end - predefined_dashboard_services - end - - def predefined_dashboard_services - ::Metrics::Dashboard::PredefinedDashboardService.descendants - PREDEFINED_DASHBOARD_EXCLUSION_LIST + PREDEFINED_DASHBOARD_LIST end def system_service diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index 7111835c85a..ff8b8bf2237 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +require 'webrick' +require 'prometheus/client/rack/exporter' + module Gitlab module Metrics module Exporter diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb index 36262b09b2d..4d38d9e67bf 100644 --- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -require 'webrick' -require 'prometheus/client/rack/exporter' - module Gitlab module Metrics module Exporter diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index 756e6b0641a..f378577f08e 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -require 'webrick' -require 'prometheus/client/rack/exporter' - module Gitlab module Metrics module Exporter diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index 757762499a9..848b73792cb 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'prometheus/client' - module Gitlab module Metrics module Prometheus diff --git a/lib/gitlab/metrics/subscribers/action_cable.rb b/lib/gitlab/metrics/subscribers/action_cable.rb index a9355eeae40..631b9209f14 100644 --- a/lib/gitlab/metrics/subscribers/action_cable.rb +++ b/lib/gitlab/metrics/subscribers/action_cable.rb @@ -12,6 +12,7 @@ module Gitlab TRANSMIT_SUBSCRIPTION_CONFIRMATION = :action_cable_subscription_confirmations_total TRANSMIT_SUBSCRIPTION_REJECTION = :action_cable_subscription_rejections_total BROADCAST = :action_cable_broadcasts_total + DATA_TRANSMITTED_BYTES = :action_cable_transmitted_bytes def transmit_subscription_confirmation(event) confirm_subscription_counter.increment @@ -23,6 +24,14 @@ module Gitlab def transmit(event) transmit_counter.increment + + if event.payload.present? + channel = event.payload[:channel_class] + operation = operation_name_from(event.payload) + data_size = ::ActiveSupport::JSON.encode(event.payload[:data]).bytesize + + transmitted_bytes_histogram.observe({ channel: channel, operation: operation }, data_size) + end end def broadcast(event) @@ -31,6 +40,13 @@ module Gitlab private + # When possible tries to query operation name + def operation_name_from(payload) + data = payload.dig(:data, 'result', 'data') || {} + + data.each_key.first + end + def transmit_counter strong_memoize("transmission_counter") do ::Gitlab::Metrics.counter( @@ -66,6 +82,12 @@ module Gitlab ) end end + + def transmitted_bytes_histogram + strong_memoize("transmitted_bytes_histogram") do + ::Gitlab::Metrics.histogram(DATA_TRANSMITTED_BYTES, 'Message size, in bytes, transmitted over action cable') + end + end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 9f7884e1364..a8fcad9ff9f 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -15,8 +15,8 @@ module Gitlab TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze DB_LOAD_BALANCING_COUNTERS = %i{ - db_replica_count db_replica_cached_count db_replica_wal_count - db_primary_count db_primary_cached_count db_primary_wal_count + db_replica_count db_replica_cached_count db_replica_wal_count db_replica_wal_cached_count + db_primary_count db_primary_cached_count db_primary_wal_count db_primary_wal_cached_count }.freeze DB_LOAD_BALANCING_DURATIONS = %i{db_primary_duration_s db_replica_duration_s}.freeze @@ -72,6 +72,14 @@ module Gitlab DB_LOAD_BALANCING_DURATIONS.each do |duration| payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) end + + if Feature.enabled?(:multiple_database_metrics, default_enabled: :yaml) + ::Gitlab::SafeRequestStore[:duration_by_database]&.each do |dbname, duration_by_role| + duration_by_role.each do |db_role, duration| + payload[:"db_#{db_role}_#{dbname}_duration_s"] = duration.to_f.round(3) + end + end + end end end end @@ -83,9 +91,14 @@ module Gitlab end def increment_db_role_counters(db_role, payload) + cached = cached_query?(payload) increment("db_#{db_role}_count".to_sym) - increment("db_#{db_role}_cached_count".to_sym) if cached_query?(payload) - increment("db_#{db_role}_wal_count".to_sym) if !cached_query?(payload) && wal_command?(payload) + increment("db_#{db_role}_cached_count".to_sym) if cached + + if wal_command?(payload) + increment("db_#{db_role}_wal_count".to_sym) + increment("db_#{db_role}_wal_cached_count".to_sym) if cached + end end def observe_db_role_duration(db_role, event) @@ -93,9 +106,18 @@ module Gitlab buckets ::Gitlab::Metrics::Subscribers::ActiveRecord::SQL_DURATION_BUCKET end + return unless ::Gitlab::SafeRequestStore.active? + duration = event.duration / 1000.0 duration_key = "db_#{db_role}_duration_s".to_sym ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration + + # Per database metrics + dbname = ::Gitlab::Database.dbname(event.payload[:connection]) + ::Gitlab::SafeRequestStore[:duration_by_database] ||= {} + ::Gitlab::SafeRequestStore[:duration_by_database][dbname] ||= {} + ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] ||= 0 + ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] += duration end def ignored_query?(payload) diff --git a/lib/gitlab/metrics/subscribers/load_balancing.rb b/lib/gitlab/metrics/subscribers/load_balancing.rb new file mode 100644 index 00000000000..333fc63ebef --- /dev/null +++ b/lib/gitlab/metrics/subscribers/load_balancing.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + class LoadBalancing < ActiveSupport::Subscriber + attach_to :load_balancing + + PROMETHEUS_COUNTER = :gitlab_transaction_caught_up_replica_pick_count_total + LOG_COUNTERS = { true => :caught_up_replica_pick_ok, false => :caught_up_replica_pick_fail }.freeze + + def caught_up_replica_pick(event) + return unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + + result = event.payload[:result] + counter_name = counter(result) + + increment(counter_name) + end + + # we want to update Prometheus counter after the controller/action are set + def web_transaction_completed(_event) + return unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + + LOG_COUNTERS.keys.each { |result| increment_prometheus_for_result_label(result) } + end + + def self.load_balancing_payload + return {} unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + + {}.tap do |payload| + LOG_COUNTERS.values.each do |counter| + value = Gitlab::SafeRequestStore[counter] + + payload[counter] = value.to_i if value + end + end + end + + private + + def increment(counter) + Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + end + + def increment_prometheus_for_result_label(label_value) + counter_name = counter(label_value) + return unless (counter_value = Gitlab::SafeRequestStore[counter_name]) + + increment_prometheus(labels: { result: label_value }, value: counter_value.to_i) + end + + def increment_prometheus(labels:, value:) + current_transaction&.increment(PROMETHEUS_COUNTER, value, labels) do + docstring 'Caught up replica pick result' + label_keys labels.keys + end + end + + def counter(result) + LOG_COUNTERS[result] + end + + def current_transaction + ::Gitlab::Metrics::WebTransaction.current + end + end + end + end +end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index e6e7d97d296..2e54e8bfc1a 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -65,32 +65,9 @@ module Gitlab # Note: By default the order is breadth-first # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - if use_distinct? - expose_depth = hierarchy_order.present? - hierarchy_order ||= :asc - - # if hierarchy_order is given, the calculated `depth` should be present in SELECT - if expose_depth - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(unscoped_model.all).distinct - read_only(unscoped_model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order)) - else - recursive_query = base_and_ancestors_cte(upto).apply_to(unscoped_model.all) - - if skip_ordering? - recursive_query = recursive_query.distinct - else - recursive_query = recursive_query.reselect(*recursive_query.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct - recursive_query = unscoped_model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)) - recursive_query = remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order) - end - - read_only(recursive_query) - end - else - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(unscoped_model.all) - recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order - read_only(recursive_query) - end + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(unscoped_model.all) + recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order + read_only(recursive_query) end # rubocop: enable CodeReuse/ActiveRecord @@ -101,27 +78,7 @@ module Gitlab # and incremented as we go down the descendant tree # rubocop: disable CodeReuse/ActiveRecord def base_and_descendants(with_depth: false) - if use_distinct? - # Always calculate `depth`, remove it later if with_depth is false - if with_depth - base_cte = base_and_descendants_cte(with_depth: true).apply_to(unscoped_model.all).distinct - read_only(unscoped_model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)).order(depth: :asc)) - else - base_cte = base_and_descendants_cte.apply_to(unscoped_model.all) - - if skip_ordering? - base_cte = base_cte.distinct - else - base_cte = base_cte.reselect(*base_cte.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct - base_cte = unscoped_model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)) - base_cte = remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc) - end - - read_only(base_cte) - end - else - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(unscoped_model.all)) - end + read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(unscoped_model.all)) end # rubocop: enable CodeReuse/ActiveRecord @@ -158,11 +115,6 @@ module Gitlab ancestors_scope = unscoped_model.from(ancestors_table) descendants_scope = unscoped_model.from(descendants_table) - if use_distinct? - ancestors_scope = ancestors_scope.distinct - descendants_scope = descendants_scope.distinct - end - relation = unscoped_model .with .recursive(ancestors.to_arel, descendants.to_arel) @@ -177,23 +129,6 @@ module Gitlab private - # Use distinct on the Namespace queries to avoid bad planner behavior in PG11. - def use_distinct? - return unless model <= Namespace - # Global use_distinct_for_all_object_hierarchy takes precedence over use_distinct_in_object_hierarchy - return true if Feature.enabled?(:use_distinct_for_all_object_hierarchy) - return options[:use_distinct] if options.key?(:use_distinct) - - false - end - - # Skips the extra ordering when using distinct on the namespace queries - def skip_ordering? - return options[:skip_ordering] if options.key?(:skip_ordering) - - false - end - # Remove the extra `depth` field using an INNER JOIN to avoid breaking UNION queries # and ordering the rows based on the `depth` column to maintain the row order. # diff --git a/lib/gitlab/pagination/keyset/column_order_definition.rb b/lib/gitlab/pagination/keyset/column_order_definition.rb index 0c8ec02a56b..0755af9587b 100644 --- a/lib/gitlab/pagination/keyset/column_order_definition.rb +++ b/lib/gitlab/pagination/keyset/column_order_definition.rb @@ -120,7 +120,7 @@ module Gitlab AREL_ORDER_CLASSES = { Arel::Nodes::Ascending => :asc, Arel::Nodes::Descending => :desc }.freeze ALLOWED_NULLABLE_VALUES = [:not_nullable, :nulls_first, :nulls_last].freeze - attr_reader :attribute_name, :column_expression, :order_expression, :add_to_projections + attr_reader :attribute_name, :column_expression, :order_expression, :add_to_projections, :order_direction def initialize(attribute_name:, order_expression:, column_expression: nil, reversed_order_expression: nil, nullable: :not_nullable, distinct: true, order_direction: nil, add_to_projections: false) @attribute_name = attribute_name @@ -175,7 +175,7 @@ module Gitlab private - attr_reader :reversed_order_expression, :nullable, :distinct, :order_direction + attr_reader :reversed_order_expression, :nullable, :distinct def calculate_reversed_order(order_expression) unless AREL_ORDER_CLASSES.has_key?(order_expression.class) # Arel can reverse simple orders diff --git a/lib/gitlab/pagination/keyset/iterator.rb b/lib/gitlab/pagination/keyset/iterator.rb index 3bc8c0bf616..c6f0014a0f4 100644 --- a/lib/gitlab/pagination/keyset/iterator.rb +++ b/lib/gitlab/pagination/keyset/iterator.rb @@ -4,8 +4,12 @@ module Gitlab module Pagination module Keyset class Iterator - def initialize(scope:, use_union_optimization: false) - @scope = scope + UnsupportedScopeOrder = Class.new(StandardError) + + def initialize(scope:, use_union_optimization: true) + @scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) + raise(UnsupportedScopeOrder, 'The order on the scope does not support keyset pagination') unless success + @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) @use_union_optimization = use_union_optimization end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index cef3a7b291a..19d44ee69dd 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -139,6 +139,8 @@ module Gitlab verify_incoming_values!(values) + return use_composite_row_comparison(values) if composite_row_comparison_possible? + where_values = [] reversed_column_definitions = column_definitions.reverse @@ -187,6 +189,28 @@ module Gitlab private + def composite_row_comparison_possible? + !column_definitions.one? && + column_definitions.all?(&:not_nullable?) && + column_definitions.map(&:order_direction).uniq.one? # all columns uses the same order direction + end + + # composite row comparison works with NOT NULL columns and may use only one index scan given a proper index setup + # Example: (created_at, id) > ('2012-09-18 01:40:01+00', 15) + def use_composite_row_comparison(values) + columns = Arel::Nodes::Grouping.new(column_definitions.map(&:column_expression)) + values = Arel::Nodes::Grouping.new(column_definitions.map do |column_definition| + value = values[column_definition.attribute_name] + Arel::Nodes.build_quoted(value, column_definition.column_expression) + end) + + if column_definitions.first.ascending_order? + [columns.gt(values)] + else + [columns.lt(values)] + end + end + # Adds extra columns to the SELECT clause def apply_custom_projections(scope) additional_projections = column_definitions.select(&:add_to_projections).map do |column_definition| diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 2805b12d95d..4f8a6ffb2cc 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -19,11 +19,10 @@ module Gitlab private def paginate_with_limit_optimization(relation) - # do not paginate relation if it is already paginated - pagination_data = if relation.respond_to?(:current_page) && relation.current_page == params[:page] && relation.limit_value == params[:per_page] - relation - else + pagination_data = if needs_pagination?(relation) relation.page(params[:page]).per(params[:per_page]) + else + relation end return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) @@ -39,6 +38,14 @@ module Gitlab end end + def needs_pagination?(relation) + return true unless relation.respond_to?(:current_page) + return true if params[:page].present? && relation.current_page != params[:page].to_i + return true if params[:per_page].present? && relation.limit_value != params[:per_page].to_i + + false + end + def add_default_order(relation) if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? relation = relation.order(:id) # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/patch/global_id.rb b/lib/gitlab/patch/global_id.rb index e99f36c7dca..145a7bfe842 100644 --- a/lib/gitlab/patch/global_id.rb +++ b/lib/gitlab/patch/global_id.rb @@ -4,7 +4,7 @@ # we alter GlobalID so it will correctly find the record with its new model name. module Gitlab module Patch - module GlobalID + module GlobalId def initialize(gid, options = {}) super diff --git a/lib/gitlab/prometheus/adapter.rb b/lib/gitlab/prometheus/adapter.rb index a977040ef6f..2c44e2cea4c 100644 --- a/lib/gitlab/prometheus/adapter.rb +++ b/lib/gitlab/prometheus/adapter.rb @@ -26,7 +26,7 @@ module Gitlab private def service_prometheus_adapter - project.find_or_initialize_service('prometheus') + project.find_or_initialize_integration('prometheus') end end end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index ce9fced9465..9d954a74948 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -10,6 +10,7 @@ module Gitlab :description, :label, :merge_when_pipeline_succeeds, + :milestone, :remove_source_branch, :target, :title, diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index e62e1172b65..7fee1d0727f 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -10,11 +10,12 @@ module Gitlab @expires_in = expires_in end - def cache_key(key) + # NOTE Remove as part of #331319 + def old_cache_key(key) "#{cache_namespace}:#{key}:set" end - def new_cache_key(key) + def cache_key(key) super(key) end diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index a08cea5a435..1e9f6c497c0 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -24,3 +24,6 @@ module Gitlab end end end + +# call prepend_mod to ensure JH-specific module run even though there is no corresponding EE-specific module +Gitlab::Recaptcha.prepend_mod diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index d7501fc7068..547549361be 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -24,8 +24,8 @@ module Gitlab super(text, context.merge(project: project)) end - def references(type) - refs = super(type, project, current_user) + def references(type, ids_only: false) + refs = super(type, project, current_user, ids_only: ids_only) @stateful_not_visible_counter += refs[:not_visible].count refs[:visible] @@ -41,6 +41,12 @@ module Gitlab define_method(type.to_s.pluralize) do @references[type] ||= references(type) end + + if %w(mentioned_user mentioned_group mentioned_project).include?(type.to_s) + define_method("#{type}_ids") do + @references[type] ||= references(type, ids_only: true) + end + end end def issues diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index a31f574fad2..0bd2ac180c3 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -118,15 +118,15 @@ module Gitlab def debian_architecture_regex # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/arch.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n43 # But we limit to lower case - @debian_architecture_regex ||= %r{\A[a-z0-9][-a-z0-9]*\z}.freeze + @debian_architecture_regex ||= %r{\A#{::Packages::Debian::ARCHITECTURE_REGEX}\z}.freeze end def debian_distribution_regex - @debian_distribution_regex ||= %r{\A[a-z0-9][a-z0-9\.-]*\z}i.freeze + @debian_distribution_regex ||= %r{\A#{::Packages::Debian::DISTRIBUTION_REGEX}\z}i.freeze end def debian_component_regex - @debian_component_regex ||= %r{#{debian_distribution_regex}}.freeze + @debian_component_regex ||= %r{\A#{::Packages::Debian::COMPONENT_REGEX}\z}.freeze end def helm_channel_regex diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 42b94d5cf3b..a4d1adf7671 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -13,7 +13,6 @@ module Gitlab # @returns [HasRepository, Project, String, String] def self.parse(path) repo_path = path.delete_prefix('/').delete_suffix('.git') - redirected_path = nil # Detect the repo type based on the path, the first one tried is the project # type, which does not have a suffix. @@ -26,9 +25,11 @@ module Gitlab # Removing the suffix (.wiki, .design, ...) from the project path full_path = repo_path.chomp(type.path_suffix) - container, project, redirected_path = find_container(type, full_path) + container, project = find_container(type, full_path) + next unless container - return [container, project, type, redirected_path] if container + redirected_path = repo_path if redirected?(container, repo_path) + return [container, project, type, redirected_path] end # When a project did not exist, the parsed repo_type would be empty. @@ -40,32 +41,28 @@ module Gitlab # Returns an array containing: # - The repository container # - The related project (if available) - # - The original container path (if redirected) # # @returns [HasRepository, Project, String] def self.find_container(type, full_path) - return [nil, nil, nil] if full_path.blank? + return [nil, nil] if full_path.blank? if type.snippet? - snippet, redirected_path = find_snippet(full_path) + snippet = find_snippet(full_path) - [snippet, snippet&.project, redirected_path] + [snippet, snippet&.project] elsif type.wiki? - wiki, redirected_path = find_wiki(full_path) + wiki = find_wiki(full_path) - [wiki, wiki.try(:project), redirected_path] + [wiki, wiki.try(:project)] else - project, redirected_path = find_project(full_path) + project = find_project(full_path) - [project, project, redirected_path] + [project, project] end end def self.find_project(project_path) - project = Project.find_by_full_path(project_path, follow_redirects: true) - redirected_path = project_path if redirected?(project, project_path) - - [project, redirected_path] + Project.find_by_full_path(project_path, follow_redirects: true) end def self.redirected?(container, container_path) @@ -77,11 +74,11 @@ module Gitlab # - h5bp/html5-boilerplate/snippets/53 def self.find_snippet(snippet_path) snippet_id, project_path = extract_snippet_info(snippet_path) - return [nil, nil] unless snippet_id + return unless snippet_id - project, redirected_path = find_project(project_path) if project_path + project = find_project(project_path) if project_path - [Snippet.find_by_id_and_project(id: snippet_id, project: project), redirected_path] + Snippet.find_by_id_and_project(id: snippet_id, project: project) end # Wiki path can be either: @@ -93,10 +90,9 @@ module Gitlab # - group/subgroup def self.find_wiki(container_path) container = Routable.find_by_full_path(container_path, follow_redirects: true) - redirected_path = container_path if redirected?(container, container_path) # In CE, Group#wiki is not available so this will return nil for a group path. - [container&.try(:wiki), redirected_path] + container&.try(:wiki) end def self.extract_snippet_info(snippet_path) diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index a20e9845fe6..7de53c4b3ff 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -13,12 +13,12 @@ module Gitlab @expires_in = expires_in end - def cache_key(type) + # NOTE Remove as part of #331319 + def old_cache_key(type) "#{type}:#{namespace}:set" end - # NOTE Remove as part of #331319 - def new_cache_key(type) + def cache_key(type) super("#{type}:#{namespace}") end diff --git a/lib/gitlab/search/sort_options.rb b/lib/gitlab/search/sort_options.rb index 2ab38147462..f8e5cf727ac 100644 --- a/lib/gitlab/search/sort_options.rb +++ b/lib/gitlab/search/sort_options.rb @@ -15,6 +15,10 @@ module Gitlab :updated_at_asc when %w[updated_at desc], [nil, 'updated_desc'] :updated_at_desc + when %w[popularity asc], [nil, 'popularity_asc'] + :popularity_asc + when %w[popularity desc], [nil, 'popularity_desc'] + :popularity_desc else :unknown end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 678c0b396ef..e6851af8264 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -7,6 +7,11 @@ module Gitlab DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 20 + SCOPE_ONLY_SORT = { + popularity_asc: %w[issues], + popularity_desc: %w[issues] + }.freeze + attr_reader :current_user, :query, :order_by, :sort, :filters # Limit search results by passed projects @@ -128,20 +133,29 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def apply_sort(scope) + def apply_sort(results, scope: nil) # Due to different uses of sort param we prefer order_by when # present - case ::Gitlab::Search::SortOptions.sort_and_direction(order_by, sort) + sort_by = ::Gitlab::Search::SortOptions.sort_and_direction(order_by, sort) + + # Reset sort to default if the chosen one is not supported by scope + sort_by = nil if SCOPE_ONLY_SORT[sort_by] && !SCOPE_ONLY_SORT[sort_by].include?(scope) + + case sort_by when :created_at_asc - scope.reorder('created_at ASC') + results.reorder('created_at ASC') when :created_at_desc - scope.reorder('created_at DESC') + results.reorder('created_at DESC') when :updated_at_asc - scope.reorder('updated_at ASC') + results.reorder('updated_at ASC') when :updated_at_desc - scope.reorder('updated_at DESC') + results.reorder('updated_at DESC') + when :popularity_asc + results.reorder('upvotes_count ASC') + when :popularity_desc + results.reorder('upvotes_count DESC') else - scope.reorder('created_at DESC') + results.reorder('created_at DESC') end end # rubocop: enable CodeReuse/ActiveRecord @@ -157,7 +171,7 @@ module Gitlab issues = issues.where(project_id: project_ids_relation) # rubocop: disable CodeReuse/ActiveRecord end - apply_sort(issues) + apply_sort(issues, scope: 'issues') end # rubocop: disable CodeReuse/ActiveRecord @@ -177,7 +191,7 @@ module Gitlab merge_requests = merge_requests.in_projects(project_ids_relation) end - apply_sort(merge_requests) + apply_sort(merge_requests, scope: 'merge_requests') end def default_scope diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 30cd63e80c0..9fc7a44ec99 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -10,12 +10,12 @@ module Gitlab @expires_in = expires_in end - def cache_key(key) + # NOTE Remove as part of https://gitlab.com/gitlab-org/gitlab/-/issues/331319 + def old_cache_key(key) "#{key}:set" end - # NOTE Remove as part of https://gitlab.com/gitlab-org/gitlab/-/issues/331319 - def new_cache_key(key) + def cache_key(key) "#{cache_namespace}:#{key}:set" end @@ -25,7 +25,7 @@ module Gitlab with do |redis| keys_to_expire = keys.map { |key| cache_key(key) } - keys_to_expire += keys.map { |key| new_cache_key(key) } # NOTE Remove as part of #331319 + keys_to_expire += keys.map { |key| old_cache_key(key) } # NOTE Remove as part of #331319 Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do redis.unlink(*keys_to_expire) diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 16a0619daf6..bd6b80530c3 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -103,6 +103,21 @@ module Gitlab queues_for_sidekiq_queues_yml != config_queues end + # Returns a hash of worker class name => mapped queue name + def worker_queue_mappings + workers + .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) } + .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] } + end + + # Like worker_queue_mappings, but only for the queues running in + # the current Sidekiq process + def current_worker_queue_mappings + worker_queue_mappings + .select { |worker, queue| Sidekiq.options[:queues].include?(queue) } + .to_h + end + private def find_workers(root, ee:) diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 32194c4926e..842e53b2ffb 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -68,7 +68,7 @@ module Gitlab message = base_message(payload) - payload['database_chosen'] = job[:database_chosen] if job[:database_chosen] + payload['load_balancing_strategy'] = job['load_balancing_strategy'] if job['load_balancing_strategy'] if job_exception payload['message'] = "#{message}: fail: #{payload['duration_s']} sec" diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 30741f29563..3422cb47516 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -12,7 +12,13 @@ module Gitlab # Size limiter should be placed at the top chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Server chain.add ::Gitlab::SidekiqMiddleware::Monitor - chain.add ::Gitlab::SidekiqMiddleware::ServerMetrics if metrics + + if metrics + chain.add ::Gitlab::SidekiqMiddleware::ServerMetrics + + ::Gitlab::SidekiqMiddleware::ServerMetrics.initialize_process_metrics + end + chain.add ::Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger chain.add ::Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer chain.add ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware diff --git a/lib/gitlab/sidekiq_middleware/client_metrics.rb b/lib/gitlab/sidekiq_middleware/client_metrics.rb index 6bc08a97c07..e3cc7b28c41 100644 --- a/lib/gitlab/sidekiq_middleware/client_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/client_metrics.rb @@ -15,6 +15,7 @@ module Gitlab # worker_class can either be the string or class of the worker being enqueued. worker_class = worker_class.safe_constantize if worker_class.respond_to?(:safe_constantize) labels = create_labels(worker_class, queue, job) + labels[:scheduling] = job.key?('at') ? 'delayed' : 'immediate' @metrics.fetch(ENQUEUED).increment(labels, 1) diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 4cf540ce3b8..c1dc616cbb2 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -133,11 +133,7 @@ module Gitlab end def idempotency_string - # TODO: dump the argument's JSON using `Sidekiq.dump_json` instead - # this should be done in the next release so all jobs are written - # with their idempotency key. - # see https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1090 - "#{worker_class_name}:#{arguments.join('-')}" + "#{worker_class_name}:#{Sidekiq.dump_json(arguments)}" end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 6d130957f36..2d9767e0266 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -9,10 +9,50 @@ module Gitlab # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze - def initialize - @metrics = init_metrics + class << self + include ::Gitlab::SidekiqMiddleware::MetricsHelper + + def metrics + { + sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), + sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), + sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), + sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all) + } + end + + def initialize_process_metrics + metrics = self.metrics + + metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) + + return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize, default_enabled: :yaml) - @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) + ::Gitlab::SidekiqConfig.current_worker_queue_mappings.each do |worker, queue| + worker_class = worker.safe_constantize + + next unless worker_class + + base_labels = create_labels(worker_class, queue, {}) + + %w[done fail].each do |status| + metrics[:sidekiq_jobs_completion_seconds].get(base_labels.merge(job_status: status)) + end + end + end + end + + def initialize + @metrics = self.class.metrics if ::Gitlab::Database::LoadBalancing.enable? @metrics[:sidekiq_load_balancing_count] = ::Gitlab::Metrics.counter(:sidekiq_load_balancing_count, 'Sidekiq jobs with load balancing') @@ -74,10 +114,10 @@ module Gitlab @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation)) @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation)) - if ::Gitlab::Database::LoadBalancing.enable? && job[:database_chosen] + with_load_balancing_settings(job) do |settings| load_balancing_labels = { - database_chosen: job[:database_chosen], - data_consistency: job[:data_consistency] + load_balancing_strategy: settings['load_balancing_strategy'], + data_consistency: settings['worker_data_consistency'] } @metrics[:sidekiq_load_balancing_count].increment(labels.merge(load_balancing_labels), 1) @@ -85,26 +125,17 @@ module Gitlab end end - def init_metrics - { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), - sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), - sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), - sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), - sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'), - sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), - sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all) - } - end - private + def with_load_balancing_settings(job) + return unless ::Gitlab::Database::LoadBalancing.enable? + + keys = %w[load_balancing_strategy worker_data_consistency] + return unless keys.all? { |k| job.key?(k) } + + yield job.slice(*keys) + end + def get_thread_cputime defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb index d86f1609f14..b37eeb8bad1 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -99,6 +99,10 @@ module Gitlab return job_args unless compress_mode? return job_args if job_args.bytesize < @compression_threshold + # When a job was scheduled in the future, it runs through the middleware + # twice. Once on scheduling and once on queueing. No need to compress twice. + return job_args if ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.compressed?(@job) + ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.compress(@job, job_args) end diff --git a/lib/gitlab/sidekiq_middleware/worker_context/client.rb b/lib/gitlab/sidekiq_middleware/worker_context/client.rb index 0eb52179db2..1a899b27ea3 100644 --- a/lib/gitlab/sidekiq_middleware/worker_context/client.rb +++ b/lib/gitlab/sidekiq_middleware/worker_context/client.rb @@ -15,7 +15,12 @@ module Gitlab context_for_args = worker_class.context_for_arguments(job['args']) - wrap_in_optional_context(context_for_args, &block) + wrap_in_optional_context(context_for_args) do + # This should be inside the context for the arguments so + # that we don't override the feature category on the worker + # with the one from the caller. + Gitlab::ApplicationContext.with_context(feature_category: worker_class.get_feature_category.to_s, &block) + end end end end diff --git a/lib/gitlab/sidekiq_queue.rb b/lib/gitlab/sidekiq_queue.rb index 4b71dfc0c1b..eb3a8e3d497 100644 --- a/lib/gitlab/sidekiq_queue.rb +++ b/lib/gitlab/sidekiq_queue.rb @@ -14,7 +14,7 @@ module Gitlab end def drop_jobs!(search_metadata, timeout:) - start_time = Gitlab::Metrics::System.monotonic_time + start_time = monotonic_time completed = true deleted_jobs = 0 @@ -62,7 +62,11 @@ module Gitlab end def timeout_exceeded?(start_time, timeout) - (Gitlab::Metrics::System.monotonic_time - start_time) > timeout + (monotonic_time - start_time) > timeout + end + + def monotonic_time + Gitlab::Metrics::System.monotonic_time end end end diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 99a056c97fc..fab016d2e1b 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -33,7 +33,7 @@ module Gitlab private def create_issue(title:, description:) - Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }).execute + Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute end def presenter(issue) diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 6afc21be4e0..df6d3eb7d0a 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -27,21 +27,18 @@ module Gitlab # connect with Spamcheck @endpoint_url = @endpoint_url.gsub(%r(^grpc:\/\/), '') - creds = + @creds = if Rails.env.development? || Rails.env.test? :this_channel_is_insecure else GRPC::Core::ChannelCredentials.new end - - @stub = ::Spamcheck::SpamcheckService::Stub.new(@endpoint_url, creds, - timeout: DEFAULT_TIMEOUT_SECS) end def issue_spam?(spam_issue:, user:, context: {}) issue = build_issue_protobuf(issue: spam_issue, user: user, context: context) - response = @stub.check_for_spam_issue(issue, + response = grpc_client.check_for_spam_issue(issue, metadata: { 'authorization' => Gitlab::CurrentSettings.spam_check_api_key }) verdict = convert_verdict_to_gitlab_constant(response.verdict) @@ -100,6 +97,16 @@ module Gitlab Google::Protobuf::Timestamp.new(seconds: ar_timestamp.to_time.to_i, nanos: ar_timestamp.to_time.nsec) end + + def grpc_client + @grpc_client ||= ::Spamcheck::SpamcheckService::Stub.new(@endpoint_url, @creds, + interceptors: interceptors, + timeout: DEFAULT_TIMEOUT_SECS) + end + + def interceptors + [Labkit::Correlation::GRPC::ClientInterceptor.instance] + end end end end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index e302865c897..da925f0f83a 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -92,4 +92,4 @@ module Gitlab end end -Gitlab::Template::GitlabCiYmlTemplate.prepend_mod_with('Gitlab::Template::GitlabCiYmlTemplate') +Gitlab::Template::GitlabCiYmlTemplate.prepend_mod diff --git a/lib/gitlab/changelog/ast.rb b/lib/gitlab/template_parser/ast.rb index 2c787d396f5..89318ee0d68 100644 --- a/lib/gitlab/changelog/ast.rb +++ b/lib/gitlab/template_parser/ast.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - module Changelog + module TemplateParser # AST nodes to evaluate when rendering a template. # # Evaluating an AST is done by walking over the nodes and calling diff --git a/lib/gitlab/template_parser/error.rb b/lib/gitlab/template_parser/error.rb new file mode 100644 index 00000000000..1dcde448749 --- /dev/null +++ b/lib/gitlab/template_parser/error.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module TemplateParser + # An error raised when a template couldn't be rendered. + Error = Class.new(StandardError) + end +end diff --git a/lib/gitlab/changelog/eval_state.rb b/lib/gitlab/template_parser/eval_state.rb index a0439df60cf..7cf2ab21f50 100644 --- a/lib/gitlab/changelog/eval_state.rb +++ b/lib/gitlab/template_parser/eval_state.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - module Changelog + module TemplateParser # A class for tracking state when evaluating a template class EvalState MAX_LOOPS = 4 diff --git a/lib/gitlab/changelog/parser.rb b/lib/gitlab/template_parser/parser.rb index fac6fc19148..157339414c4 100644 --- a/lib/gitlab/changelog/parser.rb +++ b/lib/gitlab/template_parser/parser.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true module Gitlab - module Changelog - # A parser for the template syntax used for generating changelogs. + module TemplateParser + # A parser for a simple template syntax, used for example to generate changelogs. # # As a quick primer on the template syntax, a basic template looks like # this: @@ -166,7 +166,7 @@ module Gitlab def parse_and_transform(input) AST::Transformer.new.apply(parse(input)) rescue Parslet::ParseFailed => ex - # We raise a custom error so it's easier to catch different changelog + # We raise a custom error so it's easier to catch different parser # related errors. In addition, this ensures the caller of this method # doesn't depend on a Parslet specific error class. raise Error, "Failed to parse the template: #{ex.message}" diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb index e548532e061..07a53b0892b 100644 --- a/lib/gitlab/tracking/destinations/snowplow.rb +++ b/lib/gitlab/tracking/destinations/snowplow.rb @@ -13,12 +13,13 @@ module Gitlab return unless enabled? tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) + increment_total_events_counter end private def enabled? - Gitlab::CurrentSettings.snowplow_enabled? + Gitlab::Tracking.enabled? end def tracker @@ -33,9 +34,46 @@ module Gitlab def emitter SnowplowTracker::AsyncEmitter.new( Gitlab::CurrentSettings.snowplow_collector_hostname, - protocol: 'https' + protocol: 'https', + on_success: method(:increment_successful_events_emissions), + on_failure: method(:failure_callback) ) end + + def failure_callback(success_count, failures) + increment_successful_events_emissions(success_count) + increment_failed_events_emissions(failures.size) + log_failures(failures) + end + + def increment_failed_events_emissions(value) + Gitlab::Metrics.counter( + :gitlab_snowplow_failed_events_total, + 'Number of failed Snowplow events emissions' + ).increment({}, value.to_i) + end + + def increment_successful_events_emissions(value) + Gitlab::Metrics.counter( + :gitlab_snowplow_successful_events_total, + 'Number of successful Snowplow events emissions' + ).increment({}, value.to_i) + end + + def increment_total_events_counter + Gitlab::Metrics.counter( + :gitlab_snowplow_events_total, + 'Number of Snowplow events' + ).increment + end + + def log_failures(failures) + hostname = Gitlab::CurrentSettings.snowplow_collector_hostname + + failures.each do |failure| + Gitlab::AppLogger.error("#{failure["se_ca"]} #{failure["se_ac"]} failed to be reported to collector at #{hostname}") + end + end end end end diff --git a/lib/gitlab/tracking/helpers.rb b/lib/gitlab/tracking/helpers.rb new file mode 100644 index 00000000000..bf3cefb736c --- /dev/null +++ b/lib/gitlab/tracking/helpers.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Helpers + def dnt_enabled? + Gitlab::Utils.to_boolean(request.headers['DNT']) + end + + def trackable_html_request? + request.format.html? && !dnt_enabled? + end + end + end +end diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb index c2e5d467dbb..bfe674b945e 100644 --- a/lib/gitlab/usage/docs/helper.rb +++ b/lib/gitlab/usage/docs/helper.rb @@ -51,6 +51,10 @@ module Gitlab "Tiers:#{format(:tier, object[:tier])}" end + def render_data_category(object) + "Data Category: `#{object[:data_category]}`" + end + def format(key, value) Gitlab::Usage::Docs::ValueFormatter.format(key, value) end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml index 8911ac2ed1a..83a3a5b6698 100644 --- a/lib/gitlab/usage/docs/templates/default.md.haml +++ b/lib/gitlab/usage/docs/templates/default.md.haml @@ -38,6 +38,9 @@ = render_yaml_link(object.yaml_path) \ = render_owner(object.attributes) + - if object.attributes[:data_category].present? + \ + = render_data_category(object.attributes) \ = render_status(object.attributes) \ diff --git a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb new file mode 100644 index 00000000000..dd1f9948815 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CollectedDataCategoriesMetric < GenericMetric + def value + ::ServicePing::PermitDataCategoriesService.new.execute + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index 69a288e5b6e..7b3a545185b 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -16,14 +16,20 @@ module Gitlab # end class << self def start(&block) + return @metric_start&.call unless block_given? + @metric_start = block end def finish(&block) + return @metric_finish&.call unless block_given? + @metric_finish = block end def relation(&block) + return @metric_relation&.call unless block_given? + @metric_relation = block end @@ -32,15 +38,21 @@ module Gitlab @column = column end - attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :column + def cache_start_and_finish_as(cache_key) + @cache_key = cache_key + end + + attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :column, :cache_key end def value + start, finish = get_or_cache_batch_ids + method(self.class.metric_operation) .call(relation, self.class.column, - start: self.class.metric_start&.call, - finish: self.class.metric_finish&.call) + start: start, + finish: finish) end def to_sql @@ -73,6 +85,22 @@ module Gitlab raise "Unknown time frame: #{time_frame} for DatabaseMetric" end end + + def get_or_cache_batch_ids + return [self.class.start, self.class.finish] unless self.class.cache_key.present? + + key_name = "metric_instrumentation/#{self.class.cache_key}" + + start = Gitlab::Cache.fetch_once("#{key_name}_minimum_id", expires_in: 1.day) do + self.class.start + end + + finish = Gitlab::Cache.fetch_once("#{key_name}_maximum_id", expires_in: 1.day) do + self.class.finish + end + + [start, finish] + end end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 415a5bff261..aabc706901e 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -256,7 +256,8 @@ module Gitlab settings: { ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? }, operating_system: alt_usage_data(fallback: nil) { operating_system }, - gitaly_apdex: alt_usage_data { gitaly_apdex } + gitaly_apdex: alt_usage_data { gitaly_apdex }, + collected_data_categories: alt_usage_data(fallback: []) { Gitlab::Usage::Metrics::Instrumentations::CollectedDataCategoriesMetric.new(time_frame: 'none').value } } } end @@ -403,7 +404,7 @@ module Gitlab def services_usage # rubocop: disable UsageData/LargeTable: - Integration.available_services_names(include_dev: false).each_with_object({}) do |name, response| + Integration.available_integration_names(include_dev: false).each_with_object({}) do |name, response| type = Integration.integration_name_to_type(name) response[:"projects_#{name}_active"] = count(Integration.active.where.not(project: nil).where(type: type)) @@ -426,9 +427,9 @@ module Gitlab projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) } - jira_service_data_hash = jira_service_data - results[:projects_jira_server_active] = jira_service_data_hash[:projects_jira_server_active] - results[:projects_jira_cloud_active] = jira_service_data_hash[:projects_jira_cloud_active] + jira_integration_data_hash = jira_integration_data + results[:projects_jira_server_active] = jira_integration_data_hash[:projects_jira_server_active] + results[:projects_jira_cloud_active] = jira_integration_data_hash[:projects_jira_cloud_active] results rescue ActiveRecord::StatementInvalid @@ -650,9 +651,9 @@ module Gitlab todos: distinct_count(::Todo.where(time_period), :author_id), service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period), service_desk_issues: count(::Issue.service_desk.where(time_period)), - projects_jira_active: distinct_count(::Project.with_active_jira_services.where(time_period), :creator_id), - projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_jira_services.with_jira_dvcs_cloud.where(time_period), :creator_id), - projects_jira_dvcs_server_active: distinct_count(::Project.with_active_jira_services.with_jira_dvcs_server.where(time_period), :creator_id) + projects_jira_active: distinct_count(::Project.with_active_jira_integrations.where(time_period), :creator_id), + projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_jira_integrations.with_jira_dvcs_cloud.where(time_period), :creator_id), + projects_jira_dvcs_server_active: distinct_count(::Project.with_active_jira_integrations.with_jira_dvcs_server.where(time_period), :creator_id) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index c72f487a442..21d637e7152 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -22,6 +22,7 @@ - i_package_golang_pull_package - i_package_golang_push_package - i_package_helm_pull_package +- i_package_helm_push_package - i_package_maven_delete_package - i_package_maven_pull_package - i_package_maven_push_package @@ -31,14 +32,24 @@ - i_package_nuget_delete_package - i_package_nuget_pull_package - i_package_nuget_push_package +- i_package_nuget_pull_symbol_package +- i_package_nuget_push_symbol_package - i_package_pull_package - i_package_pull_package_by_deploy_token - i_package_pull_package_by_guest - i_package_pull_package_by_user +- i_package_pull_symbol_package +- i_package_pull_symbol_package_by_deploy_token +- i_package_pull_symbol_package_by_guest +- i_package_pull_symbol_package_by_user - i_package_push_package - i_package_push_package_by_deploy_token - i_package_push_package_by_guest - i_package_push_package_by_user +- i_package_push_symbol_package +- i_package_push_symbol_package_by_deploy_token +- i_package_push_symbol_package_by_guest +- i_package_push_symbol_package_by_user - i_package_pypi_delete_package - i_package_pypi_pull_package - i_package_pypi_push_package diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 2a231f8fce0..597df9936ea 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -117,7 +117,7 @@ module Gitlab private def track(values, event_name, context: '', time: Time.zone.now) - return unless Gitlab::CurrentSettings.usage_ping_enabled? + return unless usage_ping_enabled? event = event_for(event_name) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? @@ -131,6 +131,10 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end + def usage_ping_enabled? + Gitlab::CurrentSettings.usage_ping_enabled? + end + # The array of valid context on which we allow tracking def valid_context_list Plan.all_plans diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index f2e45a52434..fe1eb090fa4 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -369,3 +369,8 @@ category: testing aggregation: weekly feature_flag: users_expanding_widgets_usage_data +# Container Security - Network Policies +- name: clusters_using_network_policies_ui + redis_slot: network_policies + category: network_policies + aggregation: weekly diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb index bc72a96a468..44d5baa42f6 100644 --- a/lib/gitlab/usage_data_non_sql_metrics.rb +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -31,7 +31,7 @@ module Gitlab def minimum_id(model, column = nil) end - def jira_service_data + def jira_integration_data { projects_jira_server_active: 0, projects_jira_cloud_active: 0 diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index da01b68e8fc..63e6cf03d1f 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -48,7 +48,7 @@ module Gitlab end end - def jira_service_data + def jira_integration_data { projects_jira_server_active: 0, projects_jira_cloud_active: 0 diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 77e0e2ca96c..0b1acaf7dd8 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -169,6 +169,16 @@ module Gitlab end end + def deep_symbolized_access(data) + if data.is_a?(Array) + data.map(&method(:deep_symbolized_access)) + elsif data.is_a?(Hash) + data.deep_symbolize_keys + else + data + end + end + def string_to_ip_object(str) return unless str diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb index 620d71a7814..ab5d18e9c8a 100644 --- a/lib/gitlab/utils/sanitize_node_link.rb +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -6,7 +6,7 @@ module Gitlab module Utils module SanitizeNodeLink UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze - ATTRS_TO_SANITIZE = %w(href src data-src).freeze + ATTRS_TO_SANITIZE = %w(href src data-src data-canonical-src).freeze def remove_unsafe_links(env, remove_invalid_links: true) node = env[:node] diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 4ea5b5a87de..faa524d171c 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -217,7 +217,7 @@ module Gitlab end # rubocop: disable UsageData/LargeTable: - def jira_service_data + def jira_integration_data data = { projects_jira_server_active: 0, projects_jira_cloud_active: 0 |