diff options
Diffstat (limited to 'lib/gitlab')
256 files changed, 4186 insertions, 1715 deletions
diff --git a/lib/gitlab/alert_management/alert_params.rb b/lib/gitlab/alert_management/alert_params.rb new file mode 100644 index 00000000000..982479784a9 --- /dev/null +++ b/lib/gitlab/alert_management/alert_params.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module AlertManagement + class AlertParams + MONITORING_TOOLS = { + prometheus: 'Prometheus' + }.freeze + + def self.from_generic_alert(project:, payload:) + parsed_payload = Gitlab::Alerting::NotificationPayloadParser.call(payload).with_indifferent_access + annotations = parsed_payload[:annotations] + + { + project_id: project.id, + title: annotations[:title], + description: annotations[:description], + monitoring_tool: annotations[:monitoring_tool], + service: annotations[:service], + hosts: Array(annotations[:hosts]), + payload: payload, + started_at: parsed_payload['startsAt'], + severity: annotations[:severity] + } + end + + def self.from_prometheus_alert(project:, parsed_alert:) + { + project_id: project.id, + title: parsed_alert.title, + description: parsed_alert.description, + monitoring_tool: MONITORING_TOOLS[:prometheus], + payload: parsed_alert.payload, + started_at: parsed_alert.starts_at, + ended_at: parsed_alert.ends_at, + fingerprint: parsed_alert.gitlab_fingerprint + } + end + end + end +end diff --git a/lib/gitlab/alert_management/alert_status_counts.rb b/lib/gitlab/alert_management/alert_status_counts.rb new file mode 100644 index 00000000000..382026236e0 --- /dev/null +++ b/lib/gitlab/alert_management/alert_status_counts.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module AlertManagement + # Represents counts of each status or category of statuses + class AlertStatusCounts + include Gitlab::Utils::StrongMemoize + + STATUSES = ::AlertManagement::Alert::STATUSES + + attr_reader :project + + def self.declarative_policy_class + 'AlertManagement::AlertPolicy' + end + + def initialize(current_user, project, params) + @project = project + @current_user = current_user + @params = params + end + + # Define method for each status + STATUSES.each_key do |status| + define_method(status) { counts[status] } + end + + def open + counts[:triggered] + counts[:acknowledged] + end + + def all + counts.values.sum # rubocop:disable CodeReuse/ActiveRecord + end + + private + + attr_reader :current_user, :params + + def counts + strong_memoize(:counts) do + Hash.new(0).merge(counts_by_status) + end + end + + def counts_by_status + ::AlertManagement::AlertsFinder + .counts_by_status(current_user, project, params) + .transform_keys { |status_id| STATUSES.key(status_id) } + end + end + end +end diff --git a/lib/gitlab/alerting/alert.rb b/lib/gitlab/alerting/alert.rb index 7d97bd1bb52..d859ca89418 100644 --- a/lib/gitlab/alerting/alert.rb +++ b/lib/gitlab/alerting/alert.rb @@ -105,6 +105,10 @@ module Gitlab metric_id.present? end + def gitlab_fingerprint + Digest::SHA1.hexdigest(plain_gitlab_fingerprint) + end + def valid? payload.respond_to?(:dig) && project && title && starts_at end @@ -115,6 +119,14 @@ module Gitlab private + def plain_gitlab_fingerprint + if gitlab_managed? + [metric_id, starts_at].join('/') + else # self managed + [starts_at, title, full_query].join('/') + end + end + def parse_environment_from_payload environment_name = payload&.dig('labels', 'gitlab_environment_name') diff --git a/lib/gitlab/alerting/notification_payload_parser.rb b/lib/gitlab/alerting/notification_payload_parser.rb index a54bb44d66a..c79d69613f3 100644 --- a/lib/gitlab/alerting/notification_payload_parser.rb +++ b/lib/gitlab/alerting/notification_payload_parser.rb @@ -6,6 +6,7 @@ module Gitlab BadPayloadError = Class.new(StandardError) DEFAULT_TITLE = 'New: Incident' + DEFAULT_SEVERITY = 'critical' def initialize(payload) @payload = payload.to_h.with_indifferent_access @@ -30,6 +31,10 @@ module Gitlab payload[:title].presence || DEFAULT_TITLE end + def severity + payload[:severity].presence || DEFAULT_SEVERITY + end + def annotations primary_params .reverse_merge(flatten_secondary_params) @@ -43,7 +48,8 @@ module Gitlab 'description' => payload[:description], 'monitoring_tool' => payload[:monitoring_tool], 'service' => payload[:service], - 'hosts' => hosts.presence + 'hosts' => hosts.presence, + 'severity' => severity } end diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb index 41883a80338..9fcaeadf351 100644 --- a/lib/gitlab/analytics/cycle_analytics/median.rb +++ b/lib/gitlab/analytics/cycle_analytics/median.rb @@ -15,7 +15,11 @@ module Gitlab @query = @query.select(median_duration_in_seconds.as('median')) result = execute_query(@query).first || {} - result['median'] ? result['median'].to_i : nil + result['median'] || nil + end + + def days + seconds ? seconds.fdiv(1.day) : nil end private diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 8e14d21f591..44e8c9c04b9 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -337,6 +337,10 @@ module Gitlab REGISTRY_SCOPES end + def resource_bot_scopes + Gitlab::Auth::API_SCOPES + Gitlab::Auth::REPOSITORY_SCOPES + Gitlab::Auth.registry_scopes - [:read_user] + end + private def non_admin_available_scopes diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index f0ca6491bd0..b7e78189d37 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -18,8 +18,6 @@ module Gitlab end module AuthFinders - prepend_if_ee('::EE::Gitlab::Auth::AuthFinders') # rubocop: disable Cop/InjectEnterpriseEditionModule - include Gitlab::Utils::StrongMemoize include ActionController::HttpAuthentication::Basic @@ -27,6 +25,7 @@ module Gitlab PRIVATE_TOKEN_PARAM = :private_token JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :job_token + DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN'.freeze RUNNER_TOKEN_PARAM = :token RUNNER_JOB_TOKEN_PARAM = :token @@ -103,6 +102,25 @@ module Gitlab access_token.user || raise(UnauthorizedError) end + # This returns a deploy token, not a user since a deploy token does not + # belong to a user. + # + # deploy tokens are accepted with deploy token headers and basic auth headers + def deploy_token_from_request + return unless route_authentication_setting[:deploy_token_allowed] + + token = current_request.env[DEPLOY_TOKEN_HEADER].presence || parsed_oauth_token + + if has_basic_credentials?(current_request) + _, token = user_name_and_password(current_request) + end + + deploy_token = DeployToken.active.find_by_token(token) + @current_authenticated_deploy_token = deploy_token # rubocop:disable Gitlab/ModuleWithInstanceVariables + + deploy_token + end + def find_runner_from_token return unless api_request? @@ -113,6 +131,9 @@ module Gitlab end def validate_access_token!(scopes: []) + # return early if we've already authenticated via a deploy token + return if @current_authenticated_deploy_token.present? # rubocop:disable Gitlab/ModuleWithInstanceVariables + return unless access_token case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) @@ -249,3 +270,5 @@ module Gitlab end end end + +Gitlab::Auth::AuthFinders.prepend_if_ee('::EE::Gitlab::Auth::AuthFinders') diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 98eec0e4a7b..66d20ee2b59 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -8,8 +8,6 @@ module Gitlab module Auth module Ldap class Access - prepend_if_ee('::EE::Gitlab::Auth::Ldap::Access') # rubocop: disable Cop/InjectEnterpriseEditionModule - attr_reader :provider, :user, :ldap_identity def self.open(user, &block) @@ -118,3 +116,5 @@ module Gitlab end end end + +Gitlab::Auth::Ldap::Access.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Access') diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index c5ec4e1981b..f64fcd822c6 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -4,8 +4,6 @@ module Gitlab module Auth module Ldap class Adapter - prepend_if_ee('::EE::Gitlab::Auth::Ldap::Adapter') # rubocop: disable Cop/InjectEnterpriseEditionModule - SEARCH_RETRY_FACTOR = [1, 1, 2, 3].freeze MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size.freeze @@ -142,3 +140,5 @@ module Gitlab end end end + +Gitlab::Auth::Ldap::Adapter.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Adapter') diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index b8874e18a0b..7677189eb9f 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -5,8 +5,6 @@ module Gitlab module Auth module Ldap class Config - prepend_if_ee('::EE::Gitlab::Auth::Ldap::Config') # rubocop: disable Cop/InjectEnterpriseEditionModule - NET_LDAP_ENCRYPTION_METHOD = { simple_tls: :simple_tls, start_tls: :start_tls, @@ -288,3 +286,5 @@ module Gitlab end end end + +Gitlab::Auth::Ldap::Config.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Config') diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index 430f94a9a28..e4a4900c37a 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -4,8 +4,6 @@ module Gitlab module Auth module Ldap class Person - prepend_if_ee('::EE::Gitlab::Auth::Ldap::Person') # rubocop: disable Cop/InjectEnterpriseEditionModule - # Active Directory-specific LDAP filter that checks if bit 2 of the # userAccountControl attribute is set. # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/ @@ -122,3 +120,5 @@ module Gitlab end end end + +Gitlab::Auth::Ldap::Person.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Person') diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index df14e5fc3dc..1405fb4ab95 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -11,8 +11,6 @@ module Gitlab module Ldap class User < Gitlab::Auth::OAuth::User extend ::Gitlab::Utils::Override - prepend_if_ee('::EE::Gitlab::Auth::Ldap::User') # rubocop: disable Cop/InjectEnterpriseEditionModule - class << self # rubocop: disable CodeReuse/ActiveRecord def find_by_uid_and_provider(uid, provider) @@ -64,3 +62,5 @@ module Gitlab end end end + +Gitlab::Auth::Ldap::User.prepend_if_ee('::EE::Gitlab::Auth::Ldap::User') diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index b37a9225dd7..46ff6b2ccab 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -6,8 +6,6 @@ module Gitlab module Auth module OAuth class AuthHash - prepend_if_ee('::EE::Gitlab::Auth::OAuth::AuthHash') # rubocop: disable Cop/InjectEnterpriseEditionModule - attr_reader :auth_hash def initialize(auth_hash) @auth_hash = auth_hash @@ -93,3 +91,5 @@ module Gitlab end end end + +Gitlab::Auth::OAuth::AuthHash.prepend_if_ee('::EE::Gitlab::Auth::OAuth::AuthHash') diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index f0811098b15..6d699d37a8c 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -66,7 +66,10 @@ module Gitlab nil end else - Gitlab.config.omniauth.providers.find { |provider| provider.name == name } + provider = Gitlab.config.omniauth.providers.find { |provider| provider.name == name } + merge_provider_args_with_defaults!(provider) + + provider end end @@ -81,6 +84,15 @@ module Gitlab config = config_for(name) config && config['icon'] end + + def self.merge_provider_args_with_defaults!(provider) + return unless provider + + provider['args'] ||= {} + + defaults = Gitlab::OmniauthInitializer.default_arguments_for(provider['name']) + provider['args'].deep_merge!(defaults.deep_stringify_keys) + end end end end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index df595da1536..8a60d6ef482 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -9,8 +9,6 @@ module Gitlab module Auth module OAuth class User - prepend_if_ee('::EE::Gitlab::Auth::OAuth::User') # rubocop: disable Cop/InjectEnterpriseEditionModule - SignupDisabledError = Class.new(StandardError) SigninDisabledForProviderError = Class.new(StandardError) @@ -275,3 +273,5 @@ module Gitlab end end end + +Gitlab::Auth::OAuth::User.prepend_if_ee('::EE::Gitlab::Auth::OAuth::User') diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 0fe91f9f3c8..757a0e671c3 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -3,8 +3,6 @@ module Gitlab module Auth Result = Struct.new(:actor, :project, :type, :authentication_abilities) do - prepend_if_ee('::EE::Gitlab::Auth::Result') # rubocop: disable Cop/InjectEnterpriseEditionModule - def ci?(for_project) type == :ci && project && @@ -26,3 +24,5 @@ module Gitlab end end end + +Gitlab::Auth::Result.prepend_if_ee('::EE::Gitlab::Auth::Result') diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index ed2f3f158c1..67a53fa3205 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -4,8 +4,6 @@ module Gitlab module Auth module Saml class Config - prepend_if_ee('::EE::Gitlab::Auth::Saml::Config') # rubocop: disable Cop/InjectEnterpriseEditionModule - class << self def options Gitlab::Auth::OAuth::Provider.config_for('saml') @@ -31,3 +29,5 @@ module Gitlab end end end + +Gitlab::Auth::Saml::Config.prepend_if_ee('::EE::Gitlab::Auth::Saml::Config') diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index 1ba36ad95b4..37bc3f9bed0 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -9,8 +9,6 @@ module Gitlab module Auth module Saml class User < Gitlab::Auth::OAuth::User - prepend_if_ee('::EE::Gitlab::Auth::Saml::User') # rubocop: disable Cop/InjectEnterpriseEditionModule - extend ::Gitlab::Utils::Override def save @@ -63,3 +61,5 @@ module Gitlab end end end + +Gitlab::Auth::Saml::User.prepend_if_ee('::EE::Gitlab::Auth::Saml::User') diff --git a/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb b/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb new file mode 100644 index 00000000000..4fd3b81fda3 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # BackfillEnvironmentIdDeploymentMergeRequests deletes duplicates + # from deployment_merge_requests table and backfills environment_id + class BackfillEnvironmentIdDeploymentMergeRequests + def perform(_start_mr_id, _stop_mr_id) + # no-op + + # Background migration removed due to + # https://gitlab.com/gitlab-org/gitlab/-/issues/217191 + end + + def backfill_range(start_mr_id, stop_mr_id) + start_mr_id = Integer(start_mr_id) + stop_mr_id = Integer(stop_mr_id) + + ActiveRecord::Base.connection.execute(<<~SQL) + DELETE FROM deployment_merge_requests + WHERE (deployment_id, merge_request_id) in ( + SELECT t.deployment_id, t.merge_request_id FROM ( + SELECT mrd.merge_request_id, mrd.deployment_id, ROW_NUMBER() OVER w AS rnum + FROM deployment_merge_requests as mrd + INNER JOIN "deployments" ON "deployments"."id" = "mrd"."deployment_id" + WHERE mrd.merge_request_id BETWEEN #{start_mr_id} AND #{stop_mr_id} + WINDOW w AS ( + PARTITION BY merge_request_id, deployments.environment_id + ORDER BY deployments.id + ) + ) t + WHERE t.rnum > 1 + ); + SQL + + ActiveRecord::Base.connection.execute(<<~SQL) + UPDATE deployment_merge_requests + SET environment_id = deployments.environment_id + FROM deployments + WHERE deployments.id = "deployment_merge_requests".deployment_id + AND "deployment_merge_requests".environment_id IS NULL + AND "deployment_merge_requests".merge_request_id BETWEEN #{start_mr_id} AND #{stop_mr_id} + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb index fa6453abefb..21538000fec 100644 --- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb +++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb @@ -8,22 +8,41 @@ module Gitlab MAX_RETRIES = 2 def perform(start_id, stop_id) - Snippet.includes(:author, snippet_repository: :shard).where(id: start_id..stop_id).find_each do |snippet| + snippets = snippet_relation.where(id: start_id..stop_id) + + migrate_snippets(snippets) + end + + def perform_by_ids(snippet_ids) + snippets = snippet_relation.where(id: snippet_ids) + + migrate_snippets(snippets) + end + + private + + def migrate_snippets(snippets) + snippets.find_each do |snippet| # We need to expire the exists? value for the cached method in case it was cached snippet.repository.expire_exists_cache next if repository_present?(snippet) retry_index = 0 + @invalid_path_error = false + @invalid_signature_error = false begin create_repository_and_files(snippet) logger.info(message: 'Snippet Migration: repository created and migrated', snippet: snippet.id) rescue => e + set_file_path_error(e) + set_signature_error(e) + retry_index += 1 - retry if retry_index < MAX_RETRIES + retry if retry_index < max_retries logger.error(message: "Snippet Migration: error migrating snippet. Reason: #{e.message}", snippet: snippet.id) @@ -33,7 +52,9 @@ module Gitlab end end - private + def snippet_relation + @snippet_relation ||= Snippet.includes(:author, snippet_repository: :shard) + end def repository_present?(snippet) snippet.snippet_repository && !snippet.empty_repo? @@ -44,16 +65,19 @@ module Gitlab create_commit(snippet) end + # Removing the db record def destroy_snippet_repository(snippet) - # Removing the db record - snippet.snippet_repository&.destroy + snippet.snippet_repository&.delete rescue => e logger.error(message: "Snippet Migration: error destroying snippet repository. Reason: #{e.message}", snippet: snippet.id) end + # Removing the repository in disk def delete_repository(snippet) - # Removing the repository in disk - snippet.repository.remove if snippet.repository_exists? + return unless snippet.repository_exists? + + snippet.repository.remove + snippet.repository.expire_exists_cache rescue => e logger.error(message: "Snippet Migration: error deleting repository. Reason: #{e.message}", snippet: snippet.id) end @@ -70,7 +94,10 @@ module Gitlab end def filename(snippet) - snippet.file_name.presence || empty_file_name + file_name = snippet.file_name + file_name = file_name.parameterize if @invalid_path_error + + file_name.presence || empty_file_name end def empty_file_name @@ -82,7 +109,56 @@ module Gitlab end def create_commit(snippet) - snippet.snippet_repository.multi_files_action(snippet.author, snippet_action(snippet), commit_attrs) + snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), commit_attrs) + end + + # If the user is not allowed to access git or update the snippet + # because it is blocked, internal, ghost, ... we cannot commit + # files because these users are not allowed to, but we need to + # migrate their snippets as well. + # In this scenario the migration bot user will be the one that will commit the files. + def commit_author(snippet) + return migration_bot_user if snippet_content_size_over_limit?(snippet) + return migration_bot_user if @invalid_signature_error + + if Gitlab::UserAccessSnippet.new(snippet.author, snippet: snippet).can_do_action?(:update_snippet) + snippet.author + else + migration_bot_user + end + end + + def migration_bot_user + @migration_bot_user ||= User.migration_bot + end + + # We sometimes receive invalid path errors from Gitaly if the Snippet filename + # cannot be parsed into a valid git path. + # In this situation, we need to parameterize the file name of the Snippet so that + # the migration can succeed, to achieve that, we'll identify in migration retries + # that the path is invalid + def set_file_path_error(error) + @invalid_path_error ||= error.is_a?(SnippetRepository::InvalidPathError) + end + + # We sometimes receive invalid signature from Gitaly if the commit author + # name or email is invalid to create the commit signature. + # In this situation, we set the error and use the migration_bot since + # the information used to build it is valid + def set_signature_error(error) + @invalid_signature_error ||= error.is_a?(SnippetRepository::InvalidSignatureError) + end + + # In the case where the snippet file_name is invalid and also the + # snippet author has invalid commit info, we need to increase the + # number of retries by 1, because we will receive two errors + # from Gitaly and, in the third one, we will commit successfully. + def max_retries + MAX_RETRIES + (@invalid_signature_error && @invalid_path_error ? 1 : 0) + end + + def snippet_content_size_over_limit?(snippet) + snippet.content.size > Gitlab::CurrentSettings.snippet_size_limit end end end diff --git a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb index 14e14f28439..956f9daa493 100644 --- a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb +++ b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb @@ -79,7 +79,7 @@ module Gitlab data = { 'jira_tracker_data' => [], 'issue_tracker_data' => [] } select_all(query).each do |service| begin - properties = JSON.parse(service['properties']) + properties = Gitlab::Json.parse(service['properties']) rescue JSON::ParserError logger.warn( message: 'Properties data not parsed - invalid json', diff --git a/lib/gitlab/background_migration/populate_user_highest_roles_table.rb b/lib/gitlab/background_migration/populate_user_highest_roles_table.rb index 0c9e15b5a80..16386ebf9c3 100644 --- a/lib/gitlab/background_migration/populate_user_highest_roles_table.rb +++ b/lib/gitlab/background_migration/populate_user_highest_roles_table.rb @@ -20,6 +20,8 @@ module Gitlab end def perform(from_id, to_id) + return unless User.column_names.include?('bot_type') + (from_id..to_id).each_slice(BATCH_SIZE) do |ids| execute( <<-EOF diff --git a/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb new file mode 100644 index 00000000000..3920e8dc2de --- /dev/null +++ b/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RemoveUndefinedOccurrenceConfidenceLevel + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel') diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb new file mode 100644 index 00000000000..f6ea61f4502 --- /dev/null +++ b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RemoveUndefinedVulnerabilityConfidenceLevel + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel') diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb index fc579ad8d2a..57d632afd74 100644 --- a/lib/gitlab/blob_helper.rb +++ b/lib/gitlab/blob_helper.rb @@ -3,6 +3,8 @@ # This has been extracted from https://github.com/github/linguist/blob/master/lib/linguist/blob_helper.rb module Gitlab module BlobHelper + include Gitlab::Utils::StrongMemoize + def extname File.extname(name.to_s) end @@ -120,8 +122,18 @@ module Gitlab end def encoded_newlines_re - @encoded_newlines_re ||= - Regexp.union(["\r\n", "\r", "\n"].map { |nl| nl.encode(ruby_encoding, "ASCII-8BIT").force_encoding(data.encoding) }) + strong_memoize(:encoded_newlines_re) do + newlines = ["\r\n", "\r", "\n"] + data_encoding = data&.encoding + + if ruby_encoding && data_encoding + newlines.map! do |nl| + nl.encode(ruby_encoding, "ASCII-8BIT").force_encoding(data_encoding) + end + end + + Regexp.union(newlines) + end end def ruby_encoding diff --git a/lib/gitlab/chat/responder/mattermost.rb b/lib/gitlab/chat/responder/mattermost.rb new file mode 100644 index 00000000000..0488c98e422 --- /dev/null +++ b/lib/gitlab/chat/responder/mattermost.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +module Gitlab + module Chat + module Responder + class Mattermost < Responder::Base + SUCCESS_COLOR = '#00c100' + FAILURE_COLOR = '#e40303' + + # Slack breaks messages apart if they're around 4 KB in size. We use a + # slightly smaller limit here to account for user mentions. + MESSAGE_SIZE_LIMIT = 3.5.kilobytes + + # Sends a response back to Mattermost + # + # body - The message payload to send back to Mattermost, as a Hash. + def send_response(body) + Gitlab::HTTP.post( + pipeline.chat_data.response_url, + { + headers: { 'Content-Type': 'application/json' }, + body: body.to_json + } + ) + end + + # Sends the output for a build that completed successfully. + # + # output - The output produced by the chat command. + def success(output) + return if output.empty? + + send_response( + response_type: :in_channel, + attachments: [ + { + color: SUCCESS_COLOR, + text: "ChatOps job started by #{user_ref} completed successfully", + fields: [ + { + short: true, + title: "ID", + value: "#{build_ref}" + }, + { + short: true, + title: "Name", + value: build.name + }, + { + short: false, + title: "Output", + value: success_message(output) + } + ] + } + ] + ) + end + + # Sends the output for a build that failed. + def failure + send_response( + response_type: :in_channel, + attachments: [ + { + color: FAILURE_COLOR, + text: "ChatOps job started by #{user_ref} failed!", + fields: [ + { + short: true, + title: "ID", + value: "#{build_ref}" + }, + { + short: true, + title: "Name", + value: build.name + } + ] + } + ] + ) + end + + # Returns the output to send back after a command has been scheduled. + def scheduled_output + { + response_type: :ephemeral, + text: "Your ChatOps job #{build_ref} has been created!" + } + end + + private + + def success_message(output) + <<~HEREDOC.chomp + ```shell + #{strip_ansi_colorcodes(limit_output(output))} + ``` + HEREDOC + end + + def limit_output(output) + if output.bytesize <= MESSAGE_SIZE_LIMIT + output + else + "The output is too large to be sent back directly!" + end + end + + def strip_ansi_colorcodes(output) + output.gsub(/\x1b\[[0-9;]*m/, '') + end + + def user_ref + user = pipeline.chat_data.chat_name.user + user_url = ::Gitlab::Routing.url_helpers.user_url(user) + + "[#{user.name}](#{user_url})" + end + + def build_ref + build_url = ::Gitlab::Routing.url_helpers.project_build_url(project, build) + + "[##{build.id}](#{build_url})" + end + end + end + end +end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index 8b3c5dc9e8b..9b4cb9d0134 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -16,7 +16,7 @@ module Gitlab def get Gitlab::Redis::SharedState.with do |redis| data = redis.get(redis_shared_state_key) - JSON.parse(data, symbolize_names: true) if data + Gitlab::Json.parse(data, symbolize_names: true) if data end end diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb index a14fa02c2a4..0045d8a4113 100644 --- a/lib/gitlab/checks/base_checker.rb +++ b/lib/gitlab/checks/base_checker.rb @@ -3,7 +3,6 @@ module Gitlab module Checks class BaseChecker - prepend_if_ee('EE::Gitlab::Checks::BaseChecker') # rubocop: disable Cop/InjectEnterpriseEditionModule include Gitlab::Utils::StrongMemoize attr_reader :change_access @@ -57,3 +56,5 @@ module Gitlab end end end + +Gitlab::Checks::BaseChecker.prepend_if_ee('EE::Gitlab::Checks::BaseChecker') diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 14a445fcb96..8bb5ac94e45 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -3,8 +3,6 @@ module Gitlab module Checks class ChangeAccess - prepend_if_ee('EE::Gitlab::Checks::ChangeAccess') # rubocop: disable Cop/InjectEnterpriseEditionModule - ATTRIBUTES = %i[user_access project skip_authorization skip_lfs_integrity_check protocol oldrev newrev ref branch_name tag_name logger commits].freeze @@ -55,3 +53,5 @@ module Gitlab end end end + +Gitlab::Checks::ChangeAccess.prepend_if_ee('EE::Gitlab::Checks::ChangeAccess') diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index a73f243e946..8780b410a07 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -4,7 +4,6 @@ module Gitlab module Checks class DiffCheck < BaseChecker include Gitlab::Utils::StrongMemoize - prepend_if_ee('EE::Gitlab::Checks::DiffCheck') # rubocop: disable Cop/InjectEnterpriseEditionModule LOG_MESSAGES = { validate_file_paths: "Validating diffs' file paths...", @@ -97,3 +96,5 @@ module Gitlab end end end + +Gitlab::Checks::DiffCheck.prepend_if_ee('EE::Gitlab::Checks::DiffCheck') diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 3a05feee156..e145bd2e9df 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -353,7 +353,7 @@ module Gitlab def restore_state(new_state, stream) state = Base64.urlsafe_decode64(new_state) - state = JSON.parse(state, symbolize_names: true) + state = Gitlab::Json.parse(state, symbolize_names: true) return if state[:offset].to_i > stream.size STATE_PARAMS.each do |param| diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index 7e1a8102a35..38d36e6950c 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -90,7 +90,7 @@ module Gitlab decoded_state = Base64.urlsafe_decode64(state) return unless decoded_state.present? - JSON.parse(decoded_state) + Gitlab::Json.parse(decoded_state) end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 1c3ce08be76..c5afb16ab1a 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -32,7 +32,7 @@ module Gitlab raise ParserError, 'Errors field not found!' unless errors begin - JSON.parse(errors) + Gitlab::Json.parse(errors) rescue JSON::ParserError raise ParserError, 'Invalid errors field!' end @@ -71,7 +71,7 @@ module Gitlab next unless path =~ match_pattern next if path =~ INVALID_PATH_PATTERN - entries[path] = JSON.parse(meta, symbolize_names: true) + entries[path] = Gitlab::Json.parse(meta, symbolize_names: true) rescue JSON::ParserError, Encoding::CompatibilityError next end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 80e69cdcc95..ef354832e8e 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -50,7 +50,7 @@ module Gitlab end def basename - (directory? && !blank_node?) ? name + '/' : name + directory? && !blank_node? ? name + '/' : name end def name diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 241c73db3bb..a9a9636637f 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as].freeze + ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude].freeze EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces" @@ -35,6 +35,8 @@ module Gitlab }, if: :expose_as_present? validates :expose_as, type: String, length: { maximum: 100 }, if: :expose_as_present? validates :expose_as, format: { with: EXPOSE_AS_REGEX, message: EXPOSE_AS_ERROR_MESSAGE }, if: :expose_as_present? + validates :exclude, array_of_strings: true, if: :exclude_enabled? + validates :exclude, absence: { message: 'feature is disabled' }, unless: :exclude_enabled? validates :reports, type: Hash validates :when, inclusion: { in: %w[on_success on_failure always], @@ -50,8 +52,6 @@ module Gitlab end def expose_as_present? - return false unless Feature.enabled?(:ci_expose_arbitrary_artifacts_in_mr, default_enabled: true) - # This duplicates the `validates :config, type: Hash` above, # but Validatable currently doesn't halt the validation # chain if it encounters a validation error. @@ -59,6 +59,10 @@ module Gitlab !@config[:expose_as].nil? end + + def exclude_enabled? + ::Gitlab::Ci::Features.artifacts_exclude_enabled? + end end end end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 8ccee3b5b2b..1a871e043a6 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management license_scanning metrics lsif - dotenv cobertura terraform].freeze + dotenv cobertura terraform accessibility cluster_applications].freeze attributes ALLOWED_KEYS @@ -37,6 +37,8 @@ module Gitlab validates :dotenv, array_of_strings_or_string: true validates :cobertura, array_of_strings_or_string: true validates :terraform, array_of_strings_or_string: true + validates :accessibility, array_of_strings_or_string: true + validates :cluster_applications, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index 7202784842a..c6ba53adfd7 100644 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -25,8 +25,7 @@ module Gitlab strategy :CrossProjectTrigger, if: -> (config) { !config.key?(:include) } strategy :SameProjectTrigger, if: -> (config) do - ::Feature.enabled?(:ci_parent_child_pipeline, default_enabled: true) && - config.key?(:include) + config.key?(:include) end class CrossProjectTrigger < ::Gitlab::Config::Entry::Node @@ -72,11 +71,7 @@ module Gitlab class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors - if ::Feature.enabled?(:ci_parent_child_pipeline, default_enabled: true) - ['config must specify either project or include'] - else - ['config must specify project'] - end + ['config must specify either project or include'] end end end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 1d7e7ea0f9a..efd48a9b29f 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -12,8 +12,11 @@ module Gitlab end def next_time_from(time) - @cron_line ||= try_parse_cron(@cron, @cron_timezone) - @cron_line.next_time(time).utc.in_time_zone(Time.zone) if @cron_line.present? + cron_line.next_time(time).utc.in_time_zone(Time.zone) if cron_line.present? + end + + def previous_time_from(time) + cron_line.previous_time(time).utc.in_time_zone(Time.zone) if cron_line.present? end def cron_valid? @@ -49,6 +52,10 @@ module Gitlab def try_parse_cron(cron, cron_timezone) Fugit::Cron.parse("#{cron} #{cron_timezone}") end + + def cron_line + @cron_line ||= try_parse_cron(@cron, @cron_timezone) + end end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb new file mode 100644 index 00000000000..48f3d4fdd2f --- /dev/null +++ b/lib/gitlab/ci/features.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + ## + # Ci::Features is a class that aggregates all CI/CD feature flags in one place. + # + module Features + def self.artifacts_exclude_enabled? + ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: false) + end + + def self.ensure_scheduling_type_enabled? + ::Feature.enabled?(:ci_ensure_scheduling_type, default_enabled: true) + end + end + end +end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index a44105d53c2..0e44475607b 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -3,14 +3,14 @@ module Gitlab module Ci module Parsers - prepend_if_ee('::EE::Gitlab::Ci::Parsers') # rubocop: disable Cop/InjectEnterpriseEditionModule - ParserNotFoundError = Class.new(ParserError) def self.parsers { junit: ::Gitlab::Ci::Parsers::Test::Junit, - cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura + cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura, + terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan, + accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y } end @@ -22,3 +22,5 @@ module Gitlab end end end + +Gitlab::Ci::Parsers.prepend_if_ee('::EE::Gitlab::Ci::Parsers') diff --git a/lib/gitlab/ci/parsers/accessibility/pa11y.rb b/lib/gitlab/ci/parsers/accessibility/pa11y.rb new file mode 100644 index 00000000000..953b5a91258 --- /dev/null +++ b/lib/gitlab/ci/parsers/accessibility/pa11y.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Accessibility + class Pa11y + def parse!(json_data, accessibility_report) + root = Gitlab::Json.parse(json_data).with_indifferent_access + + parse_all(root, accessibility_report) + rescue JSON::ParserError => e + accessibility_report.set_error_message("JSON parsing failed: #{e}") + rescue StandardError => e + accessibility_report.set_error_message("Pa11y parsing failed: #{e}") + end + + private + + def parse_all(root, accessibility_report) + return unless root.present? + + root.dig("results").each do |url, value| + accessibility_report.add_url(url, value) + end + + accessibility_report + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb new file mode 100644 index 00000000000..26a18c6603e --- /dev/null +++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Terraform + class Tfplan + TfplanParserError = Class.new(Gitlab::Ci::Parsers::ParserError) + + def parse!(json_data, terraform_reports, artifact:) + tfplan = Gitlab::Json.parse(json_data).tap do |parsed_data| + parsed_data['job_path'] = Gitlab::Routing.url_helpers.project_job_path( + artifact.job.project, artifact.job + ) + end + + raise TfplanParserError, 'Tfplan missing required key' unless valid_supported_keys?(tfplan) + + terraform_reports.add_plan(artifact.filename, tfplan) + rescue JSON::ParserError + raise TfplanParserError, 'JSON parsing failed' + rescue + raise TfplanParserError, 'Tfplan parsing failed' + end + + private + + def valid_supported_keys?(tfplan) + tfplan.keys == %w[create update delete job_path] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index 33140b4c7fd..5746f38ae5b 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -15,10 +15,10 @@ module Gitlab test_case = create_test_case(test_case, args) test_suite.add_test_case(test_case) end - rescue Nokogiri::XML::SyntaxError - raise JunitParserError, "XML parsing failed" - rescue - raise JunitParserError, "JUnit parsing failed" + rescue Nokogiri::XML::SyntaxError => e + test_suite.set_suite_error("JUnit XML parsing failed: #{e}") + rescue StandardError => e + test_suite.set_suite_error("JUnit data parsing failed: #{e}") end private diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index fa46114615c..73187401903 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -76,6 +76,21 @@ module Gitlab def parent_pipeline bridge&.parent_pipeline end + + def duration_histogram + strong_memoize(:duration_histogram) do + name = :gitlab_ci_pipeline_creation_duration_seconds + comment = 'Pipeline creation duration' + labels = {} + buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def observe_creation_duration(duration) + duration_histogram.observe({}, duration.seconds) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 99780409085..a7c671e76d3 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -10,6 +10,7 @@ module Gitlab @command = command @sequence = sequence @completed = [] + @start = Time.now end def build! @@ -24,6 +25,8 @@ module Gitlab @pipeline.tap do yield @pipeline, self if block_given? + + @command.observe_creation_duration(Time.now - @start) end end diff --git a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb index 3bec6d1e8b6..c0641d9ff0a 100644 --- a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb +++ b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb @@ -16,7 +16,6 @@ module Gitlab end def to_resource - return unless Feature.enabled?(:ci_resource_group, build.project, default_enabled: true) return unless resource_group_key.present? resource_group = build.project.resource_groups diff --git a/lib/gitlab/ci/reports/accessibility_reports.rb b/lib/gitlab/ci/reports/accessibility_reports.rb new file mode 100644 index 00000000000..1901ba3b102 --- /dev/null +++ b/lib/gitlab/ci/reports/accessibility_reports.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class AccessibilityReports + attr_reader :urls, :error_message + + def initialize + @urls = {} + @error_message = nil + end + + def add_url(url, data) + if url.empty? + set_error_message("Empty URL detected in gl-accessibility.json") + else + urls[url] = data + end + end + + def scans_count + @urls.size + end + + def passes_count + @urls.count { |url, errors| errors.empty? } + end + + # rubocop: disable CodeReuse/ActiveRecord + def errors_count + @urls.sum { |url, errors| errors.size } + end + # rubocop: enable CodeReuse/ActiveRecord + + def set_error_message(error) + @error_message = error + end + + def all_errors + @urls.values.flatten + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/accessibility_reports_comparer.rb b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb new file mode 100644 index 00000000000..fa6337166d5 --- /dev/null +++ b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class AccessibilityReportsComparer + include Gitlab::Utils::StrongMemoize + + STATUS_SUCCESS = 'success' + STATUS_FAILED = 'failed' + + attr_reader :base_reports, :head_reports + + def initialize(base_reports, head_reports) + @base_reports = base_reports || AccessibilityReports.new + @head_reports = head_reports + end + + def status + head_reports.errors_count.positive? ? STATUS_FAILED : STATUS_SUCCESS + end + + def existing_errors + strong_memoize(:existing_errors) do + base_reports.all_errors + end + end + + def new_errors + strong_memoize(:new_errors) do + head_reports.all_errors - base_reports.all_errors + end + end + + def resolved_errors + strong_memoize(:resolved_errors) do + base_reports.all_errors - head_reports.all_errors + end + end + + def errors_count + head_reports.errors_count + end + + def resolved_count + resolved_errors.size + end + + def total_count + existing_errors.size + new_errors.size + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/terraform_reports.rb b/lib/gitlab/ci/reports/terraform_reports.rb new file mode 100644 index 00000000000..f955d007daf --- /dev/null +++ b/lib/gitlab/ci/reports/terraform_reports.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class TerraformReports + attr_reader :plans + + def initialize + @plans = {} + end + + def pick(keys) + terraform_plans = plans.select do |key| + keys.include?(key) + end + + { plans: terraform_plans } + end + + def add_plan(name, plan) + plans[name] = plan + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb index 72323c4343d..86ba725c71e 100644 --- a/lib/gitlab/ci/reports/test_reports.rb +++ b/lib/gitlab/ci/reports/test_reports.rb @@ -42,6 +42,12 @@ module Gitlab self end + def suite_errors + test_suites.each_with_object({}) do |(name, suite), errors| + errors[suite.name] = suite.suite_error if suite.suite_error + end + end + TestCase::STATUS_TYPES.each do |status_type| define_method("#{status_type}_count") do # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index cf43c5313c0..8bbf2e0f6cf 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -7,6 +7,7 @@ module Gitlab attr_reader :name attr_reader :test_cases attr_reader :total_time + attr_reader :suite_error def initialize(name = nil) @name = name @@ -25,12 +26,16 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def total_count + return 0 if suite_error + test_cases.values.sum(&:count) end # rubocop: enable CodeReuse/ActiveRecord def total_status - if failed_count > 0 || error_count > 0 + if suite_error + TestCase::STATUS_ERROR + elsif failed_count > 0 || error_count > 0 TestCase::STATUS_FAILED else TestCase::STATUS_SUCCESS @@ -49,14 +54,22 @@ module Gitlab TestCase::STATUS_TYPES.each do |status_type| define_method("#{status_type}") do - test_cases[status_type] || {} + return {} if suite_error || test_cases[status_type].nil? + + test_cases[status_type] end define_method("#{status_type}_count") do - test_cases[status_type]&.length.to_i + return 0 if suite_error || test_cases[status_type].nil? + + test_cases[status_type].length end end + def set_suite_error(msg) + @suite_error = msg + end + private def existing_key?(test_case) diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index b0b01538a30..76ad113aad9 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -29,8 +29,6 @@ module Gitlab private_constant :REASONS - prepend_if_ee('::EE::Gitlab::Ci::Status::Build::Failed') # rubocop: disable Cop/InjectEnterpriseEditionModule - def status_tooltip base_message end @@ -65,3 +63,5 @@ module Gitlab end end end + +Gitlab::Ci::Status::Build::Failed.prepend_if_ee('::EE::Gitlab::Ci::Status::Build::Failed') diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index a9f29bda9b9..5017037fb5a 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -48,7 +48,6 @@ variables: POSTGRES_PASSWORD: testing-password POSTGRES_ENABLED: "true" POSTGRES_DB: $CI_ENVIRONMENT_SLUG - POSTGRES_VERSION: 9.6.2 DOCKER_DRIVER: overlay2 @@ -159,5 +158,5 @@ include: - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml index a41b399032f..82b2f5c035e 100644 --- a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml @@ -5,32 +5,9 @@ stages: - deploy - production +variables: + AUTO_DEVOPS_PLATFORM_TARGET: ECS + include: - template: Jobs/Build.gitlab-ci.yml - -.deploy_to_ecs: - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest - script: - - ecs update-task-definition - -review: - extends: .deploy_to_ecs - stage: review - environment: - name: review/$CI_COMMIT_REF_NAME - only: - refs: - - branches - - tags - except: - refs: - - master - -production: - extends: .deploy_to_ecs - stage: production - environment: - name: production - only: - refs: - - master + - template: Jobs/Deploy/ECS.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index d85078c0a40..adbf9731e43 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -30,11 +30,9 @@ performance: paths: - performance.json - sitespeed-results/ - only: - refs: - - branches - - tags - kubernetes: active - except: - variables: - - $PERFORMANCE_DISABLED + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$PERFORMANCE_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 3949b87bbda..787f07521e0 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -15,6 +15,5 @@ build: export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} fi - /build/build.sh - only: - - branches - - tags + rules: + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 9c4699f1f44..24e75c56a75 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -26,10 +26,7 @@ code_quality: codequality: gl-code-quality-report.json expire_in: 1 week dependencies: [] - only: - refs: - - branches - - tags - except: - variables: - - $CODE_QUALITY_DISABLED + rules: + - if: '$CODE_QUALITY_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 3cf4910fe86..5174aed04ba 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.10.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0" dast_environment_deploy: extends: .dast-auto-deploy @@ -18,17 +18,16 @@ dast_environment_deploy: on_stop: stop_dast_environment artifacts: paths: [environment_url.txt] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bdast\b/ - kubernetes: active - except: - variables: - - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - - $DAST_WEBSITE # we don't need to create a review app if a URL is already given + rules: + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME + when: never + - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH + when: never + - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given + when: never + - if: $CI_COMMIT_BRANCH && + $CI_KUBERNETES_ACTIVE && + $GITLAB_FEATURES =~ /\bdast\b/ stop_dast_environment: extends: .dast-auto-deploy @@ -42,14 +41,13 @@ stop_dast_environment: name: dast-default action: stop needs: ["dast"] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bdast\b/ - kubernetes: active - except: - variables: - - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - - $DAST_WEBSITE + rules: + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME + when: never + - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH + when: never + - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given + when: never + - if: $CI_COMMIT_BRANCH && + $CI_KUBERNETES_ACTIVE && + $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 9bf0d31409a..b4e5a41a34d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,8 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.13.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0" + +include: + - template: Jobs/Deploy/ECS.gitlab-ci.yml review: extends: .auto-deploy @@ -18,16 +21,14 @@ review: on_stop: stop_review artifacts: paths: [environment_url.txt] - only: - refs: - - branches - - tags - kubernetes: active - except: - refs: - - master - variables: - - $REVIEW_DISABLED + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' stop_review: extends: .auto-deploy @@ -41,18 +42,16 @@ stop_review: name: review/$CI_COMMIT_REF_NAME action: stop dependencies: [] - when: manual allow_failure: true - only: - refs: - - branches - - tags - kubernetes: active - except: - refs: - - master - variables: - - $REVIEW_DISABLED + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + when: manual # Staging deploys are disabled by default since # continuous deployment to production is enabled by default @@ -73,12 +72,12 @@ staging: environment: name: staging url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN - only: - refs: - - master - kubernetes: active - variables: - - $STAGING_ENABLED + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$STAGING_ENABLED' # Canaries are disabled by default, but if you want them, # and know what the downsides are, you can enable this by setting @@ -97,13 +96,13 @@ canary: environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN - when: manual - only: - refs: - - master - kubernetes: active - variables: - - $CANARY_ENABLED + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$CANARY_ENABLED' + when: manual .production: &production_template extends: .auto-deploy @@ -126,32 +125,33 @@ canary: production: <<: *production_template - only: - refs: - - master - kubernetes: active - except: - variables: - - $STAGING_ENABLED - - $CANARY_ENABLED - - $INCREMENTAL_ROLLOUT_ENABLED - - $INCREMENTAL_ROLLOUT_MODE + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$STAGING_ENABLED' + when: never + - if: '$CANARY_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' production_manual: <<: *production_template - when: manual allow_failure: false - only: - refs: - - master - kubernetes: active - variables: - - $STAGING_ENABLED - - $CANARY_ENABLED - except: - variables: - - $INCREMENTAL_ROLLOUT_ENABLED - - $INCREMENTAL_ROLLOUT_MODE + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE' + when: never + - if: '$CI_COMMIT_BRANCH == "master" && $STAGING_ENABLED' + when: manual + - if: '$CI_COMMIT_BRANCH == "master" && $CANARY_ENABLED' + when: manual # This job implements incremental rollout on for every push to `master`. @@ -176,29 +176,29 @@ production_manual: .manual_rollout_template: &manual_rollout_template <<: *rollout_template stage: production - when: manual - # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4) - only: - refs: - - master - kubernetes: active - variables: - - $INCREMENTAL_ROLLOUT_MODE == "manual" - - $INCREMENTAL_ROLLOUT_ENABLED - except: - variables: - - $INCREMENTAL_ROLLOUT_MODE == "timed" + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + # $INCREMENTAL_ROLLOUT_ENABLED is for compamtibilty with pre-GitLab 11.4 syntax + - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' + when: manual .timed_rollout_template: &timed_rollout_template <<: *rollout_template - when: delayed - start_in: 5 minutes - only: - refs: - - master - kubernetes: active - variables: - - $INCREMENTAL_ROLLOUT_MODE == "timed" + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' + when: delayed + start_in: 5 minutes timed rollout 10%: <<: *timed_rollout_template diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml new file mode 100644 index 00000000000..642f0ebeaf7 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -0,0 +1,30 @@ +.deploy_to_ecs: + image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + script: + - ecs update-task-definition + +review_ecs: + extends: .deploy_to_ecs + stage: review + environment: + name: review/$CI_COMMIT_REF_NAME + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + +production_ecs: + extends: .deploy_to_ecs + stage: production + environment: + name: production + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' diff --git a/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml index a0ddd273552..3b87d53f165 100644 --- a/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml @@ -1,10 +1,12 @@ test: - services: - - "postgres:${POSTGRES_VERSION}" variables: + POSTGRES_VERSION: 9.6.16 POSTGRES_DB: test + services: + - "postgres:${POSTGRES_VERSION}" stage: test image: gliderlabs/herokuish:latest + needs: [] script: - | if [ -z ${KUBERNETES_PORT+x} ]; then @@ -15,9 +17,7 @@ test: - export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}" - cp -R . /tmp/app - /bin/herokuish buildpack test - only: - - branches - - tags - except: - variables: - - $TEST_DISABLED + rules: + - if: '$TEST_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index b4208ed9d7d..e081e20564a 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -1,7 +1,7 @@ -# Official Java image. Look for the different tagged releases at -# https://hub.docker.com/r/library/java/tags/ . A Java image is not required +# Official OpenJDK Java image. Look for the different tagged releases at +# https://hub.docker.com/_/openjdk/ . A Java image is not required # but an image with a JVM speeds up the build a bit. -image: java:8 +image: openjdk:8 before_script: # Enable the usage of sources over https @@ -14,7 +14,7 @@ before_script: - apt-get update -yqq - apt-get install sbt -yqq # Log the sbt version - - sbt sbt-version + - sbt sbtVersion test: script: diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 6efb6b4e273..21bcdd8d9b5 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,16 +1,20 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/ variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + CS_MAJOR_VERSION: 2 container_scanning: stage: test - image: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CS_MAJOR_VERSION + image: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION variables: # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes CLAIR_DB_IMAGE_TAG: "latest" - CLAIR_DB_IMAGE: "arminc/clair-db:$CLAIR_DB_IMAGE_TAG" + CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml` # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details @@ -25,11 +29,8 @@ container_scanning: reports: container_scanning: gl-container-scanning-report.json dependencies: [] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ - except: - variables: - - $CONTAINER_SCANNING_DISABLED + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 0e3d7660bdf..07399216597 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -12,11 +12,14 @@ stages: variables: DAST_VERSION: 1 + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" dast: stage: dast image: - name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" + name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION" variables: GIT_STRATEGY: none allow_failure: true @@ -27,12 +30,15 @@ dast: artifacts: reports: dast: gl-dast-report.json - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bdast\b/ - except: - variables: - - $DAST_DISABLED - - $DAST_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + rules: + - if: $DAST_DISABLED + when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED && $DAST_WEBSITE == null && + $DAST_API_SPECIFICATION == null + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 0ecf37b37a3..616966b4f04 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -5,11 +5,16 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - SECURITY_SCANNER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products" - DS_ANALYZER_IMAGE_PREFIX: "$SECURITY_SCANNER_IMAGE_PREFIX/analyzers" + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + + # Deprecated, use SECURE_ANALYZERS_PREFIX instead + DS_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX" + DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_MAJOR_VERSION: 2 - DS_DISABLE_DIND: "false" + DS_DISABLE_DIND: "true" dependency_scanning: stage: test @@ -21,7 +26,6 @@ dependency_scanning: services: - docker:stable-dind script: - - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | if ! docker info &>/dev/null; then if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then @@ -68,28 +72,25 @@ dependency_scanning: ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ - "$SECURITY_SCANNER_IMAGE_PREFIX/dependency-scanning:$DS_VERSION" /code + "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_MAJOR_VERSION" /code artifacts: reports: dependency_scanning: gl-dependency-scanning-report.json dependencies: [] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ - except: - variables: - - $DEPENDENCY_SCANNING_DISABLED - - $DS_DISABLE_DIND == 'true' + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'true' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ .ds-analyzer: extends: dependency_scanning services: [] - except: - variables: - - $DEPENDENCY_SCANNING_DISABLED - - $DS_DISABLE_DIND == 'false' + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ script: - /analyzer run @@ -97,48 +98,81 @@ gemnasium-dependency_scanning: extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium:$DS_MAJOR_VERSION" - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby|javascript|php|\bgo\b/ + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ + exists: + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + - '{composer.lock,*/composer.lock,*/*/composer.lock}' + - '{gems.locked,*/gems.locked,*/*/gems.locked}' + - '{go.sum,*/go.sum,*/*/go.sum}' + - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' + - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' + - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' gemnasium-maven-dependency_scanning: extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(java|scala)\b/ + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ + exists: + - '{build.gradle,*/build.gradle,*/*/build.gradle}' + - '{build.sbt,*/build.sbt,*/*/build.sbt}' + - '{pom.xml,*/pom.xml,*/*/pom.xml}' gemnasium-python-dependency_scanning: extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/ + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ + exists: + - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' + - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' + - '{Pipfile,*/Pipfile,*/*/Pipfile}' + - '{requires.txt,*/requires.txt,*/*/requires.txt}' + - '{setup.py,*/setup.py,*/*/setup.py}' + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ && + $PIP_REQUIREMENTS_FILE bundler-audit-dependency_scanning: extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/bundler-audit:$DS_MAJOR_VERSION" - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/ + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ + exists: + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' retire-js-dependency_scanning: extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/retire.js:$DS_MAJOR_VERSION" - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /retire.js/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/ + rules: + - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /retire.js/ + exists: + - '{package.json,*/package.json,*/*/package.json}' diff --git a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml index 58fd018a82d..87f78d0c887 100644 --- a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml @@ -1,29 +1,13 @@ # Deprecated: https://gitlab.com/gitlab-org/gitlab/issues/14624 # Please, use License-Scanning.gitlab-ci.yml template instead -variables: - LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. +include: + - template: License-Scanning.gitlab-ci.yml -license_management: - stage: test - image: - name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" - entrypoint: [""] - variables: - SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD - allow_failure: true - script: - - echo "This template is deprecated, please use License-Scanning.gitlab-ci.yml template instead." - - /run.sh analyze . - artifacts: - reports: - license_management: gl-license-management-report.json - dependencies: [] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\blicense_management\b/ - except: - variables: - - $LICENSE_MANAGEMENT_DISABLED +license_scanning: + before_script: + - | + echo "As of GitLab 12.8, we deprecated the License-Management.gitlab.ci.yml template. + Please replace it with the License-Scanning.gitlab-ci.yml template instead. + For more details visit + https://docs.gitlab.com/ee/user/compliance/license_compliance/#migration-from-license_management-to-license_scanning" diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index 2333fb4e947..b86014c1ebc 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -5,29 +5,30 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. + LICENSE_MANAGEMENT_VERSION: 3 license_scanning: stage: test image: - name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" + name: "$SECURE_ANALYZERS_PREFIX/license-finder:$LICENSE_MANAGEMENT_VERSION" entrypoint: [""] variables: + LM_REPORT_FILE: gl-license-scanning-report.json SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD allow_failure: true script: - /run.sh analyze . - after_script: - - mv gl-license-management-report.json gl-license-scanning-report.json artifacts: reports: - license_scanning: gl-license-scanning-report.json + license_scanning: $LM_REPORT_FILE dependencies: [] - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\blicense_scanning\b/ - except: - variables: - - $LICENSE_MANAGEMENT_DISABLED + rules: + - if: $LICENSE_MANAGEMENT_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\blicense_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 03b9720747d..47f68118ee0 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -5,10 +5,16 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + + # Deprecated, use SECURE_ANALYZERS_PREFIX instead + SAST_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec" SAST_ANALYZER_IMAGE_TAG: 2 - SAST_DISABLE_DIND: "false" + SAST_DISABLE_DIND: "true" SCAN_KUBERNETES_MANIFESTS: "false" sast: @@ -17,19 +23,18 @@ sast: artifacts: reports: sast: gl-sast-report.json - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'true' + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bsast\b/ image: docker:stable variables: + SEARCH_MAX_DEPTH: 4 DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: - - export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | if ! docker info &>/dev/null; then if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then @@ -41,19 +46,16 @@ sast: $(awk 'BEGIN{for(v in ENVIRON) print v}' | grep -v -E '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | awk '{printf " -e %s", $0}') \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code - except: - variables: - - $SAST_DISABLED - - $SAST_DISABLE_DIND == 'true' + "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_ANALYZER_IMAGE_TAG" /app/bin/run /code .sast-analyzer: extends: sast services: [] - except: - variables: - - $SAST_DISABLED - - $SAST_DISABLE_DIND == 'false' + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ script: - /analyzer run @@ -61,49 +63,65 @@ bandit-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /bandit/&& - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bpython\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /bandit/ + exists: + - '**/*.py' brakeman-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /brakeman/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bruby\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /brakeman/ + exists: + - '**/*.rb' eslint-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /eslint/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bjavascript\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /eslint/ + exists: + - '**/*.html' + - '**/*.js' flawfinder-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /(c(\+\+)?,)|(c(\+\+)?$)/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ + exists: + - '**/*.c' + - '**/*.cpp' kubesec-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && $SCAN_KUBERNETES_MANIFESTS == 'true' @@ -111,87 +129,117 @@ gosec-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /gosec/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bgo\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /gosec/ + exists: + - '**/*.go' nodejs-scan-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bjavascript\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ + exists: + - '**/*.js' phpcs-security-audit-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bphp\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ + exists: + - '**/*.php' pmd-apex-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bapex\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ + exists: + - '**/*.cls' secrets-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /secrets/ security-code-scan-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\#|visual basic\b)/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ + exists: + - '**/*.csproj' + - '**/*.vbproj' sobelow-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /sobelow/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\belixir\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /sobelow/ + exists: + - '**/*.ex' + - '**/*.exs' spotbugs-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(groovy|java|scala)\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ + exists: + - '**/*.groovy' + - '**/*.java' + - '**/*.scala' tslint-sast: extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint:$SAST_ANALYZER_IMAGE_TAG" - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ && - $SAST_DEFAULT_ANALYZERS =~ /tslint/ && - $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\btypescript\b/ + rules: + - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /tslint/ + exists: + - '**/*.ts' diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml new file mode 100644 index 00000000000..b6c05c61db1 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -0,0 +1,246 @@ +# This template should be used when Security Products (https://about.gitlab.com/handbook/engineering/development/secure/#security-products) +# have to be downloaded and stored locally. +# +# Usage: +# +# ``` +# include: +# - template: Secure-Binaries.gitlab-ci.yml +# ``` +# +# Docs: https://docs.gitlab.com/ee/topics/airgap/ + + +variables: + SECURE_BINARIES_ANALYZERS: >- + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec, + bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, + klar, clair-vulnerabilities-db, + license-finder, + dast + + SECURE_BINARIES_DOWNLOAD_IMAGES: "true" + SECURE_BINARIES_PUSH_IMAGES: "true" + SECURE_BINARIES_SAVE_ARTIFACTS: "false" + + SECURE_BINARIES_ANALYZER_VERSION: "2" + +.download_images: + allow_failure: true + image: docker:stable + only: + refs: + - branches + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + services: + - docker:stable-dind + script: + - docker info + - env + - if [ -z "$SECURE_BINARIES_IMAGE" ]; then export SECURE_BINARIES_IMAGE=${SECURE_BINARIES_IMAGE:-"registry.gitlab.com/gitlab-org/security-products/analyzers/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"}; fi + - docker pull ${SECURE_BINARIES_IMAGE} + - mkdir -p output/$(dirname ${CI_JOB_NAME}) + - | + if [ "$SECURE_BINARIES_SAVE_ARTIFACTS" = "true" ]; then + docker save ${SECURE_BINARIES_IMAGE} | gzip > output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar.gz + sha256sum output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar.gz > output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar.gz.sha256sum + fi + - | + if [ "$SECURE_BINARIES_PUSH_IMAGES" = "true" ]; then + docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + docker tag ${SECURE_BINARIES_IMAGE} ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION} + docker push ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION} + fi + + artifacts: + paths: + - output/ + +# +# SAST jobs +# + +bandit: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bbandit\b/ + +brakeman: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bbrakeman\b/ + +gosec: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgosec\b/ + +spotbugs: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bspotbugs\b/ + +flawfinder: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bflawfinder\b/ + +phpcs-security-audit: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bphpcs-security-audit\b/ + +security-code-scan: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsecurity-code-scan\b/ + +nodejs-scan: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bnodejs-scan\b/ + +eslint: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\beslint\b/ + +tslint: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\btslint\b/ + +secrets: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/ + +sobelow: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsobelow\b/ + +pmd-apex: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/ + +kubesec: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bkubesec\b/ +# +# Container Scanning jobs +# + +klar: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bklar\b/ + +clair-vulnerabilities-db: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bclair-vulnerabilities-db\b/ + variables: + SECURE_BINARIES_IMAGE: arminc/clair-db + SECURE_BINARIES_ANALYZER_VERSION: latest + +# +# Dependency Scanning jobs +# + +bundler-audit: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bbundler-audit\b/ + +retire.js: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bretire\.js\b/ + +gemnasium: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium\b/ + +gemnasium-maven: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium-maven\b/ + +gemnasium-python: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium-python\b/ + +# +# License Scanning +# + +license-finder: + extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\blicense-finder\b/ + +# +# DAST +# + +dast: + extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "1" + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 83483108fde..a0832718214 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -19,7 +19,7 @@ cache: - .terraform before_script: - - alias convert_report="jq -r '([.resource_changes[].change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" + - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" - terraform --version - terraform init diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 5d9d3c74def..e8a99a6ea06 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -8,12 +8,14 @@ stages: a11y: stage: accessibility - image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.2 + image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.3 script: /gitlab-accessibility.sh $a11y_urls allow_failure: true artifacts: when: always expose_as: 'Accessibility Reports' paths: ['reports/'] + reports: + accessibility: reports/gl-accessibility.json rules: - if: $a11y_urls diff --git a/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml b/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml new file mode 100644 index 00000000000..05635cf71be --- /dev/null +++ b/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml @@ -0,0 +1,7 @@ +# Read more on when to use this template at +# https://docs.gitlab.com/ee/ci/yaml/#workflowrules + +workflow: + rules: + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml b/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml new file mode 100644 index 00000000000..50ff4c1f60b --- /dev/null +++ b/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml @@ -0,0 +1,8 @@ +# Read more on when to use this template at +# https://docs.gitlab.com/ee/ci/yaml/#workflowrules + +workflow: + rules: + - if: $CI_MERGE_REQUEST_IID + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 933504ea82f..5816ac3bc54 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -157,7 +157,7 @@ module Gitlab return unless job[:stage] unless job[:stage].is_a?(String) && job[:stage].in?(@stages) - raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" + raise ValidationError, "#{name} job: chosen stage does not exist; available stages are #{@stages.join(", ")}" end end diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb index a9961cb8968..3df243e319e 100644 --- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb +++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb @@ -35,6 +35,8 @@ module Gitlab count += relation.delete_all end + ProjectCacheWorker.perform_async(project.id, [], [:lfs_objects_size]) + log_info("Removed invalid references: #{count}") end end diff --git a/lib/gitlab/code_navigation_path.rb b/lib/gitlab/code_navigation_path.rb index 8dd2e9cb1bb..57aeb6c4fb2 100644 --- a/lib/gitlab/code_navigation_path.rb +++ b/lib/gitlab/code_navigation_path.rb @@ -5,7 +5,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize include Gitlab::Routing - CODE_NAVIGATION_JOB_NAME = 'code_navigation' + LATEST_COMMITS_LIMIT = 10 def initialize(project, commit_sha) @project = project @@ -16,7 +16,7 @@ module Gitlab return if Feature.disabled?(:code_navigation, project) return unless build - raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json") + raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json", file_type: :lsif) end private @@ -25,10 +25,14 @@ module Gitlab def build strong_memoize(:build) do - artifact = ::Ci::JobArtifact - .for_sha(commit_sha, project.id) - .for_job_name(CODE_NAVIGATION_JOB_NAME) - .last + latest_commits_shas = + project.repository.commits(commit_sha, limit: LATEST_COMMITS_LIMIT).map(&:sha) + + artifact = + ::Ci::JobArtifact + .with_file_types(['lsif']) + .for_sha(latest_commits_shas, project.id) + .last artifact&.job end diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb new file mode 100644 index 00000000000..795082a10a0 --- /dev/null +++ b/lib/gitlab/config_checker/external_database_checker.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module ConfigChecker + module ExternalDatabaseChecker + extend self + + # DB is considered deprecated if it is below version 11 + def db_version_deprecated? + Gitlab::Database.version.to_f < 11 + end + + def check + return [] unless db_version_deprecated? + + [ + { + type: 'warning', + message: _('Note that PostgreSQL 11 will become the minimum required PostgreSQL version in GitLab 13.0 (May 2020). '\ + 'PostgreSQL 9.6 and PostgreSQL 10 will no longer be supported in GitLab 13.0. '\ + 'Please consider upgrading your PostgreSQL version (%{db_version}) soon.') % { db_version: Gitlab::Database.version.to_s } + } + ] + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb deleted file mode 100644 index 09b33d01846..00000000000 --- a/lib/gitlab/cycle_analytics/group_stage_summary.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class GroupStageSummary - attr_reader :group, :current_user, :options - - def initialize(group, options:) - @group = group - @current_user = options[:current_user] - @options = options - end - - def data - [issue_stats, - deploy_stats, - deployment_frequency_stats] - end - - private - - def issue_stats - serialize( - Summary::Group::Issue.new( - group: group, current_user: current_user, options: options) - ) - end - - def deployments_summary - @deployments_summary ||= - Summary::Group::Deploy.new(group: group, options: options) - end - - def deploy_stats - serialize deployments_summary - end - - def deployment_frequency_stats - serialize( - Summary::Group::DeploymentFrequency.new( - deployments: deployments_summary.value, - group: group, - options: options), - with_unit: true - ) - end - - def serialize(summary_object, with_unit: false) - AnalyticsSummarySerializer.new.represent( - summary_object, with_unit: with_unit) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb index 564feb0319f..7559cd376bf 100644 --- a/lib/gitlab/cycle_analytics/stage_summary.rb +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -28,8 +28,7 @@ module Gitlab end def deployments_summary - @deployments_summary ||= - Summary::Deploy.new(project: @project, from: @from, to: @to) + @deployments_summary ||= Summary::Deploy.new(project: @project, from: @from, to: @to) end def deploy_stats @@ -39,7 +38,7 @@ module Gitlab def deployment_frequency_stats serialize( Summary::DeploymentFrequency.new( - deployments: deployments_summary.value, + deployments: deployments_summary.value.raw_value, from: @from, to: @to), with_unit: true diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index 76049c6b742..1f426b81800 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -9,7 +9,7 @@ module Gitlab end def value - @value ||= count_commits + @value ||= commits_count ? Value::PrettyNumeric.new(commits_count) : Value::None.new end private @@ -18,10 +18,10 @@ module Gitlab # a limit. Since we need a commit count, we _can't_ enforce a limit, so # the easiest way forward is to replicate the relevant portions of the # `log` function here. - def count_commits + def commits_count return unless ref - gitaly_commit_client.commit_count(ref, after: @from, before: @to) + @commits_count ||= gitaly_commit_client.commit_count(ref, after: @from, before: @to) end def gitaly_commit_client diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index 5ff8d881143..8544ea1a91e 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -4,18 +4,20 @@ module Gitlab module CycleAnalytics module Summary class Deploy < Base - include Gitlab::Utils::StrongMemoize - def title n_('Deploy', 'Deploys', value) end def value - strong_memoize(:value) do - query = @project.deployments.success.where("created_at >= ?", @from) - query = query.where("created_at <= ?", @to) if @to - query.count - end + @value ||= Value::PrettyNumeric.new(deployments_count) + end + + private + + def deployments_count + query = @project.deployments.success.where("created_at >= ?", @from) + query = query.where("created_at <= ?", @to) if @to + query.count end end end diff --git a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb index 436dc91bd6b..00676a02a6f 100644 --- a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb +++ b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb @@ -17,8 +17,7 @@ module Gitlab end def value - @value ||= - frequency(@deployments, @from, @to || Time.now) + @value ||= frequency(@deployments, @from, @to || Time.now) end def unit diff --git a/lib/gitlab/cycle_analytics/summary/group/base.rb b/lib/gitlab/cycle_analytics/summary/group/base.rb deleted file mode 100644 index f1d20d5aefa..00000000000 --- a/lib/gitlab/cycle_analytics/summary/group/base.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module Summary - module Group - class Base - attr_reader :group, :options - - def initialize(group:, options:) - @group = group - @options = options - end - - def title - raise NotImplementedError.new("Expected #{self.name} to implement title") - end - - def value - raise NotImplementedError.new("Expected #{self.name} to implement value") - end - end - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb deleted file mode 100644 index 11a9152cf0c..00000000000 --- a/lib/gitlab/cycle_analytics/summary/group/deploy.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module Summary - module Group - class Deploy < Group::Base - include GroupProjectsProvider - - def title - n_('Deploy', 'Deploys', value) - end - - def value - @value ||= find_deployments - end - - private - - def find_deployments - deployments = Deployment.joins(:project).merge(Project.inside_path(group.full_path)) - deployments = deployments.where(projects: { id: options[:projects] }) if options[:projects] - deployments = deployments.where("deployments.created_at > ?", options[:from]) - deployments = deployments.where("deployments.created_at < ?", options[:to]) if options[:to] - deployments.success.count - end - end - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/summary/group/deployment_frequency.rb b/lib/gitlab/cycle_analytics/summary/group/deployment_frequency.rb deleted file mode 100644 index 9fbbbb5a1ec..00000000000 --- a/lib/gitlab/cycle_analytics/summary/group/deployment_frequency.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module Summary - module Group - class DeploymentFrequency < Group::Base - include GroupProjectsProvider - include SummaryHelper - - def initialize(deployments:, group:, options:) - @deployments = deployments - - super(group: group, options: options) - end - - def title - _('Deployment Frequency') - end - - def value - @value ||= - frequency(@deployments, options[:from], options[:to] || Time.now) - end - - def unit - _('per day') - end - end - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/summary/group/issue.rb b/lib/gitlab/cycle_analytics/summary/group/issue.rb deleted file mode 100644 index 4d5ee1d43ca..00000000000 --- a/lib/gitlab/cycle_analytics/summary/group/issue.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module Summary - module Group - class Issue < Group::Base - attr_reader :group, :current_user, :options - - def initialize(group:, current_user:, options:) - @group = group - @current_user = current_user - @options = options - end - - def title - n_('New Issue', 'New Issues', value) - end - - def value - @value ||= find_issues - end - - private - - def find_issues - issues = IssuesFinder.new(current_user, finder_params).execute - issues = issues.where(projects: { id: options[:projects] }) if options[:projects] - issues.count - end - - def finder_params - { - group_id: group.id, - include_subgroups: true, - created_after: options[:from], - created_before: options[:to] - }.compact - end - end - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb index 52892eb5a1a..ce7788590b9 100644 --- a/lib/gitlab/cycle_analytics/summary/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -16,7 +16,16 @@ module Gitlab end def value - @value ||= IssuesFinder.new(@current_user, project_id: @project.id, created_after: @from, created_before: @to).execute.count + @value ||= Value::PrettyNumeric.new(issues_count) + end + + private + + def issues_count + IssuesFinder + .new(@current_user, project_id: @project.id, created_after: @from, created_before: @to) + .execute + .count end end end diff --git a/lib/gitlab/cycle_analytics/summary/value.rb b/lib/gitlab/cycle_analytics/summary/value.rb new file mode 100644 index 00000000000..ce32132e048 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/value.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + class Value + attr_reader :value + + def raw_value + value + end + + def to_s + raise NotImplementedError + end + + class None < self + def to_s + '-' + end + end + + class Numeric < self + def initialize(value) + @value = value + end + + def to_s + value.zero? ? '0' : value.to_s + end + end + + class PrettyNumeric < Numeric + def to_s + # 0 is shown as - + (value || 0).nonzero? ? super : None.new.to_s + end + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary_helper.rb b/lib/gitlab/cycle_analytics/summary_helper.rb index 06abcd151d4..3cf9f463024 100644 --- a/lib/gitlab/cycle_analytics/summary_helper.rb +++ b/lib/gitlab/cycle_analytics/summary_helper.rb @@ -4,10 +4,11 @@ module Gitlab module CycleAnalytics module SummaryHelper def frequency(count, from, to) - return count if count.zero? + return Summary::Value::None.new if count.zero? freq = (count / days(from, to)).round(1) - freq.zero? ? '0' : freq + + Summary::Value::Numeric.new(freq) end def days(from, to) diff --git a/lib/gitlab/danger/changelog.rb b/lib/gitlab/danger/changelog.rb index d64177f9565..85f386594be 100644 --- a/lib/gitlab/danger/changelog.rb +++ b/lib/gitlab/danger/changelog.rb @@ -14,10 +14,6 @@ module Gitlab @found ||= git.added_files.find { |path| path =~ %r{\A(ee/)?(changelogs/unreleased)(-ee)?/} } end - def presented_no_changelog_labels - NO_CHANGELOG_LABELS.map { |label| "~#{label}" }.join(', ') - end - def sanitized_mr_title gitlab.mr_json["title"].gsub(/^WIP: */, '').gsub(/`/, '\\\`') end diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb index 616c05d0a02..58db2b58560 100644 --- a/lib/gitlab/danger/commit_linter.rb +++ b/lib/gitlab/danger/commit_linter.rb @@ -18,7 +18,7 @@ module Gitlab PROBLEMS = { subject_too_short: "The %s must contain at least #{MIN_SUBJECT_WORDS_COUNT} words", subject_too_long: "The %s may not be longer than #{MAX_LINE_LENGTH} characters", - subject_above_warning: "The %s length is acceptable, but please try to [reduce it to #{WARN_SUBJECT_LENGTH} characters](#{URL_LIMIT_SUBJECT}).", + subject_above_warning: "The %s length is acceptable, but please try to [reduce it to #{WARN_SUBJECT_LENGTH} characters](#{URL_LIMIT_SUBJECT})", subject_starts_with_lowercase: "The %s must start with a capital letter", subject_ends_with_a_period: "The %s must not end with a period", separator_missing: "The commit subject and body must be separated by a blank line", diff --git a/lib/gitlab/danger/emoji_checker.rb b/lib/gitlab/danger/emoji_checker.rb index e31a6ae5011..a2867087428 100644 --- a/lib/gitlab/danger/emoji_checker.rb +++ b/lib/gitlab/danger/emoji_checker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'json' +require_relative '../json' module Gitlab module Danger @@ -25,8 +25,8 @@ module Gitlab )}x.freeze def initialize - names = JSON.parse(File.read(DIGESTS)).keys + - JSON.parse(File.read(ALIASES)).keys + names = Gitlab::Json.parse(File.read(DIGESTS)).keys + + Gitlab::Json.parse(File.read(ALIASES)).keys @emoji = names.map { |name| ":#{name}:" } end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index aa2737262be..0f0af5f777b 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -191,6 +191,23 @@ module Gitlab gitlab_helper.mr_json['web_url'].include?('/gitlab-org/security/') end + def mr_has_labels?(*labels) + return false unless gitlab_helper + + labels = labels.flatten.uniq + (labels & gitlab_helper.mr_labels) == labels + end + + def labels_list(labels, sep: ', ') + labels.map { |label| %Q{~"#{label}"} }.join(sep) + end + + def prepare_labels_for_mr(labels) + return '' unless labels.any? + + "/label #{labels_list(labels, sep: ' ')}" + end + private def has_database_scoped_labels?(current_mr_labels) diff --git a/lib/gitlab/danger/request_helper.rb b/lib/gitlab/danger/request_helper.rb index 06da4ed9ad3..ef51c3f2052 100644 --- a/lib/gitlab/danger/request_helper.rb +++ b/lib/gitlab/danger/request_helper.rb @@ -16,7 +16,7 @@ module Gitlab raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}" end - JSON.parse(rsp.body) + Gitlab::Json.parse(rsp.body) end end end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index 55476cd9789..651b002d2bf 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -1,12 +1,19 @@ # frozen_string_literal: true require 'cgi' +require 'set' module Gitlab module Danger class Teammate attr_reader :name, :username, :role, :projects + AT_CAPACITY_EMOJI = Set.new(%w[red_circle]).freeze + OOO_EMOJI = Set.new(%w[ + palm_tree + beach beach_umbrella beach_with_umbrella + ]).freeze + def initialize(options = {}) @username = options['username'] @name = options['name'] || @username @@ -37,10 +44,14 @@ module Gitlab end def status - api_endpoint = "https://gitlab.com/api/v4/users/#{CGI.escape(username)}/status" - @status ||= Gitlab::Danger::RequestHelper.http_get_json(api_endpoint) - rescue Gitlab::Danger::RequestHelper::HTTPError, JSON::ParserError - nil # better no status than a crashing Danger + return @status if defined?(@status) + + @status ||= + begin + Gitlab::Danger::RequestHelper.http_get_json(status_api_endpoint) + rescue Gitlab::Danger::RequestHelper::HTTPError, JSON::ParserError + nil # better no status than a crashing Danger + end end # @return [Boolean] @@ -50,14 +61,22 @@ module Gitlab private + def status_api_endpoint + "https://gitlab.com/api/v4/users/#{CGI.escape(username)}/status" + end + + def status_emoji + status&.dig("emoji") + end + # @return [Boolean] def out_of_office? - status&.dig("message")&.match?(/OOO/i) || false + status&.dig("message")&.match?(/OOO/i) || OOO_EMOJI.include?(status_emoji) end # @return [Boolean] def has_capacity? - status&.dig("emoji") != 'red_circle' + !AT_CAPACITY_EMOJI.include?(status_emoji) end def has_capability?(project, category, kind, labels) diff --git a/lib/gitlab/data_builder/wiki_page.rb b/lib/gitlab/data_builder/wiki_page.rb index 9368446fa59..8aee25e9fe6 100644 --- a/lib/gitlab/data_builder/wiki_page.rb +++ b/lib/gitlab/data_builder/wiki_page.rb @@ -8,6 +8,9 @@ module Gitlab def build(wiki_page, user, action) wiki = wiki_page.wiki + # TODO: group hooks https://gitlab.com/gitlab-org/gitlab/-/issues/216904 + return {} if wiki.container.is_a?(Group) + { object_kind: wiki_page.class.name.underscore, user: user.hook_attrs, diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 2359dceae48..ab069ce1da1 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -91,11 +91,17 @@ module Gitlab def batch_fetch(start, finish, mode) # rubocop:disable GitlabSecurity/PublicSend - @relation.select(@column).public_send(mode).where(@column => start..(finish - 1)).count + @relation.select(@column).public_send(mode).where(between_condition(start, finish)).count end private + def between_condition(start, finish) + return @column.between(start..(finish - 1)) if @column.is_a?(Arel::Attributes::Attribute) + + { @column => start..(finish - 1) } + end + def actual_start(start) start || @relation.minimum(@column) || 0 end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 6cd90c01ab2..e226ed7613a 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -72,7 +72,7 @@ module Gitlab # @param [Array] table names # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) def get_statistics(table_names, check_statistics: true) - time = 1.hour.ago + time = 6.hours.ago query = PgClass.joins("LEFT JOIN pg_stat_user_tables USING (relname)") .where(relname: table_names) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index cf5ff8ddb7b..96be057f77e 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -265,12 +265,19 @@ module Gitlab # or `RESET ALL` is executed def disable_statement_timeout if block_given? - begin - execute('SET statement_timeout TO 0') - + if statement_timeout_disabled? + # Don't do anything if the statement_timeout is already disabled + # Allows for nested calls of disable_statement_timeout without + # resetting the timeout too early (before the outer call ends) yield - ensure - execute('RESET ALL') + else + begin + execute('SET statement_timeout TO 0') + + yield + ensure + execute('RESET ALL') + end end else unless transaction_open? @@ -378,7 +385,7 @@ module Gitlab # make things _more_ complex). # # `batch_column_name` option is for tables without primary key, in this - # case an other unique integer column can be used. Example: :user_id + # case another unique integer column can be used. Example: :user_id # # rubocop: disable Metrics/AbcSize def update_column_in_batches(table, column, value, batch_size: nil, batch_column_name: :id) @@ -444,66 +451,13 @@ module Gitlab # Adds a column with a default value without locking an entire table. # - # This method runs the following steps: - # - # 1. Add the column with a default value of NULL. - # 2. Change the default value of the column to the specified value. - # 3. Update all existing rows in batches. - # 4. Set a `NOT NULL` constraint on the column if desired (the default). - # - # These steps ensure a column can be added to a large and commonly used - # table without locking the entire table for the duration of the table - # modification. - # - # table - The name of the table to update. - # column - The name of the column to add. - # type - The column type (e.g. `:integer`). - # default - The default value for the column. - # limit - Sets a column limit. For example, for :integer, the default is - # 4-bytes. Set `limit: 8` to allow 8-byte integers. - # allow_null - When set to `true` the column will allow NULL values, the - # default is to not allow NULL values. - # - # This method can also take a block which is passed directly to the - # `update_column_in_batches` method. - def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false, update_column_in_batches_args: {}, &block) - if transaction_open? - raise 'add_column_with_default can not be run inside a transaction, ' \ - 'you can disable transactions by calling disable_ddl_transaction! ' \ - 'in the body of your migration class' - end - - disable_statement_timeout do - transaction do - if limit - add_column(table, column, type, default: nil, limit: limit) - else - add_column(table, column, type, default: nil) - end - - # Changing the default before the update ensures any newly inserted - # rows already use the proper default value. - change_column_default(table, column, default) - end - - begin - default_after_type_cast = connection.type_cast(default, column_for(table, column)) + # @deprecated With PostgreSQL 11, adding columns with a default does not lead to a table rewrite anymore. + # As such, this method is not needed anymore and the default `add_column` helper should be used. + # This helper is subject to be removed in a >13.0 release. + def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false) + raise 'Deprecated: add_column_with_default does not support being passed blocks anymore' if block_given? - if update_column_in_batches_args.any? - update_column_in_batches(table, column, default_after_type_cast, **update_column_in_batches_args, &block) - else - update_column_in_batches(table, column, default_after_type_cast, &block) - end - - change_column_null(table, column, false) unless allow_null - # We want to rescue _all_ exceptions here, even those that don't inherit - # from StandardError. - rescue Exception => error # rubocop: disable all - remove_column(table, column) - - raise error - end - end + add_column(table, column, type, default: default, limit: limit, null: allow_null) end # Renames a column without requiring downtime. @@ -519,14 +473,20 @@ module Gitlab # new - The new column name. # type - The type of the new column. If no type is given the old column's # type is used. - def rename_column_concurrently(table, old, new, type: nil) + # batch_column_name - option is for tables without primary key, in this + # case another unique integer column can be used. Example: :user_id + def rename_column_concurrently(table, old, new, type: nil, batch_column_name: :id) + unless column_exists?(table, batch_column_name) + raise "Column #{batch_column_name} does not exist on #{table}" + end + if transaction_open? raise 'rename_column_concurrently can not be run inside a transaction' end check_trigger_permissions!(table) - create_column_from(table, old, new, type: type) + create_column_from(table, old, new, type: type, batch_column_name: batch_column_name) install_rename_triggers(table, old, new) end @@ -626,14 +586,20 @@ module Gitlab # new - The new column name. # type - The type of the old column. If no type is given the new column's # type is used. - def undo_cleanup_concurrent_column_rename(table, old, new, type: nil) + # batch_column_name - option is for tables without primary key, in this + # case another unique integer column can be used. Example: :user_id + def undo_cleanup_concurrent_column_rename(table, old, new, type: nil, batch_column_name: :id) + unless column_exists?(table, batch_column_name) + raise "Column #{batch_column_name} does not exist on #{table}" + end + if transaction_open? raise 'undo_cleanup_concurrent_column_rename can not be run inside a transaction' end check_trigger_permissions!(table) - create_column_from(table, new, old, type: type) + create_column_from(table, new, old, type: type, batch_column_name: batch_column_name) install_rename_triggers(table, old, new) end @@ -1063,6 +1029,8 @@ into similar problems in the future (e.g. when new tables are created). # batch_size - The maximum number of rows per job # other_arguments - Other arguments to send to the job # + # *Returns the final migration delay* + # # Example: # # class Route < ActiveRecord::Base @@ -1079,7 +1047,7 @@ into similar problems in the future (e.g. when new tables are created). # # do something # end # end - def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_arguments: []) + def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_job_arguments: [], initial_delay: 0) raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') # To not overload the worker too much we enforce a minimum interval both @@ -1088,14 +1056,19 @@ into similar problems in the future (e.g. when new tables are created). delay_interval = BackgroundMigrationWorker.minimum_interval end + final_delay = 0 + model_class.each_batch(of: batch_size) do |relation, index| start_id, end_id = relation.pluck(Arel.sql('MIN(id), MAX(id)')).first # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for # the same time, which is not helpful in most cases where we wish to # spread the work over time. - migrate_in(delay_interval * index, job_class_name, [start_id, end_id] + other_arguments) + final_delay = initial_delay + delay_interval * index + migrate_in(final_delay, job_class_name, [start_id, end_id] + other_job_arguments) end + + final_delay end # Fetches indexes on a column by name for postgres. @@ -1315,12 +1288,73 @@ into similar problems in the future (e.g. when new tables are created). check_constraint_exists?(table, text_limit_name(table, column, name: constraint_name)) end + # Migration Helpers for managing not null constraints + def add_not_null_constraint(table, column, constraint_name: nil, validate: true) + if column_is_nullable?(table, column) + add_check_constraint( + table, + "#{column} IS NOT NULL", + not_null_constraint_name(table, column, name: constraint_name), + validate: validate + ) + else + warning_message = <<~MESSAGE + NOT NULL check constraint was not created: + column #{table}.#{column} is already defined as `NOT NULL` + MESSAGE + + Rails.logger.warn warning_message + end + end + + def validate_not_null_constraint(table, column, constraint_name: nil) + validate_check_constraint( + table, + not_null_constraint_name(table, column, name: constraint_name) + ) + end + + def remove_not_null_constraint(table, column, constraint_name: nil) + remove_check_constraint( + table, + not_null_constraint_name(table, column, name: constraint_name) + ) + end + + def check_not_null_constraint_exists?(table, column, constraint_name: nil) + check_constraint_exists?( + table, + not_null_constraint_name(table, column, name: constraint_name) + ) + end + private + def statement_timeout_disabled? + # This is a string of the form "100ms" or "0" when disabled + connection.select_value('SHOW statement_timeout') == "0" + end + + def column_is_nullable?(table, column) + # Check if table.column has not been defined with NOT NULL + check_sql = <<~SQL + SELECT c.is_nullable + FROM information_schema.columns c + WHERE c.table_name = '#{table}' + AND c.column_name = '#{column}' + SQL + + connection.select_value(check_sql) == 'YES' + end + def text_limit_name(table, column, name: nil) name.presence || check_constraint_name(table, column, 'max_length') end + def not_null_constraint_name(table, column, name: nil) + name.presence || check_constraint_name(table, column, 'not_null') + end + def missing_schema_object_message(table, type, name) <<~MESSAGE Could not find #{type} "#{name}" on table "#{table}" which was referenced during the migration. @@ -1348,7 +1382,7 @@ into similar problems in the future (e.g. when new tables are created). "ON DELETE #{on_delete.upcase}" end - def create_column_from(table, old, new, type: nil) + def create_column_from(table, old, new, type: nil, batch_column_name: :id) old_col = column_for(table, old) new_type = type || old_col.type @@ -1362,9 +1396,9 @@ into similar problems in the future (e.g. when new tables are created). # necessary since we copy over old values further down. change_column_default(table, new, old_col.default) unless old_col.default.nil? - update_column_in_batches(table, new, Arel::Table.new(table)[old]) + update_column_in_batches(table, new, Arel::Table.new(table)[old], batch_column_name: batch_column_name) - change_column_null(table, new, false) unless old_col.null + add_not_null_constraint(table, new) unless old_col.null copy_indexes(table, old, new) copy_foreign_keys(table, old, new) diff --git a/lib/gitlab/database/partitioning_migration_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers.rb new file mode 100644 index 00000000000..55649ebbf8a --- /dev/null +++ b/lib/gitlab/database/partitioning_migration_helpers.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PartitioningMigrationHelpers + include SchemaHelpers + + def add_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id, on_delete: :cascade) + cascade_delete = extract_cascade_option(on_delete) + + update_foreign_keys(from_table, to_table, column, primary_key, cascade_delete) do |current_keys, existing_key, specified_key| + if existing_key.nil? + unless specified_key.save + raise "failed to create foreign key: #{specified_key.errors.full_messages.to_sentence}" + end + + current_keys << specified_key + else + Rails.logger.warn "foreign key not added because it already exists: #{specified_key}" # rubocop:disable Gitlab/RailsLogger + current_keys + end + end + end + + def remove_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id) + update_foreign_keys(from_table, to_table, column, primary_key) do |current_keys, existing_key, specified_key| + if existing_key + existing_key.destroy! + current_keys.delete(existing_key) + else + Rails.logger.warn "foreign key not removed because it doesn't exist: #{specified_key}" # rubocop:disable Gitlab/RailsLogger + end + + current_keys + end + end + + def fk_function_name(table) + object_name(table, 'fk_cascade_function') + end + + def fk_trigger_name(table) + object_name(table, 'fk_cascade_trigger') + end + + private + + def fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete) + PartitionedForeignKey.new(from_table: from_table.to_s, to_table: to_table.to_s, from_column: from_column.to_s, + to_column: to_column.to_s, cascade_delete: cascade_delete) + end + + def update_foreign_keys(from_table, to_table, from_column, to_column, cascade_delete = nil) + if transaction_open? + raise 'partitioned foreign key operations can not be run inside a transaction block, ' \ + 'you can disable transaction blocks by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + from_column ||= "#{to_table.to_s.singularize}_id" + specified_key = fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete) + + current_keys = PartitionedForeignKey.by_referenced_table(to_table).to_a + existing_key = find_existing_key(current_keys, specified_key) + + final_keys = yield current_keys, existing_key, specified_key + + fn_name = fk_function_name(to_table) + trigger_name = fk_trigger_name(to_table) + + with_lock_retries do + drop_trigger(to_table, trigger_name, if_exists: true) + + if final_keys.empty? + drop_function(fn_name, if_exists: true) + else + create_or_replace_fk_function(fn_name, final_keys) + create_function_trigger(trigger_name, fn_name, fires: "AFTER DELETE ON #{to_table}") + end + end + end + + def extract_cascade_option(on_delete) + case on_delete + when :cascade then true + when :nullify then false + else raise ArgumentError, "invalid option #{on_delete} for :on_delete" + end + end + + def with_lock_retries(&block) + Gitlab::Database::WithLockRetries.new({ + klass: self.class, + logger: Gitlab::BackgroundMigration::Logger + }).run(&block) + end + + def find_existing_key(keys, key) + keys.find { |k| k.from_table == key.from_table && k.from_column == key.from_column } + end + + def create_or_replace_fk_function(fn_name, fk_specs) + create_trigger_function(fn_name, replace: true) do + cascade_statements = build_cascade_statements(fk_specs) + cascade_statements << 'RETURN OLD;' + + cascade_statements.join("\n") + end + end + + def build_cascade_statements(foreign_keys) + foreign_keys.map do |fks| + if fks.cascade_delete? + "DELETE FROM #{fks.from_table} WHERE #{fks.from_column} = OLD.#{fks.to_column};" + else + "UPDATE #{fks.from_table} SET #{fks.from_column} = NULL WHERE #{fks.from_column} = OLD.#{fks.to_column};" + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb new file mode 100644 index 00000000000..f9a90511f9b --- /dev/null +++ b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PartitioningMigrationHelpers + class PartitionedForeignKey < ApplicationRecord + validates_with PartitionedForeignKeyValidator + + scope :by_referenced_table, ->(table) { where(to_table: table) } + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb new file mode 100644 index 00000000000..089cf2b8931 --- /dev/null +++ b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PartitioningMigrationHelpers + class PartitionedForeignKeyValidator < ActiveModel::Validator + def validate(record) + validate_key_part(record, :from_table, :from_column) + validate_key_part(record, :to_table, :to_column) + end + + private + + def validate_key_part(record, table_field, column_field) + if !connection.table_exists?(record[table_field]) + record.errors.add(table_field, 'must be a valid table') + elsif !connection.column_exists?(record[table_field], record[column_field]) + record.errors.add(column_field, 'must be a valid column') + end + end + + def connection + ActiveRecord::Base.connection + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 565f34b78b7..2c9d0d6c0d1 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -157,7 +157,7 @@ module Gitlab failed_reverts = [] while rename_info = redis.lpop(key) - path_before_rename, path_after_rename = JSON.parse(rename_info) + path_before_rename, path_after_rename = Gitlab::Json.parse(rename_info) say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}" begin yield(path_before_rename, path_after_rename) diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb new file mode 100644 index 00000000000..f8d01c78ae8 --- /dev/null +++ b/lib/gitlab/database/schema_helpers.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaHelpers + def create_trigger_function(name, replace: true) + replace_clause = optional_clause(replace, "OR REPLACE") + execute(<<~SQL) + CREATE #{replace_clause} FUNCTION #{name}() + RETURNS TRIGGER AS + $$ + BEGIN + #{yield} + END + $$ LANGUAGE PLPGSQL + SQL + end + + def create_function_trigger(name, fn_name, fires: nil) + execute(<<~SQL) + CREATE TRIGGER #{name} + #{fires} + FOR EACH ROW + EXECUTE PROCEDURE #{fn_name}() + SQL + end + + def drop_function(name, if_exists: true) + exists_clause = optional_clause(if_exists, "IF EXISTS") + execute("DROP FUNCTION #{exists_clause} #{name}()") + end + + def drop_trigger(table_name, name, if_exists: true) + exists_clause = optional_clause(if_exists, "IF EXISTS") + execute("DROP TRIGGER #{exists_clause} #{name} ON #{table_name}") + end + + def object_name(table, type) + identifier = "#{table}_#{type}" + hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) + + "#{type}_#{hashed_identifier}" + end + + private + + def optional_clause(flag, clause) + flag ? clause : "" + end + end + end +end diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb index 2f36bfa1480..bebcba6f42e 100644 --- a/lib/gitlab/database/with_lock_retries.rb +++ b/lib/gitlab/database/with_lock_retries.rb @@ -78,12 +78,18 @@ module Gitlab run_block_with_transaction rescue ActiveRecord::LockWaitTimeout if retry_with_lock_timeout? + disable_idle_in_transaction_timeout wait_until_next_retry + reset_db_settings retry else + reset_db_settings run_block_without_lock_timeout end + + ensure + reset_db_settings end end @@ -153,6 +159,14 @@ module Gitlab def current_sleep_time_in_seconds timing_configuration[current_iteration - 1][1].to_f end + + def disable_idle_in_transaction_timeout + execute("SET LOCAL idle_in_transaction_session_timeout TO '0'") + end + + def reset_db_settings + execute('RESET idle_in_transaction_session_timeout; RESET lock_timeout') + end end end end diff --git a/lib/gitlab/dependency_linker/json_linker.rb b/lib/gitlab/dependency_linker/json_linker.rb index 298d214df61..86dc7efb0d9 100644 --- a/lib/gitlab/dependency_linker/json_linker.rb +++ b/lib/gitlab/dependency_linker/json_linker.rb @@ -39,7 +39,7 @@ module Gitlab end def json - @json ||= JSON.parse(plain_text) rescue nil + @json ||= Gitlab::Json.parse(plain_text) rescue nil end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 4fc5bfddf0c..d1398ddb642 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -314,6 +314,10 @@ module Gitlab @rich_viewer = rich_viewer_class&.new(self) end + def alternate_viewer + alternate_viewer_class&.new(self) + end + def rendered_as_text?(ignore_errors: true) simple_viewer.is_a?(DiffViewer::Text) && (ignore_errors || simple_viewer.render_error.nil?) end @@ -353,7 +357,7 @@ module Gitlab def fetch_blob(sha, path) return unless sha - Blob.lazy(repository.project, sha, path) + Blob.lazy(repository, sha, path) end def total_blob_lines(blob) @@ -419,6 +423,17 @@ module Gitlab return if collapsed? return unless diffable? return unless modified_file? + + find_renderable_viewer_class(classes) + end + + def alternate_viewer_class + return unless viewer.class == DiffViewer::Renamed + + find_renderable_viewer_class(RICH_VIEWERS) || (DiffViewer::Text if text?) + end + + def find_renderable_viewer_class(classes) return if different_type? || external_storage_error? verify_binary = !stored_externally? diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb index 728457b3139..9ea9bdfdf15 100644 --- a/lib/gitlab/diff/formatters/text_formatter.rb +++ b/lib/gitlab/diff/formatters/text_formatter.rb @@ -45,7 +45,8 @@ module Gitlab def ==(other) other.is_a?(self.class) && new_line == other.new_line && - old_line == other.old_line + old_line == other.old_line && + line_range == other.line_range end end end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 01ec9798fe4..ccf09b37b9b 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -156,7 +156,7 @@ module Gitlab end results.map! do |result| - JSON.parse(extract_data(result), symbolize_names: true) unless result.nil? + Gitlab::Json.parse(extract_data(result), symbolize_names: true) unless result.nil? end file_paths.zip(results).to_h diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 8b99fd5cd42..10ad23b7774 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -68,7 +68,7 @@ module Gitlab end def to_json(opts = nil) - JSON.generate(formatter.to_h, opts) + Gitlab::Json.generate(formatter.to_h, opts) end def as_json(opts = nil) diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index 1f64883cb69..75d5a5df74b 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -42,7 +42,7 @@ module Gitlab content.map! do |lines| next unless lines - JSON.parse(lines).map! do |line| + Gitlab::Json.parse(lines).map! do |line| Gitlab::Diff::Line.safe_init_from_hash(line) end end diff --git a/lib/gitlab/elasticsearch/logs/lines.rb b/lib/gitlab/elasticsearch/logs/lines.rb index fb32a6c9fcd..ff9185dd331 100644 --- a/lib/gitlab/elasticsearch/logs/lines.rb +++ b/lib/gitlab/elasticsearch/logs/lines.rb @@ -13,7 +13,7 @@ module Gitlab @client = client end - def pod_logs(namespace, pod_name: nil, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil) + def pod_logs(namespace, pod_name: nil, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil, chart_above_v2: true) query = { bool: { must: [] } }.tap do |q| filter_pod_name(q, pod_name) filter_namespace(q, namespace) @@ -22,7 +22,7 @@ module Gitlab filter_times(q, start_time, end_time) end - body = build_body(query, cursor) + body = build_body(query, cursor, chart_above_v2) response = @client.search body: body format_response(response) @@ -30,13 +30,14 @@ module Gitlab private - def build_body(query, cursor = nil) + def build_body(query, cursor = nil, chart_above_v2 = true) + offset_field = chart_above_v2 ? "log.offset" : "offset" body = { query: query, # reverse order so we can query N-most recent records sort: [ { "@timestamp": { order: :desc } }, - { "offset": { order: :desc } } + { "#{offset_field}": { order: :desc } } ], # only return these fields in the response _source: ["@timestamp", "message", "kubernetes.pod.name"], diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index e9a7c9bcf5c..7f8dd815103 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -3,18 +3,16 @@ module Gitlab module Email module Handler - prepend_if_ee('::EE::Gitlab::Email::Handler') # rubocop: disable Cop/InjectEnterpriseEditionModule - def self.handlers @handlers ||= load_handlers end def self.load_handlers [ - UnsubscribeHandler, CreateNoteHandler, - CreateMergeRequestHandler, - CreateIssueHandler + CreateIssueHandler, + UnsubscribeHandler, + CreateMergeRequestHandler ] end @@ -27,3 +25,5 @@ module Gitlab end end end + +Gitlab::Email::Handler.prepend_if_ee('::EE::Gitlab::Email::Handler') diff --git a/lib/gitlab/email/hook/smime_signature_interceptor.rb b/lib/gitlab/email/hook/smime_signature_interceptor.rb index 61c9c984f8e..fe39589d019 100644 --- a/lib/gitlab/email/hook/smime_signature_interceptor.rb +++ b/lib/gitlab/email/hook/smime_signature_interceptor.rb @@ -10,6 +10,7 @@ module Gitlab signed_message = Gitlab::Email::Smime::Signer.sign( cert: certificate.cert, key: certificate.key, + ca_certs: certificate.ca_certs, data: message.encoded) signed_email = Mail.new(signed_message) @@ -21,7 +22,7 @@ module Gitlab private def certificate - @certificate ||= Gitlab::Email::Smime::Certificate.from_files(key_path, cert_path) + @certificate ||= Gitlab::Email::Smime::Certificate.from_files(key_path, cert_path, ca_certs_path) end def key_path @@ -32,6 +33,10 @@ module Gitlab Gitlab.config.gitlab.email_smime.cert_file end + def ca_certs_path + Gitlab.config.gitlab.email_smime.ca_certs_file + end + def overwrite_body(message, signed_email) # since this is a multipart email, assignment to nil is important, # otherwise Message#body will add a new mail part diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index ec412e7a8b1..d55cf3202a6 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -52,7 +52,7 @@ module Gitlab end def compare - @opts[:compare] if @opts[:compare] + @opts[:compare] end def diff_refs diff --git a/lib/gitlab/email/smime/certificate.rb b/lib/gitlab/email/smime/certificate.rb index 59d7b0c3c5b..3607b95b4bc 100644 --- a/lib/gitlab/email/smime/certificate.rb +++ b/lib/gitlab/email/smime/certificate.rb @@ -4,29 +4,53 @@ module Gitlab module Email module Smime class Certificate - attr_reader :key, :cert + CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze + + attr_reader :key, :cert, :ca_certs def key_string - @key.to_s + key.to_s end def cert_string - @cert.to_pem + cert.to_pem + end + + def ca_certs_string + ca_certs.map(&:to_pem).join('\n') unless ca_certs.blank? end - def self.from_strings(key_string, cert_string) + def self.from_strings(key_string, cert_string, ca_certs_string = nil) key = OpenSSL::PKey::RSA.new(key_string) cert = OpenSSL::X509::Certificate.new(cert_string) - new(key, cert) + ca_certs = load_ca_certs_bundle(ca_certs_string) + + new(key, cert, ca_certs) end - def self.from_files(key_path, cert_path) - from_strings(File.read(key_path), File.read(cert_path)) + def self.from_files(key_path, cert_path, ca_certs_path = nil) + ca_certs_string = File.read(ca_certs_path) if ca_certs_path + + from_strings(File.read(key_path), File.read(cert_path), ca_certs_string) + end + + # Returns an array of OpenSSL::X509::Certificate objects, empty array if none found + # + # Ruby OpenSSL::X509::Certificate.new will only load the first + # certificate if a bundle is presented, this allows to parse multiple certs + # in the same file + def self.load_ca_certs_bundle(ca_certs_string) + return [] unless ca_certs_string + + ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string| + OpenSSL::X509::Certificate.new(ca_cert_string) + end end - def initialize(key, cert) + def initialize(key, cert, ca_certs = nil) @key = key @cert = cert + @ca_certs = ca_certs end end end diff --git a/lib/gitlab/email/smime/signer.rb b/lib/gitlab/email/smime/signer.rb index db03e383ecf..6a445730463 100644 --- a/lib/gitlab/email/smime/signer.rb +++ b/lib/gitlab/email/smime/signer.rb @@ -7,19 +7,32 @@ module Gitlab module Smime # Tooling for signing and verifying data with SMIME class Signer - def self.sign(cert:, key:, data:) - signed_data = OpenSSL::PKCS7.sign(cert, key, data, nil, OpenSSL::PKCS7::DETACHED) + # The `ca_certs` parameter, if provided, is an array of CA certificates + # that will be attached in the signature together with the main `cert`. + # This will be typically intermediate CAs + def self.sign(cert:, key:, ca_certs: nil, data:) + signed_data = OpenSSL::PKCS7.sign(cert, key, data, Array.wrap(ca_certs), OpenSSL::PKCS7::DETACHED) OpenSSL::PKCS7.write_smime(signed_data) end - # return nil if data cannot be verified, otherwise the signed content data - def self.verify_signature(cert:, ca_cert: nil, signed_data:) + # Return nil if data cannot be verified, otherwise the signed content data + # + # Be careful with the `ca_certs` parameter, it will implicitly trust all the CAs + # in the array by creating a trusted store, stopping validation at the first match + # This is relevant when using intermediate CAs, `ca_certs` should only + # include the trusted, root CA + def self.verify_signature(ca_certs: nil, signed_data:) store = OpenSSL::X509::Store.new store.set_default_paths - store.add_cert(ca_cert) if ca_cert + Array.wrap(ca_certs).compact.each { |ca_cert| store.add_cert(ca_cert) } signed_smime = OpenSSL::PKCS7.read_smime(signed_data) - signed_smime if signed_smime.verify([cert], store) + + # The S/MIME certificate(s) are included in the message and the trusted + # CAs are in the store parameter, so we pass no certs as parameters + # to `PKCS7.verify` + # See https://www.openssl.org/docs/manmaster/man3/PKCS7_verify.html + signed_smime if signed_smime.verify(nil, store) end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index 305bcada2f5..bcf92b35720 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -21,7 +21,7 @@ module Gitlab end def emojis_aliases - @emoji_aliases ||= JSON.parse(File.read(Rails.root.join('fixtures', 'emojis', 'aliases.json'))) + @emoji_aliases ||= Gitlab::Json.parse(File.read(Rails.root.join('fixtures', 'emojis', 'aliases.json'))) end def emoji_filename(name) @@ -63,7 +63,7 @@ module Gitlab def emoji_unicode_versions_by_name @emoji_unicode_versions_by_name ||= - JSON.parse(File.read(Rails.root.join('fixtures', 'emojis', 'emoji-unicode-version-map.json'))) + Gitlab::Json.parse(File.read(Rails.root.join('fixtures', 'emojis', 'emoji-unicode-version-map.json'))) end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 425ef5d738a..1d2c1c69423 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -12,6 +12,9 @@ module Gitlab # ExclusiveLease. # class ExclusiveLease + PREFIX = 'gitlab:exclusive_lease' + NoKey = Class.new(ArgumentError) + LUA_CANCEL_SCRIPT = <<~EOS.freeze local key, uuid = KEYS[1], ARGV[1] if redis.call("get", key) == uuid then @@ -34,13 +37,21 @@ module Gitlab end def self.cancel(key, uuid) + return unless key.present? + Gitlab::Redis::SharedState.with do |redis| - redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) + redis.eval(LUA_CANCEL_SCRIPT, keys: [ensure_prefixed_key(key)], argv: [uuid]) end end def self.redis_shared_state_key(key) - "gitlab:exclusive_lease:#{key}" + "#{PREFIX}:#{key}" + end + + def self.ensure_prefixed_key(key) + raise NoKey unless key.present? + + key.start_with?(PREFIX) ? key : redis_shared_state_key(key) end # Removes any existing exclusive_lease from redis @@ -94,6 +105,11 @@ module Gitlab ttl if ttl.positive? end end + + # Gives up this lease, allowing it to be obtained by others. + def cancel + self.class.cancel(@redis_shared_state_key, @uuid) + end end end diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb index 61eb030563d..10762d83588 100644 --- a/lib/gitlab/exclusive_lease_helpers.rb +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -6,29 +6,38 @@ module Gitlab FailedToObtainLockError = Class.new(StandardError) ## - # This helper method blocks a process/thread until the other process cancel the obrainted lease key. + # This helper method blocks a process/thread until the lease can be acquired, either due to + # the lease TTL expiring, or due to the current holder explicitly releasing + # their hold. # - # Note: It's basically discouraged to use this method in the unicorn's thread, - # because it holds the connection until all `retries` is consumed. + # If the lease cannot be obtained, raises `FailedToObtainLockError`. + # + # @param [String] key The lock the thread will try to acquire. Only one thread + # in one process across all Rails instances can hold this named lock at any + # one time. + # @param [Float] ttl: The length of time the lock will be valid for. The lock + # will be automatically be released after this time, so any work should be + # completed within this time. + # @param [Integer] retries: The maximum number of times we will re-attempt + # to acquire the lock. The maximum number of attempts will be `retries + 1`: + # one for the initial attempt, and then one for every re-try. + # @param [Float|Proc] sleep_sec: Either a number of seconds to sleep, or + # a proc that computes the sleep time given the number of preceding attempts + # (from 1 to retries - 1) + # + # Note: It's basically discouraged to use this method in a unicorn thread, + # because this ties up all thread related resources until all `retries` are consumed. # This could potentially eat up all connection pools. def in_lock(key, ttl: 1.minute, retries: 10, sleep_sec: 0.01.seconds) raise ArgumentError, 'Key needs to be specified' unless key - lease = Gitlab::ExclusiveLease.new(key, timeout: ttl) - retried = false - - until uuid = lease.try_obtain - # Keep trying until we obtain the lease. To prevent hammering Redis too - # much we'll wait for a bit. - sleep(sleep_sec) - (retries -= 1) < 0 ? break : retried ||= true - end + lease = SleepingLock.new(key, timeout: ttl, delay: sleep_sec) - raise FailedToObtainLockError, 'Failed to obtain a lock' unless uuid + lease.obtain(1 + retries) - yield(retried) + yield(lease.retried?) ensure - Gitlab::ExclusiveLease.cancel(key, uuid) + lease&.cancel end end end diff --git a/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb b/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb new file mode 100644 index 00000000000..8213c9bc042 --- /dev/null +++ b/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module ExclusiveLeaseHelpers + # Wrapper around ExclusiveLease that adds retry logic + class SleepingLock + delegate :cancel, to: :@lease + + def initialize(key, timeout:, delay:) + @lease = ::Gitlab::ExclusiveLease.new(key, timeout: timeout) + @delay = delay + @attempts = 0 + end + + def obtain(max_attempts) + until held? + raise FailedToObtainLockError, 'Failed to obtain a lock' if attempts >= max_attempts + + sleep(sleep_sec) unless first_attempt? + try_obtain + end + end + + def retried? + attempts > 1 + end + + private + + attr_reader :delay, :attempts + + def held? + @uuid.present? + end + + def try_obtain + @uuid ||= @lease.try_obtain + @attempts += 1 + end + + def first_attempt? + attempts.zero? + end + + def sleep_sec + delay.respond_to?(:call) ? delay.call(attempts) : delay + end + end + end +end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 68f3c9e332e..3495b4a0b72 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -2,48 +2,53 @@ # == Experimentation # -# Utility module used for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant. -# The feature_toggle and environment keys are optional. If the feature_toggle is not set, a feature with the name of -# the experiment will be checked, with a default value of true. The enabled_ratio is required and should be -# the ratio for the number of users for which this experiment is enabled. For example: a ratio of 0.1 will -# enable the experiment for 10% of the users (determined by the `experimentation_subject_index`). +# Utility module for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant. +# Experiment options: +# - environment (optional, defaults to enabled for development and GitLab.com) +# - tracking_category (optional, used to set the category when tracking an experiment event) +# +# The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html), +# which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes. +# +# To enable the experiment for 10% of the users: +# +# chatops: `/chatops run feature set experiment_key_experiment_percentage 10` +# console: `Feature.get(:experiment_key_experiment_percentage).enable_percentage_of_time(10)` +# +# To disable the experiment: +# +# chatops: `/chatops run feature delete experiment_key_experiment_percentage` +# console: `Feature.get(:experiment_key_experiment_percentage).remove` +# +# To check the current rollout percentage: +# +# chatops: `/chatops run feature get experiment_key_experiment_percentage` +# console: `Feature.get(:experiment_key_experiment_percentage).percentage_of_time_value` # module Gitlab module Experimentation EXPERIMENTS = { signup_flow: { - feature_toggle: :experimental_separate_sign_up_flow, - environment: ::Gitlab.dev_env_or_com?, - enabled_ratio: 1, tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow' }, - paid_signup_flow: { - feature_toggle: :paid_signup_flow, - environment: ::Gitlab.dev_env_or_com?, - enabled_ratio: 1, - tracking_category: 'Growth::Acquisition::Experiment::PaidSignUpFlow' + onboarding_issues: { + tracking_category: 'Growth::Conversion::Experiment::OnboardingIssues' }, suggest_pipeline: { - feature_toggle: :suggest_pipeline, - environment: ::Gitlab.dev_env_or_com?, - enabled_ratio: 0.1, tracking_category: 'Growth::Expansion::Experiment::SuggestPipeline' }, ci_notification_dot: { - feature_toggle: :ci_notification_dot, - environment: ::Gitlab.dev_env_or_com?, - enabled_ratio: 0.1, tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot' }, buy_ci_minutes_version_a: { - feature_toggle: :buy_ci_minutes_version_a, - environment: ::Gitlab.dev_env_or_com?, - enabled_ratio: 0.2, tracking_category: 'Growth::Expansion::Experiment::BuyCiMinutesVersionA' + }, + upgrade_link_in_user_menu_a: { + tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA' } }.freeze - # Controller concern that checks if an experimentation_subject_id cookie is present and sets it if absent. + # Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent. # Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method # to controllers and views. It returns true when the experiment is enabled and the user is selected as part # of the experimental group. @@ -144,7 +149,7 @@ module Gitlab return false unless EXPERIMENTS.key?(experiment_key) experiment = experiment(experiment_key) - experiment.feature_toggle_enabled? && experiment.enabled_for_environment? + experiment.enabled? && experiment.enabled_for_environment? end def enabled_for_user?(experiment_key, experimentation_subject_index) @@ -153,23 +158,28 @@ module Gitlab end end - Experiment = Struct.new(:key, :feature_toggle, :environment, :enabled_ratio, :tracking_category, keyword_init: true) do - def feature_toggle_enabled? - return Feature.enabled?(key, default_enabled: true) if feature_toggle.nil? - - Feature.enabled?(feature_toggle) + Experiment = Struct.new(:key, :environment, :tracking_category, keyword_init: true) do + def enabled? + experiment_percentage.positive? end def enabled_for_environment? - return true if environment.nil? + return ::Gitlab.dev_env_or_com? if environment.nil? environment end def enabled_for_experimentation_subject?(experimentation_subject_index) - return false if enabled_ratio.nil? || experimentation_subject_index.blank? + return false if experimentation_subject_index.blank? + + experimentation_subject_index <= experiment_percentage + end + + private - experimentation_subject_index <= enabled_ratio * 100 + # When a feature does not exist, the `percentage_of_time_value` method will return 0 + def experiment_percentage + @experiment_percentage ||= Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value end end end diff --git a/lib/gitlab/external_authorization/response.rb b/lib/gitlab/external_authorization/response.rb index 4f3fe5882db..04f9688fad0 100644 --- a/lib/gitlab/external_authorization/response.rb +++ b/lib/gitlab/external_authorization/response.rb @@ -28,7 +28,7 @@ module Gitlab end def parse_response! - JSON.parse(@excon_response.body) + Gitlab::Json.parse(@excon_response.body) rescue JSON::JSONError # The JSON response is optional, so don't fail when it's missing nil diff --git a/lib/gitlab/git/attributes_parser.rb b/lib/gitlab/git/attributes_parser.rb index 8b9d74ae8e7..630b1aba2f5 100644 --- a/lib/gitlab/git/attributes_parser.rb +++ b/lib/gitlab/git/attributes_parser.rb @@ -85,6 +85,8 @@ module Gitlab yield line.strip end + # Catch invalid byte sequences + rescue ArgumentError end private diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 605084f1ec2..a554dc0b667 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -57,11 +57,8 @@ module Gitlab # Already a commit? return commit_id if commit_id.is_a?(Gitlab::Git::Commit) - # Some weird thing? - return unless commit_id.is_a?(String) - # This saves us an RPC round trip. - return if commit_id.include?(':') + return unless valid?(commit_id) commit = find_commit(repo, commit_id) @@ -431,6 +428,15 @@ module Gitlab def fetch_body_from_gitaly self.class.get_message(@repository, id) end + + def self.valid?(commit_id) + commit_id.is_a?(String) && !( + commit_id.start_with?('-') || + commit_id.include?(':') || + commit_id.include?("\x00") || + commit_id.match?(/\s/) + ) + end end end end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 08dbd52e3fb..da86d6baf4a 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -66,6 +66,27 @@ module Gitlab @raw_tag.tagger end + def has_signature? + signature_type != :NONE + end + + def signature_type + @raw_tag.signature_type || :NONE + end + + def signature + return unless has_signature? + + case signature_type + when :PGP + nil # not implemented, see https://gitlab.com/gitlab-org/gitlab/issues/19260 + when :X509 + X509::Tag.new(@raw_tag).signature + else + nil + end + end + private def message_from_gitaly_tag diff --git a/lib/gitlab/git_access_design.rb b/lib/gitlab/git_access_design.rb new file mode 100644 index 00000000000..36604bd0b3b --- /dev/null +++ b/lib/gitlab/git_access_design.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + class GitAccessDesign < GitAccess + def check(_cmd, _changes) + check_protocol! + check_can_create_design! + + success_result + end + + private + + def check_protocol! + if protocol != 'web' + raise ::Gitlab::GitAccess::ForbiddenError, "Designs are only accessible using the web interface" + end + end + + def check_can_create_design! + unless user&.can?(:create_design, project) + raise ::Gitlab::GitAccess::ForbiddenError, "You are not allowed to manage designs of this project" + end + end + end +end + +Gitlab::GitAccessDesign.prepend_if_ee('EE::Gitlab::GitAccessDesign') diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index d05ffe9b508..70db4271469 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -61,6 +61,13 @@ module Gitlab end end + override :can_read_project? + def can_read_project? + return true if user&.migration_bot? + + super + end + override :check_download_access! def check_download_access! passed = guest_can_download_code? || user_can_download_code? @@ -99,7 +106,7 @@ module Gitlab def check_single_change_access(change) Checks::SnippetCheck.new(change, logger: logger).validate! - Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet::MAX_FILE_COUNT, logger: logger).validate! + Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit(user), logger: logger).validate! rescue Checks::TimedLogger::TimeoutError raise TimeoutError, logger.full_message end @@ -121,5 +128,12 @@ module Gitlab def check_custom_action(cmd) nil end + + override :check_size_limit? + def check_size_limit? + return false if user&.migration_bot? + + super + end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 697c943b4ec..3aaed0edb87 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -130,7 +130,7 @@ module Gitlab end def self.address_metadata(storage) - Base64.strict_encode64(JSON.dump(storage => connection_data(storage))) + Base64.strict_encode64(Gitlab::Json.dump(storage => connection_data(storage))) end def self.connection_data(storage) @@ -209,7 +209,8 @@ module Gitlab end def self.query_time - SafeRequestStore[:gitaly_query_time] ||= 0 + query_time = SafeRequestStore[:gitaly_query_time] ||= 0 + query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.query_time=(duration) @@ -457,7 +458,7 @@ module Gitlab def self.filesystem_id_from_disk(storage) metadata_file = File.read(storage_metadata_file_path(storage)) - metadata_hash = JSON.parse(metadata_file) + metadata_hash = Gitlab::Json.parse(metadata_file) metadata_hash['gitaly_filesystem_id'] rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError nil diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index 8fb99c996b3..1b4750da868 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -5,8 +5,6 @@ module Gitlab # The ParallelImporter schedules the importing of a GitHub project using # Sidekiq. class ParallelImporter - prepend_if_ee('::EE::Gitlab::GithubImport::ParallelImporter') # rubocop: disable Cop/InjectEnterpriseEditionModule - attr_reader :project def self.async? @@ -41,3 +39,5 @@ module Gitlab end end end + +Gitlab::GithubImport::ParallelImporter.prepend_if_ee('::EE::Gitlab::GithubImport::ParallelImporter') diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index 26440e6f82d..8abefad1ef3 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -23,11 +23,18 @@ module Gitlab project_resolver: -> (snippet) { snippet&.project }, guest_read_ability: :read_snippet ).freeze + DESIGN = ::Gitlab::GlRepository::RepoType.new( + name: :design, + access_checker_class: ::Gitlab::GitAccessDesign, + repository_resolver: -> (project) { ::DesignManagement::Repository.new(project) }, + suffix: :design + ).freeze TYPES = { PROJECT.name.to_s => PROJECT, WIKI.name.to_s => WIKI, - SNIPPET.name.to_s => SNIPPET + SNIPPET.name.to_s => SNIPPET, + DESIGN.name.to_s => DESIGN }.freeze def self.types @@ -58,5 +65,3 @@ module Gitlab private_class_method :instance end end - -Gitlab::GlRepository.prepend_if_ee('::EE::Gitlab::GlRepository') diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 052ce578881..64c329b15ae 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -57,6 +57,10 @@ module Gitlab self == SNIPPET end + def design? + self == DESIGN + end + def path_suffix suffix ? ".#{suffix}" : '' end @@ -87,5 +91,3 @@ module Gitlab end end end - -Gitlab::GlRepository::RepoType.prepend_if_ee('EE::Gitlab::GlRepository::RepoType') diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index e4e69241bd9..fbbfed7279d 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -42,11 +42,11 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend - push_frontend_feature_flag(:snippets_vue, default_enabled: false) - push_frontend_feature_flag(:monaco_snippets, default_enabled: false) + push_frontend_feature_flag(:snippets_vue, default_enabled: true) push_frontend_feature_flag(:monaco_blobs, default_enabled: false) push_frontend_feature_flag(:monaco_ci, default_enabled: false) push_frontend_feature_flag(:snippets_edit_vue, default_enabled: false) + push_frontend_feature_flag(:webperf_experiment, default_enabled: false) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/grape_logging/loggers/cloudflare_logger.rb b/lib/gitlab/grape_logging/loggers/cloudflare_logger.rb new file mode 100644 index 00000000000..3abb0100a86 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/cloudflare_logger.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module GrapeLogging + module Loggers + class CloudflareLogger < ::GrapeLogging::Loggers::Base + include ::Gitlab::Logging::CloudflareHelper + + def parameters(request, _response) + data = {} + store_cloudflare_headers!(data, request) + + data + end + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/context_logger.rb b/lib/gitlab/grape_logging/loggers/context_logger.rb new file mode 100644 index 00000000000..0a8f0872fbe --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/context_logger.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# This module adds additional correlation id the grape logger +module Gitlab + module GrapeLogging + module Loggers + class ContextLogger < ::GrapeLogging::Loggers::Base + def parameters(_, _) + Labkit::Context.current.to_h + end + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/exception_logger.rb b/lib/gitlab/grape_logging/loggers/exception_logger.rb index 606b7c0dbce..14147769422 100644 --- a/lib/gitlab/grape_logging/loggers/exception_logger.rb +++ b/lib/gitlab/grape_logging/loggers/exception_logger.rb @@ -4,14 +4,16 @@ module Gitlab module GrapeLogging module Loggers class ExceptionLogger < ::GrapeLogging::Loggers::Base - def parameters(request, _) + def parameters(request, response_body) + data = {} + data[:api_error] = format_body(response_body) if bad_request?(request) + # grape-logging attempts to pass the logger the exception # (https://github.com/aserafin/grape_logging/blob/v1.7.0/lib/grape_logging/middleware/request_logger.rb#L63), # but it appears that the rescue_all in api.rb takes # precedence so the logger never sees it. We need to # store and retrieve the exception from the environment. exception = request.env[::API::Helpers::API_EXCEPTION_ENV] - data = {} return data unless exception.is_a?(Exception) @@ -19,6 +21,28 @@ module Gitlab data end + + private + + def format_body(response_body) + # https://github.com/rack/rack/blob/master/SPEC.rdoc#label-The+Body: + # The response_body must respond to each, but just in case we + # guard against errors here. + response_body = Array(response_body) unless response_body.respond_to?(:each) + + # To avoid conflicting types in Elasticsearch, convert every + # element into an Array of strings. A response body is usually + # an array of Strings so that the response can be sent in + # chunks. + body = [] + # each_with_object doesn't work with Rack::BodyProxy + response_body.each { |chunk| body << chunk.to_s } + body + end + + def bad_request?(request) + request.env[::API::Helpers::API_RESPONSE_STATUS_CODE] == 400 + end end end end diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb index c7f430490d6..61668b634fd 100644 --- a/lib/gitlab/graphql/authorize/authorize_field_service.rb +++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb @@ -70,7 +70,10 @@ module Gitlab end def filter_allowed(current_user, resolved_type, authorizing_object) - if authorizing_object + if resolved_type.nil? + # We're not rendering anything, for example when a record was not found + # no need to do anything + elsif authorizing_object # Authorizing fields representing scalars, or a simple field with an object resolved_type if allowed_access?(current_user, authorizing_object) elsif @field.connection? @@ -83,9 +86,6 @@ module Gitlab resolved_type.select do |single_object_type| allowed_access?(current_user, single_object_type.object) end - elsif resolved_type.nil? - # We're not rendering anything, for example when a record was not found - # no need to do anything else raise "Can't authorize #{@field}" end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index 5466924a794..1a32ab468b1 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -128,7 +128,7 @@ module Gitlab end def ordering_from_encoded_json(cursor) - JSON.parse(decode(cursor)) + Gitlab::Json.parse(decode(cursor)) rescue JSON::ParserError raise Gitlab::Graphql::Errors::ArgumentError, "Please provide a valid cursor" end diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb index 876d6114f3c..12bcc4993b5 100644 --- a/lib/gitlab/graphql/pagination/keyset/order_info.rb +++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb @@ -40,6 +40,8 @@ module Gitlab end if order_list.count > 2 + # Keep in mind an order clause for primary key is added if one is not present + # lib/gitlab/graphql/pagination/keyset/connection.rb:97 raise ArgumentError.new('A maximum of 2 ordering fields are allowed') end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb index 327a9c549d5..6f705239fa3 100644 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -34,7 +34,7 @@ module Gitlab memo[:depth] = depth memo[:complexity] = complexity - memo[:duration] = duration(memo[:time_started]).round(1) + memo[:duration_s] = duration(memo[:time_started]).round(1) GraphqlLogger.info(memo.except!(:time_started, :query)) rescue => e @@ -62,7 +62,7 @@ module Gitlab query_string: nil, query: query, variables: nil, - duration: nil + duration_s: nil } end end diff --git a/lib/gitlab/graphql/variables.rb b/lib/gitlab/graphql/variables.rb index b13ea37c21f..1c6fb011012 100644 --- a/lib/gitlab/graphql/variables.rb +++ b/lib/gitlab/graphql/variables.rb @@ -20,7 +20,7 @@ module Gitlab case ambiguous_param when String if ambiguous_param.present? - ensure_hash(JSON.parse(ambiguous_param)) + ensure_hash(Gitlab::Json.parse(ambiguous_param)) else {} end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 8597903ad00..eb4361cdc53 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -4,8 +4,8 @@ module Gitlab class GroupSearchResults < SearchResults attr_reader :group - def initialize(current_user, limit_projects, group, query, default_project_filter: false, per_page: 20) - super(current_user, limit_projects, query, default_project_filter: default_project_filter, per_page: per_page) + def initialize(current_user, limit_projects, group, query, default_project_filter: false) + super(current_user, limit_projects, query, default_project_filter: default_project_filter) @group = group end diff --git a/lib/gitlab/health_checks/puma_check.rb b/lib/gitlab/health_checks/puma_check.rb index 9f09070a57d..2dc8a093572 100644 --- a/lib/gitlab/health_checks/puma_check.rb +++ b/lib/gitlab/health_checks/puma_check.rb @@ -21,7 +21,7 @@ module Gitlab return unless Gitlab::Runtime.puma? stats = Puma.stats - stats = JSON.parse(stats) + stats = Gitlab::Json.parse(stats) # If `workers` is missing this means that # Puma server is running in single mode diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 7e0398f09af..18f4cb559c5 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -5,28 +5,28 @@ module Gitlab extend self AVAILABLE_LANGUAGES = { + 'bg' => 'Bulgarian - български', + 'cs_CZ' => 'Czech - čeština', + 'de' => 'German - Deutsch', 'en' => 'English', - 'es' => 'Español', - 'gl_ES' => 'Galego', - 'de' => 'Deutsch', - 'fr' => 'Français', - 'pt_BR' => 'Português (Brasil)', - 'zh_CN' => '简体中文', - 'zh_HK' => '繁體中文 (香港)', - 'zh_TW' => '繁體中文 (臺灣)', - 'bg' => 'български', - 'ru' => 'Русский', - 'eo' => 'Esperanto', - 'it' => 'Italiano', - 'uk' => 'Українська', - 'ja' => '日本語', - 'ko' => '한국어', - 'nl_NL' => 'Nederlands', - 'tr_TR' => 'Türkçe', - 'id_ID' => 'Bahasa Indonesia', + 'eo' => 'Esperanto - esperanto', + 'es' => 'Spanish - español', 'fil_PH' => 'Filipino', - 'pl_PL' => 'Polski', - 'cs_CZ' => 'Čeština' + 'fr' => 'French - français', + 'gl_ES' => 'Galician - galego', + 'id_ID' => 'Indonesian - Bahasa Indonesia', + 'it' => 'Italian - italiano', + 'ja' => 'Japanese - 日本語', + 'ko' => 'Korean - 한국어', + 'nl_NL' => 'Dutch - Nederlands', + 'pl_PL' => 'Polish - polski', + 'pt_BR' => 'Portuguese (Brazil) - português (Brasil)', + 'ru' => 'Russian - Русский', + 'tr_TR' => 'Turkish - Türkçe', + 'uk' => 'Ukrainian - українська', + 'zh_CN' => 'Chinese, Simplified - 简体中文', + 'zh_HK' => 'Chinese, Traditional (Hong Kong) - 繁體中文 (香港)', + 'zh_TW' => 'Chinese, Traditional (Taiwan) - 繁體中文 (台灣)' }.freeze def available_locales diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 52102b6f508..921072a4970 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -42,6 +42,10 @@ module Gitlab "project.wiki.bundle" end + def design_repo_bundle_filename + 'project.design.bundle' + end + def snippet_repo_bundle_dir 'snippets' end @@ -88,6 +92,10 @@ module Gitlab 'group.json' end + def legacy_group_config_file + Rails.root.join('lib/gitlab/import_export/group/legacy_import_export.yml') + end + def group_config_file Rails.root.join('lib/gitlab/import_export/group/import_export.yml') end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index fd98bc2caad..e2dba831661 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -52,7 +52,10 @@ module Gitlab end def headers - { 'Content-Length' => export_size.to_s } + { + 'Content-Type' => 'application/gzip', + 'Content-Length' => export_size.to_s + } end def export_size diff --git a/lib/gitlab/import_export/design_repo_restorer.rb b/lib/gitlab/import_export/design_repo_restorer.rb new file mode 100644 index 00000000000..a702c58a7c2 --- /dev/null +++ b/lib/gitlab/import_export/design_repo_restorer.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class DesignRepoRestorer < RepoRestorer + def initialize(project:, shared:, path_to_bundle:) + super(project: project, shared: shared, path_to_bundle: path_to_bundle) + + @repository = project.design_repository + end + + # `restore` method is handled in super class + end + end +end diff --git a/lib/gitlab/import_export/design_repo_saver.rb b/lib/gitlab/import_export/design_repo_saver.rb new file mode 100644 index 00000000000..db9ebee6a13 --- /dev/null +++ b/lib/gitlab/import_export/design_repo_saver.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class DesignRepoSaver < RepoSaver + def save + @repository = project.design_repository + + super + end + + private + + def bundle_full_path + File.join(shared.export_path, ::Gitlab::ImportExport.design_repo_bundle_filename) + end + end + end +end diff --git a/lib/gitlab/import_export/group/group_restorer.rb b/lib/gitlab/import_export/group/group_restorer.rb new file mode 100644 index 00000000000..b338950fb71 --- /dev/null +++ b/lib/gitlab/import_export/group/group_restorer.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class GroupRestorer + def initialize( + user:, + shared:, + group:, + attributes:, + importable_path:, + relation_reader:, + reader: + ) + @user = user + @shared = shared + @group = group + @group_attributes = attributes + @importable_path = importable_path + @relation_reader = relation_reader + @reader = reader + end + + def restore + # consume_relation returns a list of [relation, index] + @group_members = @relation_reader + .consume_relation(@importable_path, 'members') + .map(&:first) + + return unless members_mapper.map + + restorer.restore + end + + private + + def restorer + @relation_tree_restorer ||= RelationTreeRestorer.new( + user: @user, + shared: @shared, + relation_reader: @relation_reader, + members_mapper: members_mapper, + object_builder: object_builder, + relation_factory: relation_factory, + reader: @reader, + importable: @group, + importable_attributes: @group_attributes, + importable_path: @importable_path + ) + end + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new( + exported_members: @group_members, + user: @user, + importable: @group + ) + end + + def relation_factory + Gitlab::ImportExport::Group::RelationFactory + end + + def object_builder + Gitlab::ImportExport::Group::ObjectBuilder + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index 5008639077c..e30206dc509 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -27,9 +27,7 @@ included_attributes: excluded_attributes: group: - - :id - :owner_id - - :parent_id - :created_at - :updated_at - :runners_token diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml new file mode 100644 index 00000000000..5008639077c --- /dev/null +++ b/lib/gitlab/import_export/group/legacy_import_export.yml @@ -0,0 +1,86 @@ +# Model relationships to be included in the group import/export +# +# This list _must_ only contain relationships that are available to both FOSS and +# Enterprise editions. EE specific relationships must be defined in the `ee` section further +# down below. +tree: + group: + - :milestones + - :badges + - labels: + - :priorities + - boards: + - lists: + - label: + - :priorities + - :board + - members: + - :user + +included_attributes: + user: + - :id + - :email + - :username + author: + - :name + +excluded_attributes: + group: + - :id + - :owner_id + - :parent_id + - :created_at + - :updated_at + - :runners_token + - :runners_token_encrypted + - :saml_discovery_token + - :visibility_level + - :trial_ends_on + - :shared_runners_minute_limit + - :extra_shared_runners_minutes_limit + epics: + - :state_id + +methods: + labels: + - :type + label: + - :type + badges: + - :type + notes: + - :type + events: + - :action + lists: + - :list_type + epics: + - :state + +preloads: + +# EE specific relationships and settings to include. All of this will be merged +# into the previous structures if EE is used. +ee: + tree: + group: + - epics: + - :parent + - :award_emoji + - events: + - :push_event_payload + - notes: + - :author + - :award_emoji + - events: + - :push_event_payload + - boards: + - :board_assignee + - :milestone + - labels: + - :priorities + - lists: + - milestone: + - events: + - :push_event_payload diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb index 5d96a0f3c0a..5499b79cee6 100644 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb @@ -122,7 +122,7 @@ module Gitlab @reader ||= Gitlab::ImportExport::Reader.new( shared: @shared, config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.group_config_file + config: Gitlab::ImportExport.legacy_group_config_file ).to_h ) end diff --git a/lib/gitlab/import_export/group/legacy_tree_saver.rb b/lib/gitlab/import_export/group/legacy_tree_saver.rb index 3776ef0d8f5..7ab81c09885 100644 --- a/lib/gitlab/import_export/group/legacy_tree_saver.rb +++ b/lib/gitlab/import_export/group/legacy_tree_saver.rb @@ -43,7 +43,7 @@ module Gitlab @reader ||= Gitlab::ImportExport::Reader.new( shared: @shared, config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.group_config_file + config: Gitlab::ImportExport.legacy_group_config_file ).to_h ) end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb new file mode 100644 index 00000000000..d0c0999f291 --- /dev/null +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class TreeRestorer + include Gitlab::Utils::StrongMemoize + + attr_reader :user, :shared + + def initialize(user:, shared:, group:) + @user = user + @shared = shared + @top_level_group = group + @groups_mapping = {} + end + + def restore + group_ids = relation_reader.consume_relation('groups', '_all').map { |value, _idx| Integer(value) } + root_group_id = group_ids.delete_at(0) + + process_root(root_group_id) + + group_ids.each do |group_id| + process_child(group_id) + end + + true + rescue => e + shared.error(e) + false + end + + class GroupAttributes + attr_reader :attributes, :group_id, :id, :path + + def initialize(group_id, relation_reader) + @group_id = group_id + + @path = "groups/#{group_id}" + @attributes = relation_reader.consume_attributes(@path) + @id = @attributes.delete('id') + + unless @id == @group_id + raise ArgumentError, "Invalid group_id for #{group_id}" + end + end + + def delete_attribute(name) + attributes.delete(name) + end + + def delete_attributes(*names) + names.map(&method(:delete_attribute)) + end + end + private_constant :GroupAttributes + + private + + def process_root(group_id) + group_attributes = GroupAttributes.new(group_id, relation_reader) + + # name and path are not imported on the root group to avoid conflict + # with existing groups name and/or path. + group_attributes.delete_attributes('name', 'path') + + restore_group(@top_level_group, group_attributes) + end + + def process_child(group_id) + group_attributes = GroupAttributes.new(group_id, relation_reader) + + group = create_group(group_attributes) + + restore_group(group, group_attributes) + end + + def create_group(group_attributes) + parent_id = group_attributes.delete_attribute('parent_id') + name = group_attributes.delete_attribute('name') + path = group_attributes.delete_attribute('path') + + parent_group = @groups_mapping.fetch(parent_id) { raise(ArgumentError, 'Parent group not found') } + + ::Groups::CreateService.new( + user, + name: name, + path: path, + parent_id: parent_group.id, + visibility_level: sub_group_visibility_level(group_attributes.attributes, parent_group) + ).execute + end + + def restore_group(group, group_attributes) + @groups_mapping[group_attributes.id] = group + + Group::GroupRestorer.new( + user: user, + shared: shared, + group: group, + attributes: group_attributes.attributes, + importable_path: group_attributes.path, + relation_reader: relation_reader, + reader: reader + ).restore + end + + def relation_reader + strong_memoize(:relation_reader) do + ImportExport::JSON::NdjsonReader.new( + File.join(shared.export_path, 'tree') + ) + end + end + + def sub_group_visibility_level(group_hash, parent_group) + original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE + + if parent_group && parent_group.visibility_level < original_visibility_level + Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) + else + original_visibility_level + end + end + + def reader + strong_memoize(:reader) do + Gitlab::ImportExport::Reader.new( + shared: @shared, + config: Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + ) + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb new file mode 100644 index 00000000000..d538de33c51 --- /dev/null +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class TreeSaver + attr_reader :full_path, :shared + + def initialize(group:, current_user:, shared:, params: {}) + @params = params + @current_user = current_user + @shared = shared + @group = group + @full_path = File.join(@shared.export_path, 'tree') + end + + def save + all_groups = Enumerator.new do |group_ids| + groups.each do |group| + serialize(group) + group_ids << group.id + end + end + + json_writer.write_relation_array('groups', '_all', all_groups) + + true + rescue => e + @shared.error(e) + false + ensure + json_writer&.close + end + + private + + def groups + @groups ||= Gitlab::ObjectHierarchy + .new(::Group.where(id: @group.id)) + .base_and_descendants(with_depth: true) + .order_by(:depth) + end + + def serialize(group) + ImportExport::JSON::StreamingSerializer.new( + group, + group_tree, + json_writer, + exportable_path: "groups/#{group.id}" + ).execute + end + + def group_tree + @group_tree ||= Gitlab::ImportExport::Reader.new( + shared: @shared, + config: group_config + ).group_tree + end + + def group_config + Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + end + + def json_writer + @json_writer ||= ImportExport::JSON::NdjsonWriter.new(@full_path) + end + end + end + end +end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 4b761eb86ae..b1219384732 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -34,7 +34,7 @@ module Gitlab attr_accessor :archive_file, :current_user, :project, :shared def restorers - [repo_restorer, wiki_restorer, project_tree, avatar_restorer, + [repo_restorer, wiki_restorer, project_tree, avatar_restorer, design_repo_restorer, uploads_restorer, lfs_restorer, statistics_restorer, snippets_repo_restorer] end @@ -71,6 +71,12 @@ module Gitlab wiki_enabled: project.wiki_enabled?) end + def design_repo_restorer + Gitlab::ImportExport::DesignRepoRestorer.new(path_to_bundle: design_repo_path, + shared: shared, + project: project) + end + def uploads_restorer Gitlab::ImportExport::UploadsRestorer.new(project: project, shared: shared) end @@ -101,6 +107,10 @@ module Gitlab File.join(shared.export_path, Gitlab::ImportExport.wiki_repo_bundle_filename) end + def design_repo_path + File.join(shared.export_path, Gitlab::ImportExport.design_repo_bundle_filename) + end + def remove_import_file upload = project.import_export_upload @@ -111,13 +121,17 @@ module Gitlab end def overwrite_project - return unless can?(current_user, :admin_namespace, project.namespace) + return true unless overwrite_project? - if overwrite_project? - ::Projects::OverwriteProjectService.new(project, current_user) - .execute(project_to_overwrite) + unless can?(current_user, :admin_namespace, project.namespace) + message = "User #{current_user&.username} (#{current_user&.id}) cannot overwrite a project in #{project.namespace.path}" + @shared.error(::Projects::ImportService::PermissionError.new(message)) + return false end + ::Projects::OverwriteProjectService.new(project, current_user) + .execute(project_to_overwrite) + true end @@ -137,5 +151,3 @@ module Gitlab end end end - -Gitlab::ImportExport::Importer.prepend_if_ee('EE::Gitlab::ImportExport::Importer') diff --git a/lib/gitlab/import_export/project/base_task.rb b/lib/gitlab/import_export/project/base_task.rb index 6a7b24421c9..356e261e251 100644 --- a/lib/gitlab/import_export/project/base_task.rb +++ b/lib/gitlab/import_export/project/base_task.rb @@ -11,17 +11,27 @@ module Gitlab @file_path = opts.fetch(:file_path) @namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path)) @current_user = User.find_by_username(opts.fetch(:username)) - @measurement_enabled = opts.fetch(:measurement_enabled) - @measurement = Gitlab::Utils::Measuring.new(logger: logger) if @measurement_enabled @logger = logger end private - attr_reader :measurement, :project, :namespace, :current_user, :file_path, :project_path, :logger + attr_reader :project, :namespace, :current_user, :file_path, :project_path, :logger - def measurement_enabled? - @measurement_enabled + def disable_upload_object_storage + overwrite_uploads_setting('enabled', false) do + yield + end + end + + def overwrite_uploads_setting(key, value) + old_value = Settings.uploads.object_store[key] + Settings.uploads.object_store[key] = value + + yield + + ensure + Settings.uploads.object_store[key] = old_value end def success(message) diff --git a/lib/gitlab/import_export/project/export_task.rb b/lib/gitlab/import_export/project/export_task.rb index ec287380c48..5e105b4653d 100644 --- a/lib/gitlab/import_export/project/export_task.rb +++ b/lib/gitlab/import_export/project/export_task.rb @@ -19,7 +19,11 @@ module Gitlab .execute(Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy.new(archive_path: file_path)) end + return error(project.import_export_shared.errors.join(', ')) if project.import_export_shared.errors.any? + success('Done!') + rescue Gitlab::ImportExport::Error => e + error(e.message) end private @@ -32,8 +36,13 @@ module Gitlab def with_export with_request_store do - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - measurement_enabled? ? measurement.with_measuring { yield } : yield + # We are disabling ObjectStorage for `export` + # since when direct upload is enabled, remote storage will be used + # and Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy will fail to copy exported archive + disable_upload_object_storage do + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + yield + end end end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 3cbd0d144e6..8851b106ad5 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -29,6 +29,14 @@ tree: - resource_label_events: - label: - :priorities + - designs: + - notes: + - :author + - events: + - :push_event_payload + - design_versions: + - actions: + - :design # Duplicate export of issues.designs in order to link the record to both Issue and Action - :issue_assignees - :zoom_meetings - :sentry_issue @@ -160,6 +168,7 @@ excluded_attributes: - :marked_for_deletion_at - :marked_for_deletion_by_user_id - :compliance_framework_setting + - :show_default_award_emojis namespaces: - :runners_token - :runners_token_encrypted @@ -190,6 +199,7 @@ excluded_attributes: - :merge_request_diff_id issues: - :milestone_id + - :sprint_id - :moved_to_id - :sent_notifications - :state_id @@ -197,6 +207,7 @@ excluded_attributes: - :promoted_to_epic_id merge_request: - :milestone_id + - :sprint_id - :ref_fetched - :merge_jid - :rebase_jid @@ -205,6 +216,7 @@ excluded_attributes: - :state_id merge_requests: - :milestone_id + - :sprint_id - :ref_fetched - :merge_jid - :rebase_jid @@ -250,8 +262,9 @@ excluded_attributes: - :token - :token_encrypted services: - - :template + - :inherit_from_id - :instance + - :template error_tracking_setting: - :encrypted_token - :encrypted_token_iv @@ -284,6 +297,7 @@ excluded_attributes: actions: - :design_id - :version_id + - image_v432x230 links: - :release_id project_members: @@ -376,14 +390,6 @@ ee: tree: project: - issues: - - designs: - - notes: - - :author - - events: - - :push_event_payload - - design_versions: - - actions: - - :design # Duplicate export of issues.designs in order to link the record to both Issue and Action - epic_issue: - :epic - protected_branches: @@ -391,6 +397,3 @@ ee: - protected_environments: - :deploy_access_levels - :service_desk_setting - excluded_attributes: - actions: - - image_v432x230 diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb index ae654ddbeaf..59bb8af750e 100644 --- a/lib/gitlab/import_export/project/import_task.rb +++ b/lib/gitlab/import_export/project/import_task.rb @@ -32,7 +32,7 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635 # For development setups, this code-path will be excluded from n+1 detection. ::Gitlab::GitalyClient.allow_n_plus_1_calls do - measurement_enabled? ? measurement.with_measuring { yield } : yield + yield end end @@ -56,11 +56,7 @@ module Gitlab disable_upload_object_storage do service = Projects::GitlabProjectsImportService.new( current_user, - { - namespace_id: namespace.id, - path: project_path, - file: File.open(file_path) - } + import_params ) service.execute @@ -71,24 +67,6 @@ module Gitlab Sidekiq::Worker.drain_all end - def disable_upload_object_storage - overwrite_uploads_setting('background_upload', false) do - overwrite_uploads_setting('direct_upload', false) do - yield - end - end - end - - def overwrite_uploads_setting(key, value) - old_value = Settings.uploads.object_store[key] - Settings.uploads.object_store[key] = value - - yield - - ensure - Settings.uploads.object_store[key] = old_value - end - def full_path "#{namespace.full_path}/#{project_path}" end @@ -99,6 +77,14 @@ module Gitlab " as #{current_user.name}" end + def import_params + { + namespace_id: namespace.id, + path: project_path, + file: File.open(file_path) + } + end + def show_import_failures_count return unless project.import_failures.exists? diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index c3637b1c115..831e38f3034 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -57,6 +57,8 @@ module Gitlab # Returns Arel clause for a particular model or `nil`. def where_clause_for_klass + return attrs_to_arel(attributes.slice('filename')).and(table[:issue_id].eq(nil)) if design? + attrs_to_arel(attributes.slice('iid')) if merge_request? end @@ -95,6 +97,10 @@ module Gitlab klass == Epic end + def design? + klass == DesignManagement::Design + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: @@ -115,5 +121,3 @@ module Gitlab end end end - -Gitlab::ImportExport::Project::ObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::Project::ObjectBuilder') diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index f7f1195f2f1..3ab9f2c4bfa 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -4,8 +4,6 @@ module Gitlab module ImportExport module Project class RelationFactory < Base::RelationFactory - prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule - OVERRIDES = { snippets: :project_snippets, ci_pipelines: 'Ci::Pipeline', pipelines: 'Ci::Pipeline', @@ -19,6 +17,10 @@ module Gitlab merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', create_access_levels: 'ProtectedTag::CreateAccessLevel', + design: 'DesignManagement::Design', + designs: 'DesignManagement::Design', + design_versions: 'DesignManagement::Version', + actions: 'DesignManagement::Action', labels: :project_labels, priorities: :label_priorities, auto_devops: :project_auto_devops, @@ -53,6 +55,7 @@ module Gitlab container_expiration_policy external_pull_request external_pull_requests + DesignManagement::Design ].freeze def create @@ -161,3 +164,5 @@ module Gitlab end end end + +Gitlab::ImportExport::Project::RelationFactory.prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index e9c89b803ba..a16ffe36054 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -54,7 +54,7 @@ module Gitlab end def ndjson_relation_reader - return unless Feature.enabled?(:project_import_ndjson, project.namespace) + return unless Feature.enabled?(:project_import_ndjson, project.namespace, default_enabled: true) ImportExport::JSON::NdjsonReader.new( File.join(shared.export_path, 'tree') diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 0017aa523c1..7cca3596da6 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -54,7 +54,7 @@ module Gitlab def json_writer @json_writer ||= begin - if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) + if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace, default_enabled: true) full_path = File.join(@shared.export_path, 'tree') Gitlab::ImportExport::JSON::NdjsonWriter.new(full_path) else diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 056945d0294..ea16d978127 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -37,9 +37,7 @@ module Gitlab ActiveRecord::Base.no_touching do update_params! - bulk_inserts_enabled = @importable.class == ::Project && - Feature.enabled?(:import_bulk_inserts, @importable.group, default_enabled: true) - BulkInsertableAssociations.with_bulk_insert(enabled: bulk_inserts_enabled) do + BulkInsertableAssociations.with_bulk_insert(enabled: @importable.class == ::Project) do fix_ci_pipelines_not_sorted_on_legacy_project_json! create_relations! end diff --git a/lib/gitlab/import_export/snippets_repo_restorer.rb b/lib/gitlab/import_export/snippets_repo_restorer.rb index 8fe83225812..9ff3e74a6b1 100644 --- a/lib/gitlab/import_export/snippets_repo_restorer.rb +++ b/lib/gitlab/import_export/snippets_repo_restorer.rb @@ -10,7 +10,6 @@ module Gitlab end def restore - return true unless Feature.enabled?(:version_snippets, @user) return true unless Dir.exist?(snippets_repo_bundle_path) @project.snippets.find_each.all? do |snippet| diff --git a/lib/gitlab/import_export/snippets_repo_saver.rb b/lib/gitlab/import_export/snippets_repo_saver.rb index 85e094c0d15..d3b0fe1c18c 100644 --- a/lib/gitlab/import_export/snippets_repo_saver.rb +++ b/lib/gitlab/import_export/snippets_repo_saver.rb @@ -12,8 +12,6 @@ module Gitlab end def save - return true unless Feature.enabled?(:version_snippets, @current_user) - create_snippets_repo_directory @project.snippets.find_each.all? do |snippet| diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index 6b066b800a5..cc99e828251 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -38,7 +38,8 @@ module Gitlab end def self.query_time - ::RequestStore[REDIS_CALL_DURATION] || 0 + query_time = ::RequestStore[REDIS_CALL_DURATION] || 0 + query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.add_duration(duration) diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 308c3007720..7c5a601cd5b 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -5,27 +5,28 @@ module Gitlab extend self KEYS = %i(gitaly_calls gitaly_duration_s rugged_calls rugged_duration_s redis_calls redis_duration_s).freeze + DURATION_PRECISION = 6 # microseconds def add_instrumentation_data(payload) gitaly_calls = Gitlab::GitalyClient.get_request_count if gitaly_calls > 0 payload[:gitaly_calls] = gitaly_calls - payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time.round(2) + payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time end rugged_calls = Gitlab::RuggedInstrumentation.query_count if rugged_calls > 0 payload[:rugged_calls] = rugged_calls - payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time.round(2) + payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time end redis_calls = Gitlab::Instrumentation::Redis.get_request_count if redis_calls > 0 payload[:redis_calls] = redis_calls - payload[:redis_duration_s] = Gitlab::Instrumentation::Redis.query_time.round(2) + payload[:redis_duration_s] = Gitlab::Instrumentation::Redis.query_time end end @@ -47,7 +48,7 @@ module Gitlab # Its possible that if theres clock-skew between two nodes # this value may be less than zero. In that event, we record the value # as zero. - [elapsed_by_absolute_time(enqueued_at_time), 0].max.round(2) + [elapsed_by_absolute_time(enqueued_at_time), 0].max.round(DURATION_PRECISION) end # Calculates the time in seconds, as a float, from diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb index 5381812186d..306736df30f 100644 --- a/lib/gitlab/jira_import/base_importer.rb +++ b/lib/gitlab/jira_import/base_importer.rb @@ -3,12 +3,13 @@ module Gitlab module JiraImport class BaseImporter - attr_reader :project, :client, :formatter, :jira_project_key + attr_reader :project, :client, :formatter, :jira_project_key, :running_import def initialize(project) project.validate_jira_import_settings! - @jira_project_key = project.latest_jira_import&.jira_project_key + @running_import = project.latest_jira_import + @jira_project_key = running_import&.jira_project_key raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key diff --git a/lib/gitlab/jira_import/handle_labels_service.rb b/lib/gitlab/jira_import/handle_labels_service.rb new file mode 100644 index 00000000000..1b00515cced --- /dev/null +++ b/lib/gitlab/jira_import/handle_labels_service.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + class HandleLabelsService + def initialize(project, jira_labels) + @project = project + @jira_labels = jira_labels + end + + def execute + return if jira_labels.blank? + + existing_labels = LabelsFinder.new(nil, project: project, title: jira_labels) + .execute(skip_authorization: true).select(:id, :name) + new_labels = create_missing_labels(existing_labels) + + label_ids = existing_labels.map(&:id) + label_ids += new_labels if new_labels.present? + label_ids + end + + private + + attr_reader :project, :jira_labels + + def create_missing_labels(existing_labels) + labels_to_create = jira_labels - existing_labels.map(&:name) + return if labels_to_create.empty? + + new_labels_hash = labels_to_create.map do |title| + { project_id: project.id, title: title, type: 'ProjectLabel' } + end + + Label.insert_all(new_labels_hash).rows.flatten + end + end + end +end diff --git a/lib/gitlab/jira_import/issue_serializer.rb b/lib/gitlab/jira_import/issue_serializer.rb index cdcb62ac6e9..df57680073e 100644 --- a/lib/gitlab/jira_import/issue_serializer.rb +++ b/lib/gitlab/jira_import/issue_serializer.rb @@ -3,15 +3,14 @@ module Gitlab module JiraImport class IssueSerializer - attr_reader :jira_issue, :project, :params, :formatter - attr_accessor :metadata + attr_reader :jira_issue, :project, :import_owner_id, :params, :formatter - def initialize(project, jira_issue, params = {}) + def initialize(project, jira_issue, import_owner_id, params = {}) @jira_issue = jira_issue @project = project + @import_owner_id = import_owner_id @params = params @formatter = Gitlab::ImportFormatter.new - @metadata = [] end def execute @@ -23,7 +22,9 @@ module Gitlab state_id: map_status(jira_issue.status.statusCategory), updated_at: jira_issue.updated, created_at: jira_issue.created, - author_id: project.creator_id # TODO: map actual author: https://gitlab.com/gitlab-org/gitlab/-/issues/210580 + author_id: reporter, + assignee_ids: assignees, + label_ids: label_ids } end @@ -35,10 +36,8 @@ module Gitlab def description body = [] - body << formatter.author_line(jira_issue.reporter.displayName) - body << formatter.assignee_line(jira_issue.assignee.displayName) if jira_issue.assignee body << jira_issue.description - body << add_metadata + body << MetadataCollector.new(jira_issue).execute body.join end @@ -52,48 +51,33 @@ module Gitlab end end - def add_metadata - add_field(%w(issuetype name), 'Issue type') - add_field(%w(priority name), 'Priority') - add_labels - add_field('environment', 'Environment') - add_field('duedate', 'Due date') - add_parent - add_versions - - return if metadata.empty? - - metadata.join("\n").prepend("\n\n---\n\n**Issue metadata**\n\n") + def map_user_id(jira_user) + Gitlab::JiraImport::UserMapper.new(project, jira_user).execute&.id end - def add_field(keys, field_label) - value = fields.dig(*keys) - return if value.blank? - - metadata << "- #{field_label}: #{value}" + def reporter + map_user_id(jira_issue.reporter&.attrs) || import_owner_id end - def add_labels - return if fields['labels'].blank? - - metadata << "- Labels: #{fields['labels'].join(', ')}" - end + def assignees + found_user_id = map_user_id(jira_issue.assignee&.attrs) - def add_parent - parent_issue_key = fields.dig('parent', 'key') - return if parent_issue_key.blank? + return unless found_user_id - metadata << "- Parent issue: [#{parent_issue_key}] #{fields['parent']['fields']['summary']}" + [found_user_id] end - def add_versions - return if fields['fixVersions'].blank? + # We already create labels in Gitlab::JiraImport::LabelsImporter stage but + # there is a possibility it may fail or + # new labels were created on the Jira in the meantime + def label_ids + return if jira_issue.fields['labels'].blank? - metadata << "- Fix versions: #{fields['fixVersions'].map { |version| version['name'] }.join(', ')}" + Gitlab::JiraImport::HandleLabelsService.new(project, jira_issue.fields['labels']).execute end - def fields - jira_issue.fields + def logger + @logger ||= Gitlab::Import::Logger.build end end end diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 6543b633ddf..8c18e58d9df 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -57,7 +57,7 @@ module Gitlab # For such cases we exit early if issue was already imported. next if already_imported?(jira_issue.id) - issue_attrs = IssueSerializer.new(project, jira_issue, { iid: next_iid }).execute + issue_attrs = IssueSerializer.new(project, jira_issue, running_import.user_id, { iid: next_iid }).execute Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key) job_waiter.jobs_remaining += 1 diff --git a/lib/gitlab/jira_import/labels_importer.rb b/lib/gitlab/jira_import/labels_importer.rb index 35c434e48a4..6e6842e06bf 100644 --- a/lib/gitlab/jira_import/labels_importer.rb +++ b/lib/gitlab/jira_import/labels_importer.rb @@ -5,6 +5,8 @@ module Gitlab class LabelsImporter < BaseImporter attr_reader :job_waiter + MAX_LABELS = 500 + def initialize(project) super @job_waiter = JobWaiter.new @@ -19,15 +21,35 @@ module Gitlab def cache_import_label(project) label = project.jira_imports.by_jira_project_key(jira_project_key).last.label - raise Projects::ImportService::Error, _('Failed to find import label for jira import.') unless label + raise Projects::ImportService::Error, _('Failed to find import label for Jira import.') unless label JiraImport.cache_import_label_id(project.id, label.id) end def import_jira_labels - # todo: import jira labels, see https://gitlab.com/gitlab-org/gitlab/-/issues/212651 + start_at = 0 + loop do + break if process_jira_page(start_at) + + start_at += MAX_LABELS + end + job_waiter end + + def process_jira_page(start_at) + request = "/rest/api/2/label?maxResults=#{MAX_LABELS}&startAt=#{start_at}" + response = client.get(request) + + return true if response['values'].blank? + return true unless response.key?('isLast') + + Gitlab::JiraImport::HandleLabelsService.new(project, response['values']).execute + + response['isLast'] + rescue => e + Gitlab::ErrorTracking.track_exception(e, project_id: project.id, request: request) + end end end end diff --git a/lib/gitlab/jira_import/metadata_collector.rb b/lib/gitlab/jira_import/metadata_collector.rb new file mode 100644 index 00000000000..4551f38ba98 --- /dev/null +++ b/lib/gitlab/jira_import/metadata_collector.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + class MetadataCollector + attr_accessor :jira_issue, :metadata + + def initialize(jira_issue) + @jira_issue = jira_issue + @metadata = [] + end + + def execute + add_field(%w(issuetype name), 'Issue type') + add_field(%w(priority name), 'Priority') + add_field('environment', 'Environment') + add_field('duedate', 'Due date') + add_parent + add_versions + + return if metadata.empty? + + metadata.join("\n").prepend("\n\n---\n\n**Issue metadata**\n\n") + end + + private + + def add_field(keys, field_label) + value = fields.dig(*keys) + return if value.blank? + + metadata << "- #{field_label}: #{value}" + end + + def add_parent + parent_issue_key = fields.dig('parent', 'key') + + return if parent_issue_key.blank? + + parent_summary_key = fields.dig('parent', 'fields', 'summary') + + metadata << "- Parent issue: [#{parent_issue_key}] #{parent_summary_key}".strip + end + + def add_versions + return if fields['fixVersions'].blank? || !fields['fixVersions'].is_a?(Array) + + versions = fields['fixVersions'].map { |version| version['name'] }.compact.join(', ') + metadata << "- Fix versions: #{versions}" + end + + def fields + jira_issue.fields + end + end + end +end diff --git a/lib/gitlab/jira_import/user_mapper.rb b/lib/gitlab/jira_import/user_mapper.rb new file mode 100644 index 00000000000..208ee49b724 --- /dev/null +++ b/lib/gitlab/jira_import/user_mapper.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module JiraImport + class UserMapper + include ::Gitlab::Utils::StrongMemoize + + def initialize(project, jira_user) + @project = project + @jira_user = jira_user + end + + def execute + return unless jira_user + + email = jira_user['emailAddress'] + + # We also include emails that are not yet confirmed + users = User.by_any_email(email).to_a + + user = users.first + + # this event should never happen but we should log it in case we have invalid data + log_user_mapping_message('Multiple users found for an email address', email) if users.count > 1 + + unless project.project_member(user) || project.group&.group_member(user) + log_user_mapping_message('Jira user not found', email) + + return + end + + user + end + + private + + attr_reader :project, :jira_user, :params + + def log_user_mapping_message(message, email) + logger.info( + project_id: project.id, + project_path: project.full_path, + user_email: email, + message: message + ) + end + + def logger + @logger ||= Gitlab::Import::Logger.build + end + end + end +end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 5ebda67e2ae..5b6689dbefe 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -2,13 +2,25 @@ module Gitlab module Json + INVALID_LEGACY_TYPES = [String, TrueClass, FalseClass].freeze + class << self - def parse(*args) - adapter.parse(*args) + def parse(string, *args, **named_args) + legacy_mode = legacy_mode_enabled?(named_args.delete(:legacy_mode)) + data = adapter.parse(string, *args, **named_args) + + handle_legacy_mode!(data) if legacy_mode + + data end - def parse!(*args) - adapter.parse!(*args) + def parse!(string, *args, **named_args) + legacy_mode = legacy_mode_enabled?(named_args.delete(:legacy_mode)) + data = adapter.parse!(string, *args, **named_args) + + handle_legacy_mode!(data) if legacy_mode + + data end def dump(*args) @@ -28,6 +40,20 @@ module Gitlab def adapter ::JSON end + + def parser_error + ::JSON::ParserError + end + + def legacy_mode_enabled?(arg_value) + arg_value.nil? ? false : arg_value + end + + def handle_legacy_mode!(data) + return data unless Feature.enabled?(:json_wrapper_legacy_mode, default_enabled: true) + + raise parser_error if INVALID_LEGACY_TYPES.any? { |type| data.is_a?(type) } + end end end end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 3e201d68297..00ab7109267 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -3,7 +3,7 @@ module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.16.3' + HELM_VERSION = '2.16.6' KUBECTL_VERSION = '1.13.12' NAMESPACE = 'gitlab-managed-apps' NAMESPACE_LABELS = { 'app.gitlab.com/managed_by' => :gitlab }.freeze diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index 3b843799d66..ceda18442d6 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -99,11 +99,7 @@ module Gitlab command.cluster_role_binding_resource.tap do |cluster_role_binding_resource| break unless cluster_role_binding_resource - if cluster_role_binding_exists?(cluster_role_binding_resource) - kubeclient.update_cluster_role_binding(cluster_role_binding_resource) - else - kubeclient.create_cluster_role_binding(cluster_role_binding_resource) - end + kubeclient.update_cluster_role_binding(cluster_role_binding_resource) end end diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb index 9d0fd30ba8f..771444ee9ee 100644 --- a/lib/gitlab/kubernetes/helm/delete_command.rb +++ b/lib/gitlab/kubernetes/helm/delete_command.rb @@ -36,8 +36,6 @@ module Gitlab @rbac end - private - def delete_command command = ['helm', 'delete', '--purge', name] + tls_flags_if_remote_tiller diff --git a/lib/gitlab/kubernetes/helm/parsers/list_v2.rb b/lib/gitlab/kubernetes/helm/parsers/list_v2.rb new file mode 100644 index 00000000000..c5c5d198a6c --- /dev/null +++ b/lib/gitlab/kubernetes/helm/parsers/list_v2.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module Parsers + # Parses Helm v2 list (JSON) output + class ListV2 + ParserError = Class.new(StandardError) + + attr_reader :contents, :json + + def initialize(contents) + @contents = contents + @json = Gitlab::Json.parse(contents) + rescue JSON::ParserError => e + raise ParserError, e.message + end + + def releases + @releases = helm_releases + end + + private + + def helm_releases + helm_releases = json['Releases'] || [] + + raise ParserError, 'Invalid format for Releases' unless helm_releases.all? { |item| item.is_a?(Hash) } + + helm_releases + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 7c5525b982c..2110d586d30 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -19,7 +19,9 @@ module Gitlab apps: { group: 'apis/apps', version: 'v1' }, extensions: { group: 'apis/extensions', version: 'v1beta1' }, istio: { group: 'apis/networking.istio.io', version: 'v1alpha3' }, - knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' } + knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' }, + metrics: { group: 'apis/metrics.k8s.io', version: 'v1beta1' }, + networking: { group: 'apis/networking.k8s.io', version: 'v1' } }.freeze SUPPORTED_API_GROUPS.each do |name, params| @@ -33,7 +35,8 @@ module Gitlab end # Core API methods delegates to the core api group client - delegate :get_pods, + delegate :get_nodes, + :get_pods, :get_secrets, :get_config_map, :get_namespace, @@ -56,9 +59,7 @@ module Gitlab # RBAC methods delegates to the apis/rbac.authorization.k8s.io api # group client - delegate :create_cluster_role_binding, - :get_cluster_role_binding, - :update_cluster_role_binding, + delegate :update_cluster_role_binding, to: :rbac_client # RBAC methods delegates to the apis/rbac.authorization.k8s.io api @@ -70,9 +71,7 @@ module Gitlab # RBAC methods delegates to the apis/rbac.authorization.k8s.io api # group client - delegate :create_role_binding, - :get_role_binding, - :update_role_binding, + delegate :update_role_binding, to: :rbac_client # non-entity methods that can only work with the core client @@ -88,6 +87,14 @@ module Gitlab :update_gateway, to: :istio_client + # NetworkPolicy methods delegate to the apis/networking.k8s.io api + # group client + delegate :create_network_policy, + :get_network_policies, + :update_network_policy, + :delete_network_policy, + to: :networking_client + attr_reader :api_prefix, :kubeclient_options DEFAULT_KUBECLIENT_OPTIONS = { @@ -97,6 +104,31 @@ module Gitlab } }.freeze + def self.graceful_request(cluster_id) + { status: :connected, response: yield } + rescue *Gitlab::Kubernetes::Errors::CONNECTION + { status: :unreachable } + rescue *Gitlab::Kubernetes::Errors::AUTHENTICATION + { status: :authentication_failure } + rescue Kubeclient::HttpError => e + { status: kubeclient_error_status(e.message) } + rescue => e + Gitlab::ErrorTracking.track_exception(e, cluster_id: cluster_id) + + { status: :unknown_failure } + end + + # KubeClient uses the same error class + # For connection errors (eg. timeout) and + # for Kubernetes errors. + def self.kubeclient_error_status(message) + if message&.match?(/timed out|timeout/i) + :unreachable + else + :authentication_failure + end + end + # We disable redirects through 'http_max_redirects: 0', # so that KubeClient does not follow redirects and # expose internal services. @@ -125,19 +157,11 @@ module Gitlab end def create_or_update_cluster_role_binding(resource) - if cluster_role_binding_exists?(resource) - update_cluster_role_binding(resource) - else - create_cluster_role_binding(resource) - end + update_cluster_role_binding(resource) end def create_or_update_role_binding(resource) - if role_binding_exists?(resource) - update_role_binding(resource) - else - create_role_binding(resource) - end + update_role_binding(resource) end def create_or_update_service_account(resource) @@ -164,18 +188,6 @@ module Gitlab Gitlab::UrlBlocker.validate!(api_prefix, allow_local_network: false) end - def cluster_role_binding_exists?(resource) - get_cluster_role_binding(resource.metadata.name) - rescue ::Kubeclient::ResourceNotFoundError - false - end - - def role_binding_exists?(resource) - get_role_binding(resource.metadata.name, resource.metadata.namespace) - rescue ::Kubeclient::ResourceNotFoundError - false - end - def service_account_exists?(resource) get_service_account(resource.metadata.name, resource.metadata.namespace) rescue ::Kubeclient::ResourceNotFoundError diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb new file mode 100644 index 00000000000..ea25d81cbd2 --- /dev/null +++ b/lib/gitlab/kubernetes/network_policy.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class NetworkPolicy + def initialize(name:, namespace:, pod_selector:, ingress:, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil) + @name = name + @namespace = namespace + @creation_timestamp = creation_timestamp + @pod_selector = pod_selector + @policy_types = policy_types + @ingress = ingress + @egress = egress + end + + def self.from_yaml(manifest) + return unless manifest + + policy = YAML.safe_load(manifest, symbolize_names: true) + return if !policy[:metadata] || !policy[:spec] + + metadata = policy[:metadata] + spec = policy[:spec] + self.new( + name: metadata[:name], + namespace: metadata[:namespace], + pod_selector: spec[:podSelector], + policy_types: spec[:policyTypes], + ingress: spec[:ingress], + egress: spec[:egress] + ) + rescue Psych::SyntaxError, Psych::DisallowedClass + nil + end + + def self.from_resource(resource) + return unless resource + return if !resource[:metadata] || !resource[:spec] + + metadata = resource[:metadata] + spec = resource[:spec].to_h + self.new( + name: metadata[:name], + namespace: metadata[:namespace], + creation_timestamp: metadata[:creationTimestamp], + pod_selector: spec[:podSelector], + policy_types: spec[:policyTypes], + ingress: spec[:ingress], + egress: spec[:egress] + ) + end + + def generate + ::Kubeclient::Resource.new.tap do |resource| + resource.metadata = metadata + resource.spec = spec + end + end + + def as_json(opts = nil) + { + name: name, + namespace: namespace, + creation_timestamp: creation_timestamp, + manifest: manifest + } + end + + private + + attr_reader :name, :namespace, :creation_timestamp, :pod_selector, :policy_types, :ingress, :egress + + def metadata + { name: name, namespace: namespace } + end + + def spec + { + podSelector: pod_selector, + policyTypes: policy_types, + ingress: ingress, + egress: egress + } + end + + def manifest + YAML.dump({ metadata: metadata, spec: spec }.deep_stringify_keys) + end + end + end +end diff --git a/lib/gitlab/logging/cloudflare_helper.rb b/lib/gitlab/logging/cloudflare_helper.rb new file mode 100644 index 00000000000..5cffe335bb5 --- /dev/null +++ b/lib/gitlab/logging/cloudflare_helper.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Logging + module CloudflareHelper + CLOUDFLARE_CUSTOM_HEADERS = { 'Cf-Ray' => :cf_ray, 'Cf-Request-Id' => :cf_request_id }.freeze + + def store_cloudflare_headers!(payload, request) + CLOUDFLARE_CUSTOM_HEADERS.each do |header, value| + payload[value] = request.headers[header] if valid_cloudflare_header?(request.headers[header]) + end + end + + def valid_cloudflare_header?(value) + return false unless value.present? + return false if value.length > 64 + return false if value.index(/[^[A-Za-z0-9-]]/) + + true + end + end + end +end diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index 145d67d7101..55c46c365f6 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -3,6 +3,8 @@ module Gitlab module Lograge module CustomOptions + include ::Gitlab::Logging::CloudflareHelper + LIMITED_ARRAY_SENTINEL = { key: 'truncated', value: '...' }.freeze IGNORE_PARAMS = Set.new(%w(controller action format)).freeze @@ -21,6 +23,8 @@ module Gitlab queue_duration_s: event.payload[:queue_duration_s] } + payload.merge!(event.payload[:metadata]) if event.payload[:metadata] + ::Gitlab::InstrumentationHelper.add_instrumentation_data(payload) payload[:response] = event.payload[:response] if event.payload[:response] @@ -31,6 +35,10 @@ module Gitlab payload[:cpu_s] = cpu_s.round(2) end + CLOUDFLARE_CUSTOM_HEADERS.each do |_, value| + payload[value] = event.payload[value] if event.payload[value] + end + # https://github.com/roidrage/lograge#logging-errors--exceptions exception = event.payload[:exception_object] diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index bd69843adf1..0633efc6b0c 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -19,7 +19,8 @@ module Gitlab start_tls: false, mailbox: 'inbox', idle_timeout: 60, - log_path: RAILS_ROOT_DIR.join('log', 'mail_room_json.log') + log_path: RAILS_ROOT_DIR.join('log', 'mail_room_json.log'), + expunge_deleted: false }.freeze # Email specific configuration which is merged with configuration diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb index d7a0a9b6518..489fc6fddac 100644 --- a/lib/gitlab/markdown_cache.rb +++ b/lib/gitlab/markdown_cache.rb @@ -3,7 +3,7 @@ module Gitlab module MarkdownCache # Increment this number every time the renderer changes its output - CACHE_COMMONMARK_VERSION = 20 + CACHE_COMMONMARK_VERSION = 21 CACHE_COMMONMARK_VERSION_START = 10 BaseError = Class.new(StandardError) diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index d759ae24051..5fed3d38d7c 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -2,17 +2,106 @@ module Gitlab module Metrics - include Gitlab::Metrics::InfluxDb include Gitlab::Metrics::Prometheus + include Gitlab::Metrics::Methods + + EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze @error = false def self.enabled? - influx_metrics_enabled? || prometheus_metrics_enabled? + prometheus_metrics_enabled? end def self.error? @error end + + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def self.add_event(*args) + current_transaction&.add_event(*args) + end + + # Allow access from other metrics related middlewares + def self.current_transaction + Transaction.current + end + + # Returns the prefix to use for the name of a series. + def self.series_prefix + @series_prefix ||= Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_' + end + + def self.settings + @settings ||= begin + current_settings = Gitlab::CurrentSettings.current_application_settings + + { + + method_call_threshold: current_settings[:metrics_method_call_threshold] + + } + end + end + + def self.method_call_threshold + # This is memoized since this method is called for every instrumented + # method. Loading data from an external cache on every method call slows + # things down too much. + # in milliseconds + @method_call_threshold ||= settings[:method_call_threshold] + end + + # Measures the execution time of a block. + # + # Example: + # + # Gitlab::Metrics.measure(:find_by_username_duration) do + # UserFinder.new(some_username).find_by_username + # end + # + # name - The name of the field to store the execution time in. + # + # Returns the value yielded by the supplied block. + def self.measure(name) + trans = current_transaction + + return yield unless trans + + real_start = System.monotonic_time + cpu_start = System.cpu_time + + retval = yield + + cpu_stop = System.cpu_time + real_stop = System.monotonic_time + + real_time = (real_stop - real_start) + cpu_time = cpu_stop - cpu_start + + real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do + docstring "Measure #{name}" + base_labels Transaction::BASE_LABELS + buckets EXECUTION_MEASUREMENT_BUCKETS + end + + real_duration_seconds.observe(trans.labels, real_time) + + cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do + docstring "Measure #{name}" + base_labels Transaction::BASE_LABELS + buckets EXECUTION_MEASUREMENT_BUCKETS + with_feature "prometheus_metrics_measure_#{name}_cpu_duration" + end + cpu_duration_seconds.observe(trans.labels, cpu_time) + + trans.increment("#{name}_real_time", real_time.in_milliseconds, false) + trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false) + trans.increment("#{name}_call_count", 1, false) + + retval + end end end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index 1d948883151..31670a3f533 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -23,7 +23,7 @@ module Gitlab %r{ /environments /(?<environment>\d+) - /metrics + /(metrics_dashboard|metrics) }x ) end diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb index 5ba7b29734b..054b4949dd6 100644 --- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -32,7 +32,7 @@ module Gitlab Sidekiq.logger.error( class: self.class.to_s, message: 'Cannot start sidekiq_exporter', - exception: e.message + 'exception.message' => e.message ) false diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb deleted file mode 100644 index 1f252572461..00000000000 --- a/lib/gitlab/metrics/influx_db.rb +++ /dev/null @@ -1,183 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module InfluxDb - extend ActiveSupport::Concern - include Gitlab::Metrics::Methods - - EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze - - MUTEX = Mutex.new - private_constant :MUTEX - - class_methods do - def influx_metrics_enabled? - settings[:enabled] || false - end - - # Prometheus histogram buckets used for arbitrary code measurements - - def settings - @settings ||= begin - current_settings = Gitlab::CurrentSettings.current_application_settings - - { - enabled: current_settings[:metrics_enabled], - pool_size: current_settings[:metrics_pool_size], - timeout: current_settings[:metrics_timeout], - method_call_threshold: current_settings[:metrics_method_call_threshold], - host: current_settings[:metrics_host], - port: current_settings[:metrics_port], - sample_interval: current_settings[:metrics_sample_interval] || 15, - packet_size: current_settings[:metrics_packet_size] || 1 - } - end - end - - def mri? - RUBY_ENGINE == 'ruby' - end - - def method_call_threshold - # This is memoized since this method is called for every instrumented - # method. Loading data from an external cache on every method call slows - # things down too much. - # in milliseconds - @method_call_threshold ||= settings[:method_call_threshold] - end - - def submit_metrics(metrics) - prepared = prepare_metrics(metrics) - - pool&.with do |connection| - prepared.each_slice(settings[:packet_size]) do |slice| - connection.write_points(slice) - rescue StandardError - end - end - rescue Errno::EADDRNOTAVAIL, SocketError => ex - Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') - Gitlab::EnvironmentLogger.error(ex) - end - - def prepare_metrics(metrics) - metrics.map do |hash| - new_hash = hash.symbolize_keys - - new_hash[:tags].each do |key, value| - if value.blank? - new_hash[:tags].delete(key) - else - new_hash[:tags][key] = escape_value(value) - end - end - - new_hash - end - end - - def escape_value(value) - value.to_s.gsub('=', '\\=') - end - - # Measures the execution time of a block. - # - # Example: - # - # Gitlab::Metrics.measure(:find_by_username_duration) do - # UserFinder.new(some_username).find_by_username - # end - # - # name - The name of the field to store the execution time in. - # - # Returns the value yielded by the supplied block. - def measure(name) - trans = current_transaction - - return yield unless trans - - real_start = Time.now.to_f - cpu_start = System.cpu_time - - retval = yield - - cpu_stop = System.cpu_time - real_stop = Time.now.to_f - - real_time = (real_stop - real_start) - cpu_time = cpu_stop - cpu_start - - real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do - docstring "Measure #{name}" - base_labels Transaction::BASE_LABELS - buckets EXECUTION_MEASUREMENT_BUCKETS - end - - real_duration_seconds.observe(trans.labels, real_time) - - cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do - docstring "Measure #{name}" - base_labels Transaction::BASE_LABELS - buckets EXECUTION_MEASUREMENT_BUCKETS - with_feature "prometheus_metrics_measure_#{name}_cpu_duration" - end - cpu_duration_seconds.observe(trans.labels, cpu_time) - - # InfluxDB stores the _real_time and _cpu_time time values as milliseconds - trans.increment("#{name}_real_time", real_time.in_milliseconds, false) - trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false) - trans.increment("#{name}_call_count", 1, false) - - retval - end - - # Sets the action of the current transaction (if any) - # - # action - The name of the action. - def action=(action) - trans = current_transaction - - trans&.action = action - end - - # Tracks an event. - # - # See `Gitlab::Metrics::Transaction#add_event` for more details. - def add_event(*args) - current_transaction&.add_event(*args) - end - - # Returns the prefix to use for the name of a series. - def series_prefix - @series_prefix ||= Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_' - end - - # Allow access from other metrics related middlewares - def current_transaction - Transaction.current - end - - # When enabled this should be set before being used as the usual pattern - # "@foo ||= bar" is _not_ thread-safe. - def pool - if influx_metrics_enabled? - if @pool.nil? - MUTEX.synchronize do - @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do - host = settings[:host] - port = settings[:port] - - InfluxDB::Client - .new(udp: { host: host, port: port }) - end - end - end - - @pool - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index d0c63a862c2..fbeda3b75e0 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -49,19 +49,6 @@ module Gitlab retval end - # Returns a Metric instance of the current method call. - def to_metric - Metric.new( - Instrumentation.series, - { - duration: real_time.in_milliseconds.to_i, - cpu_duration: cpu_time.in_milliseconds.to_i, - call_count: call_count - }, - method: @name - ) - end - # Returns true if the total runtime of this method exceeds the method call # threshold. def above_threshold? diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb deleted file mode 100644 index 30f181542be..00000000000 --- a/lib/gitlab/metrics/metric.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - # Class for storing details of a single metric (label, value, etc). - class Metric - JITTER_RANGE = (0.000001..0.001).freeze - - attr_reader :series, :values, :tags, :type - - # series - The name of the series (as a String) to store the metric in. - # values - A Hash containing the values to store. - # tags - A Hash containing extra tags to add to the metrics. - def initialize(series, values, tags = {}, type = :metric) - @values = values - @series = series - @tags = tags - @type = type - end - - def event? - type == :event - end - - # Returns a Hash in a format that can be directly written to InfluxDB. - def to_hash - # InfluxDB overwrites an existing point if a new point has the same - # series, tag set, and timestamp. In a highly concurrent environment - # this means that using the number of seconds since the Unix epoch is - # inevitably going to collide with another timestamp. For example, two - # Rails requests processed by different processes may end up generating - # metrics using the _exact_ same timestamp (in seconds). - # - # Due to the way InfluxDB is set up there's no solution to this problem, - # all we can do is lower the amount of collisions. We do this by using - # System.real_time which returns the nanoseconds as a Float providing - # greater accuracy. We then add a small random value that is large - # enough to distinguish most timestamps but small enough to not alter - # the timestamp significantly. - # - # See https://gitlab.com/gitlab-com/operations/issues/175 for more - # information. - time = System.real_time(:nanosecond) + rand(JITTER_RANGE) - - { - series: @series, - tags: @tags, - values: @values, - timestamp: time.to_i - } - end - end - end -end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 9aa97515961..c6a0457ffe5 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -20,10 +20,6 @@ module Gitlab trans.add_event(:rails_exception) raise error - # Even in the event of an error we want to submit any metrics we - # might've gathered up to this point. - ensure - trans.finish end retval diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb new file mode 100644 index 00000000000..9ee4b0960c5 --- /dev/null +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Samplers + class DatabaseSampler < BaseSampler + SAMPLING_INTERVAL_SECONDS = 5 + + METRIC_PREFIX = 'gitlab_database_connection_pool_' + + METRIC_DESCRIPTIONS = { + size: 'Total connection pool capacity', + connections: 'Current connections in the pool', + busy: 'Connections in use where the owner is still alive', + dead: 'Connections in use where the owner is not alive', + idle: 'Connections not in use', + waiting: 'Threads currently waiting on this queue' + }.freeze + + def metrics + @metrics ||= init_metrics + end + + def sample + host_stats.each do |host_stat| + METRIC_DESCRIPTIONS.each_key do |metric| + metrics[metric].set(host_stat[:labels], host_stat[:stats][metric]) + end + end + end + + private + + def init_metrics + METRIC_DESCRIPTIONS.map do |name, description| + [name, ::Gitlab::Metrics.gauge(:"#{METRIC_PREFIX}#{name}", description)] + end.to_h + end + + def host_stats + return [] unless ActiveRecord::Base.connected? + + [{ labels: labels_for_class(ActiveRecord::Base), stats: ActiveRecord::Base.connection_pool.stat }] + end + + def labels_for_class(klass) + { + host: klass.connection_config[:host], + port: klass.connection_config[:port], + class: klass.to_s + } + end + end + end + end +end + +Gitlab::Metrics::Samplers::DatabaseSampler.prepend_if_ee('EE::Gitlab::Metrics::Samplers::DatabaseSampler') diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb deleted file mode 100644 index 4e16e335bee..00000000000 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Samplers - # Class that sends certain metrics to InfluxDB at a specific interval. - # - # This class is used to gather statistics that can't be directly associated - # with a transaction such as system memory usage, garbage collection - # statistics, etc. - class InfluxSampler < BaseSampler - # interval - The sampling interval in seconds. - def initialize(interval = ::Gitlab::Metrics.settings[:sample_interval]) - super(interval) - @last_step = nil - - @metrics = [] - end - - def sample - sample_memory_usage - sample_file_descriptors - - flush - ensure - @metrics.clear - end - - def flush - ::Gitlab::Metrics.submit_metrics(@metrics.map(&:to_hash)) - end - - def sample_memory_usage - add_metric('memory_usage', value: System.memory_usage) - end - - def sample_file_descriptors - add_metric('file_descriptors', value: System.file_descriptor_count) - end - - def add_metric(series, values, tags = {}) - prefix = Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_' - - @metrics << Metric.new("#{prefix}#{series}", values, tags) - end - end - end - end -end diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index f788f51b1ce..98dd517ee3b 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -26,7 +26,7 @@ module Gitlab json_stats = puma_stats return unless json_stats - stats = JSON.parse(json_stats) + stats = Gitlab::Json.parse(json_stats) if cluster?(stats) sample_cluster(stats) diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index c38769f39a9..df59c06911b 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -34,14 +34,15 @@ module Gitlab def init_metrics metrics = { - file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels), - memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels), - process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), - process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), - process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels), - process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), - sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), - gc_duration_seconds: ::Gitlab::Metrics.histogram(with_prefix(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS) + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used (RSS)', labels), + process_unique_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :unique_memory_bytes), 'Memory used (USS)', labels), + process_proportional_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :proportional_memory_bytes), 'Memory used (PSS)', labels), + process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), + sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), + gc_duration_seconds: ::Gitlab::Metrics.histogram(with_prefix(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS) } GC.stat.keys.each do |key| @@ -85,10 +86,13 @@ module Gitlab end def set_memory_usage_metrics - memory_usage = System.memory_usage + metrics[:process_resident_memory_bytes].set(labels, System.memory_usage_rss) - metrics[:memory_bytes].set(labels, memory_usage) - metrics[:process_resident_memory_bytes].set(labels, memory_usage) + if Gitlab::Utils.to_boolean(ENV['enable_memory_uss_pss'] || '1') + memory_uss_pss = System.memory_usage_uss_pss + metrics[:process_unique_memory_bytes].set(labels, memory_uss_pss[:uss]) + metrics[:process_proportional_memory_bytes].set(labels, memory_uss_pss[:pss]) + end end end end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index 0b4485feea9..8dfb61046c4 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -17,8 +17,6 @@ module Gitlab trans.add_event(:sidekiq_exception) raise error - ensure - trans.finish end end end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 5bd21b8e5d1..24107e42aa9 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -26,23 +26,17 @@ module Gitlab private def track(event) - values = values_for(event) - tags = tags_for(event) + tags = tags_for(event) self.class.gitlab_view_rendering_duration_seconds.observe(current_transaction.labels.merge(tags), event.duration) current_transaction.increment(:view_duration, event.duration) - current_transaction.add_metric(SERIES, values, tags) end def relative_path(path) path.gsub(%r{^#{Rails.root}/?}, '') end - def values_for(event) - { duration: event.duration } - end - def tags_for(event) path = relative_path(event.payload[:identifier]) diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 2a61b3de405..43005303dec 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -7,47 +7,37 @@ module Gitlab # This module relies on the /proc filesystem being available. If /proc is # not available the methods of this module will be stubbed. module System - if File.exist?('/proc') - # Returns the current process' memory usage in bytes. - def self.memory_usage - mem = 0 - match = File.read('/proc/self/status').match(/VmRSS:\s+(\d+)/) - - if match && match[1] - mem = match[1].to_f * 1024 - end - - mem - end - - def self.file_descriptor_count - Dir.glob('/proc/self/fd/*').length - end - - def self.max_open_file_descriptors - match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/) - - return unless match && match[1] + PROC_STATUS_PATH = '/proc/self/status' + PROC_SMAPS_ROLLUP_PATH = '/proc/self/smaps_rollup' + PROC_LIMITS_PATH = '/proc/self/limits' + PROC_FD_GLOB = '/proc/self/fd/*' + + PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/.freeze + PSS_PATTERN = /^Pss:\s+(?<value>\d+)/.freeze + RSS_PATTERN = /VmRSS:\s+(?<value>\d+)/.freeze + MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/.freeze + + # Returns the current process' RSS (resident set size) in bytes. + def self.memory_usage_rss + sum_matches(PROC_STATUS_PATH, rss: RSS_PATTERN)[:rss].kilobytes + end - match[1].to_i - end - else - def self.memory_usage - 0.0 - end + # Returns the current process' USS/PSS (unique/proportional set size) in bytes. + def self.memory_usage_uss_pss + sum_matches(PROC_SMAPS_ROLLUP_PATH, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) + .transform_values(&:kilobytes) + end - def self.file_descriptor_count - 0 - end + def self.file_descriptor_count + Dir.glob(PROC_FD_GLOB).length + end - def self.max_open_file_descriptors - 0 - end + def self.max_open_file_descriptors + sum_matches(PROC_LIMITS_PATH, max_fds: MAX_OPEN_FILES_PATTERN)[:max_fds] end def self.cpu_time - Process - .clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :float_second) + Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :float_second) end # Returns the current real time in a given precision. @@ -78,6 +68,27 @@ module Gitlab end_time - start_time end + + # Given a path to a file in /proc and a hash of (metric, pattern) pairs, + # sums up all values found for those patterns under the respective metric. + def self.sum_matches(proc_file, **patterns) + results = patterns.transform_values { 0 } + + begin + File.foreach(proc_file) do |line| + patterns.each do |metric, pattern| + match = line.match(pattern) + value = match&.named_captures&.fetch('value', 0) + results[metric] += value.to_i + end + end + rescue Errno::ENOENT + # This means the procfile we're reading from did not exist; + # this is safe to ignore, since we initialize each metric to 0 + end + + results + end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 552eae639e6..b126efd2dd5 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -16,20 +16,18 @@ module Gitlab # The series to store events (e.g. Git pushes) in. EVENT_SERIES = 'events' - attr_reader :tags, :values, :method, :metrics + attr_reader :tags, :method def self.current Thread.current[THREAD_KEY] end def initialize - @metrics = [] @methods = {} @started_at = nil @finished_at = nil - @values = Hash.new(0) @tags = {} @memory_before = 0 @@ -40,10 +38,6 @@ module Gitlab @finished_at ? (@finished_at - @started_at) : 0.0 end - def duration_milliseconds - duration.in_milliseconds.to_i - end - def thread_cpu_duration System.thread_cpu_duration(@thread_cputime_start) end @@ -55,13 +49,13 @@ module Gitlab def run Thread.current[THREAD_KEY] = self - @memory_before = System.memory_usage + @memory_before = System.memory_usage_rss @started_at = System.monotonic_time @thread_cputime_start = System.thread_cpu_time yield ensure - @memory_after = System.memory_usage + @memory_after = System.memory_usage_rss @finished_at = System.monotonic_time self.class.gitlab_transaction_cputime_seconds.observe(labels, thread_cpu_duration) @@ -71,10 +65,6 @@ module Gitlab Thread.current[THREAD_KEY] = nil end - def add_metric(series, values, tags = {}) - @metrics << Metric.new("#{::Gitlab::Metrics.series_prefix}#{series}", values, filter_tags(tags)) - end - # Tracks a business level event # # Business level events including events such as Git pushes, Emails being @@ -85,7 +75,6 @@ module Gitlab def add_event(event_name, tags = {}) filtered_tags = filter_tags(tags) self.class.transaction_metric(event_name, :counter, prefix: 'event_', tags: filtered_tags).increment(filtered_tags.merge(labels)) - @metrics << Metric.new(EVENT_SERIES, { count: 1 }, filtered_tags.merge(event: event_name), :event) end # Returns a MethodCall object for the given name. @@ -99,55 +88,16 @@ module Gitlab def increment(name, value, use_prometheus = true) self.class.transaction_metric(name, :counter).increment(labels, value) if use_prometheus - @values[name] += value end def set(name, value, use_prometheus = true) self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus - @values[name] = value - end - - def finish - track_self - submit - end - - def track_self - values = { duration: duration_milliseconds, allocated_memory: allocated_memory } - - @values.each do |name, value| - values[name] = value - end - - add_metric('transactions', values, @tags) - end - - def submit - submit = @metrics.dup - - @methods.each do |name, method| - submit << method.to_metric if method.above_threshold? - end - - submit_hashes = submit.map do |metric| - hash = metric.to_hash - hash[:tags][:action] ||= action if action && !metric.event? - - hash - end - - ::Gitlab::Metrics.submit_metrics(submit_hashes) end def labels BASE_LABELS end - # returns string describing the action performed, usually the class plus method name. - def action - "#{labels[:controller]}##{labels[:action]}" if labels && !labels.empty? - end - define_histogram :gitlab_transaction_cputime_seconds do docstring 'Transaction thread cputime' base_labels BASE_LABELS diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 7d0de3aee1c..3c45f841653 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -110,6 +110,7 @@ module Gitlab ::FileUploader.root, Gitlab.config.uploads.storage_path, JobArtifactUploader.workhorse_upload_path, + LfsObjectUploader.workhorse_upload_path, File.join(Rails.root, 'public/uploads/tmp') ] end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index cdab86540f8..1c49379e8d2 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -4,8 +4,6 @@ module Gitlab module Middleware class ReadOnly class Controller - prepend_if_ee('EE::Gitlab::Middleware::ReadOnly::Controller') # rubocop: disable Cop/InjectEnterpriseEditionModule - DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze APPLICATION_JSON = 'application/json' APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze @@ -144,3 +142,5 @@ module Gitlab end end end + +Gitlab::Middleware::ReadOnly::Controller.prepend_if_ee('EE::Gitlab::Middleware::ReadOnly::Controller') diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index c051a581837..b60ecb6631b 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -2,7 +2,7 @@ module Gitlab class OmniauthInitializer - prepend_if_ee('::EE::Gitlab::OmniauthInitializer') # rubocop: disable Cop/InjectEnterpriseEditionModule + OAUTH2_TIMEOUT_SECONDS = 10 def initialize(devise_config) @devise_config = devise_config @@ -17,6 +17,47 @@ module Gitlab end end + class << self + def default_arguments_for(provider_name) + case provider_name + when 'cas3' + { on_single_sign_out: cas3_signout_handler } + when 'authentiq' + { remote_sign_out_handler: authentiq_signout_handler } + when 'shibboleth' + { fail_with_empty_uid: true } + when 'google_oauth2' + { client_options: { connection_opts: { request: { timeout: OAUTH2_TIMEOUT_SECONDS } } } } + else + {} + end + end + + private + + def cas3_signout_handler + lambda do |request| + ticket = request.params[:session_index] + raise "Service Ticket not found." unless Gitlab::Auth::OAuth::Session.valid?(:cas3, ticket) + + Gitlab::Auth::OAuth::Session.destroy(:cas3, ticket) + true + end + end + + def authentiq_signout_handler + lambda do |request| + authentiq_session = request.params['sid'] + if Gitlab::Auth::OAuth::Session.valid?(:authentiq, authentiq_session) + Gitlab::Auth::OAuth::Session.destroy(:authentiq, authentiq_session) + true + else + false + end + end + end + end + private def add_provider_to_devise(*args) @@ -35,7 +76,8 @@ module Gitlab # An Array from the configuration will be expanded. provider_arguments.concat provider['args'] when Hash - hash_arguments = provider['args'].merge(provider_defaults(provider)) + defaults = provider_defaults(provider) + hash_arguments = provider['args'].deep_symbolize_keys.deep_merge(defaults) # A Hash from the configuration will be passed as is. provider_arguments << normalize_hash_arguments(hash_arguments) @@ -45,7 +87,7 @@ module Gitlab end def normalize_hash_arguments(args) - args.symbolize_keys! + args.deep_symbolize_keys! # Rails 5.1 deprecated the use of string names in the middleware # (https://github.com/rails/rails/commit/83b767ce), so we need to @@ -68,38 +110,7 @@ module Gitlab end def provider_defaults(provider) - case provider['name'] - when 'cas3' - { on_single_sign_out: cas3_signout_handler } - when 'authentiq' - { remote_sign_out_handler: authentiq_signout_handler } - when 'shibboleth' - { fail_with_empty_uid: true } - else - {} - end - end - - def cas3_signout_handler - lambda do |request| - ticket = request.params[:session_index] - raise "Service Ticket not found." unless Gitlab::Auth::OAuth::Session.valid?(:cas3, ticket) - - Gitlab::Auth::OAuth::Session.destroy(:cas3, ticket) - true - end - end - - def authentiq_signout_handler - lambda do |request| - authentiq_session = request.params['sid'] - if Gitlab::Auth::OAuth::Session.valid?(:authentiq, authentiq_session) - Gitlab::Auth::OAuth::Session.destroy(:authentiq, authentiq_session) - true - else - false - end - end + self.class.default_arguments_for(provider['name']) end def omniauth_customized_providers @@ -121,3 +132,5 @@ module Gitlab end end end + +Gitlab::OmniauthInitializer.prepend_if_ee('::EE::Gitlab::OmniauthInitializer') diff --git a/lib/gitlab/pagination/keyset.rb b/lib/gitlab/pagination/keyset.rb index 8692f30e165..67a5530d46c 100644 --- a/lib/gitlab/pagination/keyset.rb +++ b/lib/gitlab/pagination/keyset.rb @@ -3,11 +3,18 @@ module Gitlab module Pagination module Keyset + SUPPORTED_TYPES = [ + Project + ].freeze + + def self.available_for_type?(relation) + SUPPORTED_TYPES.include?(relation.klass) + end + def self.available?(request_context, relation) order_by = request_context.page.order_by - # This is only available for Project and order-by id (asc/desc) - return false unless relation.klass == Project + return false unless available_for_type?(relation) return false unless order_by.size == 1 && order_by[:id] true diff --git a/lib/gitlab/patch/draw_route.rb b/lib/gitlab/patch/draw_route.rb index 4d1b57fbbbb..f5fcd5c6093 100644 --- a/lib/gitlab/patch/draw_route.rb +++ b/lib/gitlab/patch/draw_route.rb @@ -5,8 +5,6 @@ module Gitlab module Patch module DrawRoute - prepend_if_ee('EE::Gitlab::Patch::DrawRoute') # rubocop: disable Cop/InjectEnterpriseEditionModule - RoutesNotFound = Class.new(StandardError) def draw(routes_name) @@ -38,3 +36,5 @@ module Gitlab end end end + +Gitlab::Patch::DrawRoute.prepend_if_ee('EE::Gitlab::Patch::DrawRoute') diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 4445c876e7a..e26309b5dfd 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -44,7 +44,7 @@ module Gitlab end def self.l1_cache_backend - Gitlab::ThreadMemoryCache.cache_backend + Gitlab::ProcessMemoryCache.cache_backend end def self.l2_cache_backend diff --git a/lib/gitlab/phabricator_import/conduit/response.rb b/lib/gitlab/phabricator_import/conduit/response.rb index 6053ecfbd5e..1b03cfa05e6 100644 --- a/lib/gitlab/phabricator_import/conduit/response.rb +++ b/lib/gitlab/phabricator_import/conduit/response.rb @@ -9,7 +9,7 @@ module Gitlab "Phabricator responded with #{http_response.status}" end - response = new(JSON.parse(http_response.body)) + response = new(Gitlab::Json.parse(http_response.body)) unless response.success? raise ResponseError, diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index eb7ca80dd60..fbdfe166645 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -2,30 +2,29 @@ module Gitlab class ProjectSearchResults < SearchResults - attr_reader :project, :repository_ref, :per_page + attr_reader :project, :repository_ref - def initialize(current_user, project, query, repository_ref = nil, per_page: 20) + def initialize(current_user, project, query, repository_ref = nil) @current_user = current_user @project = project @repository_ref = repository_ref.presence @query = query - @per_page = per_page end - def objects(scope, page = nil) + def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE) case scope when 'notes' notes.page(page).per(per_page) when 'blobs' - paginated_blobs(blobs(page), page) + paginated_blobs(blobs(limit: limit_up_to_page(page, per_page)), page, per_page) when 'wiki_blobs' - paginated_blobs(wiki_blobs, page) + paginated_wiki_blobs(wiki_blobs(limit: limit_up_to_page(page, per_page)), page, per_page) when 'commits' Kaminari.paginate_array(commits).page(page).per(per_page) when 'users' users.page(page).per(per_page) else - super(scope, page, false) + super(scope, page: page, per_page: per_page, without_count: false) end end @@ -49,7 +48,7 @@ module Gitlab end def limited_blobs_count - @limited_blobs_count ||= blobs.count + @limited_blobs_count ||= blobs(limit: count_limit).count end # rubocop: disable CodeReuse/ActiveRecord @@ -69,7 +68,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def wiki_blobs_count - @wiki_blobs_count ||= wiki_blobs.count + @wiki_blobs_count ||= wiki_blobs(limit: count_limit).count end def commits_count @@ -87,7 +86,7 @@ module Gitlab private - def paginated_blobs(blobs, page) + def paginated_blobs(blobs, page, per_page) results = Kaminari.paginate_array(blobs).page(page).per(per_page) Gitlab::Search::FoundBlob.preload_blobs(results) @@ -95,19 +94,26 @@ module Gitlab results end - def limit_up_to_page(page) + def paginated_wiki_blobs(blobs, page, per_page) + blob_array = paginated_blobs(blobs, page, per_page) + blob_array.map! do |blob| + Gitlab::Search::FoundWikiPage.new(blob) + end + end + + def limit_up_to_page(page, per_page) current_page = page&.to_i || 1 offset = per_page * (current_page - 1) count_limit + offset end - def blobs(page = 1) + def blobs(limit: count_limit) return [] unless Ability.allowed?(@current_user, :download_code, @project) - @blobs ||= Gitlab::FileFinder.new(project, repository_project_ref).find(query, content_match_cutoff: limit_up_to_page(page)) + @blobs ||= Gitlab::FileFinder.new(project, repository_project_ref).find(query, content_match_cutoff: limit) end - def wiki_blobs + def wiki_blobs(limit: count_limit) return [] unless Ability.allowed?(@current_user, :read_wiki, @project) @wiki_blobs ||= begin @@ -115,7 +121,7 @@ module Gitlab if project.wiki.empty? [] else - Gitlab::WikiFileFinder.new(project, repository_wiki_ref).find(query) + Gitlab::WikiFileFinder.new(project, repository_wiki_ref).find(query, content_match_cutoff: limit) end else [] diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 1b6f7282eb3..4a39260a340 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -3,7 +3,6 @@ module Gitlab module Prometheus class MetricGroup - prepend_if_ee('EE::Gitlab::Prometheus::MetricGroup') # rubocop: disable Cop/InjectEnterpriseEditionModule include ActiveModel::Model attr_accessor :name, :priority, :metrics @@ -31,3 +30,5 @@ module Gitlab end end end + +Gitlab::Prometheus::MetricGroup.prepend_if_ee('EE::Gitlab::Prometheus::MetricGroup') diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index a5e7d0ac9d5..d24b98e790b 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -4,8 +4,6 @@ module Gitlab module Prometheus module Queries module QueryAdditionalMetrics - prepend_if_ee('EE::Gitlab::Prometheus::Queries::QueryAdditionalMetrics') # rubocop: disable Cop/InjectEnterpriseEditionModule - def query_metrics(project, environment, query_context) matched_metrics(project).map(&query_group(query_context)) .select(&method(:group_with_any_metrics)) @@ -99,3 +97,5 @@ module Gitlab end end end + +Gitlab::Prometheus::Queries::QueryAdditionalMetrics.prepend_if_ee('EE::Gitlab::Prometheus::Queries::QueryAdditionalMetrics') diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 71a0d528bd7..b03d8a4d254 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -163,7 +163,7 @@ module Gitlab end def parse_json(response_body) - JSON.parse(response_body) + Gitlab::Json.parse(response_body, legacy_mode: true) rescue JSON::ParserError raise PrometheusClient::Error, 'Parsing response failed' end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 942f90e8040..6aa3f515ef0 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -109,7 +109,7 @@ module Gitlab quick_action_target.labels.any? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", parent) end - command :unlabel do |labels_param = nil| + command :unlabel, :remove_label do |labels_param = nil| if labels_param.present? labels = find_labels(labels_param) label_ids = labels.map(&:id) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index db531f06f11..c8b04ce2a5c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -79,6 +79,12 @@ module Gitlab "Must start with a letter, and cannot end with '-'" end + # Pod name adheres to DNS Subdomain Names(RFC 1123) naming convention + # https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names + def kubernetes_dns_subdomain_regex + /\A[a-z0-9]([a-z0-9\-\.]*[a-z0-9])?\z/ + end + def environment_slug_regex @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze end @@ -121,7 +127,7 @@ module Gitlab # Based on Jira's project key format # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html def jira_issue_key_regex - @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+\b/ + @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+/ end def jira_transition_id_regex diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index 9da6732796a..952ae55d90a 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -24,11 +24,12 @@ module Gitlab end def ensure_deadline_not_exceeded! + return unless enabled? return unless request_deadline return if Gitlab::Metrics::System.real_time < request_deadline raise RequestDeadlineExceeded, - "Request takes longer than #{max_request_duration_seconds}" + "Request takes longer than #{max_request_duration_seconds} seconds" end private @@ -36,5 +37,9 @@ module Gitlab def max_request_duration_seconds Settings.gitlab.max_request_duration_seconds end + + def enabled? + !Rails.env.test? + end end end diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb index c2b55431547..9a5917ffba9 100644 --- a/lib/gitlab/rugged_instrumentation.rb +++ b/lib/gitlab/rugged_instrumentation.rb @@ -3,7 +3,8 @@ module Gitlab module RuggedInstrumentation def self.query_time - SafeRequestStore[:rugged_query_time] ||= 0 + query_time = SafeRequestStore[:rugged_query_time] ||= 0 + query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) end def self.query_time=(duration) diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index bf579dd3b77..abf6ee07d53 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -37,7 +37,7 @@ module Gitlab end def puma? - !!defined?(::Puma) + !!defined?(::Puma) && !defined?(ACTION_CABLE_SERVER) end # For unicorn, we need to check for actual server instances to avoid false positives. @@ -70,25 +70,31 @@ module Gitlab end def web_server? - puma? || unicorn? + puma? || unicorn? || action_cable? + end + + def action_cable? + !!defined?(ACTION_CABLE_SERVER) end def multi_threaded? - puma? || sidekiq? + puma? || sidekiq? || action_cable? end def max_threads main_thread = 1 - if puma? - Puma.cli_config.options[:max_threads] + main_thread + if action_cable? + Gitlab::Application.config.action_cable.worker_pool_size + elsif puma? + Puma.cli_config.options[:max_threads] elsif sidekiq? # An extra thread for the poller in Sidekiq Cron: # https://github.com/ondrejbartas/sidekiq-cron#under-the-hood - Sidekiq.options[:concurrency] + main_thread + 1 + Sidekiq.options[:concurrency] + 1 else - main_thread - end + 0 + end + main_thread end end end diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index 5eeb8b00ff3..7e22bf4d7df 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -152,7 +152,7 @@ module Gitlab raise "failed to get exif tags: #{output}" if status != 0 - JSON.parse(output).first + Gitlab::Json.parse(output).first end end end diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb index f3136fff294..1f6e0519b4c 100644 --- a/lib/gitlab/search/parsed_query.rb +++ b/lib/gitlab/search/parsed_query.rb @@ -3,8 +3,6 @@ module Gitlab module Search class ParsedQuery - prepend_if_ee('EE::Gitlab::Search::ParsedQuery') # rubocop: disable Cop/InjectEnterpriseEditionModule - attr_reader :term, :filters def initialize(term, filters) @@ -25,3 +23,5 @@ module Gitlab end end end + +Gitlab::Search::ParsedQuery.prepend_if_ee('EE::Gitlab::Search::ParsedQuery') diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 0473fa89a0d..c35ee62163a 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -4,8 +4,10 @@ module Gitlab class SearchResults COUNT_LIMIT = 100 COUNT_LIMIT_MESSAGE = "#{COUNT_LIMIT - 1}+" + DEFAULT_PAGE = 1 + DEFAULT_PER_PAGE = 20 - attr_reader :current_user, :query, :per_page + attr_reader :current_user, :query # Limit search results by passed projects # It allows us to search only for projects user has access to @@ -17,15 +19,14 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, limit_projects, query, default_project_filter: false, per_page: 20) + def initialize(current_user, limit_projects, query, default_project_filter: false) @current_user = current_user @limit_projects = limit_projects || Project.all @query = query @default_project_filter = default_project_filter - @per_page = per_page end - def objects(scope, page = nil, without_count = true) + def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, without_count: true) collection = case scope when 'projects' projects @@ -39,7 +40,9 @@ module Gitlab users else Kaminari.paginate_array([]) - end.page(page).per(per_page) + end + + collection = collection.page(page).per(per_page) without_count ? collection.without_count : collection end diff --git a/lib/gitlab/services/logger.rb b/lib/gitlab/services/logger.rb new file mode 100644 index 00000000000..4e7ef73922c --- /dev/null +++ b/lib/gitlab/services/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Services + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'service_measurement' + end + end + end +end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 196dc0e3447..3f36725cb66 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -96,7 +96,8 @@ module Gitlab class << self def configuration_toml(gitaly_dir, storage_paths) nodes = [{ storage: 'default', address: "unix:#{gitaly_dir}/gitaly.socket", primary: true, token: 'secret' }] - config = { socket_path: "#{gitaly_dir}/praefect.socket", virtual_storage_name: 'default', token: 'secret', node: nodes } + storages = [{ name: 'default', node: nodes }] + config = { socket_path: "#{gitaly_dir}/praefect.socket", virtual_storage: storages } config[:token] = 'secret' if Rails.env.test? TomlRB.dump(config) diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb index c49432f0fc6..0d0efe8ffbd 100644 --- a/lib/gitlab/sidekiq_config/cli_methods.rb +++ b/lib/gitlab/sidekiq_config/cli_methods.rb @@ -48,7 +48,6 @@ module Gitlab # rubocop:enable Gitlab/ModuleWithInstanceVariables def worker_queues(rails_path = Rails.root.to_s) - # https://gitlab.com/gitlab-org/gitlab/issues/199230 worker_names(all_queues(rails_path)) end @@ -75,7 +74,7 @@ module Gitlab private def worker_names(workers) - workers.map { |queue| queue.is_a?(Hash) ? queue[:name] : queue } + workers.map { |queue| queue[:name] } end def query_string_to_lambda(query_string) diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb index 0723b514c90..1f1d63877b5 100644 --- a/lib/gitlab/sidekiq_daemon/monitor.rb +++ b/lib/gitlab/sidekiq_daemon/monitor.rb @@ -134,7 +134,7 @@ module Gitlab end def safe_parse(message) - JSON.parse(message) + Gitlab::Json.parse(message) rescue JSON::ParserError end diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb index 45c6842c59b..64782e1e1d1 100644 --- a/lib/gitlab/sidekiq_logging/json_formatter.rb +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -19,6 +19,7 @@ module Gitlab output[:message] = data when Hash convert_to_iso8601!(data) + convert_retry_to_integer!(data) stringify_args!(data) output.merge!(data) end @@ -41,6 +42,20 @@ module Gitlab Time.at(timestamp).utc.iso8601(3) end + def convert_retry_to_integer!(payload) + payload['retry'] = + case payload['retry'] + when Integer + payload['retry'] + when false, nil + 0 + when true + Sidekiq::JobRetry::DEFAULT_MAX_RETRY_ATTEMPTS + else + -1 + end + end + def stringify_args!(payload) payload['args'] = Gitlab::Utils::LogLimitedArray.log_limited_array(payload['args'].map(&:to_s)) if payload['args'] end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index ea60190353e..4e39120f8a7 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -30,6 +30,12 @@ module Gitlab output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS)) end + def add_logging_extras!(job, output_payload) + output_payload.merge!( + job.select { |key, _| key.to_s.start_with?("#{ApplicationWorker::LOGGING_EXTRA_KEY}.") } + ) + end + def log_job_start(payload) payload['message'] = "#{base_message(payload)}: start" payload['job_status'] = 'start' @@ -43,6 +49,7 @@ module Gitlab def log_job_done(job, started_time, payload, job_exception = nil) payload = payload.dup add_instrumentation_keys!(job, payload) + add_logging_extras!(job, payload) elapsed_time = elapsed(started_time) add_time_keys!(elapsed_time, payload) @@ -66,11 +73,11 @@ module Gitlab end def add_time_keys!(time, payload) - payload['duration_s'] = time[:duration].round(2) + payload['duration_s'] = time[:duration].round(Gitlab::InstrumentationHelper::DURATION_PRECISION) # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0) # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime - payload['cpu_s'] = time[:cputime].round(2) if time[:cputime] > 0 + payload['cpu_s'] = time[:cputime].round(Gitlab::InstrumentationHelper::DURATION_PRECISION) if time[:cputime] > 0 payload['completed_at'] = Time.now.utc.to_f end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 1b155570f18..4eef3fbd12e 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -7,13 +7,14 @@ module Gitlab # The result of this method should be passed to # Sidekiq's `config.server_middleware` method # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)` - def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true, request_store: true) + def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true) lambda do |chain| chain.add ::Gitlab::SidekiqMiddleware::Monitor chain.add ::Gitlab::SidekiqMiddleware::ServerMetrics if metrics chain.add ::Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger chain.add ::Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer - chain.add ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware if request_store + chain.add ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware + chain.add ::Gitlab::SidekiqMiddleware::ExtraDoneLogMetadata chain.add ::Gitlab::SidekiqMiddleware::BatchLoader chain.add ::Labkit::Middleware::Sidekiq::Server chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb index e227ee654ee..36204e1bee0 100644 --- a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb +++ b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb @@ -8,7 +8,7 @@ module Gitlab # If enabled then it injects a job field that persists through the job execution class Client def call(_worker_class, job, _queue, _redis_pool) - return yield unless Feature.enabled?(:user_mode_in_session) + return yield unless ::Feature.enabled?(:user_mode_in_session) # Admin mode enabled in the original request or in a nested sidekiq job admin_mode_user_id = find_admin_user_id @@ -16,7 +16,7 @@ module Gitlab if admin_mode_user_id job['admin_mode_user_id'] ||= admin_mode_user_id - Gitlab::AppLogger.debug("AdminMode::Client injected admin mode for job: #{job.inspect}") + ::Gitlab::AppLogger.debug("AdminMode::Client injected admin mode for job: #{job.inspect}") end yield @@ -25,8 +25,8 @@ module Gitlab private def find_admin_user_id - Gitlab::Auth::CurrentUserMode.current_admin&.id || - Gitlab::Auth::CurrentUserMode.bypass_session_admin_id + ::Gitlab::Auth::CurrentUserMode.current_admin&.id || + ::Gitlab::Auth::CurrentUserMode.bypass_session_admin_id end end end diff --git a/lib/gitlab/sidekiq_middleware/arguments_logger.rb b/lib/gitlab/sidekiq_middleware/arguments_logger.rb index 2859aa5f4a6..fe5213fc5d7 100644 --- a/lib/gitlab/sidekiq_middleware/arguments_logger.rb +++ b/lib/gitlab/sidekiq_middleware/arguments_logger.rb @@ -4,7 +4,7 @@ module Gitlab module SidekiqMiddleware class ArgumentsLogger def call(worker, job, queue) - Sidekiq.logger.info "arguments: #{JSON.dump(job['args'])}" + Sidekiq.logger.info "arguments: #{Gitlab::Json.dump(job['args'])}" yield end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 79bbb99752e..fa742d07af2 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -67,7 +67,7 @@ module Gitlab end def droppable? - idempotent? && duplicate? + idempotent? && duplicate? && ::Feature.disabled?("disable_#{queue_name}_deduplication") end private diff --git a/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb b/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb new file mode 100644 index 00000000000..93c3131d50e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class ExtraDoneLogMetadata + def call(worker, job, queue) + yield + + # We needed a way to pass state from a worker in to the + # Gitlab::SidekiqLogging::StructuredLogger . Unfortunately the + # StructuredLogger itself is not a middleware so cannot access the + # worker object. We also tried to use SafeRequestStore but to pass the + # data up but that doesn't work either because this is reset in + # Gitlab::SidekiqMiddleware::RequestStoreMiddleware inside yield for + # the StructuredLogger so it's cleared before we get to logging the + # done statement. As such the only way to do this is to pass the data + # up in the `job` object. Since `job` is just a Hash we can add this + # extra metadata there. + if worker.respond_to?(:logging_extras) + job.merge!(worker.logging_extras) + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/issue_base.rb b/lib/gitlab/slash_commands/presenters/issue_base.rb index 4bc05d1f318..017fb8a62c4 100644 --- a/lib/gitlab/slash_commands/presenters/issue_base.rb +++ b/lib/gitlab/slash_commands/presenters/issue_base.rb @@ -4,8 +4,6 @@ module Gitlab module SlashCommands module Presenters module IssueBase - prepend_if_ee('EE::Gitlab::SlashCommands::Presenters::IssueBase') # rubocop: disable Cop/InjectEnterpriseEditionModule - def color(issuable) issuable.open? ? '#38ae67' : '#d22852' end @@ -51,3 +49,5 @@ module Gitlab end end end + +Gitlab::SlashCommands::Presenters::IssueBase.prepend_if_ee('EE::Gitlab::SlashCommands::Presenters::IssueBase') diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index e955ccd35da..9911f9e62a6 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -11,36 +11,18 @@ module Gitlab @query = query end - def objects(scope, page = nil) - case scope - when 'snippet_titles' - paginated_objects(snippet_titles, page) - when 'snippet_blobs' - paginated_objects(snippet_blobs, page) - else - super(scope, nil, false) - end + def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE) + paginated_objects(snippet_titles, page, per_page) end def formatted_count(scope) - case scope - when 'snippet_titles' - formatted_limited_count(limited_snippet_titles_count) - when 'snippet_blobs' - formatted_limited_count(limited_snippet_blobs_count) - else - super - end + formatted_limited_count(limited_snippet_titles_count) end def limited_snippet_titles_count @limited_snippet_titles_count ||= limited_count(snippet_titles) end - def limited_snippet_blobs_count - @limited_snippet_blobs_count ||= limited_count(snippet_blobs) - end - private # rubocop: disable CodeReuse/ActiveRecord @@ -56,15 +38,7 @@ module Gitlab snippets.search(query) end - def snippet_blobs - snippets.search_code(query) - end - - def default_scope - 'snippet_blobs' - end - - def paginated_objects(relation, page) + def paginated_objects(relation, page, per_page) relation.page(page).per(per_page) end diff --git a/lib/gitlab/static_site_editor/config.rb b/lib/gitlab/static_site_editor/config.rb index 41d54ee0a92..c931cdecbeb 100644 --- a/lib/gitlab/static_site_editor/config.rb +++ b/lib/gitlab/static_site_editor/config.rb @@ -22,7 +22,8 @@ module Gitlab project: project.path, namespace: project.namespace.path, return_url: return_url, - is_supported_content: supported_content? + is_supported_content: supported_content?.to_s, + base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path) } end @@ -47,6 +48,10 @@ module Gitlab def file_exists? commit_id.present? && repository.blob_at(commit_id, file_path).present? end + + def full_path + "#{ref}/#{file_path}" + end end end end diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index ac02ec635e4..6ccb442b1e0 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -157,8 +157,8 @@ module Gitlab Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home end - def checkout_or_clone_version(version:, repo:, target_dir:) - clone_repo(repo, target_dir) unless Dir.exist?(target_dir) + def checkout_or_clone_version(version:, repo:, target_dir:, clone_opts: []) + clone_repo(repo, target_dir, clone_opts: clone_opts) unless Dir.exist?(target_dir) checkout_version(get_version(version), target_dir) end @@ -171,8 +171,8 @@ module Gitlab "v#{component_version}" end - def clone_repo(repo, target_dir) - run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) + def clone_repo(repo, target_dir, clone_opts: []) + run_command!(%W[#{Gitlab.config.git.bin_path} clone] + clone_opts + %W[-- #{repo} #{target_dir}]) end def checkout_version(version, target_dir) diff --git a/lib/gitlab/testing/clear_thread_memory_cache_middleware.rb b/lib/gitlab/testing/clear_process_memory_cache_middleware.rb index 6f54038ae22..1e69e5e142d 100644 --- a/lib/gitlab/testing/clear_thread_memory_cache_middleware.rb +++ b/lib/gitlab/testing/clear_process_memory_cache_middleware.rb @@ -2,13 +2,13 @@ module Gitlab module Testing - class ClearThreadMemoryCacheMiddleware + class ClearProcessMemoryCacheMiddleware def initialize(app) @app = app end def call(env) - Gitlab::ThreadMemoryCache.cache_backend.clear + Gitlab::ProcessMemoryCache.cache_backend.clear @app.call(env) end diff --git a/lib/gitlab/thread_memory_cache.rb b/lib/gitlab/thread_memory_cache.rb deleted file mode 100644 index 7f363dc7feb..00000000000 --- a/lib/gitlab/thread_memory_cache.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class ThreadMemoryCache - THREAD_KEY = :thread_memory_cache - - def self.cache_backend - # Note ActiveSupport::Cache::MemoryStore is thread-safe. Since - # each backend is local per thread we probably don't need to worry - # about synchronizing access, but this is a drop-in replacement - # for ActiveSupport::Cache::RedisStore. - Thread.current[THREAD_KEY] ||= ActiveSupport::Cache::MemoryStore.new - end - end -end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 9a7b4cc65f6..37688d6e0e7 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -9,13 +9,6 @@ module Gitlab module ControllerConcern extend ActiveSupport::Concern - included do - # Tracking events from the template is not ideal and we are moving this to the client in https://gitlab.com/gitlab-org/gitlab/-/issues/213712 - # In the meantime, using this method from the view is frowned upon and this line will likely be removed - # in the near future - helper_method :track_event - end - protected def track_event(action = action_name, **args) diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index 5df53b5adde..4ec43e62c19 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -2,21 +2,24 @@ module Gitlab class TreeSummary - prepend_if_ee('::EE::Gitlab::TreeSummary') # rubocop: disable Cop/InjectEnterpriseEditionModule - include ::Gitlab::Utils::StrongMemoize + include ::MarkupHelper + + CACHE_EXPIRE_IN = 1.hour + MAX_OFFSET = 2**31 - attr_reader :commit, :project, :path, :offset, :limit + attr_reader :commit, :project, :path, :offset, :limit, :user attr_reader :resolved_commits private :resolved_commits - def initialize(commit, project, params = {}) + def initialize(commit, project, user, params = {}) @commit = commit @project = project + @user = user @path = params.fetch(:path, nil).presence - @offset = params.fetch(:offset, 0).to_i + @offset = [params.fetch(:offset, 0).to_i, MAX_OFFSET].min @limit = (params.fetch(:limit, 25) || 25).to_i # Ensure that if multiple tree entries share the same last commit, they share @@ -43,6 +46,17 @@ module Gitlab [summary, commits] end + def fetch_logs + cache_key = ['projects', project.id, 'logs', commit.id, path, offset] + Rails.cache.fetch(cache_key, expires_in: CACHE_EXPIRE_IN) do + logs, _ = summarize + + new_offset = next_offset if more? + + [logs.as_json, new_offset] + end + end + # Does the tree contain more entries after the given offset + limit? def more? all_contents[next_offset].present? @@ -84,6 +98,7 @@ module Gitlab end commits_hsh = repository.list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit) + prerender_commit_full_titles!(commits_hsh.values) entries.each do |entry| path_key = entry_path(entry) @@ -92,6 +107,7 @@ module Gitlab if commit entry[:commit] = commit entry[:commit_path] = commit_path(commit) + entry[:commit_title_html] = markdown_field(commit, :full_title) end end end @@ -119,5 +135,15 @@ module Gitlab def tree strong_memoize(:tree) { repository.tree(commit.id, path) } end + + def prerender_commit_full_titles!(commits) + # Preload commit authors as they are used in rendering + commits.each(&:lazy_author) + + renderer = Banzai::ObjectRenderer.new(user: user, default_project: project) + renderer.render(commits, :full_title) + end end end + +Gitlab::TreeSummary.prepend_if_ee('::EE::Gitlab::TreeSummary') diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb index 96ee6f0e8e6..9377ccfec1e 100644 --- a/lib/gitlab/uploads/migration_helper.rb +++ b/lib/gitlab/uploads/migration_helper.rb @@ -15,6 +15,7 @@ module Gitlab %w(FileUploader Project), %w(PersonalFileUploader Snippet), %w(NamespaceFileUploader Snippet), + %w(DesignManagement::DesignV432x230Uploader DesignManagement::Action :image_v432x230), %w(FileUploader MergeRequest)].freeze def initialize(args, logger) @@ -74,5 +75,3 @@ module Gitlab end end end - -Gitlab::Uploads::MigrationHelper.prepend_if_ee('EE::Gitlab::Uploads::MigrationHelper') diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 4c56b9bb3c9..329f87d8be8 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -11,6 +11,10 @@ module Gitlab class << self include ActionView::RecordIdentifier + # Using a case statement here is preferable for readability and maintainability. + # See discussion in https://gitlab.com/gitlab-org/gitlab/-/issues/217397 + # + # rubocop:disable Metrics/CyclomaticComplexity def build(object, **options) # Objects are sometimes wrapped in a BatchLoader instance case object.itself @@ -34,14 +38,17 @@ module Gitlab snippet_url(object, **options) when User instance.user_url(object, **options) - when ProjectWiki - instance.project_wiki_url(object.project, :home, **options) + when Wiki + wiki_url(object, **options) when WikiPage instance.project_wiki_url(object.wiki.project, object.slug, **options) + when ::DesignManagement::Design + design_url(object, **options) else raise NotImplementedError.new("No URL builder defined for #{object.inspect}") end end + # rubocop:enable Metrics/CyclomaticComplexity def commit_url(commit, **options) return '' unless commit.project @@ -70,6 +77,25 @@ module Gitlab instance.gitlab_snippet_url(snippet, **options) end end + + def wiki_url(object, **options) + if object.container.is_a?(Project) + instance.project_wiki_url(object.container, Wiki::HOMEPAGE, **options) + else + raise NotImplementedError.new("No URL builder defined for #{object.inspect}") + end + end + + def design_url(design, **options) + size, ref = options.values_at(:size, :ref) + options.except!(:size, :ref) + + if size + instance.project_design_management_designs_resized_image_url(design.project, design, ref, size, **options) + else + instance.project_design_management_designs_raw_image_url(design.project, design, ref, **options) + end + end end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index f8ee0ca6877..e60c786b52c 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -1,15 +1,25 @@ # frozen_string_literal: true -# For hardening usage ping and make it easier to add measures there is in place alt_usage_data method -# which handles StandardError and fallbacks into -1 -# this way not all measures fail if we encounter one exception +# For hardening usage ping and make it easier to add measures there is in place +# * alt_usage_data method +# handles StandardError and fallbacks into -1 this way not all measures fail if we encounter one exception # -# Examples: -# alt_usage_data { Gitlab::VERSION } -# alt_usage_data { Gitlab::CurrentSettings.uuid } +# Examples: +# alt_usage_data { Gitlab::VERSION } +# alt_usage_data { Gitlab::CurrentSettings.uuid } +# +# * redis_usage_data method +# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent +# returns -1 when a block is sent or hash with all values -1 when a counter is sent +# different behaviour due to 2 different implementations of redis counter +# +# Examples: +# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) +# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } module Gitlab class UsageData BATCH_SIZE = 100 + FALLBACK = -1 class << self def data(force_refresh: false) @@ -24,6 +34,8 @@ module Gitlab .merge(features_usage_data) .merge(components_usage_data) .merge(cycle_analytics_usage_data) + .merge(object_store_usage_data) + .merge(recording_ce_finish_data) end def to_json(force_refresh: false) @@ -32,19 +44,27 @@ module Gitlab def license_usage_data { + recorded_at: Time.now, # should be calculated very first uuid: alt_usage_data { Gitlab::CurrentSettings.uuid }, hostname: alt_usage_data { Gitlab.config.gitlab.host }, version: alt_usage_data { Gitlab::VERSION }, installation_type: alt_usage_data { installation_type }, active_user_count: count(User.active), - recorded_at: Time.now, edition: 'CE' } end + def recording_ce_finish_data + { + recording_ce_finished_at: Time.now + } + end + # rubocop: disable Metrics/AbcSize # rubocop: disable CodeReuse/ActiveRecord def system_usage_data + alert_bot_incident_count = count(::Issue.authored(::User.alert_bot)) + { counts: { assignee_lists: count(List.assignee), @@ -94,7 +114,10 @@ module Gitlab issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue), issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data, - incident_issues: count(::Issue.authored(::User.alert_bot)), + issues_created_gitlab_alerts: count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot)), + incident_issues: alert_bot_incident_count, + alert_bot_incident_issues: alert_bot_incident_count, + incident_labeled_issues: count(::Issue.with_label_attributes(IncidentManagement::CreateIssueService::INCIDENT_LABEL)), keys: count(Key), label_lists: count(List.label), lfs_objects: count(LfsObject), @@ -123,7 +146,8 @@ module Gitlab services_usage, usage_counters, user_preferences_usage, - ingress_modsecurity_usage + ingress_modsecurity_usage, + container_expiration_policies_usage ) } end @@ -154,7 +178,6 @@ module Gitlab dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled, gitlab_shared_runners_enabled: alt_usage_data { Gitlab.config.gitlab_ci.shared_runners_enabled }, gravatar_enabled: alt_usage_data { Gitlab::CurrentSettings.gravatar_enabled? }, - influxdb_metrics_enabled: alt_usage_data { Gitlab::Metrics.influx_metrics_enabled? }, ldap_enabled: alt_usage_data { Gitlab.config.ldap.enabled }, mattermost_enabled: alt_usage_data { Gitlab.config.mattermost.enabled }, omniauth_enabled: alt_usage_data { Gitlab::Auth.omniauth_enabled? }, @@ -162,36 +185,14 @@ module Gitlab reply_by_email_enabled: alt_usage_data { Gitlab::IncomingEmail.enabled? }, signup_enabled: alt_usage_data { Gitlab::CurrentSettings.allow_signup? }, web_ide_clientside_preview_enabled: alt_usage_data { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? }, - ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity) - }.merge(features_usage_data_container_expiration_policies) - end - - # rubocop: disable CodeReuse/ActiveRecord - def features_usage_data_container_expiration_policies - results = {} - start = ::Project.minimum(:id) - finish = ::Project.maximum(:id) - - results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish) - base = ::ContainerExpirationPolicy.active - results[:projects_with_expiration_policy_enabled] = distinct_count(base, :project_id, start: start, finish: finish) - - %i[keep_n cadence older_than].each do |option| - ::ContainerExpirationPolicy.public_send("#{option}_options").keys.each do |value| # rubocop: disable GitlabSecurity/PublicSend - results["projects_with_expiration_policy_enabled_with_#{option}_set_to_#{value}".to_sym] = distinct_count(base.where(option => value), :project_id, start: start, finish: finish) - end - end - - results[:projects_with_expiration_policy_enabled_with_keep_n_unset] = distinct_count(base.where(keep_n: nil), :project_id, start: start, finish: finish) - results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish) - - results + ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity), + grafana_link_enabled: alt_usage_data { Gitlab::CurrentSettings.grafana_enabled? } + } end - # rubocop: enable CodeReuse/ActiveRecord # @return [Hash<Symbol, Integer>] def usage_counters - usage_data_counters.map(&:totals).reduce({}) { |a, b| a.merge(b) } + usage_data_counters.map { |counter| redis_usage_data(counter) }.reduce({}, :merge) end # @return [Array<#totals>] An array of objects that respond to `#totals` @@ -205,7 +206,8 @@ module Gitlab Gitlab::UsageDataCounters::CycleAnalyticsCounter, Gitlab::UsageDataCounters::ProductivityAnalyticsCounter, Gitlab::UsageDataCounters::SourceCodeCounter, - Gitlab::UsageDataCounters::MergeRequestCounter + Gitlab::UsageDataCounters::MergeRequestCounter, + Gitlab::UsageDataCounters::DesignsCounter ] end @@ -237,9 +239,83 @@ module Gitlab 'unknown_app_server_type' end + def object_store_config(component) + config = alt_usage_data(fallback: nil) do + Settings[component]['object_store'] + end + + if config + { + enabled: alt_usage_data { Settings[component]['enabled'] }, + object_store: { + enabled: alt_usage_data { config['enabled'] }, + direct_upload: alt_usage_data { config['direct_upload'] }, + background_upload: alt_usage_data { config['background_upload'] }, + provider: alt_usage_data { config['connection']['provider'] } + } + } + else + { + enabled: alt_usage_data { Settings[component]['enabled'] } + } + end + end + + def object_store_usage_data + { + object_store: { + artifacts: object_store_config('artifacts'), + external_diffs: object_store_config('external_diffs'), + lfs: object_store_config('lfs'), + uploads: object_store_config('uploads'), + packages: object_store_config('packages') + } + } + end + def ingress_modsecurity_usage - ::Clusters::Applications::IngressModsecurityUsageService.new.execute + ## + # This method measures usage of the Modsecurity Web Application Firewall across the entire + # instance's deployed environments. + # + # NOTE: this service is an approximation as it does not yet take into account if environment + # is enabled and only measures applications installed using GitLab Managed Apps (disregards + # CI-based managed apps). + # + # More details: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28331#note_318621786 + ## + + column = ::Deployment.arel_table[:environment_id] + { + ingress_modsecurity_logging: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_enabled.logging), column), + ingress_modsecurity_blocking: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_enabled.blocking), column), + ingress_modsecurity_disabled: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_disabled), column), + ingress_modsecurity_not_installed: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_not_installed), column) + } + end + + # rubocop: disable CodeReuse/ActiveRecord + def container_expiration_policies_usage + results = {} + start = ::Project.minimum(:id) + finish = ::Project.maximum(:id) + + results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish) + base = ::ContainerExpirationPolicy.active + results[:projects_with_expiration_policy_enabled] = distinct_count(base, :project_id, start: start, finish: finish) + + %i[keep_n cadence older_than].each do |option| + ::ContainerExpirationPolicy.public_send("#{option}_options").keys.each do |value| # rubocop: disable GitlabSecurity/PublicSend + results["projects_with_expiration_policy_enabled_with_#{option}_set_to_#{value}".to_sym] = distinct_count(base.where(option => value), :project_id, start: start, finish: finish) + end + end + + results[:projects_with_expiration_policy_enabled_with_keep_n_unset] = distinct_count(base.where(keep_n: nil), :project_id, start: start, finish: finish) + results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish) + + results end + # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def services_usage @@ -251,7 +327,7 @@ module Gitlab results[:projects_slack_notifications_active] = results[:projects_slack_active] results[:projects_slack_slash_active] = results[:projects_slack_slash_commands_active] - results.merge(jira_usage) + results.merge(jira_usage).merge(jira_import_usage) end def jira_usage @@ -281,35 +357,52 @@ module Gitlab results rescue ActiveRecord::StatementInvalid - { projects_jira_server_active: -1, projects_jira_cloud_active: -1, projects_jira_active: -1 } + { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK, projects_jira_active: FALLBACK } + end + + def successful_deployments_with_cluster(scope) + scope + .joins(cluster: :deployments) + .merge(Clusters::Cluster.enabled) + .merge(Deployment.success) end # rubocop: enable CodeReuse/ActiveRecord + def jira_import_usage + finished_jira_imports = JiraImportState.finished + + { + jira_imports_total_imported_count: count(finished_jira_imports), + jira_imports_projects_count: distinct_count(finished_jira_imports, :project_id), + jira_imports_total_imported_issues_count: alt_usage_data { JiraImportState.finished_imports_count } + } + end + def user_preferences_usage {} # augmented in EE end - def count(relation, column = nil, fallback: -1, batch: true, start: nil, finish: nil) + def count(relation, column = nil, batch: true, start: nil, finish: nil) if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true) Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish) else relation.count end rescue ActiveRecord::StatementInvalid - fallback + FALLBACK end - def distinct_count(relation, column = nil, fallback: -1, batch: true, start: nil, finish: nil) + def distinct_count(relation, column = nil, batch: true, start: nil, finish: nil) if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true) Gitlab::Database::BatchCount.batch_distinct_count(relation, column, start: start, finish: finish) else relation.distinct_count_by(column) end rescue ActiveRecord::StatementInvalid - fallback + FALLBACK end - def alt_usage_data(value = nil, fallback: -1, &block) + def alt_usage_data(value = nil, fallback: FALLBACK, &block) if block_given? yield else @@ -319,8 +412,28 @@ module Gitlab fallback end + def redis_usage_data(counter = nil, &block) + if block_given? + redis_usage_counter(&block) + elsif counter.present? + redis_usage_data_totals(counter) + end + end + private + def redis_usage_counter + yield + rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent + FALLBACK + end + + def redis_usage_data_totals(counter) + counter.totals + rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent + counter.fallback_totals + end + def installation_type if Rails.env.production? Gitlab::INSTALLATION_TYPE diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb index 33111b46381..96898e5189c 100644 --- a/lib/gitlab/usage_data_counters/base_counter.rb +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -22,11 +22,19 @@ module Gitlab::UsageDataCounters end def totals - known_events.map { |e| ["#{prefix}_#{e}".to_sym, read(e)] }.to_h + known_events.map { |event| [counter_key(event), read(event)] }.to_h + end + + def fallback_totals + known_events.map { |event| [counter_key(event), -1] }.to_h end private + def counter_key(event) + "#{prefix}_#{event}".to_sym + end + def known_events self::KNOWN_EVENTS end diff --git a/lib/gitlab/usage_data_counters/designs_counter.rb b/lib/gitlab/usage_data_counters/designs_counter.rb new file mode 100644 index 00000000000..801fb8f3b3d --- /dev/null +++ b/lib/gitlab/usage_data_counters/designs_counter.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class DesignsCounter + extend Gitlab::UsageDataCounters::RedisCounter + + KNOWN_EVENTS = %w[create update delete].map(&:freeze).freeze + + UnknownEvent = Class.new(StandardError) + + class << self + # Each event gets a unique Redis key + def redis_key(event) + raise UnknownEvent, event unless KNOWN_EVENTS.include?(event.to_s) + + "USAGE_DESIGN_MANAGEMENT_DESIGNS_#{event}".upcase + end + + def count(event) + increment(redis_key(event)) + end + + def read(event) + total_count(redis_key(event)) + end + + def totals + KNOWN_EVENTS.map { |event| [counter_key(event), read(event)] }.to_h + end + + def fallback_totals + KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h + end + + private + + def counter_key(event) + "design_management_designs_#{event}".to_sym + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/note_counter.rb b/lib/gitlab/usage_data_counters/note_counter.rb index 672450ec82b..7a76180cb08 100644 --- a/lib/gitlab/usage_data_counters/note_counter.rb +++ b/lib/gitlab/usage_data_counters/note_counter.rb @@ -25,12 +25,20 @@ module Gitlab::UsageDataCounters def totals COUNTABLE_TYPES.map do |countable_type| - [:"#{countable_type.underscore}_comment", read(:create, countable_type)] + [counter_key(countable_type), read(:create, countable_type)] end.to_h end + def fallback_totals + COUNTABLE_TYPES.map { |counter_key| [counter_key(counter_key), -1] }.to_h + end + private + def counter_key(countable_type) + "#{countable_type.underscore}_comment".to_sym + end + def countable?(noteable_type) COUNTABLE_TYPES.include?(noteable_type.to_s) end diff --git a/lib/gitlab/usage_data_counters/search_counter.rb b/lib/gitlab/usage_data_counters/search_counter.rb index 5f0735347e1..b9e3a5c0104 100644 --- a/lib/gitlab/usage_data_counters/search_counter.rb +++ b/lib/gitlab/usage_data_counters/search_counter.rb @@ -21,6 +21,10 @@ module Gitlab navbar_searches: total_navbar_searches_count } end + + def fallback_totals + { navbar_searches: -1 } + end end end end diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb index c012a6c96df..00fcd42a9af 100644 --- a/lib/gitlab/usage_data_counters/web_ide_counter.rb +++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb @@ -4,54 +4,52 @@ module Gitlab module UsageDataCounters class WebIdeCounter extend RedisCounter - - COMMITS_COUNT_KEY = 'WEB_IDE_COMMITS_COUNT' - MERGE_REQUEST_COUNT_KEY = 'WEB_IDE_MERGE_REQUESTS_COUNT' - VIEWS_COUNT_KEY = 'WEB_IDE_VIEWS_COUNT' - PREVIEW_COUNT_KEY = 'WEB_IDE_PREVIEWS_COUNT' + KNOWN_EVENTS = %i[commits views merge_requests previews terminals pipelines].freeze + PREFIX = 'web_ide' class << self def increment_commits_count - increment(COMMITS_COUNT_KEY) - end - - def total_commits_count - total_count(COMMITS_COUNT_KEY) + increment(redis_key('commits')) end def increment_merge_requests_count - increment(MERGE_REQUEST_COUNT_KEY) + increment(redis_key('merge_requests')) end - def total_merge_requests_count - total_count(MERGE_REQUEST_COUNT_KEY) + def increment_views_count + increment(redis_key('views')) end - def increment_views_count - increment(VIEWS_COUNT_KEY) + def increment_terminals_count + increment(redis_key('terminals')) end - def total_views_count - total_count(VIEWS_COUNT_KEY) + def increment_pipelines_count + increment(redis_key('pipelines')) end def increment_previews_count return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? - increment(PREVIEW_COUNT_KEY) + increment(redis_key('previews')) + end + + def totals + KNOWN_EVENTS.map { |event| [counter_key(event), total_count(redis_key(event))] }.to_h end - def total_previews_count - total_count(PREVIEW_COUNT_KEY) + def fallback_totals + KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h end - def totals - { - web_ide_commits: total_commits_count, - web_ide_views: total_views_count, - web_ide_merge_requests: total_merge_requests_count, - web_ide_previews: total_previews_count - } + private + + def redis_key(event) + "#{PREFIX}_#{event}_count".upcase + end + + def counter_key(event) + "#{PREFIX}_#{event}".to_sym end end end diff --git a/lib/gitlab/user_access_snippet.rb b/lib/gitlab/user_access_snippet.rb index bfed86c4df4..dcd45f9350d 100644 --- a/lib/gitlab/user_access_snippet.rb +++ b/lib/gitlab/user_access_snippet.rb @@ -17,7 +17,14 @@ module Gitlab @project = snippet&.project end + def allowed? + return true if snippet_migration? + + super + end + def can_do_action?(action) + return true if snippet_migration? return false unless can_access_git? permission_cache[action] = @@ -35,7 +42,10 @@ module Gitlab end def can_push_to_branch?(ref) + return true if snippet_migration? + super + return false unless snippet return false unless can_do_action?(:update_snippet) @@ -45,5 +55,9 @@ module Gitlab def can_merge_to_branch?(ref) false end + + def snippet_migration? + user&.migration_bot? && snippet + end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 2e8a3ca4242..d46601fa2e8 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -75,12 +75,12 @@ module Gitlab str.gsub(/\r?\n/, '') end - def to_boolean(value) + def to_boolean(value, default: nil) return value if [true, false].include?(value) return true if value =~ /^(true|t|yes|y|1|on)$/i return false if value =~ /^(false|f|no|n|0|off)$/i - nil + default end def boolean_to_yes_no(bool) @@ -123,7 +123,7 @@ module Gitlab end def ms_to_round_sec(ms) - (ms.to_f / 1000).round(2) + (ms.to_f / 1000).round(6) end # Used in EE diff --git a/lib/gitlab/utils/measuring.rb b/lib/gitlab/utils/measuring.rb index 0680cefd249..febe489f1f8 100644 --- a/lib/gitlab/utils/measuring.rb +++ b/lib/gitlab/utils/measuring.rb @@ -5,38 +5,51 @@ require 'prometheus/pid_provider' module Gitlab module Utils class Measuring - def initialize(logger: Logger.new($stdout)) - @logger = logger + class << self + attr_writer :logger + + def logger + @logger ||= Logger.new(STDOUT) + end + end + + def initialize(base_log_data = {}) + @base_log_data = base_log_data end def with_measuring - logger.info "Measuring enabled..." + result = nil with_gc_stats do with_count_queries do with_measure_time do - yield + result = yield end end end - logger.info "Memory usage: #{Gitlab::Metrics::System.memory_usage.to_f / 1024 / 1024} MiB" - logger.info "Label: #{::Prometheus::PidProvider.worker_id}" + log_info( + gc_stats: gc_stats, + time_to_finish: time_to_finish, + number_of_sql_calls: sql_calls_count, + memory_usage: "#{Gitlab::Metrics::System.memory_usage_rss.to_f / 1024 / 1024} MiB", + label: ::Prometheus::PidProvider.worker_id + ) + + result end private - attr_reader :logger + attr_reader :gc_stats, :time_to_finish, :sql_calls_count, :base_log_data def with_count_queries(&block) - count = 0 + @sql_calls_count = 0 counter_f = ->(_name, _started, _finished, _unique_id, payload) { - count += 1 unless payload[:name].in? %w[CACHE SCHEMA] + @sql_calls_count += 1 unless payload[:name].in? %w[CACHE SCHEMA] } ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block) - - logger.info "Number of sql calls: #{count}" end def with_gc_stats @@ -44,33 +57,22 @@ module Gitlab stats_before = GC.stat yield stats_after = GC.stat - stats_diff = stats_after.map do |key, after_value| + @gc_stats = stats_after.map do |key, after_value| before_value = stats_before[key] [key, before: before_value, after: after_value, diff: after_value - before_value] end.to_h - logger.info "GC stats:" - logger.info JSON.pretty_generate(stats_diff) end def with_measure_time - timing = Benchmark.realtime do + @time_to_finish = Benchmark.realtime do yield end - - logger.info "Time to finish: #{duration_in_numbers(timing)}" end - def duration_in_numbers(duration_in_seconds) - milliseconds = duration_in_seconds.in_milliseconds % 1.second.in_milliseconds - seconds = duration_in_seconds % 1.minute - minutes = (duration_in_seconds / 1.minute) % (1.hour / 1.minute) - hours = duration_in_seconds / 1.hour - - if hours == 0 - "%02d:%02d:%03d" % [minutes, seconds, milliseconds] - else - "%02d:%02d:%02d:%03d" % [hours, minutes, seconds, milliseconds] - end + def log_info(details) + details = base_log_data.merge(details) + details = details.to_yaml if ActiveSupport::Logger.logger_outputs_to?(Measuring.logger, STDOUT) + Measuring.logger.info(details) end end end diff --git a/lib/gitlab/wiki_pages.rb b/lib/gitlab/wiki_pages.rb index 47f9aa1117f..dee885e74d1 100644 --- a/lib/gitlab/wiki_pages.rb +++ b/lib/gitlab/wiki_pages.rb @@ -11,5 +11,8 @@ module Gitlab # through the GitLab web interface and API: MAX_TITLE_BYTES = 245 # reserving 10 bytes for the file extension MAX_DIRECTORY_BYTES = 255 + + # Limit the number of pages displayed in the wiki sidebar. + MAX_SIDEBAR_PAGES = 15 end end diff --git a/lib/gitlab/with_request_store.rb b/lib/gitlab/with_request_store.rb index d6c05e1e256..d13cd9a72f7 100644 --- a/lib/gitlab/with_request_store.rb +++ b/lib/gitlab/with_request_store.rb @@ -2,12 +2,24 @@ module Gitlab module WithRequestStore - def with_request_store + def with_request_store(&block) + # Skip enabling the request store if it was already active. Whatever + # instantiated the request store first is responsible for clearing it + return yield if RequestStore.active? + + enabling_request_store(&block) + end + + private + + def enabling_request_store RequestStore.begin! yield ensure RequestStore.end! RequestStore.clear! end + + extend self end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index b375602a5fe..c91d1b05440 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -62,9 +62,6 @@ module Gitlab end def send_git_archive(repository, ref:, format:, append_sha:, path: nil) - path_enabled = Feature.enabled?(:git_archive_path, default_enabled: true) - path = nil unless path_enabled - format ||= 'tar.gz' format = format.downcase @@ -78,12 +75,7 @@ module Gitlab raise "Repository or ref not found" if metadata.empty? - params = - if path_enabled - send_git_archive_params(repository, metadata, path, archive_format(format)) - else - metadata - end + params = send_git_archive_params(repository, metadata, path, archive_format(format)) # If present, DisableCache must be a Boolean. Otherwise # workhorse ignores it. @@ -138,8 +130,7 @@ module Gitlab ] end - def send_artifacts_entry(build, entry) - file = build.artifacts_file + def send_artifacts_entry(file, entry) archive = file.file_storage? ? file.path : file.url params = { @@ -213,7 +204,7 @@ module Gitlab # This is the outermost encoding of a senddata: header. It is safe for # inclusion in HTTP response headers def encode(hash) - Base64.urlsafe_encode64(JSON.dump(hash)) + Base64.urlsafe_encode64(Gitlab::Json.dump(hash)) end # This is for encoding individual fields inside the senddata JSON that diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index ed248e29211..7d4d4d9d13a 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -22,6 +22,10 @@ module Gitlab X509Certificate.safe_create!(certificate_attributes) unless verified_signature.nil? end + def user + User.find_by_any_email(@email) + end + def verified_signature strong_memoize(:verified_signature) { verified_signature? } end diff --git a/lib/gitlab/x509/tag.rb b/lib/gitlab/x509/tag.rb new file mode 100644 index 00000000000..48582c17764 --- /dev/null +++ b/lib/gitlab/x509/tag.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true +require 'openssl' +require 'digest' + +module Gitlab + module X509 + class Tag + include Gitlab::Utils::StrongMemoize + + def initialize(raw_tag) + @raw_tag = raw_tag + end + + def signature + signature = X509::Signature.new(signature_text, signed_text, @raw_tag.tagger.email, Time.at(@raw_tag.tagger.date.seconds)) + + return if signature.verified_signature.nil? + + signature + end + + private + + def signature_text + @raw_tag.message.slice(@raw_tag.message.index("-----BEGIN SIGNED MESSAGE-----")..-1) + rescue + nil + end + + def signed_text + # signed text is reconstructed as long as there is no specific gitaly function + %{object #{@raw_tag.target_commit.id} +type commit +tag #{@raw_tag.name} +tagger #{@raw_tag.tagger.name} <#{@raw_tag.tagger.email}> #{@raw_tag.tagger.date.seconds} #{@raw_tag.tagger.timezone} + +#{@raw_tag.message.gsub(/-----BEGIN SIGNED MESSAGE-----(.*)-----END SIGNED MESSAGE-----/m, "")}} + end + end + end +end |