diff options
Diffstat (limited to 'lib/gitlab')
257 files changed, 5340 insertions, 1384 deletions
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index bf4438fb518..830980f0997 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -9,12 +9,13 @@ module Gitlab module Access AccessDeniedError = Class.new(StandardError) - NO_ACCESS = 0 - GUEST = 10 - REPORTER = 20 - DEVELOPER = 30 - MAINTAINER = 40 - OWNER = 50 + NO_ACCESS = 0 + MINIMAL_ACCESS = 5 + GUEST = 10 + REPORTER = 20 + DEVELOPER = 30 + MAINTAINER = 40 + OWNER = 50 # Branch protection settings PROTECTION_NONE = 0 diff --git a/lib/gitlab/alert_management/alert_params.rb b/lib/gitlab/alert_management/alert_params.rb index 84a75e62ecf..3bb839c1114 100644 --- a/lib/gitlab/alert_management/alert_params.rb +++ b/lib/gitlab/alert_management/alert_params.rb @@ -20,8 +20,10 @@ module Gitlab hosts: Array(annotations[:hosts]), payload: payload, started_at: parsed_payload['startsAt'], + ended_at: parsed_payload['endsAt'], severity: annotations[:severity], - fingerprint: annotations[:fingerprint] + fingerprint: annotations[:fingerprint], + environment: annotations[:environment] } end diff --git a/lib/gitlab/alert_management/payload.rb b/lib/gitlab/alert_management/payload.rb new file mode 100644 index 00000000000..177d544d720 --- /dev/null +++ b/lib/gitlab/alert_management/payload.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module AlertManagement + module Payload + MONITORING_TOOLS = { + prometheus: 'Prometheus' + }.freeze + + class << self + # Instantiates an instance of a subclass of + # Gitlab::AlertManagement::Payload::Base. This can + # be used to create new alerts or read content from + # the payload of an existing AlertManagement::Alert + # + # @param project [Project] + # @param payload [Hash] + # @param monitoring_tool [String] + def parse(project, payload, monitoring_tool: nil) + payload_class = payload_class_for( + monitoring_tool: monitoring_tool || payload&.dig('monitoring_tool'), + payload: payload + ) + + payload_class.new(project: project, payload: payload) + end + + private + + def payload_class_for(monitoring_tool:, payload:) + if monitoring_tool == MONITORING_TOOLS[:prometheus] + if gitlab_managed_prometheus?(payload) + ::Gitlab::AlertManagement::Payload::ManagedPrometheus + else + ::Gitlab::AlertManagement::Payload::Prometheus + end + else + ::Gitlab::AlertManagement::Payload::Generic + end + end + + def gitlab_managed_prometheus?(payload) + payload&.dig('labels', 'gitlab_alert_id').present? + end + end + end + end +end diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb new file mode 100644 index 00000000000..74e47e5226e --- /dev/null +++ b/lib/gitlab/alert_management/payload/base.rb @@ -0,0 +1,167 @@ +# frozen_string_literal: true + +# Representation of a payload of an alert. Defines a constant +# API so that payloads from various sources can be treated +# identically. Subclasses should define how to parse payload +# based on source of alert. +module Gitlab + module AlertManagement + module Payload + class Base + include ActiveModel::Model + include Gitlab::Utils::StrongMemoize + include Gitlab::Routing + + attr_accessor :project, :payload + + # Any attribute expected to be specifically read from + # or derived from an alert payload should be defined. + EXPECTED_PAYLOAD_ATTRIBUTES = [ + :alert_markdown, + :alert_title, + :annotations, + :description, + :ends_at, + :environment, + :environment_name, + :full_query, + :generator_url, + :gitlab_alert, + :gitlab_fingerprint, + :gitlab_prometheus_alert_id, + :gitlab_y_label, + :has_required_attributes?, + :hosts, + :metric_id, + :metrics_dashboard_url, + :monitoring_tool, + :resolved?, + :runbook, + :service, + :severity, + :starts_at, + :status, + :title + ].freeze + + # Define expected API for a payload + EXPECTED_PAYLOAD_ATTRIBUTES.each do |key| + define_method(key) {} + end + + # Defines a method which allows access to a given + # value within an alert payload + # + # @param key [Symbol] Name expected to be used to reference value + # @param paths [String, Array<String>, Array<Array<String>>,] + # List of (nested) keys at value can be found, the + # first to yield a result will be used + # @param type [Symbol] If value should be converted to another type, + # that should be specified here + # @param fallback [Proc] Block to be executed to yield a value if + # a value cannot be idenitied at any provided paths + # Example) + # attribute :title + # paths: [['title'], + # ['details', 'title']] + # fallback: Proc.new { 'New Alert' } + # + # The above sample definition will define a method + # called #title which will return the value from the + # payload under the key `title` if available, otherwise + # looking under `details.title`. If neither returns a + # value, the return value will be `'New Alert'` + def self.attribute(key, paths:, type: nil, fallback: -> { nil }) + define_method(key) do + strong_memoize(key) do + paths = Array(paths).first.is_a?(String) ? [Array(paths)] : paths + value = value_for_paths(paths) + value = parse_value(value, type) if value + + value.presence || fallback.call + end + end + end + + # Attributes of an AlertManagement::Alert as read + # directly from a payload. Prefer accessing + # AlertManagement::Alert directly for read operations. + def alert_params + { + description: description, + ended_at: ends_at, + environment: environment, + fingerprint: gitlab_fingerprint, + hosts: Array(hosts), + monitoring_tool: monitoring_tool, + payload: payload, + project_id: project.id, + prometheus_alert: gitlab_alert, + service: service, + severity: severity, + started_at: starts_at, + title: title + }.transform_values(&:presence).compact + end + + def gitlab_fingerprint + strong_memoize(:gitlab_fingerprint) do + next unless plain_gitlab_fingerprint + + Gitlab::AlertManagement::Fingerprint.generate(plain_gitlab_fingerprint) + end + end + + def environment + strong_memoize(:environment) do + next unless environment_name + + EnvironmentsFinder + .new(project, nil, { name: environment_name }) + .find + .first + end + end + + def resolved? + status == 'resolved' + end + + def has_required_attributes? + true + end + + private + + def plain_gitlab_fingerprint; end + + def value_for_paths(paths) + target_path = paths.find { |path| payload&.dig(*path) } + + payload&.dig(*target_path) if target_path + end + + def parse_value(value, type) + case type + when :time + parse_time(value) + when :integer + parse_integer(value) + else + value + end + end + + def parse_time(value) + Time.parse(value).utc + rescue ArgumentError + end + + def parse_integer(value) + Integer(value) + rescue ArgumentError, TypeError + end + end + end + end +end diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb new file mode 100644 index 00000000000..7efdfac75dc --- /dev/null +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# Attribute mapping for alerts via generic alerting integration. +module Gitlab + module AlertManagement + module Payload + class Generic < Base + DEFAULT_TITLE = 'New: Incident' + DEFAULT_SEVERITY = 'critical' + + attribute :environment_name, paths: 'gitlab_environment_name' + attribute :hosts, paths: 'hosts' + attribute :monitoring_tool, paths: 'monitoring_tool' + attribute :runbook, paths: 'runbook' + attribute :service, paths: 'service' + attribute :severity, paths: 'severity', fallback: -> { DEFAULT_SEVERITY } + attribute :starts_at, paths: 'start_time', type: :time, fallback: -> { Time.current.utc } + attribute :title, paths: 'title', fallback: -> { DEFAULT_TITLE } + + attribute :plain_gitlab_fingerprint, paths: 'fingerprint' + private :plain_gitlab_fingerprint + end + end + end +end diff --git a/lib/gitlab/alert_management/payload/managed_prometheus.rb b/lib/gitlab/alert_management/payload/managed_prometheus.rb new file mode 100644 index 00000000000..2236e60a0c6 --- /dev/null +++ b/lib/gitlab/alert_management/payload/managed_prometheus.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# Attribute mapping for alerts via prometheus alerting integration, +# and for which payload includes gitlab-controlled attributes. +module Gitlab + module AlertManagement + module Payload + class ManagedPrometheus < ::Gitlab::AlertManagement::Payload::Prometheus + attribute :gitlab_prometheus_alert_id, + paths: %w(labels gitlab_prometheus_alert_id), + type: :integer + attribute :metric_id, + paths: %w(labels gitlab_alert_id), + type: :integer + + def gitlab_alert + strong_memoize(:gitlab_alert) do + next unless metric_id || gitlab_prometheus_alert_id + + alerts = Projects::Prometheus::AlertsFinder + .new(project: project, metric: metric_id, id: gitlab_prometheus_alert_id) + .execute + + next if alerts.blank? || alerts.size > 1 + + alerts.first + end + end + + def full_query + gitlab_alert&.full_query || super + end + + def environment + gitlab_alert&.environment || super + end + + def metrics_dashboard_url + return unless gitlab_alert + + metrics_dashboard_project_prometheus_alert_url( + project, + gitlab_alert.prometheus_metric_id, + environment_id: environment.id, + embedded: true, + **alert_embed_window_params + ) + end + + private + + def plain_gitlab_fingerprint + [metric_id, starts_at_raw].join('/') + end + end + end + end +end diff --git a/lib/gitlab/alert_management/payload/prometheus.rb b/lib/gitlab/alert_management/payload/prometheus.rb new file mode 100644 index 00000000000..336e9b319e8 --- /dev/null +++ b/lib/gitlab/alert_management/payload/prometheus.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +# Attribute mapping for alerts via prometheus alerting integration. +module Gitlab + module AlertManagement + module Payload + class Prometheus < Base + attribute :alert_markdown, paths: %w(annotations gitlab_incident_markdown) + attribute :annotations, paths: 'annotations' + attribute :description, paths: %w(annotations description) + attribute :ends_at, paths: 'endsAt', type: :time + attribute :environment_name, paths: %w(labels gitlab_environment_name) + attribute :generator_url, paths: %w(generatorURL) + attribute :gitlab_y_label, + paths: [%w(annotations gitlab_y_label), + %w(annotations title), + %w(annotations summary), + %w(labels alertname)] + attribute :runbook, paths: %w(annotations runbook) + attribute :starts_at, + paths: 'startsAt', + type: :time, + fallback: -> { Time.current.utc } + attribute :status, paths: 'status' + attribute :title, + paths: [%w(annotations title), + %w(annotations summary), + %w(labels alertname)] + + attribute :starts_at_raw, + paths: [%w(startsAt)] + private :starts_at_raw + + METRIC_TIME_WINDOW = 30.minutes + + def monitoring_tool + Gitlab::AlertManagement::Payload::MONITORING_TOOLS[:prometheus] + end + + # Parses `g0.expr` from `generatorURL`. + # + # Example: http://localhost:9090/graph?g0.expr=vector%281%29&g0.tab=1 + def full_query + return unless generator_url + + uri = URI(generator_url) + + Rack::Utils.parse_query(uri.query).fetch('g0.expr') + rescue URI::InvalidURIError, KeyError + end + + def metrics_dashboard_url + return unless environment && full_query && title + + metrics_dashboard_project_environment_url( + project, + environment, + embed_json: dashboard_json, + embedded: true, + **alert_embed_window_params + ) + end + + def has_required_attributes? + project && title && starts_at_raw + end + + private + + def plain_gitlab_fingerprint + [starts_at_raw, title, full_query].join('/') + end + + # Formatted for parsing by JS + def alert_embed_window_params + { + start: (starts_at - METRIC_TIME_WINDOW).utc.strftime('%FT%TZ'), + end: (starts_at + METRIC_TIME_WINDOW).utc.strftime('%FT%TZ') + } + end + + def dashboard_json + { + panel_groups: [{ + panels: [{ + type: 'area-chart', + title: title, + y_label: gitlab_y_label, + metrics: [{ + query_range: full_query + }] + }] + }] + }.to_json + end + end + end + end +end diff --git a/lib/gitlab/alerting/notification_payload_parser.rb b/lib/gitlab/alerting/notification_payload_parser.rb index f285dcf507f..348f851f551 100644 --- a/lib/gitlab/alerting/notification_payload_parser.rb +++ b/lib/gitlab/alerting/notification_payload_parser.rb @@ -20,7 +20,8 @@ module Gitlab def call { 'annotations' => annotations, - 'startsAt' => starts_at + 'startsAt' => starts_at, + 'endsAt' => ends_at }.compact end @@ -55,7 +56,8 @@ module Gitlab 'service' => payload[:service], 'hosts' => hosts.presence, 'severity' => severity, - 'fingerprint' => fingerprint + 'fingerprint' => fingerprint, + 'environment' => environment } end @@ -73,8 +75,24 @@ module Gitlab current_time end + def ends_at + Time.parse(payload[:end_time].to_s).rfc3339 + rescue ArgumentError + nil + end + + def environment + environment_name = payload[:gitlab_environment_name] + + return unless environment_name + + EnvironmentsFinder.new(project, nil, { name: environment_name }) + .find + &.first + end + def secondary_params - payload.except(:start_time) + payload.except(:start_time, :end_time) end def flatten_secondary_params diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb index 79e60e28fc7..fc91dd6e138 100644 --- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb +++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb @@ -18,8 +18,7 @@ module Gitlab params_for_code_stage, params_for_test_stage, params_for_review_stage, - params_for_staging_stage, - params_for_production_stage + params_for_staging_stage ] end @@ -86,16 +85,6 @@ module Gitlab end_event_identifier: :merge_request_first_deployed_to_production } end - - def self.params_for_production_stage - { - name: 'production', - custom: false, - relative_position: 7, - start_event_identifier: :issue_created, - end_event_identifier: :production_stage_end - } - end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb index cf05ebeb706..b778364a917 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb @@ -6,7 +6,7 @@ module Gitlab module StageEvents class ProductionStageEnd < StageEvent def self.name - _("Issue first depoloyed to production") + _("Issue first deployed to production") end def self.identifier diff --git a/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb b/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb new file mode 100644 index 00000000000..636bba22c23 --- /dev/null +++ b/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module InstanceStatistics + class WorkersArgumentBuilder + def initialize(measurement_identifiers: [], recorded_at: Time.zone.now) + @measurement_identifiers = measurement_identifiers + @recorded_at = recorded_at + end + + def execute + measurement_identifiers.map do |measurement_identifier| + query_scope = ::Analytics::InstanceStatistics::Measurement::IDENTIFIER_QUERY_MAPPING[measurement_identifier]&.call + + next if query_scope.nil? + + # Determining the query range (id range) as early as possible in order to get more accurate counts. + start = query_scope.minimum(:id) + finish = query_scope.maximum(:id) + + [measurement_identifier, start, finish, recorded_at] + end.compact + end + + private + + attr_reader :measurement_identifiers, :recorded_at + end + end + end +end diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb index ad746ebbd42..292048dcad9 100644 --- a/lib/gitlab/analytics/unique_visits.rb +++ b/lib/gitlab/analytics/unique_visits.rb @@ -14,23 +14,23 @@ module Gitlab # @param [ActiveSupport::TimeWithZone] end_date end of time frame # @return [Integer] number of unique visitors def unique_visits_for(targets:, start_date: 7.days.ago, end_date: start_date + 1.week) - target_ids = if targets == :analytics - self.class.analytics_ids - elsif targets == :compliance - self.class.compliance_ids - else - Array(targets) - end + events = if targets == :analytics + self.class.analytics_events + elsif targets == :compliance + self.class.compliance_events + else + Array(targets) + end - Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: target_ids, start_date: start_date, end_date: end_date) + Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: start_date, end_date: end_date) end class << self - def analytics_ids + def analytics_events Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('analytics') end - def compliance_ids + def compliance_events Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('compliance') end end diff --git a/lib/gitlab/anonymous_session.rb b/lib/gitlab/anonymous_session.rb index 148b6d3310d..911825eef3a 100644 --- a/lib/gitlab/anonymous_session.rb +++ b/lib/gitlab/anonymous_session.rb @@ -2,35 +2,34 @@ module Gitlab class AnonymousSession - def initialize(remote_ip, session_id: nil) + def initialize(remote_ip) @remote_ip = remote_ip - @session_id = session_id end - def store_session_id_per_ip + def count_session_ip Gitlab::Redis::SharedState.with do |redis| redis.pipelined do - redis.sadd(session_lookup_name, session_id) + redis.incr(session_lookup_name) redis.expire(session_lookup_name, 24.hours) end end end - def stored_sessions + def session_count Gitlab::Redis::SharedState.with do |redis| - redis.scard(session_lookup_name) + redis.get(session_lookup_name).to_i end end - def cleanup_session_per_ip_entries + def cleanup_session_per_ip_count Gitlab::Redis::SharedState.with do |redis| - redis.srem(session_lookup_name, session_id) + redis.del(session_lookup_name) end end private - attr_reader :remote_ip, :session_id + attr_reader :remote_ip def session_lookup_name @session_lookup_name ||= "#{Gitlab::Redis::SharedState::IP_SESSIONS_LOOKUP_NAMESPACE}:#{remote_ip}" diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index a3feda9bb59..30cb74bcf54 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -29,7 +29,7 @@ module Gitlab Labkit::Context.current.to_h.include?(Labkit::Context.log_key(attribute_name)) end - def initialize(**args) + def initialize(args) unknown_attributes = args.keys - APPLICATION_ATTRIBUTES.map(&:name) raise ArgumentError, "#{unknown_attributes} are not known keys" if unknown_attributes.any? diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index ece4946383d..609eef5e365 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -50,7 +50,7 @@ module Gitlab build_access_token_check(login, password) || lfs_token_check(login, password, project) || oauth_access_token_check(login, password) || - personal_access_token_check(password) || + personal_access_token_check(password, project) || deploy_token_check(login, password, project) || user_with_password_for_git(login, password) || Gitlab::Auth::Result.new @@ -117,7 +117,6 @@ module Gitlab private - # rubocop:disable Gitlab/RailsLogger def rate_limit!(rate_limiter, success:, login:) return if skip_rate_limit?(login: login) @@ -132,12 +131,11 @@ module Gitlab # This returns true when the failures are over the threshold and the IP # is banned. if rate_limiter.register_fail! - Rails.logger.info "IP #{rate_limiter.ip} failed to login " \ + Gitlab::AppLogger.info "IP #{rate_limiter.ip} failed to login " \ "as #{login} but has been temporarily banned from Git auth" end end end - # rubocop:enable Gitlab/RailsLogger def skip_rate_limit?(login:) CI_JOB_USER == login @@ -191,12 +189,18 @@ module Gitlab end end - def personal_access_token_check(password) + def personal_access_token_check(password, project) return unless password.present? token = PersonalAccessTokensFinder.new(state: 'active').find_by_token(password) - if token && valid_scoped_token?(token, all_available_scopes) && token.user.can?(:log_in) + return unless token + + return if project && token.user.project_bot? && !project.bots.include?(token.user) + + return unless valid_scoped_token?(token, all_available_scopes) + + if token.user.project_bot? || token.user.can?(:log_in) Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end diff --git a/lib/gitlab/auth/atlassian/auth_hash.rb b/lib/gitlab/auth/atlassian/auth_hash.rb new file mode 100644 index 00000000000..047e4eabc51 --- /dev/null +++ b/lib/gitlab/auth/atlassian/auth_hash.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Atlassian + class AuthHash < Gitlab::Auth::OAuth::AuthHash + def token + credentials[:token] + end + + def refresh_token + credentials[:refresh_token] + end + + def expires? + credentials[:expires] + end + + def expires_at + credentials[:expires_at] + end + + private + + def credentials + auth_hash[:credentials] + end + end + end + end +end diff --git a/lib/gitlab/auth/atlassian/identity_linker.rb b/lib/gitlab/auth/atlassian/identity_linker.rb new file mode 100644 index 00000000000..4dec54d44d6 --- /dev/null +++ b/lib/gitlab/auth/atlassian/identity_linker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Atlassian + class IdentityLinker < OmniauthIdentityLinkerBase + extend ::Gitlab::Utils::Override + include ::Gitlab::Utils::StrongMemoize + + private + + override :identity + def identity + strong_memoize(:identity) do + current_user.atlassian_identity || build_atlassian_identity + end + end + + def build_atlassian_identity + identity = current_user.build_atlassian_identity + ::Gitlab::Auth::Atlassian::User.assign_identity_from_auth_hash!(identity, auth_hash) + end + + def auth_hash + ::Gitlab::Auth::Atlassian::AuthHash.new(oauth) + end + end + end + end +end diff --git a/lib/gitlab/auth/atlassian/user.rb b/lib/gitlab/auth/atlassian/user.rb new file mode 100644 index 00000000000..6ab7741cc54 --- /dev/null +++ b/lib/gitlab/auth/atlassian/user.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Atlassian + class User < Gitlab::Auth::OAuth::User + def self.assign_identity_from_auth_hash!(identity, auth_hash) + identity.extern_uid = auth_hash.uid + identity.token = auth_hash.token + identity.refresh_token = auth_hash.refresh_token + identity.expires_at = Time.at(auth_hash.expires_at).utc.to_datetime if auth_hash.expires? + + identity + end + + protected + + def find_by_uid_and_provider + ::Atlassian::Identity.find_by_extern_uid(auth_hash.uid)&.user + end + + def add_or_update_user_identities + return unless gl_user + + identity = gl_user.atlassian_identity || gl_user.build_atlassian_identity + self.class.assign_identity_from_auth_hash!(identity, auth_hash) + end + + def auth_hash=(auth_hash) + @auth_hash = ::Gitlab::Auth::Atlassian::AuthHash.new(auth_hash) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 4f448211abf..b7bb61f0677 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -55,7 +55,7 @@ module Gitlab response = ldap.get_operation_result unless response.code == 0 - Rails.logger.warn("LDAP search error: #{response.message}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn("LDAP search error: #{response.message}") end [] @@ -67,7 +67,7 @@ module Gitlab retries += 1 error_message = connection_error_message(error) - Rails.logger.warn(error_message) # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn(error_message) if retries < MAX_SEARCH_RETRIES renew_connection_adapter diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 7677189eb9f..88cc840c395 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -248,7 +248,7 @@ module Gitlab begin custom_options[:cert] = OpenSSL::X509::Certificate.new(custom_options[:cert]) rescue OpenSSL::X509::CertificateError => e - Rails.logger.error "LDAP TLS Options 'cert' is invalid for provider #{provider}: #{e.message}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error "LDAP TLS Options 'cert' is invalid for provider #{provider}: #{e.message}" end end @@ -256,7 +256,7 @@ module Gitlab begin custom_options[:key] = OpenSSL::PKey.read(custom_options[:key]) rescue OpenSSL::PKey::PKeyError => e - Rails.logger.error "LDAP TLS Options 'key' is invalid for provider #{provider}: #{e.message}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error "LDAP TLS Options 'key' is invalid for provider #{provider}: #{e.message}" end end diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index 8c5000147c4..102820d6bd5 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -45,7 +45,7 @@ module Gitlab def self.normalize_dn(dn) ::Gitlab::Auth::Ldap::DN.new(dn).to_normalized_s rescue ::Gitlab::Auth::Ldap::DN::FormatError => e - Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") dn end @@ -57,13 +57,13 @@ module Gitlab def self.normalize_uid(uid) ::Gitlab::Auth::Ldap::DN.normalize_value(uid) rescue ::Gitlab::Auth::Ldap::DN::FormatError => e - Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") uid end def initialize(entry, provider) - Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.debug "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" @entry = entry @provider = provider end diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 1ca59aa827b..1eae7af442d 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -5,10 +5,11 @@ module Gitlab module OAuth class Provider LABELS = { - "github" => "GitHub", - "gitlab" => "GitLab.com", - "google_oauth2" => "Google", - "azure_oauth2" => "Azure AD" + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google", + "azure_oauth2" => "Azure AD", + 'atlassian_oauth2' => 'Atlassian' }.freeze def self.authentication(user, provider) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 086f4a2e91c..3211d2ffaea 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -273,7 +273,11 @@ module Gitlab end def auto_link_user? - Gitlab.config.omniauth.auto_link_user + auto_link = Gitlab.config.omniauth.auto_link_user + return auto_link if [true, false].include?(auto_link) + + auto_link = Array(auto_link) + auto_link.include?(auth_hash.provider) end end end diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index ea0307e8bd6..d1b9062a23c 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -9,7 +9,7 @@ module Gitlab # Begins stealing jobs from the background migrations queue, blocking the # caller until all jobs have been completed. # - # When a migration raises a StandardError is is going to be retries up to + # When a migration raises a StandardError it is going to retry up to # three times, for example, to recover from a deadlock. # # When Exception is being raised, it enqueues the migration again, and diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb index c912628d0fc..5b9ee8a0ee2 100644 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -9,7 +9,7 @@ module Gitlab end def perform(start_id, stop_id) - Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") update = ' commits_count = ( diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb index e62f5edd0e7..76598f6e2a6 100644 --- a/lib/gitlab/background_migration/calculate_wiki_sizes.rb +++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb @@ -10,7 +10,7 @@ module Gitlab .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| statistics.refresh!(only: [:wiki_size]) rescue => e - Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" end end end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb index 4016b807f21..c0099d44b5a 100644 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -25,7 +25,7 @@ module Gitlab certificate_valid_not_after: domain.x509&.not_after&.iso8601 ) rescue => e - Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" end end end diff --git a/lib/gitlab/background_migration/fix_pages_access_level.rb b/lib/gitlab/background_migration/fix_pages_access_level.rb index 31d2e78b2d2..8e46021bd93 100644 --- a/lib/gitlab/background_migration/fix_pages_access_level.rb +++ b/lib/gitlab/background_migration/fix_pages_access_level.rb @@ -103,8 +103,8 @@ module Gitlab end # Private projects are not allowed to have enabled access level, only `private` and `public` - # If access control is enabled, these projects currently behave as if the have `private` pages_access_level - # if access control is disabled, these projects currently behave as if the have `public` pages_access_level + # If access control is enabled, these projects currently behave as if they have `private` pages_access_level + # if access control is disabled, these projects currently behave as if they have `public` pages_access_level # so we preserve this behaviour for projects with pages already deployed # for project without pages we always set `private` access_level def fix_private_access_level(start_id, stop_id) diff --git a/lib/gitlab/background_migration/migrate_to_hashed_storage.rb b/lib/gitlab/background_migration/migrate_to_hashed_storage.rb new file mode 100644 index 00000000000..4054db4fb87 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_to_hashed_storage.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration to move any legacy project to Hashed Storage + class MigrateToHashedStorage + def perform + batch_size = helper.batch_size + legacy_projects_count = Project.with_unmigrated_storage.count + + if storage_migrator.rollback_pending? + logger.warn( + migrator: 'MigrateToHashedStorage', + message: 'Aborting an storage rollback operation currently in progress' + ) + + storage_migrator.abort_rollback! + end + + if legacy_projects_count == 0 + logger.info( + migrator: 'MigrateToHashedStorage', + message: 'There are no projects requiring migration to Hashed Storage' + ) + + return + end + + logger.info( + migrator: 'MigrateToHashedStorage', + message: "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}" + ) + + helper.project_id_batches_migration do |start, finish| + storage_migrator.bulk_schedule_migration(start: start, finish: finish) + + logger.info( + migrator: 'MigrateToHashedStorage', + message: "Enqueuing migration of projects in batches of #{batch_size} from ID=#{start} to ID=#{finish}", + batch_from: start, + batch_to: finish + ) + end + end + + private + + def helper + Gitlab::HashedStorage::RakeHelper + end + + def storage_migrator + @storage_migrator ||= Gitlab::HashedStorage::Migrator.new + end + + def logger + @logger ||= ::Gitlab::BackgroundMigration::Logger.build + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb new file mode 100644 index 00000000000..eb72ef1de33 --- /dev/null +++ b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class PopulateResolvedOnDefaultBranchColumn + def perform(*); end + end + end +end + +Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn') diff --git a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb new file mode 100644 index 00000000000..a0c89cc4664 --- /dev/null +++ b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class creates/updates those project historical vulnerability statistics + # that haven't been created nor initialized. It should only be executed in EE. + class PopulateVulnerabilityHistoricalStatistics + def perform(project_ids) + end + end + end +end + +Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics') diff --git a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb new file mode 100644 index 00000000000..cc9b0329556 --- /dev/null +++ b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RemoveDuplicateCsFindings + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::RemoveDuplicateCsFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicateCsFindings') diff --git a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb index 9f765d03d62..527dd2a0a83 100644 --- a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb +++ b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb @@ -4,13 +4,18 @@ module Gitlab module BackgroundMigration # Sets the MergeRequestDiff#files_count value for old rows class SetMergeRequestDiffFilesCount - COUNT_SUBQUERY = <<~SQL - files_count = ( - SELECT count(*) - FROM merge_request_diff_files - WHERE merge_request_diff_files.merge_request_diff_id = merge_request_diffs.id - ) - SQL + # Some historic data has a *lot* of files. Apply a sentinel to these cases + FILES_COUNT_SENTINEL = 2**15 - 1 + + def self.count_subquery + <<~SQL + files_count = ( + SELECT LEAST(#{FILES_COUNT_SENTINEL}, count(*)) + FROM merge_request_diff_files + WHERE merge_request_diff_files.merge_request_diff_id = merge_request_diffs.id + ) + SQL + end class MergeRequestDiff < ActiveRecord::Base # rubocop:disable Style/Documentation include EachBatch @@ -20,7 +25,7 @@ module Gitlab def perform(start_id, end_id) MergeRequestDiff.where(id: start_id..end_id).each_batch do |relation| - relation.update_all(COUNT_SUBQUERY) + relation.update_all(self.class.count_subquery) end end end diff --git a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb index 9ac92aab637..c485c23f3be 100644 --- a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb +++ b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb @@ -11,9 +11,11 @@ module Gitlab class SetNullPackageFilesFileStoreToLocalValue LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL - # Temporary AR class for package files - class PackageFile < ActiveRecord::Base - self.table_name = 'packages_package_files' + module Packages + # Temporary AR class for package files + class PackageFile < ActiveRecord::Base + self.table_name = 'packages_package_files' + end end def perform(start_id, stop_id) diff --git a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb new file mode 100644 index 00000000000..651df36fcfd --- /dev/null +++ b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class UpdateLocationFingerprintForContainerScanningFindings + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings') diff --git a/lib/gitlab/backtrace_cleaner.rb b/lib/gitlab/backtrace_cleaner.rb index d04f0983d12..caea05c720d 100644 --- a/lib/gitlab/backtrace_cleaner.rb +++ b/lib/gitlab/backtrace_cleaner.rb @@ -31,7 +31,7 @@ module Gitlab return unless backtrace Array(Rails.backtrace_cleaner.clean(backtrace)).reject do |line| - line.match(IGNORED_BACKTRACES_REGEXP) + IGNORED_BACKTRACES_REGEXP.match?(line) end end end diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb index 6b78825aefd..1b985f83b22 100644 --- a/lib/gitlab/badge/coverage/template.rb +++ b/lib/gitlab/badge/coverage/template.rb @@ -25,7 +25,7 @@ module Gitlab end def key_text - if @key_text && @key_text.size <= MAX_KEY_SIZE + if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE @key_text else @entity.to_s @@ -37,7 +37,7 @@ module Gitlab end def key_width - if @key_width && @key_width.between?(1, MAX_KEY_SIZE) + if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) @key_width else 62 diff --git a/lib/gitlab/badge/pipeline/status.rb b/lib/gitlab/badge/pipeline/status.rb index 17f179f027d..f061ba22688 100644 --- a/lib/gitlab/badge/pipeline/status.rb +++ b/lib/gitlab/badge/pipeline/status.rb @@ -12,6 +12,7 @@ module Gitlab def initialize(project, ref, opts: {}) @project = project @ref = ref + @ignore_skipped = Gitlab::Utils.to_boolean(opts[:ignore_skipped], default: false) @customization = { key_width: opts[:key_width].to_i, key_text: opts[:key_text] @@ -26,9 +27,11 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def status - @project.ci_pipelines + pipelines = @project.ci_pipelines .where(sha: @sha) - .latest_status(@ref) || 'unknown' + + relation = @ignore_skipped ? pipelines.without_statuses([:skipped]) : pipelines + relation.latest_status(@ref) || 'unknown' end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/badge/pipeline/template.rb b/lib/gitlab/badge/pipeline/template.rb index 781897fab4b..af8e318395b 100644 --- a/lib/gitlab/badge/pipeline/template.rb +++ b/lib/gitlab/badge/pipeline/template.rb @@ -29,7 +29,7 @@ module Gitlab end def key_text - if @key_text && @key_text.size <= MAX_KEY_SIZE + if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE @key_text else @entity.to_s @@ -41,7 +41,7 @@ module Gitlab end def key_width - if @key_width && @key_width.between?(1, MAX_KEY_SIZE) + if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) @key_width else 62 diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/badge/template.rb index 97103e3f42c..9ac8f1c17f2 100644 --- a/lib/gitlab/badge/template.rb +++ b/lib/gitlab/badge/template.rb @@ -6,7 +6,8 @@ module Gitlab # Abstract template class for badges # class Template - MAX_KEY_SIZE = 128 + MAX_KEY_TEXT_SIZE = 64 + MAX_KEY_WIDTH = 512 def initialize(badge) @entity = badge.entity diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 18a1b64729e..aca5a63a424 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -61,17 +61,18 @@ module Gitlab }.to_json) end - def gitlab_user_id(email) - find_user_id(email) || project.creator_id - end + def find_user_id(by:, value:) + return unless value - def find_user_id(email) - return unless email + return users[value] if users.key?(value) - return users[email] if users.key?(email) + user = if by == :email + User.find_by_any_email(value, confirmed: true) + else + User.find_by_username(value) + end - user = User.find_by_any_email(email, confirmed: true) - users[email] = user&.id + users[value] = user&.id user&.id end @@ -197,9 +198,8 @@ module Gitlab log_info(stage: 'import_bitbucket_pull_requests', message: 'starting', iid: pull_request.iid) description = '' - description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author_email) + description += author_line(pull_request) description += pull_request.description if pull_request.description - author_id = gitlab_user_id(pull_request.author_email) attributes = { iid: pull_request.iid, @@ -212,7 +212,7 @@ module Gitlab target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name), target_branch_sha: pull_request.target_branch_sha, state_id: MergeRequest.available_states[pull_request.state], - author_id: author_id, + author_id: author_id(pull_request), created_at: pull_request.created_at, updated_at: pull_request.updated_at } @@ -254,7 +254,7 @@ module Gitlab committer = merge_event.committer_email - user_id = gitlab_user_id(committer) + user_id = find_user_id(by: :email, value: committer) || project.creator_id timestamp = merge_event.merge_timestamp merge_request.update({ merge_commit_sha: merge_event.merge_commit }) metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request) @@ -353,7 +353,7 @@ module Gitlab end def pull_request_comment_attributes(comment) - author = find_user_id(comment.author_email) + author = uid(comment) note = '' unless author @@ -397,6 +397,23 @@ module Gitlab def metrics @metrics ||= Gitlab::Import::Metrics.new(:bitbucket_server_importer, @project) end + + def author_line(rep_object) + return '' if uid(rep_object) + + @formatter.author_line(rep_object.author) + end + + def author_id(rep_object) + uid(rep_object) || project.creator_id + end + + def uid(rep_object) + find_user_id(by: :email, value: rep_object.author_email) unless Feature.enabled?(:bitbucket_server_user_mapping_by_username) + + find_user_id(by: :username, value: rep_object.author_username) || + find_user_id(by: :email, value: rep_object.author_email) + end end end end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index 6e48ca90054..3ad919fbba8 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -55,7 +55,7 @@ module Gitlab .join(':') end - private cache_key_method_name + private cache_key_method_name # rubocop: disable Style/AccessModifierDeclarations end end end diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index e18cf6ff8f2..78952db7a3e 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -17,7 +17,7 @@ module Gitlab return false unless new_lfs_pointers.present? - existing_count = @project.all_lfs_objects + existing_count = @project.lfs_objects .for_oids(new_lfs_pointers.map(&:lfs_oid)) .count diff --git a/lib/gitlab/checks/snippet_check.rb b/lib/gitlab/checks/snippet_check.rb index bcecd0fc251..8c61b782baa 100644 --- a/lib/gitlab/checks/snippet_check.rb +++ b/lib/gitlab/checks/snippet_check.rb @@ -3,7 +3,6 @@ module Gitlab module Checks class SnippetCheck < BaseChecker - DEFAULT_BRANCH = 'master'.freeze ERROR_MESSAGES = { create_delete_branch: 'You can not create or delete branches.' }.freeze @@ -11,17 +10,18 @@ module Gitlab ATTRIBUTES = %i[oldrev newrev ref branch_name tag_name logger].freeze attr_reader(*ATTRIBUTES) - def initialize(change, logger:) + def initialize(change, default_branch:, logger:) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @tag_name = Gitlab::Git.tag_name(@ref) + @default_branch = default_branch @logger = logger @logger.append_message("Running checks for ref: #{@branch_name || @tag_name}") end def validate! - if creation? || deletion? + if !@default_branch || creation? || deletion? raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_delete_branch] end @@ -31,7 +31,7 @@ module Gitlab private def creation? - @branch_name != DEFAULT_BRANCH && super + @branch_name != @default_branch && super end end end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index e145bd2e9df..1fac00337a3 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -31,105 +31,205 @@ module Gitlab end class Converter - def on_0(_) reset end + def on_0(_) + reset + end - def on_1(_) enable(STYLE_SWITCHES[:bold]) end + def on_1(_) + enable(STYLE_SWITCHES[:bold]) + end - def on_3(_) enable(STYLE_SWITCHES[:italic]) end + def on_3(_) + enable(STYLE_SWITCHES[:italic]) + end - def on_4(_) enable(STYLE_SWITCHES[:underline]) end + def on_4(_) + enable(STYLE_SWITCHES[:underline]) + end - def on_8(_) enable(STYLE_SWITCHES[:conceal]) end + def on_8(_) + enable(STYLE_SWITCHES[:conceal]) + end - def on_9(_) enable(STYLE_SWITCHES[:cross]) end + def on_9(_) + enable(STYLE_SWITCHES[:cross]) + end - def on_21(_) disable(STYLE_SWITCHES[:bold]) end + def on_21(_) + disable(STYLE_SWITCHES[:bold]) + end - def on_22(_) disable(STYLE_SWITCHES[:bold]) end + def on_22(_) + disable(STYLE_SWITCHES[:bold]) + end - def on_23(_) disable(STYLE_SWITCHES[:italic]) end + def on_23(_) + disable(STYLE_SWITCHES[:italic]) + end - def on_24(_) disable(STYLE_SWITCHES[:underline]) end + def on_24(_) + disable(STYLE_SWITCHES[:underline]) + end - def on_28(_) disable(STYLE_SWITCHES[:conceal]) end + def on_28(_) + disable(STYLE_SWITCHES[:conceal]) + end - def on_29(_) disable(STYLE_SWITCHES[:cross]) end + def on_29(_) + disable(STYLE_SWITCHES[:cross]) + end - def on_30(_) set_fg_color(0) end + def on_30(_) + set_fg_color(0) + end - def on_31(_) set_fg_color(1) end + def on_31(_) + set_fg_color(1) + end - def on_32(_) set_fg_color(2) end + def on_32(_) + set_fg_color(2) + end - def on_33(_) set_fg_color(3) end + def on_33(_) + set_fg_color(3) + end - def on_34(_) set_fg_color(4) end + def on_34(_) + set_fg_color(4) + end - def on_35(_) set_fg_color(5) end + def on_35(_) + set_fg_color(5) + end - def on_36(_) set_fg_color(6) end + def on_36(_) + set_fg_color(6) + end - def on_37(_) set_fg_color(7) end + def on_37(_) + set_fg_color(7) + end - def on_38(stack) set_fg_color_256(stack) end + def on_38(stack) + set_fg_color_256(stack) + end - def on_39(_) set_fg_color(9) end + def on_39(_) + set_fg_color(9) + end - def on_40(_) set_bg_color(0) end + def on_40(_) + set_bg_color(0) + end - def on_41(_) set_bg_color(1) end + def on_41(_) + set_bg_color(1) + end - def on_42(_) set_bg_color(2) end + def on_42(_) + set_bg_color(2) + end - def on_43(_) set_bg_color(3) end + def on_43(_) + set_bg_color(3) + end - def on_44(_) set_bg_color(4) end + def on_44(_) + set_bg_color(4) + end - def on_45(_) set_bg_color(5) end + def on_45(_) + set_bg_color(5) + end - def on_46(_) set_bg_color(6) end + def on_46(_) + set_bg_color(6) + end - def on_47(_) set_bg_color(7) end + def on_47(_) + set_bg_color(7) + end - def on_48(stack) set_bg_color_256(stack) end + def on_48(stack) + set_bg_color_256(stack) + end - def on_49(_) set_bg_color(9) end + def on_49(_) + set_bg_color(9) + end - def on_90(_) set_fg_color(0, 'l') end + def on_90(_) + set_fg_color(0, 'l') + end - def on_91(_) set_fg_color(1, 'l') end + def on_91(_) + set_fg_color(1, 'l') + end - def on_92(_) set_fg_color(2, 'l') end + def on_92(_) + set_fg_color(2, 'l') + end - def on_93(_) set_fg_color(3, 'l') end + def on_93(_) + set_fg_color(3, 'l') + end - def on_94(_) set_fg_color(4, 'l') end + def on_94(_) + set_fg_color(4, 'l') + end - def on_95(_) set_fg_color(5, 'l') end + def on_95(_) + set_fg_color(5, 'l') + end - def on_96(_) set_fg_color(6, 'l') end + def on_96(_) + set_fg_color(6, 'l') + end - def on_97(_) set_fg_color(7, 'l') end + def on_97(_) + set_fg_color(7, 'l') + end - def on_99(_) set_fg_color(9, 'l') end + def on_99(_) + set_fg_color(9, 'l') + end - def on_100(_) set_bg_color(0, 'l') end + def on_100(_) + set_bg_color(0, 'l') + end - def on_101(_) set_bg_color(1, 'l') end + def on_101(_) + set_bg_color(1, 'l') + end - def on_102(_) set_bg_color(2, 'l') end + def on_102(_) + set_bg_color(2, 'l') + end - def on_103(_) set_bg_color(3, 'l') end + def on_103(_) + set_bg_color(3, 'l') + end - def on_104(_) set_bg_color(4, 'l') end + def on_104(_) + set_bg_color(4, 'l') + end - def on_105(_) set_bg_color(5, 'l') end + def on_105(_) + set_bg_color(5, 'l') + end - def on_106(_) set_bg_color(6, 'l') end + def on_106(_) + set_bg_color(6, 'l') + end - def on_107(_) set_bg_color(7, 'l') end + def on_107(_) + set_bg_color(7, 'l') + end - def on_109(_) set_bg_color(9, 'l') end + def on_109(_) + set_bg_color(9, 'l') + end attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask, :sections, :lineno_in_section diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb index c2d17cc176e..6395a20ca99 100644 --- a/lib/gitlab/ci/artifact_file_reader.rb +++ b/lib/gitlab/ci/artifact_file_reader.rb @@ -45,6 +45,31 @@ module Gitlab end def read_zip_file!(file_path) + if ::Gitlab::Ci::Features.new_artifact_file_reader_enabled?(job.project) + read_with_new_artifact_file_reader(file_path) + else + read_with_legacy_artifact_file_reader(file_path) + end + end + + def read_with_new_artifact_file_reader(file_path) + job.artifacts_file.use_open_file do |file| + zip_file = Zip::File.new(file, false, true) + entry = zip_file.find_entry(file_path) + + unless entry + raise Error, "Path `#{file_path}` does not exist inside the `#{job.name}` artifacts archive!" + end + + if entry.name_is_directory? + raise Error, "Path `#{file_path}` was expected to be a file but it was a directory!" + end + + zip_file.read(entry) + end + end + + def read_with_legacy_artifact_file_reader(file_path) job.artifacts_file.use_file do |archive_path| Zip::File.open(archive_path) do |zip_file| entry = zip_file.find_entry(file_path) diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb index b64990d6a7a..72ef0a8d067 100644 --- a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb +++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb @@ -33,7 +33,7 @@ module Gitlab def kubernetes_namespace strong_memoize(:kubernetes_namespace) do - Clusters::KubernetesNamespaceFinder.new( + ::Clusters::KubernetesNamespaceFinder.new( deployment_cluster, project: environment.project, environment_name: environment.name, @@ -47,7 +47,7 @@ module Gitlab return if conflicting_ci_namespace_requested?(namespace) - Clusters::Kubernetes::CreateOrUpdateNamespaceService.new( + ::Clusters::Kubernetes::CreateOrUpdateNamespaceService.new( cluster: deployment_cluster, kubernetes_namespace: namespace ).execute @@ -71,7 +71,7 @@ module Gitlab end def build_namespace_record - Clusters::BuildKubernetesNamespaceService.new( + ::Clusters::BuildKubernetesNamespaceService.new( deployment_cluster, environment: environment ).execute diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index d81a3fef1f5..9d269831679 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -62,6 +62,10 @@ module Gitlab root.jobs_value end + def normalized_jobs + @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs + end + private def expand_config(config) diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index f960cec1f26..ecc2c5cb729 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -122,39 +122,9 @@ module Gitlab :needs, :retry, :parallel, :start_in, :interruptible, :timeout, :resource_group, :release - Matcher = Struct.new(:name, :config) do - def applies? - job_is_not_hidden? && - config_is_a_hash? && - has_job_keys? - end - - private - - def job_is_not_hidden? - !name.to_s.start_with?('.') - end - - def config_is_a_hash? - config.is_a?(Hash) - end - - def has_job_keys? - if name == :default - config.key?(:script) - else - (ALLOWED_KEYS & config.keys).any? - end - end - end - def self.matching?(name, config) - if Gitlab::Ci::Features.job_entry_matches_all_keys? - Matcher.new(name, config).applies? - else - !name.to_s.start_with?('.') && - config.is_a?(Hash) && config.key?(:script) - end + !name.to_s.start_with?('.') && + config.is_a?(Hash) && config.key?(:script) end def self.visible? diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 1d3036189b0..b5ce42969a5 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -14,8 +14,8 @@ module Gitlab validates :config, type: Hash validate do - unless has_valid_jobs? - errors.add(:config, 'should contain valid jobs') + each_unmatched_job do |name| + errors.add(name, 'config should implement a script: or a trigger: keyword') end unless has_visible_job? @@ -23,9 +23,9 @@ module Gitlab end end - def has_valid_jobs? - config.all? do |name, value| - Jobs.find_type(name, value) + def each_unmatched_job + config.each do |name, value| + yield(name) unless Jobs.find_type(name, value) end end diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 19d6a470941..2d93f1ab06e 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -134,7 +134,7 @@ module Gitlab @jobs_config = @config .except(*self.class.reserved_nodes_names) .select do |name, config| - Entry::Jobs.find_type(name, config).present? + Entry::Jobs.find_type(name, config).present? || ALLOWED_KEYS.exclude?(name) end @config = @config.except(*@jobs_config.keys) diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 451ba14bb89..22fcd84c968 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -11,6 +11,7 @@ module Gitlab end def normalize_jobs + return {} unless @jobs_config return @jobs_config if parallelized_jobs.empty? expand_parallelize_jobs do |job_name, config| diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb index db21274a9ed..5a23836d8a0 100644 --- a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb +++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb @@ -48,14 +48,13 @@ module Gitlab } end - def name_with_details - vars = variables.map { |key, value| "#{key}=#{value}"}.join('; ') - - "#{job_name} (#{vars})" - end - def name - "#{job_name} #{instance}/#{total}" + vars = variables + .values + .compact + .join(', ') + + "#{job_name}: [#{vars}]" end private diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 2f6667d3600..e770187b124 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -31,56 +31,49 @@ module Gitlab ::Feature.enabled?(:ci_store_pipeline_messages, project, default_enabled: true) end - # Remove in https://gitlab.com/gitlab-org/gitlab/-/issues/227052 - def self.variables_api_filter_environment_scope? - ::Feature.enabled?(:ci_variables_api_filter_environment_scope, default_enabled: true) - end - def self.raise_job_rules_without_workflow_rules_warning? ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) end - def self.keep_latest_artifacts_for_ref_enabled?(project) - ::Feature.enabled?(:keep_latest_artifacts_for_ref, project, default_enabled: false) - end - - def self.destroy_only_unlocked_expired_artifacts_enabled? - ::Feature.enabled?(:destroy_only_unlocked_expired_artifacts, default_enabled: false) - end - def self.bulk_insert_on_create?(project) ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true) end - def self.ci_if_parenthesis_enabled? - ::Feature.enabled?(:ci_if_parenthesis_enabled, default_enabled: true) + # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` + # is a safe switch to disable the feature for a parituclar project when something went wrong, + # therefore it's not supposed to be enabled by default. + def self.disallow_to_create_merge_request_pipelines_in_target_project?(target_project) + ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) end - def self.allow_to_create_merge_request_pipelines_in_target_project?(target_project) - ::Feature.enabled?(:ci_allow_to_create_merge_request_pipelines_in_target_project, target_project, default_enabled: true) + def self.lint_creates_pipeline_with_dry_run?(project) + ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project, default_enabled: true) end - def self.ci_plan_needs_size_limit?(project) - ::Feature.enabled?(:ci_plan_needs_size_limit, project, default_enabled: true) + def self.project_transactionless_destroy?(project) + Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) end - def self.job_entry_matches_all_keys? - ::Feature.enabled?(:ci_job_entry_matches_all_keys) + def self.coverage_report_view?(project) + ::Feature.enabled?(:coverage_report_view, project, default_enabled: true) end - def self.lint_creates_pipeline_with_dry_run?(project) - ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project, default_enabled: true) + def self.child_of_child_pipeline_enabled?(project) + ::Feature.enabled?(:ci_child_of_child_pipeline, project, default_enabled: true) end - def self.reset_ci_minutes_for_all_namespaces? - ::Feature.enabled?(:reset_ci_minutes_for_all_namespaces, default_enabled: false) + def self.trace_overwrite? + ::Feature.enabled?(:ci_trace_overwrite, type: :ops, default_enabled: false) end - def self.expand_names_for_cross_pipeline_artifacts?(project) - ::Feature.enabled?(:ci_expand_names_for_cross_pipeline_artifacts, project) + def self.accept_trace?(project) + ::Feature.enabled?(:ci_enable_live_trace, project) && + ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: false) + end + + def self.new_artifact_file_reader_enabled?(project) + ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: false) end end end end - -::Gitlab::Ci::Features.prepend_if_ee('::EE::Gitlab::Ci::Features') diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb new file mode 100644 index 00000000000..86a9ebfa451 --- /dev/null +++ b/lib/gitlab/ci/lint.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Lint + class Result + attr_reader :jobs, :errors, :warnings + + def initialize(jobs:, errors:, warnings:) + @jobs = jobs + @errors = errors + @warnings = warnings + end + + def valid? + @errors.empty? + end + end + + def initialize(project:, current_user:) + @project = project + @current_user = current_user + end + + def validate(content, dry_run: false) + if dry_run && Gitlab::Ci::Features.lint_creates_pipeline_with_dry_run?(@project) + simulate_pipeline_creation(content) + else + static_validation(content) + end + end + + private + + def simulate_pipeline_creation(content) + pipeline = ::Ci::CreatePipelineService + .new(@project, @current_user, ref: @project.default_branch) + .execute(:push, dry_run: true, content: content) + + Result.new( + jobs: dry_run_convert_to_jobs(pipeline.stages), + errors: pipeline.error_messages.map(&:content), + warnings: pipeline.warning_messages(limit: ::Gitlab::Ci::Warnings::MAX_LIMIT).map(&:content) + ) + end + + def static_validation(content) + result = Gitlab::Ci::YamlProcessor.new( + content, + project: @project, + user: @current_user, + sha: @project.repository.commit.sha + ).execute + + Result.new( + jobs: static_validation_convert_to_jobs(result), + errors: result.errors, + warnings: result.warnings.take(::Gitlab::Ci::Warnings::MAX_LIMIT) # rubocop: disable CodeReuse/ActiveRecord + ) + end + + def dry_run_convert_to_jobs(stages) + stages.reduce([]) do |jobs, stage| + jobs + stage.statuses.map do |job| + { + name: job.name, + stage: stage.name, + before_script: job.options[:before_script].to_a, + script: job.options[:script].to_a, + after_script: job.options[:after_script].to_a, + tag_list: (job.tag_list if job.is_a?(::Ci::Build)).to_a, + environment: job.options.dig(:environment, :name), + when: job.when, + allow_failure: job.allow_failure + } + end + end + end + + def static_validation_convert_to_jobs(result) + jobs = [] + return jobs unless result.valid? + + result.stages.each do |stage_name| + result.builds.each do |job| + next unless job[:stage] == stage_name + + jobs << { + name: job[:name], + stage: stage_name, + before_script: job.dig(:options, :before_script).to_a, + script: job.dig(:options, :script).to_a, + after_script: job.dig(:options, :after_script).to_a, + tag_list: job[:tag_list].to_a, + only: job[:only], + except: job[:except], + environment: job[:environment], + when: job[:when], + allow_failure: job[:allow_failure] + } + end + end + + jobs + end + end + end +end diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb index 58d55b1bd6f..e5a7151b823 100644 --- a/lib/gitlab/ci/mask_secret.rb +++ b/lib/gitlab/ci/mask_secret.rb @@ -8,6 +8,11 @@ module Gitlab # We assume 'value' must be mutable, given # that frozen string is enabled. + + ## + # TODO We need to remove this because it is going to change checksum of + # a trace. + # value.gsub!(token, 'x' * token.length) value end diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 4190c40eb66..9662209f88e 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -20,11 +20,7 @@ module Gitlab pipeline_schedule: @command.schedule, merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, - variables_attributes: Array(@command.variables_attributes), - # This should be removed and set on the database column default - # level when the keep_latest_artifacts_for_ref feature flag is - # removed. - locked: ::Gitlab::Ci::Features.keep_latest_artifacts_for_ref_enabled?(@command.project) ? :artifacts_locked : :unlocked + variables_attributes: Array(@command.variables_attributes) ) end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index dbaa6951e64..d1882059dd8 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -12,7 +12,7 @@ module Gitlab :seeds_block, :variables_attributes, :push_options, :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, # These attributes are set by Chains during processing: - :config_content, :config_processor, :stage_seeds + :config_content, :yaml_processor_result, :stage_seeds ) do include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb index dcc336b8929..4990a5a6eb5 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb @@ -9,7 +9,7 @@ module Gitlab class Remote < Source def content strong_memoize(:content) do - next unless ci_config_path =~ URI.regexp(%w[http https]) + next unless ci_config_path =~ URI::DEFAULT_PARSER.make_regexp(%w[http https]) YAML.dump('include' => [{ 'remote' => ci_config_path }]) end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 2cfcb295407..8ccb33ffd34 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -11,20 +11,23 @@ module Gitlab def perform! raise ArgumentError, 'missing config content' unless @command.config_content - @command.config_processor = ::Gitlab::Ci::YamlProcessor.new( + result = ::Gitlab::Ci::YamlProcessor.new( @command.config_content, { project: project, sha: @pipeline.sha, user: current_user, parent_pipeline: parent_pipeline } - ) + ).execute + + add_warnings_to_pipeline(result.warnings) - add_warnings_to_pipeline(@command.config_processor.warnings) - rescue Gitlab::Ci::YamlProcessor::ValidationError => ex - add_warnings_to_pipeline(ex.warnings) + if result.valid? + @command.yaml_processor_result = result + else + error(result.errors.first, config_error: true) + end - error(ex.message, config_error: true) rescue => ex Gitlab::ErrorTracking.track_exception(ex, project_id: project.id, diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb index a793ae9cc24..3c910963a2a 100644 --- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -39,7 +39,7 @@ module Gitlab end def workflow_config - @command.config_processor.workflow_attributes || {} + @command.yaml_processor_result.workflow_attributes || {} end end end diff --git a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb index 9267c72efa4..71f22c52869 100644 --- a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb +++ b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb @@ -6,13 +6,13 @@ module Gitlab module Chain class RemoveUnwantedChatJobs < Chain::Base def perform! - raise ArgumentError, 'missing config processor' unless @command.config_processor + raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result return unless pipeline.chat? # When scheduling a chat pipeline we only want to run the build # that matches the chat command. - @command.config_processor.jobs.select! do |name, _| + @command.yaml_processor_result.jobs.select! do |name, _| name.to_s == command.chat_data[:command].to_s end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index e48e79d561b..e10a0bc3718 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize def perform! - raise ArgumentError, 'missing config processor' unless @command.config_processor + raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result # Allocate next IID. This operation must be outside of transactions of pipeline creations. pipeline.ensure_project_iid! @@ -56,7 +56,7 @@ module Gitlab end def stages_attributes - @command.config_processor.stages_attributes + @command.yaml_processor_result.stages_attributes end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 24628338dd2..d056501a6d3 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -51,7 +51,7 @@ module Gitlab def validate_service_request Gitlab::HTTP.post( validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT, - body: validation_service_payload(@pipeline, @command.config_processor.stages_attributes) + body: validation_service_payload(@pipeline, @command.yaml_processor_result.stages_attributes) ) end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index 5b7365cb33b..ac03ef79ccb 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -24,26 +24,8 @@ module Gitlab Expression::Lexeme::Or ].freeze - # To be removed with `ci_if_parenthesis_enabled` - LEGACY_LEXEMES = [ - Expression::Lexeme::Variable, - Expression::Lexeme::String, - Expression::Lexeme::Pattern, - Expression::Lexeme::Null, - Expression::Lexeme::Equals, - Expression::Lexeme::Matches, - Expression::Lexeme::NotEquals, - Expression::Lexeme::NotMatches, - Expression::Lexeme::And, - Expression::Lexeme::Or - ].freeze - def self.lexemes - if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled? - LEXEMES - else - LEGACY_LEXEMES - end + LEXEMES end MAX_TOKENS = 100 diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index 27d7aa2f37e..a20b0015e05 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -15,12 +15,7 @@ module Gitlab def tree results = [] - tokens = - if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled? - tokens_rpn - else - legacy_tokens_rpn - end + tokens = tokens_rpn tokens.each do |token| case token.type @@ -78,27 +73,6 @@ module Gitlab output.concat(operators.reverse) end - - # To be removed with `ci_if_parenthesis_enabled` - def legacy_tokens_rpn - output = [] - operators = [] - - @tokens.each do |token| - case token.type - when :value - output.push(token) - when :logical_operator - if operators.any? && token.lexeme.precedence >= operators.last.lexeme.precedence - output.push(operators.pop) - end - - operators.push(token) - end - end - - output.concat(operators.reverse) - end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 3be3fa63b92..91dbcc616ea 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,8 +11,6 @@ module Gitlab delegate :dig, to: :@seed_attributes - DEFAULT_NEEDS_LIMIT = 10 - def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline @seed_attributes = attributes @@ -140,11 +138,7 @@ module Gitlab end def max_needs_allowed - if ::Gitlab::Ci::Features.ci_plan_needs_size_limit?(@pipeline.project) - @pipeline.project.actual_limits.ci_needs_size_limit - else - DEFAULT_NEEDS_LIMIT - end + @pipeline.project.actual_limits.ci_needs_size_limit end def pipeline_attributes diff --git a/lib/gitlab/ci/pipeline_object_hierarchy.rb b/lib/gitlab/ci/pipeline_object_hierarchy.rb new file mode 100644 index 00000000000..de3262b10e0 --- /dev/null +++ b/lib/gitlab/ci/pipeline_object_hierarchy.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class PipelineObjectHierarchy < ::Gitlab::ObjectHierarchy + private + + def middle_table + ::Ci::Sources::Pipeline.arel_table + end + + def from_tables(cte) + [objects_table, cte.table, middle_table] + end + + def parent_id_column(_cte) + middle_table[:source_pipeline_id] + end + + def ancestor_conditions(cte) + middle_table[:source_pipeline_id].eq(objects_table[:id]).and( + middle_table[:pipeline_id].eq(cte.table[:id]) + ).and( + same_project_condition + ) + end + + def descendant_conditions(cte) + middle_table[:pipeline_id].eq(objects_table[:id]).and( + middle_table[:source_pipeline_id].eq(cte.table[:id]) + ).and( + same_project_condition + ) + end + + def same_project_condition + if options[:same_project] + middle_table[:source_project_id].eq(middle_table[:project_id]) + else + Arel.sql('TRUE') + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb index 75898745366..15a3c862c9e 100644 --- a/lib/gitlab/ci/reports/test_case.rb +++ b/lib/gitlab/ci/reports/test_case.rb @@ -8,7 +8,7 @@ module Gitlab STATUS_FAILED = 'failed' STATUS_SKIPPED = 'skipped' STATUS_ERROR = 'error' - STATUS_TYPES = [STATUS_SUCCESS, STATUS_FAILED, STATUS_SKIPPED, STATUS_ERROR].freeze + STATUS_TYPES = [STATUS_ERROR, STATUS_FAILED, STATUS_SUCCESS, STATUS_SKIPPED].freeze attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index 5ee779227ec..e9b78b841e4 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -78,11 +78,27 @@ module Gitlab end end + def sorted + sort_by_status + sort_by_execution_time_desc + self + end + private def existing_key?(test_case) @test_cases[test_case.status]&.key?(test_case.key) end + + def sort_by_status + @test_cases = @test_cases.sort_by { |status, _| Gitlab::Ci::Reports::TestCase::STATUS_TYPES.index(status) }.to_h + end + + def sort_by_execution_time_desc + @test_cases = @test_cases.keys.each_with_object({}) do |key, hash| + hash[key] = @test_cases[key].sort_by { |_key, test_case| -test_case.execution_time }.to_h + end + end end end end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index 4746195c618..b95565b5e09 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -10,14 +10,28 @@ module Gitlab end def has_details? - false + !!details_path + end + + def details_path + return unless Feature.enabled?(:ci_bridge_pipeline_details, subject.project, default_enabled: true) + return unless can?(user, :read_pipeline, downstream_pipeline) + + project_pipeline_path(downstream_project, downstream_pipeline) end def has_action? false end - def details_path + private + + def downstream_pipeline + subject.downstream_pipeline + end + + def downstream_project + downstream_pipeline&.project end end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 88846f724e7..f6562737838 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -25,7 +25,8 @@ module Gitlab insufficient_bridge_permissions: 'no permissions to trigger downstream pipeline', bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline', downstream_pipeline_creation_failed: 'downstream pipeline can not be created', - secrets_provider_not_found: 'secrets provider can not be found' + secrets_provider_not_found: 'secrets provider can not be found', + reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 04a9fc29802..9a4f5644f7d 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -7,7 +7,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize # This class accepts an array of arrays/hashes/or objects - def initialize(all_statuses, with_allow_failure: true) + def initialize(all_statuses, with_allow_failure: true, dag: false) unless all_statuses.respond_to?(:pluck) raise ArgumentError, "all_statuses needs to respond to `.pluck`" end @@ -15,6 +15,7 @@ module Gitlab @status_set = Set.new @status_key = 0 @allow_failure_key = 1 if with_allow_failure + @dag = dag consume_all_statuses(all_statuses) end @@ -31,7 +32,13 @@ module Gitlab return if none? strong_memoize(:status) do - if only_of?(:skipped, :ignored) + if @dag && any_of?(:skipped) + # The DAG job is skipped if one of the needs does not run at all. + 'skipped' + elsif @dag && !only_of?(:success, :failed, :canceled, :skipped, :success_with_warnings) + # DAG is blocked from executing if a dependent is not "complete" + 'pending' + elsif only_of?(:skipped, :ignored) 'skipped' elsif only_of?(:success, :skipped, :success_with_warnings, :ignored) 'success' diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 968ff0fce89..6966ce88b30 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -150,16 +150,22 @@ workflow: - exists: - .static +# NOTE: These links point to the latest templates for development in GitLab canonical project, +# therefore the actual templates that were included for Auto DevOps pipelines +# could be different from the contents in the links. +# To view the actual templates, please replace `master` to the specific GitLab version when +# the Auto DevOps pipeline started running e.g. `v13.0.2-ee`. include: - - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml - - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml - - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml - - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml - - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml - - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml - - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml - - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml + - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml + - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml + - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml + - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml + - template: Jobs/Deploy/ECS.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml + - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml + - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml + - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml + - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 8553a940bd7..5edb26a0b56 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -7,7 +7,7 @@ performance: variables: DOCKER_TLS_CERTDIR: "" SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 13.3.0 + SITESPEED_VERSION: 14.1.0 SITESPEED_OPTIONS: '' services: - docker:19.03.12-dind @@ -20,15 +20,15 @@ performance: fi - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - mkdir gitlab-exporter - - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.0.1/index.js + - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | if [ -f .gitlab-urls.txt ] then sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS else - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS fi - mv sitespeed-results/data/performance.json browser-performance.json artifacts: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index cf851c875ee..568ceceeaa2 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -9,6 +9,8 @@ code_quality: DOCKER_TLS_CERTDIR: "" CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10-gitlab.1" needs: [] + before_script: + - export SOURCE_CODE=$PWD script: - | if ! docker info &>/dev/null; then @@ -16,11 +18,27 @@ code_quality: export DOCKER_HOST='tcp://localhost:2375' fi fi + - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } - docker pull --quiet "$CODE_QUALITY_IMAGE" - - docker run - --env SOURCE_CODE="$PWD" - --volume "$PWD":/code - --volume /var/run/docker.sock:/var/run/docker.sock + - | + docker run \ + $(propagate_env_vars \ + SOURCE_CODE \ + TIMEOUT_SECONDS \ + CODECLIMATE_DEBUG \ + CODECLIMATE_DEV \ + REPORT_STDOUT \ + ENGINE_MEMORY_LIMIT_BYTES \ + ) \ + --volume "$PWD":/code \ + --volume /var/run/docker.sock:/var/run/docker.sock \ "$CODE_QUALITY_IMAGE" /code artifacts: reports: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 2922e1c6e88..829fd7a722f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -2,9 +2,6 @@ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" dependencies: [] -include: - - template: Jobs/Deploy/ECS.gitlab-ci.yml - review: extends: .auto-deploy stage: review diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml new file mode 100644 index 00000000000..829fd7a722f --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -0,0 +1,249 @@ +.auto-deploy: + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" + dependencies: [] + +review: + extends: .auto-deploy + stage: review + script: + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy persist_environment_url + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN + on_stop: stop_review + artifacts: + paths: [environment_url.txt, tiller.log] + when: always + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + +stop_review: + extends: .auto-deploy + stage: cleanup + variables: + GIT_STRATEGY: none + script: + - auto-deploy initialize_tiller + - auto-deploy delete + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + allow_failure: true + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + when: manual + +# Staging deploys are disabled by default since +# continuous deployment to production is enabled by default +# If you prefer to automatically deploy to staging and +# only manually promote to production, enable this job by setting +# STAGING_ENABLED. + +staging: + extends: .auto-deploy + stage: staging + script: + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + environment: + name: staging + url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$STAGING_ENABLED' + +# Canaries are disabled by default, but if you want them, +# and know what the downsides are, you can enable this by setting +# CANARY_ENABLED. + +canary: + extends: .auto-deploy + stage: canary + allow_failure: true + script: + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy canary + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$CANARY_ENABLED' + when: manual + +.production: &production_template + extends: .auto-deploy + stage: production + script: + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy delete canary + - auto-deploy delete rollout + - auto-deploy persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN + artifacts: + paths: [environment_url.txt, tiller.log] + when: always + +production: + <<: *production_template + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$STAGING_ENABLED' + when: never + - if: '$CANARY_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + +production_manual: + <<: *production_template + allow_failure: false + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_ENABLED' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE' + when: never + - if: '$CI_COMMIT_BRANCH == "master" && $STAGING_ENABLED' + when: manual + - if: '$CI_COMMIT_BRANCH == "master" && $CANARY_ENABLED' + when: manual + +# This job implements incremental rollout on for every push to `master`. + +.rollout: &rollout_template + extends: .auto-deploy + script: + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE + - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) + - auto-deploy delete canary + - auto-deploy persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN + artifacts: + paths: [environment_url.txt, tiller.log] + when: always + +.manual_rollout_template: &manual_rollout_template + <<: *rollout_template + stage: production + resource_group: production + allow_failure: true + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + # $INCREMENTAL_ROLLOUT_ENABLED is for compamtibilty with pre-GitLab 11.4 syntax + - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' + when: manual + +.timed_rollout_template: &timed_rollout_template + <<: *rollout_template + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' + when: delayed + start_in: 5 minutes + +timed rollout 10%: + <<: *timed_rollout_template + stage: incremental rollout 10% + variables: + ROLLOUT_PERCENTAGE: 10 + +timed rollout 25%: + <<: *timed_rollout_template + stage: incremental rollout 25% + variables: + ROLLOUT_PERCENTAGE: 25 + +timed rollout 50%: + <<: *timed_rollout_template + stage: incremental rollout 50% + variables: + ROLLOUT_PERCENTAGE: 50 + +timed rollout 100%: + <<: *timed_rollout_template + <<: *production_template + stage: incremental rollout 100% + variables: + ROLLOUT_PERCENTAGE: 100 + +rollout 10%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 10 + +rollout 25%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 25 + +rollout 50%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 50 + +rollout 100%: + <<: *manual_rollout_template + <<: *production_template + allow_failure: false diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index 4a9849c85c9..9a7c513c25f 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -8,6 +8,7 @@ load_performance: K6_VERSION: 0.27.0 K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js K6_OPTIONS: '' + K6_DOCKER_OPTIONS: '' services: - docker:19.03.11-dind script: @@ -17,7 +18,7 @@ load_performance: export DOCKER_HOST='tcp://localhost:2375' fi fi - - docker run --rm -v "$(pwd)":/k6 -w /k6 $K6_IMAGE:$K6_VERSION run $K6_TEST_FILE --summary-export=load-performance.json $K6_OPTIONS + - docker run --rm -v "$(pwd)":/k6 -w /k6 $K6_DOCKER_OPTIONS $K6_IMAGE:$K6_VERSION run $K6_TEST_FILE --summary-export=load-performance.json $K6_OPTIONS artifacts: reports: load_performance: load-performance.json diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 3d0bacda853..7050b41e045 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,27 +1,11 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.24.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.29.0" environment: name: production variables: TILLER_NAMESPACE: gitlab-managed-apps GITLAB_MANAGED_APPS_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/config.yaml - INGRESS_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/ingress/values.yaml - CERT_MANAGER_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/cert-manager/values.yaml - SENTRY_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/sentry/values.yaml - GITLAB_RUNNER_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/gitlab-runner/values.yaml - CILIUM_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/cilium/values.yaml - CILIUM_HUBBLE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/cilium/hubble-values.yaml - JUPYTERHUB_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/jupyterhub/values.yaml - PROMETHEUS_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/prometheus/values.yaml - ELASTIC_STACK_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/elastic-stack/values.yaml - VAULT_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/vault/values.yaml - CROSSPLANE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/crossplane/values.yaml - FLUENTD_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/fluentd/values.yaml - KNATIVE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/knative/values.yaml - POSTHOG_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/posthog/values.yaml - FALCO_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/falco/values.yaml - APPARMOR_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/apparmor/values.yaml script: - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml only: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index e87f0f28d01..c3a92b67a8b 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -37,9 +37,6 @@ apifuzzer_fuzz: $FUZZAPI_OPENAPI == null && $FUZZAPI_D_WORKER_IMAGE == null when: never - - if: $FUZZAPI_D_WORKER_IMAGE == null && - $FUZZAPI_TARGET_URL == null - when: never - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ services: - docker:19.03.12-dind @@ -74,13 +71,15 @@ apifuzzer_fuzz: -e FUZZAPI_TIMEOUT \ -e FUZZAPI_VERBOSE \ -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ -e GITLAB_FEATURES \ -v $CI_PROJECT_DIR:/app \ -p 80:80 \ -p 8000:8000 \ -p 514:514 \ --restart=no \ - registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing-src:${FUZZAPI_VERSION}-engine + registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine # # Start target container - | @@ -119,6 +118,9 @@ apifuzzer_fuzz: # Wait for testing to complete if api fuzzer is scanning - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI" != "" ]; then echo "Waiting for API Fuzzer to exit"; docker wait apifuzzer; fi # + # Propagate exit code from api fuzzer (if any) + - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing exited with an error. Logs are available as job artifacts."; docker logs apifuzzer; exit 1; fi + # # Run user provided pre-script - sh -c "$FUZZAPI_POST_SCRIPT" # diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 3f47e575afd..4b957a8f771 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -34,5 +34,5 @@ variables: rules: - if: $COVFUZZ_DISABLED when: never - - if: $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ - if: $CI_RUNNER_EXECUTABLE_ARCH == "linux" diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index d5275c57ef8..3789f0edc1c 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -12,81 +12,24 @@ variables: DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 2 - DS_DISABLE_DIND: "true" dependency_scanning: stage: test - image: docker:stable - variables: - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - allow_failure: true - services: - - docker:stable-dind script: - - | - if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then - export DOCKER_HOST='tcp://localhost:2375' - fi - fi - - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage - function propagate_env_vars() { - CURRENT_ENV=$(printenv) - - for VAR_NAME; do - echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " - done - } - - | - docker run \ - $(propagate_env_vars \ - DS_ANALYZER_IMAGES \ - SECURE_ANALYZERS_PREFIX \ - DS_ANALYZER_IMAGE_TAG \ - DS_DEFAULT_ANALYZERS \ - DS_EXCLUDED_PATHS \ - DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - DS_PULL_ANALYZER_IMAGE_TIMEOUT \ - DS_RUN_ANALYZER_TIMEOUT \ - DS_PYTHON_VERSION \ - DS_PIP_VERSION \ - DS_PIP_DEPENDENCY_PATH \ - DS_JAVA_VERSION \ - GEMNASIUM_DB_LOCAL_PATH \ - GEMNASIUM_DB_REMOTE_URL \ - GEMNASIUM_DB_REF_NAME \ - PIP_INDEX_URL \ - PIP_EXTRA_INDEX_URL \ - PIP_REQUIREMENTS_FILE \ - MAVEN_CLI_OPTS \ - GRADLE_CLI_OPTS \ - SBT_CLI_OPTS \ - BUNDLER_AUDIT_UPDATE_DISABLED \ - BUNDLER_AUDIT_ADVISORY_DB_URL \ - BUNDLER_AUDIT_ADVISORY_DB_REF_NAME \ - RETIREJS_JS_ADVISORY_DB \ - RETIREJS_NODE_ADVISORY_DB \ - DS_REMEDIATE \ - ) \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_MAJOR_VERSION" /code + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 artifacts: reports: dependency_scanning: gl-dependency-scanning-report.json dependencies: [] rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'true' - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + - when: never .ds-analyzer: extends: dependency_scanning - services: [] + allow_failure: true rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ @@ -96,9 +39,11 @@ dependency_scanning: gemnasium-dependency_scanning: extends: .ds-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/gemnasium:$DS_MAJOR_VERSION" + name: "$DS_ANALYZER_IMAGE" + variables: + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium:$DS_MAJOR_VERSION" rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && @@ -112,13 +57,16 @@ gemnasium-dependency_scanning: - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' + - '{conan.lock,*/conan.lock,*/*/conan.lock}' gemnasium-maven-dependency_scanning: extends: .ds-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" + name: "$DS_ANALYZER_IMAGE" + variables: + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && @@ -132,9 +80,11 @@ gemnasium-maven-dependency_scanning: gemnasium-python-dependency_scanning: extends: .ds-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" + name: "$DS_ANALYZER_IMAGE" + variables: + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && @@ -155,9 +105,11 @@ gemnasium-python-dependency_scanning: bundler-audit-dependency_scanning: extends: .ds-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/bundler-audit:$DS_MAJOR_VERSION" + name: "$DS_ANALYZER_IMAGE" + variables: + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bundler-audit:$DS_MAJOR_VERSION" rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && @@ -168,9 +120,11 @@ bundler-audit-dependency_scanning: retire-js-dependency_scanning: extends: .ds-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/retire.js:$DS_MAJOR_VERSION" + name: "$DS_ANALYZER_IMAGE" + variables: + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/retire.js:$DS_MAJOR_VERSION" rules: - - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false' + - if: $DEPENDENCY_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 6eb17341472..77ea11d01d1 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,48 +9,29 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 - SAST_DISABLE_DIND: "true" SCAN_KUBERNETES_MANIFESTS: "false" sast: stage: test - allow_failure: true artifacts: reports: sast: gl-sast-report.json rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'true' - when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bsast\b/ - image: docker:stable + - when: never variables: SEARCH_MAX_DEPTH: 4 - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - services: - - docker:stable-dind script: - - | - if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then - export DOCKER_HOST='tcp://localhost:2375' - fi - fi - - | - docker run \ - $(awk 'BEGIN{for(v in ENVIRON) print v}' | grep -v -E '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | awk '{printf " -e %s", $0}') \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_ANALYZER_IMAGE_TAG" /app/bin/run /code + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 .sast-analyzer: extends: sast - services: [] + allow_failure: true rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH script: @@ -59,9 +40,11 @@ sast: bandit-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /bandit/ @@ -71,9 +54,11 @@ bandit-sast: brakeman-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /brakeman/ @@ -83,9 +68,11 @@ brakeman-sast: eslint-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /eslint/ @@ -99,9 +86,11 @@ eslint-sast: flawfinder-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ @@ -112,9 +101,11 @@ flawfinder-sast: kubesec-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && @@ -123,9 +114,11 @@ kubesec-sast: gosec-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /gosec/ @@ -135,9 +128,11 @@ gosec-sast: nodejs-scan-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ @@ -147,9 +142,11 @@ nodejs-scan-sast: phpcs-security-audit-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ @@ -159,31 +156,25 @@ phpcs-security-audit-sast: pmd-apex-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ exists: - '**/*.cls' -secrets-sast: - extends: .sast-analyzer - image: - name: "$SECURE_ANALYZERS_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /secrets/ - security-code-scan-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ @@ -194,9 +185,11 @@ security-code-scan-sast: sobelow-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /sobelow/ @@ -206,9 +199,11 @@ sobelow-sast: spotbugs-sast: extends: .sast-analyzer image: - name: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' + - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index b897c7b482f..bde6a0fbebb 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -35,6 +35,7 @@ secret_detection: - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH script: - git fetch origin $CI_DEFAULT_BRANCH $CI_BUILD_REF_NAME - - export SECRET_DETECTION_COMMIT_TO=$(git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_BUILD_REF_NAME | tail -n 1) - - export SECRET_DETECTION_COMMIT_FROM=$CI_COMMIT_SHA + - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_BUILD_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt + - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt - /analyzer run + - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index 9dbd9b679a8..e591e3cc1e2 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -12,15 +12,15 @@ performance: variables: URL: '' SITESPEED_IMAGE: sitespeedio/sitespeed.io - SITESPEED_VERSION: 13.3.0 + SITESPEED_VERSION: 14.1.0 SITESPEED_OPTIONS: '' services: - docker:stable-dind script: - mkdir gitlab-exporter - - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS - mv sitespeed-results/data/performance.json browser-performance.json artifacts: paths: diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml index f964b3b2caf..cd23af562e5 100644 --- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml @@ -14,10 +14,11 @@ load_performance: K6_VERSION: 0.27.0 K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js K6_OPTIONS: '' + K6_DOCKER_OPTIONS: '' services: - docker:stable-dind script: - - docker run --rm -v "$(pwd)":/k6 -w /k6 $K6_IMAGE:$K6_VERSION run $K6_TEST_FILE --summary-export=load-performance.json $K6_OPTIONS + - docker run --rm -v "$(pwd)":/k6 -w /k6 $K6_DOCKER_OPTIONS $K6_IMAGE:$K6_VERSION run $K6_TEST_FILE --summary-export=load-performance.json $K6_OPTIONS artifacts: reports: load_performance: load-performance.json diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index 035ba52da84..0a739cf122d 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -55,5 +55,5 @@ publish_package: npm publish && echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" } || { - echo "No new version of ${NPM_PACKAGE_NAME} published. This is most likely because version ${NPM_PACKAGE_VERSION} already exists in GitLab's NPM registry." + echo "No new version of ${NPM_PACKAGE_NAME} published. This is most likely because version ${NPM_PACKAGE_VERSION} already exists in GitLab's NPM registry."; exit 1 } diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index f76aacc2d19..348e5472cb4 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -79,22 +79,13 @@ module Gitlab job.trace_chunks.any? || current_path.present? || old_trace.present? end - def read - stream = Gitlab::Ci::Trace::Stream.new do - if trace_artifact - trace_artifact.open - elsif job.trace_chunks.any? - Gitlab::Ci::Trace::ChunkedIO.new(job) - elsif current_path - File.open(current_path, "rb") - elsif old_trace - StringIO.new(old_trace) - end - end + def read(should_retry: true, &block) + read_stream(&block) + rescue Errno::ENOENT + raise unless should_retry - yield stream - ensure - stream&.close + job.reset + read_stream(&block) end def write(mode, &blk) @@ -141,6 +132,24 @@ module Gitlab private + def read_stream + stream = Gitlab::Ci::Trace::Stream.new do + if trace_artifact + trace_artifact.open + elsif job.trace_chunks.any? + Gitlab::Ci::Trace::ChunkedIO.new(job) + elsif current_path + File.open(current_path, "rb") + elsif old_trace + StringIO.new(old_trace) + end + end + + yield stream + ensure + stream&.close + end + def unsafe_write!(mode, &blk) stream = Gitlab::Ci::Trace::Stream.new do if trace_artifact diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb new file mode 100644 index 00000000000..82a7d5fb83c --- /dev/null +++ b/lib/gitlab/ci/trace/metrics.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + class Metrics + extend Gitlab::Utils::StrongMemoize + + OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite, + :accepted, :finalized, :discarded, :conflict].freeze + + def increment_trace_operation(operation: :unknown) + unless OPERATIONS.include?(operation) + raise ArgumentError, "unknown trace operation: #{operation}" + end + + self.class.trace_operations.increment(operation: operation) + end + + def increment_trace_bytes(size) + self.class.trace_bytes.increment(by: size.to_i) + end + + def self.trace_operations + strong_memoize(:trace_operations) do + name = :gitlab_ci_trace_operations_total + comment = 'Total amount of different operations on a build trace' + + Gitlab::Metrics.counter(name, comment) + end + end + + def self.trace_bytes + strong_memoize(:trace_bytes) do + name = :gitlab_ci_trace_bytes_total + comment = 'Total amount of build trace bytes transferred' + + Gitlab::Metrics.counter(name, comment) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 20f5620dd64..618438c8887 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -8,7 +8,7 @@ module Gitlab BUFFER_SIZE = 4096 LIMIT_SIZE = 500.kilobytes - attr_reader :stream + attr_reader :stream, :metrics delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true @@ -16,9 +16,10 @@ module Gitlab alias_method :present?, :valid? - def initialize + def initialize(metrics = Trace::Metrics.new) @stream = yield @stream&.binmode + @metrics = metrics end def valid? @@ -43,6 +44,9 @@ module Gitlab def append(data, offset) data = data.force_encoding(Encoding::BINARY) + metrics.increment_trace_operation(operation: :streamed) + metrics.increment_trace_bytes(data.bytesize) + stream.seek(offset, IO::SEEK_SET) stream.write(data) stream.truncate(offset + data.bytesize) diff --git a/lib/gitlab/ci/warnings.rb b/lib/gitlab/ci/warnings.rb new file mode 100644 index 00000000000..7138fd21b72 --- /dev/null +++ b/lib/gitlab/ci/warnings.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module Gitlab::Ci::Warnings + MAX_LIMIT = 25 +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index b7046064f44..ee55eb8b22a 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -1,183 +1,65 @@ # frozen_string_literal: true +# This is the CI Linter component that runs the syntax validations +# while parsing the YAML config into a data structure that is +# then presented to the caller as result object. +# After syntax validations (done by Ci::Config), this component also +# runs logical validation on the built data structure. module Gitlab module Ci class YamlProcessor - # ValidationError is treated like a result object in the form of an exception. - # We can return any warnings, raised during the config validation, along with - # the error object until we support multiple messages to be returned. - class ValidationError < StandardError - attr_reader :warnings - - def initialize(message, warnings: []) - @warnings = warnings - super(message) - end - end - - include Gitlab::Config::Entry::LegacyValidationHelpers + ValidationError = Class.new(StandardError) - attr_reader :stages, :jobs + def self.validation_message(content, opts = {}) + result = new(content, opts).execute - class Result - attr_reader :config, :errors, :warnings + result.errors.first + end - def initialize(config: nil, errors: [], warnings: []) - @config = config - @errors = errors - @warnings = warnings - end + def initialize(config_content, opts = {}) + @config_content = config_content + @opts = opts + end - def valid? - config.present? && errors.empty? + def execute + if @config_content.blank? + return Result.new(errors: ['Please provide content of .gitlab-ci.yml']) end - end - def initialize(config, opts = {}) - @ci_config = Gitlab::Ci::Config.new(config, **opts) - @config = @ci_config.to_hash + @ci_config = Gitlab::Ci::Config.new(@config_content, **@opts) unless @ci_config.valid? - error!(@ci_config.errors.first) + return Result.new(ci_config: @ci_config, errors: @ci_config.errors, warnings: @ci_config.warnings) end - initial_parsing - rescue Gitlab::Ci::Config::ConfigError => e - error!(e.message) - end - - def self.new_with_validation_errors(content, opts = {}) - return Result.new(errors: ['Please provide content of .gitlab-ci.yml']) if content.blank? + run_logical_validations! - config = Gitlab::Ci::Config.new(content, **opts) - return Result.new(errors: config.errors, warnings: config.warnings) unless config.valid? - - config = Gitlab::Ci::YamlProcessor.new(content, opts) - Result.new(config: config, warnings: config.warnings) - - rescue ValidationError => e - Result.new(errors: [e.message], warnings: e.warnings) + Result.new(ci_config: @ci_config, warnings: @ci_config&.warnings) rescue Gitlab::Ci::Config::ConfigError => e - Result.new(errors: [e.message]) - end - - def warnings - @ci_config&.warnings || [] - end - - def builds - @jobs.map do |name, _| - build_attributes(name) - end - end - - def build_attributes(name) - job = @jobs.fetch(name.to_sym, {}) - - { stage_idx: @stages.index(job[:stage]), - stage: job[:stage], - tag_list: job[:tags], - name: job[:name].to_s, - allow_failure: job[:ignore], - when: job[:when] || 'on_success', - environment: job[:environment_name], - coverage_regex: job[:coverage], - yaml_variables: transform_to_yaml_variables(job[:variables]), - needs_attributes: job.dig(:needs, :job), - interruptible: job[:interruptible], - only: job[:only], - except: job[:except], - rules: job[:rules], - cache: job[:cache], - resource_group_key: job[:resource_group], - scheduling_type: job[:scheduling_type], - secrets: job[:secrets], - options: { - image: job[:image], - services: job[:services], - artifacts: job[:artifacts], - dependencies: job[:dependencies], - cross_dependencies: job.dig(:needs, :cross_dependency), - job_timeout: job[:timeout], - before_script: job[:before_script], - script: job[:script], - after_script: job[:after_script], - environment: job[:environment], - retry: job[:retry], - parallel: job[:parallel], - instance: job[:instance], - start_in: job[:start_in], - trigger: job[:trigger], - bridge_needs: job.dig(:needs, :bridge)&.first, - release: release(job) - }.compact }.compact - end + Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) - def release(job) - job[:release] - end - - def stage_builds_attributes(stage) - @jobs.values - .select { |job| job[:stage] == stage } - .map { |job| build_attributes(job[:name]) } - end - - def stages_attributes - @stages.uniq.map do |stage| - seeds = stage_builds_attributes(stage) - - { name: stage, index: @stages.index(stage), builds: seeds } - end - end - - def workflow_attributes - { - rules: @config.dig(:workflow, :rules), - yaml_variables: transform_to_yaml_variables(@variables) - } - end - - def self.validation_message(content, opts = {}) - return 'Please provide content of .gitlab-ci.yml' if content.blank? - - begin - Gitlab::Ci::YamlProcessor.new(content, opts) - nil - rescue ValidationError => e - e.message - end + rescue ValidationError => e + Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) end private - def initial_parsing - ## - # Global config - # - @variables = @ci_config.variables + def run_logical_validations! @stages = @ci_config.stages - - ## - # Jobs - # - @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs + @jobs = @ci_config.normalized_jobs @jobs.each do |name, job| - # logical validation for job - validate_job_stage!(name, job) - validate_job_dependencies!(name, job) - validate_job_needs!(name, job) - validate_dynamic_child_pipeline_dependencies!(name, job) - validate_job_environment!(name, job) + validate_job!(name, job) end end - def transform_to_yaml_variables(variables) - variables.to_h.map do |key, value| - { key: key.to_s, value: value, public: true } - end + def validate_job!(name, job) + validate_job_stage!(name, job) + validate_job_dependencies!(name, job) + validate_job_needs!(name, job) + validate_dynamic_child_pipeline_dependencies!(name, job) + validate_job_environment!(name, job) end def validate_job_stage!(name, job) @@ -188,10 +70,6 @@ module Gitlab end end - def error!(message) - raise ValidationError.new(message, warnings: warnings) - end - def validate_job_dependencies!(name, job) return unless job[:dependencies] @@ -267,6 +145,10 @@ module Gitlab error!("#{name} job: on_stop job #{on_stop} needs to have action stop defined") end end + + def error!(message) + raise ValidationError.new(message) + end end end end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb new file mode 100644 index 00000000000..68f61e52df7 --- /dev/null +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +# A data object that wraps `Ci::Config` and any messages +# (errors, warnings) generated by the YamlProcessor. +module Gitlab + module Ci + class YamlProcessor + class Result + attr_reader :errors, :warnings + + def initialize(ci_config: nil, errors: [], warnings: []) + @ci_config = ci_config + @errors = errors || [] + @warnings = warnings || [] + end + + def valid? + errors.empty? + end + + def stages_attributes + stages.uniq.map do |stage| + seeds = stage_builds_attributes(stage) + + { name: stage, index: stages.index(stage), builds: seeds } + end + end + + def builds + jobs.map do |name, _| + build_attributes(name) + end + end + + def stage_builds_attributes(stage) + jobs.values + .select { |job| job[:stage] == stage } + .map { |job| build_attributes(job[:name]) } + end + + def workflow_attributes + { + rules: hash_config.dig(:workflow, :rules), + yaml_variables: transform_to_yaml_variables(variables) + } + end + + def jobs + @jobs ||= @ci_config.normalized_jobs + end + + def stages + @stages ||= @ci_config.stages + end + + def build_attributes(name) + job = jobs.fetch(name.to_sym, {}) + + { stage_idx: stages.index(job[:stage]), + stage: job[:stage], + tag_list: job[:tags], + name: job[:name].to_s, + allow_failure: job[:ignore], + when: job[:when] || 'on_success', + environment: job[:environment_name], + coverage_regex: job[:coverage], + yaml_variables: transform_to_yaml_variables(job[:variables]), + needs_attributes: job.dig(:needs, :job), + interruptible: job[:interruptible], + only: job[:only], + except: job[:except], + rules: job[:rules], + cache: job[:cache], + resource_group_key: job[:resource_group], + scheduling_type: job[:scheduling_type], + secrets: job[:secrets], + options: { + image: job[:image], + services: job[:services], + artifacts: job[:artifacts], + dependencies: job[:dependencies], + cross_dependencies: job.dig(:needs, :cross_dependency), + job_timeout: job[:timeout], + before_script: job[:before_script], + script: job[:script], + after_script: job[:after_script], + environment: job[:environment], + retry: job[:retry], + parallel: job[:parallel], + instance: job[:instance], + start_in: job[:start_in], + trigger: job[:trigger], + bridge_needs: job.dig(:needs, :bridge)&.first, + release: release(job) + }.compact }.compact + end + + private + + def variables + @variables ||= @ci_config.variables + end + + def hash_config + @hash_config ||= @ci_config.to_hash + end + + def release(job) + job[:release] + end + + def transform_to_yaml_variables(variables) + variables.to_h.map do |key, value| + { key: key.to_s, value: value, public: true } + end + end + end + end + end +end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb index 017adc7be4a..6d18f9070cc 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -18,7 +18,7 @@ module Gitlab @limit = limit @dry_run = dry_run @niceness = (niceness || DEFAULT_NICENESS).downcase - @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger + @logger = logger || Gitlab::AppLogger @total_found = @total_cleaned = 0 new_batch! diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb index 6ad05c7b2e4..4b1d16eb974 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb @@ -22,7 +22,7 @@ module Gitlab attr_reader :batch_size, :dry_run attr_accessor :artifact_files - def initialize(batch_size:, dry_run: true, logger: Rails.logger) # rubocop:disable Gitlab/RailsLogger + def initialize(batch_size:, dry_run: true, logger: Gitlab::AppLogger) @batch_size = batch_size @dry_run = dry_run @logger = logger diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb index 3df243e319e..14eac474e27 100644 --- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb +++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb @@ -12,7 +12,7 @@ module Gitlab def initialize(project, dry_run: true, logger: nil, limit: nil) @project = project @dry_run = dry_run - @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger + @logger = logger || Gitlab::AppLogger @limit = limit end @@ -25,7 +25,7 @@ module Gitlab private def remove_orphan_references - invalid_references = project.lfs_objects_projects.where(lfs_object: orphan_objects) # rubocop:disable CodeReuse/ActiveRecord + invalid_references = project.lfs_objects_projects.lfs_object_in(orphan_objects) if dry_run log_info("Found invalid references: #{invalid_references.count}") @@ -41,26 +41,22 @@ module Gitlab end end - def lfs_oids_from_repository - project.repository.gitaly_blob_client.get_all_lfs_pointers.map(&:lfs_oid) - end - - def orphan_oids - lfs_oids_from_database - lfs_oids_from_repository - end + def orphan_objects + # Get these first so racing with a git push can't remove any LFS objects + oids = project.lfs_objects_oids - def lfs_oids_from_database - oids = [] + repos = [ + project.repository, + project.design_repository, + project.wiki.repository + ].select(&:exists?) - project.lfs_objects.each_batch do |relation| - oids += relation.pluck(:oid) # rubocop:disable CodeReuse/ActiveRecord + repos.flat_map do |repo| + oids -= repo.gitaly_blob_client.get_all_lfs_pointers.map(&:lfs_oid) end - oids - end - - def orphan_objects - LfsObject.where(oid: orphan_oids) # rubocop:disable CodeReuse/ActiveRecord + # The remaining OIDs are not used by any repository, so are orphans + LfsObject.for_oids(oids) end def log_info(msg) diff --git a/lib/gitlab/cleanup/project_upload_file_finder.rb b/lib/gitlab/cleanup/project_upload_file_finder.rb index 3d35d474f5d..0f40f683354 100644 --- a/lib/gitlab/cleanup/project_upload_file_finder.rb +++ b/lib/gitlab/cleanup/project_upload_file_finder.rb @@ -49,7 +49,7 @@ module Gitlab cmd = %W[#{ionice} -c Idle] + cmd if ionice log_msg = "find command: \"#{cmd.join(' ')}\"" - Rails.logger.info log_msg # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.info log_msg cmd end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index 056e075cb21..77231665e7e 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :logger def initialize(logger: nil) - @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger + @logger = logger || Gitlab::AppLogger end def run!(dry_run: true) diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb index 42c93b7aecb..6cadb9424f7 100644 --- a/lib/gitlab/cleanup/remote_uploads.rb +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -7,7 +7,7 @@ module Gitlab BATCH_SIZE = 100 def initialize(logger: nil) - @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger + @logger = logger || Gitlab::AppLogger end def run!(dry_run: false) diff --git a/lib/gitlab/consul/internal.rb b/lib/gitlab/consul/internal.rb new file mode 100644 index 00000000000..3afc24ddab9 --- /dev/null +++ b/lib/gitlab/consul/internal.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module Consul + class Internal + Error = Class.new(StandardError) + UnexpectedResponseError = Class.new(Gitlab::Consul::Internal::Error) + SocketError = Class.new(Gitlab::Consul::Internal::Error) + SSLError = Class.new(Gitlab::Consul::Internal::Error) + ECONNREFUSED = Class.new(Gitlab::Consul::Internal::Error) + + class << self + def api_url + Gitlab.config.consul.api_url.to_s.presence if Gitlab.config.consul + rescue Settingslogic::MissingSetting + Gitlab::AppLogger.error('Consul api_url is not present in config/gitlab.yml') + + nil + end + + def discover_service(service_name:) + return unless service_name.present? && api_url + + api_path = URI.join(api_url, '/v1/catalog/service/', URI.encode_www_form_component(service_name)).to_s + services = json_get(api_path, allow_local_requests: true, open_timeout: 5, read_timeout: 10) + + # Use the first service definition + service = services&.first + + return unless service + + service_address = service['ServiceAddress'] || service['Address'] + service_port = service['ServicePort'] + + [service_address, service_port] + end + + def discover_prometheus_server_address + service_address, service_port = discover_service(service_name: 'prometheus') + + return unless service_address && service_port + + "#{service_address}:#{service_port}" + end + + private + + def json_get(path, options) + response = get(path, options) + code = response.try(:code) + body = response.try(:body) + + raise Consul::Internal::UnexpectedResponseError unless code == 200 && body + + parse_response_body(body) + end + + def parse_response_body(body) + Gitlab::Json.parse(body) + rescue + raise Consul::Internal::UnexpectedResponseError + end + + def get(path, options) + Gitlab::HTTP.get(path, options) + rescue ::SocketError + raise Consul::Internal::SocketError + rescue OpenSSL::SSL::SSLError + raise Consul::Internal::SSLError + rescue Errno::ECONNREFUSED + raise Consul::Internal::ECONNREFUSED + rescue + raise Consul::Internal::UnexpectedResponseError + end + end + end + end +end diff --git a/lib/gitlab/cross_project_access.rb b/lib/gitlab/cross_project_access.rb index 4ddc7e02d1b..93baf1e596c 100644 --- a/lib/gitlab/cross_project_access.rb +++ b/lib/gitlab/cross_project_access.rb @@ -18,7 +18,7 @@ module Gitlab end def add_check( - klass, + klass, actions: {}, positive_condition: nil, negative_condition: nil, diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb deleted file mode 100644 index d5f2e868606..00000000000 --- a/lib/gitlab/cycle_analytics/production_stage.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class ProductionStage < BaseStage - include ProductionHelper - - def start_time_attrs - @start_time_attrs ||= issue_table[:created_at] - end - - def end_time_attrs - @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at] - end - - def name - :production - end - - def title - s_('CycleAnalyticsStage|Total') - end - - def legend - _("Related Issues") - end - - def description - _("From issue creation until deploy to production") - end - - def query - # Limit to merge requests that have been deployed to production after `@from` - query.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@from)) - end - end - end -end diff --git a/lib/gitlab/danger/changelog.rb b/lib/gitlab/danger/changelog.rb index 4427c331b8e..607ca1200a0 100644 --- a/lib/gitlab/danger/changelog.rb +++ b/lib/gitlab/danger/changelog.rb @@ -11,8 +11,36 @@ module Gitlab 'meta' ].freeze NO_CHANGELOG_CATEGORIES = %i[docs none].freeze + CREATE_CHANGELOG_COMMAND = 'bin/changelog -m %<mr_iid>s "%<mr_title>s"' + CREATE_EE_CHANGELOG_COMMAND = 'bin/changelog --ee -m %<mr_iid>s "%<mr_title>s"' + CHANGELOG_MODIFIED_URL_TEXT = "**CHANGELOG.md was edited.** Please remove the additions and create a CHANGELOG entry.\n\n" + CHANGELOG_MISSING_URL_TEXT = "**[CHANGELOG missing](https://docs.gitlab.com/ee/development/changelog.html)**:\n\n" - def needed? + OPTIONAL_CHANGELOG_MESSAGE = <<~MSG + If you want to create a changelog entry for GitLab FOSS, run the following: + + #{CREATE_CHANGELOG_COMMAND} + + If you want to create a changelog entry for GitLab EE, run the following instead: + + #{CREATE_EE_CHANGELOG_COMMAND} + + If this merge request [doesn't need a CHANGELOG entry](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry), feel free to ignore this message. + MSG + + REQUIRED_CHANGELOG_MESSAGE = <<~MSG + To create a changelog entry, run the following: + + #{CREATE_CHANGELOG_COMMAND} + + This merge request requires a changelog entry because it [introduces a database migration](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry). + MSG + + def required? + git.added_files.any? { |path| path =~ %r{\Adb/(migrate|post_migrate)/} } + end + + def optional? categories_need_changelog? && without_no_changelog_label? end @@ -20,16 +48,35 @@ module Gitlab @found ||= git.added_files.find { |path| path =~ %r{\A(ee/)?(changelogs/unreleased)(-ee)?/} } end - def sanitized_mr_title - gitlab.mr_json["title"].gsub(/^WIP: */, '').gsub(/`/, '\\\`') - end - def ee_changelog? found.start_with?('ee/') end + def modified_text + CHANGELOG_MODIFIED_URL_TEXT + + format(OPTIONAL_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) + end + + def required_text + CHANGELOG_MISSING_URL_TEXT + + format(REQUIRED_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) + end + + def optional_text + CHANGELOG_MISSING_URL_TEXT + + format(OPTIONAL_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) + end + private + def mr_iid + gitlab.mr_json["iid"] + end + + def sanitized_mr_title + helper.sanitize_mr_title(gitlab.mr_json["title"]) + end + def categories_need_changelog? (helper.changes_by_category.keys - NO_CHANGELOG_CATEGORIES).any? end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 077c71f1233..3626ec5bf5b 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -44,7 +44,10 @@ module Gitlab # "+ # Test change", # "- # Old change" ] def changed_lines(changed_file) - git.diff_for_file(changed_file).patch.split("\n").select { |line| %r{^[+-]}.match?(line) } + diff = git.diff_for_file(changed_file) + return [] unless diff + + diff.patch.split("\n").select { |line| %r{^[+-]}.match?(line) } end def all_ee_changes @@ -171,6 +174,7 @@ module Gitlab %r{\A(ee/)?scripts/} => :engineering_productivity, %r{\Atooling/} => :engineering_productivity, %r{(CODEOWNERS)} => :engineering_productivity, + %r{(tests.yml)} => :engineering_productivity, %r{\A(ee/)?spec/features/} => :test, %r{\A(ee/)?spec/support/shared_examples/features/} => :test, @@ -191,6 +195,7 @@ module Gitlab # Files that don't fit into any category are marked with :none %r{\A(ee/)?changelogs/} => :none, %r{\Alocale/gitlab\.pot\z} => :none, + %r{\Adata/whats_new/} => :none, # Fallbacks in case the above patterns miss anything %r{\.rb\z} => :backend, @@ -205,16 +210,6 @@ module Gitlab usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } end - def missing_database_labels(current_mr_labels) - labels = if has_database_scoped_labels?(current_mr_labels) - ['database'] - else - ['database', 'database::review pending'] - end - - labels - current_mr_labels - end - def sanitize_mr_title(title) title.gsub(DRAFT_REGEX, '').gsub(/`/, '\\\`') end @@ -258,8 +253,6 @@ module Gitlab all_changed_files.grep(regex) end - private - def has_database_scoped_labels?(current_mr_labels) current_mr_labels.any? { |label| label.start_with?('database::') } end diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb index 2e6181d1cab..a6866868e6c 100644 --- a/lib/gitlab/danger/roulette.rb +++ b/lib/gitlab/danger/roulette.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative 'teammate' -require_relative 'request_helper' +require_relative 'request_helper' unless defined?(Gitlab::Danger::RequestHelper) module Gitlab module Danger diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index 9b389907090..ebd96be40d7 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -45,9 +45,7 @@ module Gitlab has_capability?(project, category, :maintainer, labels) end - def markdown_name(timezone_experiment: false, author: nil) - return @markdown_name unless timezone_experiment - + def markdown_name(author: nil) "#{@markdown_name} (#{utc_offset_text(author)})" end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index af363705bed..f941c57a6dd 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -86,7 +86,7 @@ module Gitlab # # rubocop:disable Metrics/ParameterLists def build( - project:, user:, ref:, oldrev: nil, newrev: nil, + project:, user:, ref:, oldrev: nil, newrev: nil, commits: [], commits_count: nil, message: nil, push_options: {}, with_changed_files: true) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 990c940d200..accc6330253 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -335,7 +335,7 @@ module Gitlab end rescue Prometheus::Client::LabelSetValidator::LabelSetError => err # Ensure that errors in recording these metrics don't affect the operation of the application - Rails.logger.error("Unable to observe database transaction duration: #{err}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error("Unable to observe database transaction duration: #{err}") end # MonkeyPatch for ActiveRecord::Base for adding observability diff --git a/lib/gitlab/database/background_migration_job.rb b/lib/gitlab/database/background_migration_job.rb index 445735b232a..1b9d7cbc9a1 100644 --- a/lib/gitlab/database/background_migration_job.rb +++ b/lib/gitlab/database/background_migration_job.rb @@ -3,6 +3,8 @@ module Gitlab module Database class BackgroundMigrationJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + include EachBatch + self.table_name = :background_migration_jobs scope :for_migration_class, -> (class_name) { where(class_name: normalize_class_name(class_name)) } diff --git a/lib/gitlab/database/concurrent_reindex.rb b/lib/gitlab/database/concurrent_reindex.rb new file mode 100644 index 00000000000..485ab35e55d --- /dev/null +++ b/lib/gitlab/database/concurrent_reindex.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class ConcurrentReindex + include Gitlab::Utils::StrongMemoize + include MigrationHelpers + + ReindexError = Class.new(StandardError) + + PG_IDENTIFIER_LENGTH = 63 + TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' + REPLACED_INDEX_PREFIX = 'old_reindex_' + + attr_reader :index_name, :logger + + def initialize(index_name, logger:) + @index_name = index_name + @logger = logger + end + + def execute + raise ReindexError, "index #{index_name} does not exist" unless index_exists? + + raise ReindexError, 'UNIQUE indexes are currently not supported' if index_unique? + + logger.debug("dropping dangling index from previous run: #{replacement_index_name}") + remove_replacement_index + + begin + create_replacement_index + + unless replacement_index_valid? + message = 'replacement index was created as INVALID' + logger.error("#{message}, cleaning up") + raise ReindexError, "failed to reindex #{index_name}: #{message}" + end + + swap_replacement_index + rescue Gitlab::Database::WithLockRetries::AttemptsExhaustedError => e + logger.error('failed to obtain the required database locks to swap the indexes, cleaning up') + raise ReindexError, e.message + rescue ActiveRecord::ActiveRecordError, PG::Error => e + logger.error("database error while attempting reindex of #{index_name}: #{e.message}") + raise ReindexError, e.message + ensure + logger.info("dropping unneeded replacement index: #{replacement_index_name}") + remove_replacement_index + end + end + + private + + def connection + @connection ||= ActiveRecord::Base.connection + end + + def replacement_index_name + @replacement_index_name ||= constrained_index_name(TEMPORARY_INDEX_PREFIX) + end + + def index + strong_memoize(:index) do + find_index(index_name) + end + end + + def index_exists? + !index.nil? + end + + def index_unique? + index.indisunique + end + + def constrained_index_name(prefix) + "#{prefix}#{index_name}".slice(0, PG_IDENTIFIER_LENGTH) + end + + def create_replacement_index + create_replacement_index_statement = index.indexdef + .sub(/CREATE INDEX/, 'CREATE INDEX CONCURRENTLY') + .sub(/#{index_name}/, replacement_index_name) + + logger.info("creating replacement index #{replacement_index_name}") + logger.debug("replacement index definition: #{create_replacement_index_statement}") + + disable_statement_timeout do + connection.execute(create_replacement_index_statement) + end + end + + def replacement_index_valid? + find_index(replacement_index_name).indisvalid + end + + def find_index(index_name) + record = connection.select_one(<<~SQL) + SELECT + pg_index.indisunique, + pg_index.indisvalid, + pg_indexes.indexdef + FROM pg_index + INNER JOIN pg_class ON pg_class.oid = pg_index.indexrelid + INNER JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid + INNER JOIN pg_indexes ON pg_class.relname = pg_indexes.indexname + WHERE pg_namespace.nspname = 'public' + AND pg_class.relname = #{connection.quote(index_name)} + SQL + + OpenStruct.new(record) if record + end + + def swap_replacement_index + replaced_index_name = constrained_index_name(REPLACED_INDEX_PREFIX) + + logger.info("swapping replacement index #{replacement_index_name} with #{index_name}") + + with_lock_retries do + rename_index(index_name, replaced_index_name) + rename_index(replacement_index_name, index_name) + rename_index(replaced_index_name, replacement_index_name) + end + end + + def rename_index(old_index_name, new_index_name) + connection.execute("ALTER INDEX #{old_index_name} RENAME TO #{new_index_name}") + end + + def remove_replacement_index + disable_statement_timeout do + connection.execute("DROP INDEX CONCURRENTLY IF EXISTS #{replacement_index_name}") + end + end + + def with_lock_retries(&block) + arguments = { klass: self.class, logger: logger } + + Gitlab::Database::WithLockRetries.new(arguments).run(raise_on_exhaustion: true, &block) + end + end + end +end diff --git a/lib/gitlab/database/custom_structure.rb b/lib/gitlab/database/custom_structure.rb index c5a76c5a787..e4404e73a63 100644 --- a/lib/gitlab/database/custom_structure.rb +++ b/lib/gitlab/database/custom_structure.rb @@ -8,8 +8,7 @@ module Gitlab def dump File.open(self.class.custom_dump_filepath, 'wb') do |io| io << "-- this file tracks custom GitLab data, such as foreign keys referencing partitioned tables\n" - io << "-- more details can be found in the issue: https://gitlab.com/gitlab-org/gitlab/-/issues/201872\n" - io << "SET search_path=public;\n\n" + io << "-- more details can be found in the issue: https://gitlab.com/gitlab-org/gitlab/-/issues/201872\n\n" dump_partitioned_foreign_keys(io) if partitioned_foreign_keys_exist? end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index b62b6e20dd5..723f0f6a308 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -87,7 +87,7 @@ module Gitlab options = options.merge({ algorithm: :concurrently }) if index_exists?(table_name, column_name, options) - Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" return end @@ -113,7 +113,7 @@ module Gitlab options = options.merge({ algorithm: :concurrently }) unless index_exists?(table_name, column_name, options) - Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" return end @@ -143,7 +143,7 @@ module Gitlab options = options.merge({ algorithm: :concurrently }) unless index_exists_by_name?(table_name, index_name) - Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, index_name: #{index_name}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, index_name: #{index_name}" return end @@ -163,7 +163,6 @@ module Gitlab # defaults to "CASCADE". # name - The name of the foreign key. # - # rubocop:disable Gitlab/RailsLogger def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, name: nil, validate: true) # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. @@ -183,7 +182,7 @@ module Gitlab "source: #{source}, target: #{target}, column: #{options[:column]}, "\ "name: #{options[:name]}, on_delete: #{options[:on_delete]}" - Rails.logger.warn warning_message + Gitlab::AppLogger.warn warning_message else # Using NOT VALID allows us to create a key without immediately # validating it. This means we keep the ALTER TABLE lock only for a @@ -217,7 +216,6 @@ module Gitlab end end end - # rubocop:enable Gitlab/RailsLogger def validate_foreign_key(source, column, name: nil) fk_name = name || concurrent_foreign_key_name(source, column) @@ -540,10 +538,10 @@ module Gitlab # table - The table containing the column. # column - The name of the column to change. # new_type - The new column type. - def change_column_type_concurrently(table, column, new_type, type_cast_function: nil) + def change_column_type_concurrently(table, column, new_type, type_cast_function: nil, batch_column_name: :id) temp_column = "#{column}_for_type_change" - rename_column_concurrently(table, column, temp_column, type: new_type, type_cast_function: type_cast_function) + rename_column_concurrently(table, column, temp_column, type: new_type, type_cast_function: type_cast_function, batch_column_name: batch_column_name) end # Performs cleanup of a concurrent type change. @@ -1085,7 +1083,6 @@ into similar problems in the future (e.g. when new tables are created). # Should be unique per table (not per column) # validate - Whether to validate the constraint in this call # - # rubocop:disable Gitlab/RailsLogger def add_check_constraint(table, check, constraint_name, validate: true) validate_check_constraint_name!(constraint_name) @@ -1102,7 +1099,7 @@ into similar problems in the future (e.g. when new tables are created). table: #{table}, check: #{check}, constraint name: #{constraint_name} MESSAGE - Rails.logger.warn warning_message + Gitlab::AppLogger.warn warning_message else # Only add the constraint without validating it # Even though it is fast, ADD CONSTRAINT requires an EXCLUSIVE lock @@ -1187,7 +1184,7 @@ into similar problems in the future (e.g. when new tables are created). column #{table}.#{column} is already defined as `NOT NULL` MESSAGE - Rails.logger.warn warning_message + Gitlab::AppLogger.warn warning_message end end diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb new file mode 100644 index 00000000000..9ec9ae684a5 --- /dev/null +++ b/lib/gitlab/database/partitioning/partition_monitoring.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class PartitionMonitoring + attr_reader :models + + def initialize(models = PartitionCreator.models) + @models = models + end + + def report_metrics + models.each do |model| + strategy = model.partitioning_strategy + + gauge_present.set({ table: model.table_name }, strategy.current_partitions.size) + gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size) + end + end + + private + + def gauge_present + @gauge_present ||= Gitlab::Metrics.gauge(:db_partitions_present, 'Number of database partitions present') + end + + def gauge_missing + @gauge_missing ||= Gitlab::Metrics.gauge(:db_partitions_missing, 'Number of database partitions currently expected, but not present') + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb index 1fb9476b7d9..2def3a4d3a9 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb @@ -31,7 +31,7 @@ module Gitlab current_keys << specified_key else - Rails.logger.warn "foreign key not added because it already exists: #{specified_key}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn "foreign key not added because it already exists: #{specified_key}" current_keys end end @@ -56,7 +56,7 @@ module Gitlab existing_key.delete current_keys.delete(existing_key) else - Rails.logger.warn "foreign key not removed because it doesn't exist: #{specified_key}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn "foreign key not removed because it doesn't exist: #{specified_key}" end current_keys diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 84b6fb9f76e..f7b0306b769 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -6,6 +6,7 @@ module Gitlab module TableManagementHelpers include ::Gitlab::Database::SchemaHelpers include ::Gitlab::Database::DynamicModelHelpers + include ::Gitlab::Database::MigrationHelpers include ::Gitlab::Database::Migrations::BackgroundMigrationHelpers ALLOWED_TABLES = %w[audit_events].freeze @@ -15,6 +16,12 @@ module Gitlab BATCH_INTERVAL = 2.minutes.freeze BATCH_SIZE = 50_000 + JobArguments = Struct.new(:start_id, :stop_id, :source_table_name, :partitioned_table_name, :source_column) do + def self.from_array(arguments) + self.new(*arguments) + end + end + # Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column. # One partition is created per month between the given `min_date` and `max_date`. Also installs a trigger on # the original table to copy writes into the partitioned table. To copy over historic data from before creation @@ -134,6 +141,42 @@ module Gitlab end end + # Executes cleanup tasks from a previous BackgroundMigration to backfill a partitioned table by finishing + # pending jobs and performing a final data synchronization. + # This performs two steps: + # 1. Wait to finish any pending BackgroundMigration jobs that have not succeeded + # 2. Inline copy any missed rows from the original table to the partitioned table + # + # **NOTE** Migrations using this method cannot be scheduled in the same release as the migration that + # schedules the background migration using the `enqueue_background_migration` helper, or else the + # background migration jobs will be force-executed. + # + # Example: + # + # finalize_backfilling_partitioned_table :audit_events + # + def finalize_backfilling_partitioned_table(table_name) + assert_table_is_allowed(table_name) + assert_not_in_transaction_block(scope: ERROR_SCOPE) + + partitioned_table_name = make_partitioned_table_name(table_name) + unless table_exists?(partitioned_table_name) + raise "could not find partitioned table for #{table_name}, " \ + "this could indicate the previous partitioning migration has been rolled back." + end + + Gitlab::BackgroundMigration.steal(MIGRATION_CLASS_NAME) do |raw_arguments| + JobArguments.from_array(raw_arguments).source_table_name == table_name.to_s + end + + primary_key = connection.primary_key(table_name) + copy_missed_records(table_name, partitioned_table_name, primary_key) + + disable_statement_timeout do + execute("VACUUM FREEZE ANALYZE #{partitioned_table_name}") + end + end + private def assert_table_is_allowed(table_name) @@ -161,10 +204,8 @@ module Gitlab def create_range_partitioned_copy(source_table_name, partitioned_table_name, partition_column, primary_key) if table_exists?(partitioned_table_name) - # rubocop:disable Gitlab/RailsLogger - Rails.logger.warn "Partitioned table not created because it already exists" \ + Gitlab::AppLogger.warn "Partitioned table not created because it already exists" \ " (this may be due to an aborted migration or similar): table_name: #{partitioned_table_name} " - # rubocop:enable Gitlab/RailsLogger return end @@ -217,10 +258,8 @@ module Gitlab def create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound) if table_exists?(table_for_range_partition(partition_name)) - # rubocop:disable Gitlab/RailsLogger - Rails.logger.warn "Partition not created because it already exists" \ + Gitlab::AppLogger.warn "Partition not created because it already exists" \ " (this may be due to an aborted migration or similar): partition_name: #{partition_name}" - # rubocop:enable Gitlab/RailsLogger return end @@ -241,10 +280,8 @@ module Gitlab def create_sync_function(name, partitioned_table_name, unique_key) if function_exists?(name) - # rubocop:disable Gitlab/RailsLogger - Rails.logger.warn "Partitioning sync function not created because it already exists" \ + Gitlab::AppLogger.warn "Partitioning sync function not created because it already exists" \ " (this may be due to an aborted migration or similar): function name: #{name}" - # rubocop:enable Gitlab/RailsLogger return end @@ -276,17 +313,15 @@ module Gitlab def create_sync_trigger(table_name, trigger_name, function_name) if trigger_exists?(table_name, trigger_name) - # rubocop:disable Gitlab/RailsLogger - Rails.logger.warn "Partitioning sync trigger not created because it already exists" \ + Gitlab::AppLogger.warn "Partitioning sync trigger not created because it already exists" \ " (this may be due to an aborted migration or similar): trigger name: #{trigger_name}" - # rubocop:enable Gitlab/RailsLogger return end create_trigger(table_name, trigger_name, function_name, fires: 'AFTER INSERT OR UPDATE OR DELETE') end - def enqueue_background_migration(source_table_name, partitioned_table_name, source_key) + def enqueue_background_migration(source_table_name, partitioned_table_name, source_column) source_model = define_batchable_model(source_table_name) queue_background_migration_jobs_by_range_at_intervals( @@ -294,13 +329,35 @@ module Gitlab MIGRATION_CLASS_NAME, BATCH_INTERVAL, batch_size: BATCH_SIZE, - other_job_arguments: [source_table_name.to_s, partitioned_table_name, source_key], + other_job_arguments: [source_table_name.to_s, partitioned_table_name, source_column], track_jobs: true) end def cleanup_migration_jobs(table_name) ::Gitlab::Database::BackgroundMigrationJob.for_partitioning_migration(MIGRATION_CLASS_NAME, table_name).delete_all end + + def copy_missed_records(source_table_name, partitioned_table_name, source_column) + backfill_table = BackfillPartitionedTable.new + relation = ::Gitlab::Database::BackgroundMigrationJob.pending + .for_partitioning_migration(MIGRATION_CLASS_NAME, source_table_name) + + relation.each_batch do |batch| + batch.each do |pending_migration_job| + job_arguments = JobArguments.from_array(pending_migration_job.arguments) + start_id = job_arguments.start_id + stop_id = job_arguments.stop_id + + say("Backfilling data into partitioned table for ids from #{start_id} to #{stop_id}") + job_updated_count = backfill_table.perform(start_id, stop_id, source_table_name, + partitioned_table_name, source_column) + + unless job_updated_count > 0 + raise "failed to update tracking record for ids from #{start_id} to #{stop_id}" + end + end + end + end end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 4fbbfdc4914..562e651cabc 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -71,7 +71,7 @@ module Gitlab unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path) message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}" - Rails.logger.error message # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error message end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 8b92b296408..5dbf30bad4e 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -56,7 +56,7 @@ module Gitlab unless gitlab_shell.mv_repository(project.repository_storage, old_path, new_path) - Rails.logger.error "Error moving #{old_path} to #{new_path}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error "Error moving #{old_path} to #{new_path}" end end diff --git a/lib/gitlab/database/schema_cleaner.rb b/lib/gitlab/database/schema_cleaner.rb index 7c415287878..8f93da2b66c 100644 --- a/lib/gitlab/database/schema_cleaner.rb +++ b/lib/gitlab/database/schema_cleaner.rb @@ -18,11 +18,18 @@ module Gitlab structure.gsub!(/^SELECT pg_catalog\.set_config\('search_path'.+/, '') structure.gsub!(/^--.*/, "\n") - structure = "SET search_path=public;\n" + structure + # We typically don't assume we're working with the public schema. + # pg_dump uses fully qualified object names though, since we have multiple schemas + # in the database. + # + # The intention here is to not introduce an assumption about the standard schema, + # unless we have a good reason to do so. + structure.gsub!(/public\.(\w+)/, '\1') + structure.gsub!(/CREATE EXTENSION IF NOT EXISTS (\w+) WITH SCHEMA public;/, 'CREATE EXTENSION IF NOT EXISTS \1;') structure.gsub!(/\n{3,}/, "\n\n") - io << structure + io << structure.strip io << <<~MSG -- schema_migrations.version information is no longer stored in this file, -- but instead tracked in the db/schema_migrations directory diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb index fb0fcc5a93b..8a5f53be20f 100644 --- a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb +++ b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb @@ -18,6 +18,7 @@ module Gitlab business: 0, response: 1, system: 2, + custom: 3, cluster_health: -100 } @@ -34,7 +35,8 @@ module Gitlab aws_elb: _('Response metrics (AWS ELB)'), nginx: _('Response metrics (NGINX)'), kubernetes: _('System metrics (Kubernetes)'), - cluster_health: _('Cluster Health') + cluster_health: _('Cluster Health'), + custom: _('Custom metrics') } end end diff --git a/lib/gitlab/database_importers/instance_administrators/create_group.rb b/lib/gitlab/database_importers/instance_administrators/create_group.rb index 5bf0e5a320d..d9425810405 100644 --- a/lib/gitlab/database_importers/instance_administrators/create_group.rb +++ b/lib/gitlab/database_importers/instance_administrators/create_group.rb @@ -6,6 +6,8 @@ module Gitlab class CreateGroup < ::BaseService include Stepable + NAME = 'GitLab Instance' + PATH_PREFIX = 'gitlab-instance' VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL steps :validate_application_settings, @@ -117,12 +119,12 @@ module Gitlab def create_group_params { - name: 'GitLab Instance Administrators', + name: NAME, visibility_level: VISIBILITY_LEVEL, # The 8 random characters at the end are so that the path does not # clash with any existing group that the user might have created. - path: "gitlab-instance-administrators-#{SecureRandom.hex(4)}" + path: "#{PATH_PREFIX}-#{SecureRandom.hex(4)}" } end end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index 07a4c3bf5e6..88f035c2d1b 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -9,7 +9,7 @@ module Gitlab include SelfMonitoring::Helpers VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL - PROJECT_NAME = 'GitLab self monitoring' + PROJECT_NAME = 'Monitoring' steps :validate_application_settings, :create_group, diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 0d027809ba8..a5259079345 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -60,7 +60,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. if rich_line - line_prefix = diff_line.text =~ /\A(.)/ ? $1 : ' ' + line_prefix = diff_line.text =~ /\A(.)/ ? Regexp.last_match(1) : ' ' "#{line_prefix}#{rich_line}".html_safe end end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 0c3b6b72313..0eb22e6b3cb 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -3,6 +3,7 @@ module Gitlab module Diff class HighlightCache + include Gitlab::Utils::Gzip include Gitlab::Utils::StrongMemoize EXPIRATION = 1.week @@ -83,7 +84,7 @@ module Gitlab redis.hset( key, diff_file_id, - compose_data(highlighted_diff_lines_hash.to_json) + gzip_compress(highlighted_diff_lines_hash.to_json) ) end @@ -145,35 +146,12 @@ module Gitlab end results.map! do |result| - Gitlab::Json.parse(extract_data(result), symbolize_names: true) unless result.nil? + Gitlab::Json.parse(gzip_decompress(result), symbolize_names: true) unless result.nil? end file_paths.zip(results).to_h end - def compose_data(json_data) - # #compress returns ASCII-8BIT, so we need to force the encoding to - # UTF-8 before caching it in redis, else we risk encoding mismatch - # errors. - # - ActiveSupport::Gzip.compress(json_data).force_encoding("UTF-8") - rescue Zlib::GzipFile::Error - json_data - end - - def extract_data(data) - # Since we could be dealing with an already populated cache full of data - # that isn't gzipped, we want to also check to see if the data is - # gzipped before we attempt to #decompress it, thus we check the first - # 2 bytes for "\x1F\x8B" to confirm it is a gzipped string. While a - # non-gzipped string will raise a Zlib::GzipFile::Error, which we're - # rescuing, we don't want to count on rescue for control flow. - # - data[0..1] == "\x1F\x8B" ? ActiveSupport::Gzip.decompress(data) : data - rescue Zlib::GzipFile::Error - data - end - def cacheable?(diff_file) diffable.present? && diff_file.text? && diff_file.diffable? end diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index 4bec6467c1a..3337aeb9262 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -3,6 +3,8 @@ module Gitlab module DiscussionsDiff class HighlightCache + extend Gitlab::Utils::Gzip + class << self VERSION = 1 EXPIRATION = 1.week @@ -17,7 +19,7 @@ module Gitlab mapping.each do |raw_key, value| key = cache_key_for(raw_key) - multi.set(key, value.to_json, ex: EXPIRATION) + multi.set(key, gzip_compress(value.to_json), ex: EXPIRATION) end end end @@ -44,7 +46,7 @@ module Gitlab content.map! do |lines| next unless lines - Gitlab::Json.parse(lines).map! do |line| + Gitlab::Json.parse(gzip_decompress(lines)).map! do |line| Gitlab::Diff::Line.safe_init_from_hash(line) end end diff --git a/lib/gitlab/email/hook/disable_email_interceptor.rb b/lib/gitlab/email/hook/disable_email_interceptor.rb index 58dc1527c7a..6e2e0201684 100644 --- a/lib/gitlab/email/hook/disable_email_interceptor.rb +++ b/lib/gitlab/email/hook/disable_email_interceptor.rb @@ -7,7 +7,7 @@ module Gitlab def self.delivering_email(message) message.perform_deliveries = false - Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" end end end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index bf6c28b9f90..f5e47b43a9a 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -54,7 +54,8 @@ module Gitlab def key_from_additional_headers(mail) find_key_from_references(mail) || find_key_from_delivered_to_header(mail) || - find_key_from_envelope_to_header(mail) + find_key_from_envelope_to_header(mail) || + find_key_from_x_envelope_to_header(mail) end def ensure_references_array(references) @@ -91,6 +92,13 @@ module Gitlab end end + def find_key_from_x_envelope_to_header(mail) + Array(mail[:x_envelope_to]).find do |header| + key = Gitlab::IncomingEmail.key_from_address(header.value) + break key if key + end + end + def ignore_auto_reply!(mail) if auto_submitted?(mail) || auto_replied?(mail) raise AutoGeneratedEmailError diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 67f8d691a77..7b79de00c66 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -59,7 +59,7 @@ module Gitlab begin CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') rescue ArgumentError => e - Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") '' end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 8d5611411c9..803acef9a40 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -26,6 +26,8 @@ module Gitlab # Sanitize fields based on those sanitized from Rails. config.sanitize_fields = Rails.application.config.filter_parameters.map(&:to_s) config.processors << ::Gitlab::ErrorTracking::Processor::SidekiqProcessor + config.processors << ::Gitlab::ErrorTracking::Processor::GrpcErrorProcessor + # Sanitize authentication headers config.sanitize_http_headers = %w[Authorization Private-Token] config.tags = extra_tags_from_env.merge(program: Gitlab.process_name) diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb new file mode 100644 index 00000000000..871e9c4b7c8 --- /dev/null +++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + module Processor + class GrpcErrorProcessor < ::Raven::Processor + DEBUG_ERROR_STRING_REGEX = RE2('(.*) debug_error_string:(.*)') + + def process(value) + process_first_exception_value(value) + process_custom_fingerprint(value) + + value + end + + # Sentry can report multiple exceptions in an event. Sanitize + # only the first one since that's what is used for grouping. + def process_first_exception_value(value) + exceptions = value.dig(:exception, :values) + + return unless exceptions.is_a?(Array) + + entry = exceptions.first + + return unless entry.is_a?(Hash) + + exception_type = entry[:type] + raw_message = entry[:value] + + return unless exception_type&.start_with?('GRPC::') + return unless raw_message.present? + + message, debug_str = split_debug_error_string(raw_message) + + entry[:value] = message if message + extra = value[:extra] || {} + extra[:grpc_debug_error_string] = debug_str if debug_str + end + + def process_custom_fingerprint(value) + fingerprint = value[:fingerprint] + + return value unless custom_grpc_fingerprint?(fingerprint) + + message, _ = split_debug_error_string(fingerprint[1]) + fingerprint[1] = message if message + end + + private + + def custom_grpc_fingerprint?(fingerprint) + fingerprint.is_a?(Array) && fingerprint.length == 2 && fingerprint[0].start_with?('GRPC::') + end + + def split_debug_error_string(message) + return unless message + + match = DEBUG_ERROR_STRING_REGEX.match(message) + + return unless match + + [match[1], match[2]] + end + end + end + end +end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 9908369426a..dca60c93fb2 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -42,9 +42,6 @@ module Gitlab ci_notification_dot: { tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot' }, - buy_ci_minutes_version_a: { - tracking_category: 'Growth::Expansion::Experiment::BuyCiMinutesVersionA' - }, upgrade_link_in_user_menu_a: { tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA' }, @@ -62,6 +59,9 @@ module Gitlab }, customize_homepage: { tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage' + }, + invite_email: { + tracking_category: 'Growth::Acquisition::Experiment::InviteEmail' } }.freeze @@ -78,7 +78,7 @@ module Gitlab included do before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled? + helper_method :experiment_enabled?, :experiment_tracking_category_and_group end def set_experimentation_subject_id_cookie @@ -118,6 +118,10 @@ module Gitlab ::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user) end + def experiment_tracking_category_and_group(experiment_key) + "#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group')}" + end + private def dnt_enabled? @@ -144,7 +148,7 @@ module Gitlab { category: tracking_category(experiment_key), action: action, - property: "#{tracking_group(experiment_key)}_group", + property: tracking_group(experiment_key, "_group"), label: experimentation_subject_id, value: value }.compact @@ -154,10 +158,12 @@ module Gitlab Experimentation.experiment(experiment_key).tracking_category end - def tracking_group(experiment_key) + def tracking_group(experiment_key, suffix = nil) return unless Experimentation.enabled?(experiment_key) - experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL + group = experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL + + suffix ? "#{group}#{suffix}" : group end def forced_enabled?(experiment_key) diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb index 475d50e37bf..38ccd2c38a9 100644 --- a/lib/gitlab/file_type_detection.rb +++ b/lib/gitlab/file_type_detection.rb @@ -20,6 +20,8 @@ module Gitlab module FileTypeDetection SAFE_IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze + SAFE_IMAGE_FOR_SCALING_EXT = %w[png jpg jpeg].freeze + PDF_EXT = 'pdf' # We recommend using the .mp4 format over .mov. Videos in .mov format can # still be used but you really need to make sure they are served with the @@ -46,6 +48,12 @@ module Gitlab extension_match?(SAFE_IMAGE_EXT) end + # For the time being, we restrict image scaling requests to the most popular and safest formats only, + # which are JPGs and PNGs. See https://gitlab.com/gitlab-org/gitlab/-/issues/237848 for more info. + def image_safe_for_scaling? + extension_match?(SAFE_IMAGE_FOR_SCALING_EXT) + end + def video? extension_match?(SAFE_VIDEO_EXT) end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 2c53f9b026d..bd5d2e53180 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -39,7 +39,7 @@ module Gitlab def user_map @user_map ||= begin - user_map = Hash.new + user_map = {} import_data = project.import_data.try(:data) stored_user_map = import_data['user_map'] if import_data user_map.update(stored_user_map) if stored_user_map diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index b6bffb11344..96f3487fd6f 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -13,7 +13,6 @@ module Gitlab TAG_REF_PREFIX = "refs/tags/" BRANCH_REF_PREFIX = "refs/heads/" - BaseError = Class.new(StandardError) CommandError = Class.new(BaseError) CommitError = Class.new(BaseError) OSError = Class.new(BaseError) diff --git a/lib/gitlab/git/base_error.rb b/lib/gitlab/git/base_error.rb new file mode 100644 index 00000000000..a7eaa82b347 --- /dev/null +++ b/lib/gitlab/git/base_error.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class BaseError < StandardError + DEBUG_ERROR_STRING_REGEX = /(.*?) debug_error_string:.*$/m.freeze + + def initialize(msg = nil) + if msg + raw_message = msg.to_s + match = DEBUG_ERROR_STRING_REGEX.match(raw_message) + raw_message = match[1] if match + + super(raw_message) + else + super + end + end + end + end +end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 46896961867..e6121d688ba 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -37,7 +37,7 @@ module Gitlab @byte_count = 0 @overflow = false @empty = true - @array = Array.new + @array = [] end def each(&block) diff --git a/lib/gitlab/git/keep_around.rb b/lib/gitlab/git/keep_around.rb index d58f10bdbb7..b6fc335c979 100644 --- a/lib/gitlab/git/keep_around.rb +++ b/lib/gitlab/git/keep_around.rb @@ -27,7 +27,7 @@ module Gitlab # This will still fail if the file is corrupted (e.g. 0 bytes) raw_repository.write_ref(keep_around_ref_name(sha), sha) rescue Gitlab::Git::CommandError => ex - Rails.logger.error "Unable to create keep-around reference for repository #{disk_path}: #{ex}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error "Unable to create keep-around reference for repository #{disk_path}: #{ex}" end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 596b4e9f692..8ace4157ad7 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -19,15 +19,15 @@ module Gitlab GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000' - NoRepository = Class.new(StandardError) - InvalidRepository = Class.new(StandardError) - InvalidBlobName = Class.new(StandardError) - InvalidRef = Class.new(StandardError) - GitError = Class.new(StandardError) - DeleteBranchError = Class.new(StandardError) - TagExistsError = Class.new(StandardError) - ChecksumError = Class.new(StandardError) - class CreateTreeError < StandardError + NoRepository = Class.new(::Gitlab::Git::BaseError) + InvalidRepository = Class.new(::Gitlab::Git::BaseError) + InvalidBlobName = Class.new(::Gitlab::Git::BaseError) + InvalidRef = Class.new(::Gitlab::Git::BaseError) + GitError = Class.new(::Gitlab::Git::BaseError) + DeleteBranchError = Class.new(::Gitlab::Git::BaseError) + TagExistsError = Class.new(::Gitlab::Git::BaseError) + ChecksumError = Class.new(::Gitlab::Git::BaseError) + class CreateTreeError < ::Gitlab::Git::BaseError attr_reader :error_code def initialize(error_code) @@ -955,7 +955,7 @@ module Gitlab gitaly_repository_client.cleanup if exists? end rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup - Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") Gitlab::Metrics.counter( :failed_repository_cleanup_total, 'Number of failed repository cleanup events' diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 76771f0417b..da2d015ca4a 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -6,7 +6,6 @@ module Gitlab include Gitlab::Git::WrapsGitalyErrors DuplicatePageError = Class.new(StandardError) - OperationError = Class.new(StandardError) DEFAULT_PAGINATION = Kaminari.config.default_per_page diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index f2b4e930707..ae83e45f2b3 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -119,7 +119,7 @@ module Gitlab override :check_single_change_access def check_single_change_access(change, _skip_lfs_integrity_check: false) - Checks::SnippetCheck.new(change, logger: logger).validate! + Checks::SnippetCheck.new(change, default_branch: snippet.default_branch, logger: logger).validate! Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit(user), logger: logger).validate! rescue Checks::TimedLogger::TimeoutError raise TimeoutError, logger.full_message diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 131c00db612..e1324530412 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -450,7 +450,7 @@ module Gitlab stack_string = Gitlab::BacktraceCleaner.clean_backtrace(caller).drop(1).join("\n") - Gitlab::SafeRequestStore[:stack_counter] ||= Hash.new + Gitlab::SafeRequestStore[:stack_counter] ||= {} count = Gitlab::SafeRequestStore[:stack_counter][stack_string] || 0 Gitlab::SafeRequestStore[:stack_counter][stack_string] = count + 1 diff --git a/lib/gitlab/gitpod.rb b/lib/gitlab/gitpod.rb new file mode 100644 index 00000000000..11b54db72ea --- /dev/null +++ b/lib/gitlab/gitpod.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + class Gitpod + class << self + def feature_conditional? + feature.conditional? + end + + def feature_available? + # The gitpod_bundle feature could be conditionally applied, so check if `!off?` + !feature.off? + end + + def feature_enabled?(actor = nil) + feature.enabled?(actor) + end + + def feature_and_settings_enabled?(actor = nil) + feature_enabled?(actor) && Gitlab::CurrentSettings.gitpod_enabled + end + + private + + def feature + Feature.get(:gitpod) # rubocop:disable Gitlab/AvoidFeatureGet + end + end + end +end diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index 7346de13626..54dca93a891 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -7,19 +7,19 @@ module Gitlab PROJECT = RepoType.new( name: :project, access_checker_class: Gitlab::GitAccessProject, - repository_resolver: -> (project) { project&.repository } + repository_resolver: -> (project) { ::Repository.new(project.full_path, project, shard: project.repository_storage, disk_path: project.disk_path) } ).freeze WIKI = RepoType.new( name: :wiki, access_checker_class: Gitlab::GitAccessWiki, - repository_resolver: -> (container) { container&.wiki&.repository }, + repository_resolver: -> (container) { ::Repository.new(container.wiki.full_path, container, shard: container.wiki.repository_storage, disk_path: container.wiki.disk_path, repo_type: WIKI) }, project_resolver: -> (container) { container.is_a?(Project) ? container : nil }, suffix: :wiki ).freeze SNIPPET = RepoType.new( name: :snippet, access_checker_class: Gitlab::GitAccessSnippet, - repository_resolver: -> (snippet) { snippet&.repository }, + repository_resolver: -> (snippet) { ::Repository.new(snippet.full_path, snippet, shard: snippet.repository_storage, disk_path: snippet.disk_path, repo_type: SNIPPET) }, container_class: Snippet, project_resolver: -> (snippet) { snippet&.project }, guest_read_ability: :read_snippet diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 2c0038b61e2..346f6be0d98 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -57,6 +57,8 @@ module Gitlab end def repository_for(container) + return unless container + repository_resolver.call(container) end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index dfba68ce899..66517ecd743 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -4,6 +4,7 @@ module Gitlab module GonHelper + include StartupCssHelper include WebpackHelper def add_gon_variables @@ -45,9 +46,12 @@ module Gitlab push_frontend_feature_flag(:snippets_vue, default_enabled: true) push_frontend_feature_flag(:monaco_blobs, default_enabled: true) push_frontend_feature_flag(:monaco_ci, default_enabled: false) - push_frontend_feature_flag(:snippets_edit_vue, default_enabled: false) + push_frontend_feature_flag(:snippets_edit_vue, default_enabled: true) push_frontend_feature_flag(:webperf_experiment, default_enabled: false) push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) + + # Startup CSS feature is a special one as it can be enabled by means of cookies and params + gon.push({ features: { 'startupCss' => use_startup_css? } }, true) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb index 0dd28b32511..dcd0e12cbfc 100644 --- a/lib/gitlab/graphql/docs/helper.rb +++ b/lib/gitlab/graphql/docs/helper.rb @@ -21,30 +21,47 @@ module Gitlab MD end - def sorted_fields(fields) - fields.sort_by { |field| field[:name] } + def render_name_and_description(object) + content = "### #{object[:name]}\n" + + if object[:description].present? + content += "\n#{object[:description]}.\n" + end + + content + end + + def sorted_by_name(objects) + objects.sort_by { |o| o[:name] } end def render_field(field) '| %s | %s | %s |' % [ - render_field_name(field), + render_name(field), render_field_type(field[:type][:info]), - render_field_description(field) + render_description(field) ] end - def render_field_name(field) - rendered_name = "`#{field[:name]}`" - rendered_name += ' **{warning-solid}**' if field[:is_deprecated] + def render_enum_value(value) + '| %s | %s |' % [ + render_name(value), + render_description(value) + ] + end + + def render_name(object) + rendered_name = "`#{object[:name]}`" + rendered_name += ' **{warning-solid}**' if object[:is_deprecated] rendered_name end - # Returns the field description. If the field has been deprecated, + # Returns the object description. If the object has been deprecated, # the deprecation reason will be returned in place of the description. - def render_field_description(field) - return field[:description] unless field[:is_deprecated] + def render_description(object) + return object[:description] unless object[:is_deprecated] - "**Deprecated:** #{field[:deprecation_reason]}" + "**Deprecated:** #{object[:deprecation_reason]}" end # Some fields types are arrays of other types and are displayed @@ -70,6 +87,13 @@ module Gitlab !object_type[:name]["__"] end end + + # We ignore the built-in enum types. + def enums + graphql_enum_types.select do |enum_type| + !enum_type[:name].in?(%w(__DirectiveLocation __TypeKind)) + end + end end end end diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index 8c033526557..ec052943589 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -15,15 +15,45 @@ CAUTION: **Caution:** Fields that are deprecated are marked with **{warning-solid}**. \ + +:plain + ## Object types + + Object types represent the resources that GitLab's GraphQL API can return. + They contain _fields_. Each field has its own type, which will either be one of the + basic GraphQL [scalar types](https://graphql.org/learn/schema/#scalar-types) + (e.g.: `String` or `Boolean`) or other object types. + + For more information, see + [Object Types and Fields](https://graphql.org/learn/schema/#object-types-and-fields) + on `graphql.org`. +\ + - objects.each do |type| - unless type[:fields].empty? - = "## #{type[:name]}" - - if type[:description]&.present? - \ - = type[:description] - \ - ~ "| Name | Type | Description |" - ~ "| --- | ---- | ---------- |" - - sorted_fields(type[:fields]).each do |field| + = render_name_and_description(type) + ~ "| Field | Type | Description |" + ~ "| ----- | ---- | ----------- |" + - sorted_by_name(type[:fields]).each do |field| = render_field(field) \ + +:plain + ## Enumeration types + + Also called _Enums_, enumeration types are a special kind of scalar that + is restricted to a particular set of allowed values. + + For more information, see + [Enumeration Types](https://graphql.org/learn/schema/#enumeration-types) + on `graphql.org`. +\ + +- enums.each do |enum| + - unless enum[:values].empty? + = render_name_and_description(enum) + ~ "| Value | Description |" + ~ "| ----- | ----------- |" + - sorted_by_name(enum[:values]).each do |value| + = render_enum_value(value) + \ diff --git a/lib/gitlab/graphql/loaders/issuable_loader.rb b/lib/gitlab/graphql/loaders/issuable_loader.rb index 1cc0fbe215f..8ac4be2b661 100644 --- a/lib/gitlab/graphql/loaders/issuable_loader.rb +++ b/lib/gitlab/graphql/loaders/issuable_loader.rb @@ -15,6 +15,7 @@ module Gitlab def batching_find_all(&with_query) if issuable_finder.params.keys == ['iids'] + issuable_finder.parent = parent batch_load_issuables(issuable_finder.params[:iids], with_query) else post_process(find_all, with_query) @@ -22,24 +23,12 @@ module Gitlab end def find_all - issuable_finder.params[parent_param] = parent if parent - + issuable_finder.parent_param = parent if parent issuable_finder.execute end private - def parent_param - case parent - when Project - :project_id - when Group - :group_id - else - raise "Unexpected parent: #{parent.class}" - end - end - def post_process(query, with_query) if with_query with_query.call(query) @@ -56,7 +45,7 @@ module Gitlab return if parent.nil? BatchLoader::GraphQL - .for([parent_param, iid.to_s]) + .for([issuable_finder.parent_param, iid.to_s]) .batch(key: batch_key) do |params, loader, args| batch_key = args[:key] user = batch_key.current_user diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb index afea7c602be..bd785880b57 100644 --- a/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb +++ b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb @@ -29,7 +29,10 @@ module Gitlab def table_condition(order_info, value, operator) if order_info.named_function target = order_info.named_function - value = value&.downcase if target&.name&.downcase == 'lower' + + if target.try(:name)&.casecmp('lower') == 0 + value = value&.downcase + end else target = arel_table[order_info.attribute_name] end diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb index 12bcc4993b5..f54695ddb9a 100644 --- a/lib/gitlab/graphql/pagination/keyset/order_info.rb +++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb @@ -71,25 +71,43 @@ module Gitlab def extract_nulls_last_order(order_value) tokens = order_value.downcase.split - [tokens.first, (tokens[1] == 'asc' ? :asc : :desc), nil] + column_reference = tokens.first + sort_direction = tokens[1] == 'asc' ? :asc : :desc + + # Handles the case when the order value is coming from another table. + # Example: table_name.column_name + # Query the value using the fully qualified column name: pass table_name.column_name as the named_function + if fully_qualified_column_reference?(column_reference) + [column_reference, sort_direction, Arel.sql(column_reference)] + else + [column_reference, sort_direction, nil] + end + end + + # Example: table_name.column_name + def fully_qualified_column_reference?(attribute) + attribute.to_s.count('.') == 1 end def extract_attribute_values(order_value) - named = nil - name = if ordering_by_lower?(order_value) - named = order_value.expr - named.expressions[0].name.to_s - else - order_value.expr.name - end - - [name, order_value.direction, named] + if ordering_by_lower?(order_value) + [order_value.expr.expressions[0].name.to_s, order_value.direction, order_value.expr] + elsif ordering_by_similarity?(order_value) + ['similarity', order_value.direction, order_value.expr] + else + [order_value.expr.name, order_value.direction, nil] + end end # determine if ordering using LOWER, eg. "ORDER BY LOWER(boards.name)" def ordering_by_lower?(order_value) order_value.expr.is_a?(Arel::Nodes::NamedFunction) && order_value.expr&.name&.downcase == 'lower' end + + # determine if ordering using SIMILARITY scoring based on Gitlab::Database::SimilarityScore + def ordering_by_similarity?(order_value) + order_value.to_sql.match?(/SIMILARITY\(.+\*/) + end end end end diff --git a/lib/gitlab/graphql/representation/submodule_tree_entry.rb b/lib/gitlab/graphql/representation/submodule_tree_entry.rb index 8d17cb9eecc..aa5e74cc837 100644 --- a/lib/gitlab/graphql/representation/submodule_tree_entry.rb +++ b/lib/gitlab/graphql/representation/submodule_tree_entry.rb @@ -24,11 +24,11 @@ module Gitlab end def web_url - @submodule_links.first + @submodule_links&.web end def tree_url - @submodule_links.last + @submodule_links&.tree end end end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index eb4361cdc53..0cc3de297ba 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -4,28 +4,32 @@ module Gitlab class GroupSearchResults < SearchResults attr_reader :group - def initialize(current_user, limit_projects, group, query, default_project_filter: false) - super(current_user, limit_projects, query, default_project_filter: default_project_filter) - + def initialize(current_user, query, limit_projects = nil, group:, default_project_filter: false, filters: {}) @group = group + + super(current_user, query, limit_projects, default_project_filter: default_project_filter, filters: filters) end # rubocop:disable CodeReuse/ActiveRecord def users - # 1: get all groups the current user has access to - groups = GroupsFinder.new(current_user).execute.joins(:users) + # get all groups the current user has access to + # ignore order inherited from GroupsFinder to improve performance + current_user_groups = GroupsFinder.new(current_user).execute.unscope(:order) + + # the hierarchy of the current group + group_groups = @group.self_and_hierarchy.unscope(:order) + + # the groups where the above hierarchies intersect + intersect_groups = group_groups.where(id: current_user_groups) - # 2: Get the group's whole hierarchy - group_users = @group.direct_and_indirect_users + # members of @group hierarchy where the user has access to the groups + members = GroupMember.where(group: intersect_groups).non_invite - # 3: get all users the current user has access to (-> - # `SearchResults#users`), which also applies the query. + # get all users the current user has access to (-> `SearchResults#users`), which also applies the query users = super - # 4: filter for users that belong to the previously selected groups - users - .where(id: group_users.select('id')) - .where(id: groups.select('members.user_id')) + # filter users that belong to the previously selected groups + users.where(id: members.select(:user_id)) end # rubocop:enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index b72d08549fe..b57560544c8 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -62,28 +62,24 @@ module Gitlab # Flag a project to be migrated to Hashed Storage # # @param [Project] project that will be migrated - # rubocop:disable Gitlab/RailsLogger def migrate(project) - Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." + Gitlab::AppLogger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." project.migrate_to_hashed_storage! rescue => err - Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") + Gitlab::AppLogger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end - # rubocop:enable Gitlab/RailsLogger # Flag a project to be rolled-back to Legacy Storage # # @param [Project] project that will be rolled-back - # rubocop:disable Gitlab/RailsLogger def rollback(project) - Rails.logger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..." + Gitlab::AppLogger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..." project.rollback_to_legacy_storage! rescue => err - Rails.logger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") + Gitlab::AppLogger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end - # rubocop:enable Gitlab/RailsLogger # Returns whether we have any pending storage migration # @@ -97,6 +93,14 @@ module Gitlab any_non_empty_queue?(::HashedStorage::RollbackerWorker, ::HashedStorage::ProjectRollbackWorker) end + # Remove all remaining scheduled rollback operations + # + def abort_rollback! + [::HashedStorage::RollbackerWorker, ::HashedStorage::ProjectRollbackWorker].each do |worker| + Sidekiq::Queue.new(worker.queue).clear + end + end + private def any_non_empty_queue?(*workers) diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index 4e0b9296819..ae99768b7b4 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -24,7 +24,7 @@ module Gitlab result, elapsed = with_timing(&method(:check)) return if result.nil? - Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) [ metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), metric("#{metric_prefix}_success", successful?(result) ? 1 : 0), diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 07c43fa4832..40dee0142b9 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -3,8 +3,8 @@ module Gitlab class Highlight TIMEOUT_BACKGROUND = 30.seconds - TIMEOUT_FOREGROUND = 3.seconds - MAXIMUM_TEXT_HIGHLIGHT_SIZE = 1.megabyte + TIMEOUT_FOREGROUND = 1.5.seconds + MAXIMUM_TEXT_HIGHLIGHT_SIZE = 512.kilobytes def self.highlight(blob_name, blob_content, language: nil, plain: false) new(blob_name, blob_content, language: language) @@ -31,8 +31,8 @@ module Gitlab def lexer @lexer ||= custom_language || begin Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new - rescue Rouge::Guesser::Ambiguous => e - e.alternatives.min_by(&:tag) + rescue Rouge::Guesser::Ambiguous => e + e.alternatives.min_by(&:tag) end end diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 559e1828a70..be87dcc0ff9 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -35,7 +35,7 @@ module Gitlab def self.perform_request(http_method, path, options, &block) log_info = options.delete(:extra_log_info) options_with_timeouts = - if !options.has_key?(:timeout) && Feature.enabled?(:http_default_timeouts) + if !options.has_key?(:timeout) options.with_defaults(DEFAULT_TIMEOUT_OPTIONS) else options diff --git a/lib/gitlab/i18n/html_todo.yml b/lib/gitlab/i18n/html_todo.yml index bfd96ba8579..91e01f8a0b8 100644 --- a/lib/gitlab/i18n/html_todo.yml +++ b/lib/gitlab/i18n/html_todo.yml @@ -16,67 +16,6 @@ # why this change has been made. # -" or <!merge request id>": - translations: - - " ወይም <!merge request id>" - - " ou <!merge request id>" - - " または <!merge request id>" - - "或 <!合併請求 id>" - - " или <!merge request id>" - - "或<!merge request id>" - - " або <!merge request id>" - - " oder <!merge request id>" - - " o <!merge request id>" - - " 또는 <!merge request id>" - - " o <!merge request id>" - - " veya <!merge request id>" - - " neu <!merge request id>" - - " neu <#issue id>" -" or <#issue id>": - translations: - - "或 <#issue id>" - - " ወይም ‹#issue id›" - - " ou <identificación #issue>" - - " ou <#issue id>" - - " または <#課題 ID>" - - " o <#issue id>" - - "或 <#議題 id>" - - " ou <#issue id>" - - " или <#issue id>" - - "或 <#issue id>" - - " або <#issue id>" - - " oder <#issue id>" - - " o <#issue id>" - - " 또는 <#issue id>" - - " ou <#issue id>" - - " o <#issue id>" - - " veya <#issue id>" - - " neu <#issue id>" -" or <&epic id>": - translations: - - " ወይም <&epic id>" - - " または <&エピックID>" - - " 或 <#史詩 id>" - - " или <&epic id>" - - " 或<#epic id>" - - " або <&epic id>" - - " oder <&epic id>" - - " o <&epic id>" - - " veya <&epic id>" - - " neu <#epic id>" - - " 또는 <&epic id>" -"< 1 hour": - translations: - - "1 時間未満" - - "< 1 小時" - - "< 1 часа" - - "< 1小时" - - "< 1 години" - - "< 1 hora" - - "< 1 saat" - - "< 1 Stunde" - - "< 1시간" - # # Strings below are fixed in the source code but the translations are still present in CrowdIn so the # locale files will fail the linter. They can be deleted after next CrowdIn sync, likely in: @@ -313,3 +252,63 @@ - "Vous êtes sur le point de d’activer la confidentialité. Cela signifie que seuls les membres de l’équipe avec <strong>au moins un accès en tant que rapporteur</strong> seront en mesure de voir et de laisser des commentaires sur le ticket." - "Va a activar la confidencialidad. Esto significa que solo los miembros del equipo con como mínimo,<strong>acceso como Reporter</strong> podrán ver y dejar comentarios sobre la incidencia." - "あなたは非公開設定をオンにしようとしています。これは、最低でも<strong>報告権限</strong>を持ったチームメンバーのみが課題を表示したりコメントを残したりすることができるようになるということです。" +" or <!merge request id>": + translations: + - " ወይም <!merge request id>" + - " ou <!merge request id>" + - " または <!merge request id>" + - "或 <!合併請求 id>" + - " или <!merge request id>" + - "或<!merge request id>" + - " або <!merge request id>" + - " oder <!merge request id>" + - " o <!merge request id>" + - " 또는 <!merge request id>" + - " o <!merge request id>" + - " veya <!merge request id>" + - " neu <!merge request id>" + - " neu <#issue id>" +" or <#issue id>": + translations: + - "或 <#issue id>" + - " ወይም ‹#issue id›" + - " ou <identificación #issue>" + - " ou <#issue id>" + - " または <#課題 ID>" + - " o <#issue id>" + - "或 <#議題 id>" + - " ou <#issue id>" + - " или <#issue id>" + - "或 <#issue id>" + - " або <#issue id>" + - " oder <#issue id>" + - " o <#issue id>" + - " 또는 <#issue id>" + - " ou <#issue id>" + - " o <#issue id>" + - " veya <#issue id>" + - " neu <#issue id>" +" or <&epic id>": + translations: + - " ወይም <&epic id>" + - " または <&エピックID>" + - " 或 <#史詩 id>" + - " или <&epic id>" + - " 或<#epic id>" + - " або <&epic id>" + - " oder <&epic id>" + - " o <&epic id>" + - " veya <&epic id>" + - " neu <#epic id>" + - " 또는 <&epic id>" +"< 1 hour": + translations: + - "1 時間未満" + - "< 1 小時" + - "< 1 часа" + - "< 1小时" + - "< 1 години" + - "< 1 hora" + - "< 1 saat" + - "< 1 Stunde" + - "< 1시간" diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 18c590e1ca9..515fd98630c 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -16,7 +16,7 @@ module Gitlab end def save - project.all_lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| + project.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| batch.each do |lfs_object| save_lfs_object(lfs_object) end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index a240c367a42..9ec5df8cde9 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -190,8 +190,20 @@ excluded_attributes: - :stored_externally - :external_diff_store - :merge_request_id + - :verification_retry_at + - :verified_at + - :verification_retry_count + - :verification_checksum + - :verification_failure merge_request_diff_commits: - :merge_request_diff_id + merge_request_diff_detail: + - :merge_request_diff_id + - :verification_retry_at + - :verified_at + - :verification_retry_count + - :verification_checksum + - :verification_failure merge_request_diff_files: - :diff - :external_diff_offset diff --git a/lib/gitlab/jira/dvcs.rb b/lib/gitlab/jira/dvcs.rb new file mode 100644 index 00000000000..4415f98fc7f --- /dev/null +++ b/lib/gitlab/jira/dvcs.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Jira + module Dvcs + ENCODED_SLASH = '@'.freeze + SLASH = '/'.freeze + ENCODED_ROUTE_REGEX = /[a-zA-Z0-9_\-\.#{ENCODED_SLASH}]+/.freeze + + def self.encode_slash(path) + path.gsub(SLASH, ENCODED_SLASH) + end + + def self.decode_slash(path) + path.gsub(ENCODED_SLASH, SLASH) + end + + # To present two types of projects stored by Jira, + # Type 1 are projects imported prior to nested group support, + # those project names are not full_path, so they are presented differently + # to maintain backwards compatibility. + # Type 2 are projects imported after nested group support, + # those project names are encoded full path + # + # @param [Project] project + def self.encode_project_name(project) + if project.namespace.has_parent? + encode_slash(project.full_path) + else + project.path + end + end + + # To interpret two types of project names stored by Jira (see `encode_project_name`) + # + # @param [String] project + # Either an encoded full path, or just project name + # @param [String] namespace + def self.restore_full_path(namespace:, project:) + if project.include?(ENCODED_SLASH) + project.gsub(ENCODED_SLASH, SLASH) + else + "#{namespace}/#{project}" + end + end + end + end +end diff --git a/lib/gitlab/jira/middleware.rb b/lib/gitlab/jira/middleware.rb new file mode 100644 index 00000000000..8a74729da49 --- /dev/null +++ b/lib/gitlab/jira/middleware.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Jira + class Middleware + def self.jira_dvcs_connector?(env) + env['HTTP_USER_AGENT']&.downcase&.start_with?('jira dvcs connector') + end + + def initialize(app) + @app = app + end + + def call(env) + if self.class.jira_dvcs_connector?(env) + env['HTTP_AUTHORIZATION'] = env['HTTP_AUTHORIZATION']&.sub('token', 'Bearer') + end + + @app.call(env) + end + end + end +end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index d6681347f42..29cfec443e8 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -22,7 +22,7 @@ module Gitlab string = string.to_s unless string.is_a?(String) legacy_mode = legacy_mode_enabled?(opts.delete(:legacy_mode)) - data = adapter_load(string, opts) + data = adapter_load(string, **opts) handle_legacy_mode!(data) if legacy_mode diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb new file mode 100644 index 00000000000..08dde98e965 --- /dev/null +++ b/lib/gitlab/kas.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Kas + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Kas-Api-Request' + JWT_ISSUER = 'gitlab-kas' + + include JwtAuthenticatable + + class << self + def verify_api_request(request_headers) + decode_jwt_for_issuer(JWT_ISSUER, request_headers[INTERNAL_API_REQUEST_HEADER]) + rescue JWT::DecodeError + nil + end + + def secret_path + Gitlab.config.gitlab_kas.secret_file + end + + def ensure_secret! + return if File.exist?(secret_path) + + write_secret + end + end + end +end diff --git a/lib/gitlab/kubernetes/cilium_network_policy.rb b/lib/gitlab/kubernetes/cilium_network_policy.rb index 55afd2b586e..9043932bbe5 100644 --- a/lib/gitlab/kubernetes/cilium_network_policy.rb +++ b/lib/gitlab/kubernetes/cilium_network_policy.rb @@ -9,8 +9,12 @@ module Gitlab API_VERSION = "cilium.io/v2" KIND = 'CiliumNetworkPolicy' - def initialize(name:, namespace:, selector:, ingress:, resource_version:, labels: nil, creation_timestamp: nil, egress: nil) + # We are modeling existing kubernetes resource and don't have + # control over amount of parameters. + # rubocop:disable Metrics/ParameterLists + def initialize(name:, namespace:, selector:, ingress:, resource_version: nil, description: nil, labels: nil, creation_timestamp: nil, egress: nil) @name = name + @description = description @namespace = namespace @labels = labels @creation_timestamp = creation_timestamp @@ -19,15 +23,7 @@ module Gitlab @ingress = ingress @egress = egress end - - def generate - ::Kubeclient::Resource.new.tap do |resource| - resource.kind = KIND - resource.apiVersion = API_VERSION - resource.metadata = metadata - resource.spec = spec - end - end + # rubocop:enable Metrics/ParameterLists def self.from_yaml(manifest) return unless manifest @@ -39,6 +35,7 @@ module Gitlab spec = policy[:spec] self.new( name: metadata[:name], + description: policy[:description], namespace: metadata[:namespace], resource_version: metadata[:resourceVersion], labels: metadata[:labels], @@ -58,6 +55,7 @@ module Gitlab spec = resource[:spec].to_h self.new( name: metadata[:name], + description: resource[:description], namespace: metadata[:namespace], resource_version: metadata[:resourceVersion], labels: metadata[:labels]&.to_h, @@ -68,21 +66,39 @@ module Gitlab ) end + override :resource + def resource + resource = { + apiVersion: API_VERSION, + kind: KIND, + metadata: metadata, + spec: spec + } + resource[:description] = description if description + resource + end + private - attr_reader :name, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress + attr_reader :name, :description, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress def selector @selector ||= {} end - override :spec + def metadata + meta = { name: name, namespace: namespace } + meta[:labels] = labels if labels + meta[:resourceVersion] = resource_version if resource_version + meta + end + def spec { endpointSelector: selector, ingress: ingress, egress: egress - } + }.compact end end end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 9e3cf58bb1e..fa68afd39f5 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -92,6 +92,7 @@ module Gitlab # group client delegate :create_network_policy, :get_network_policies, + :get_network_policy, :update_network_policy, :delete_network_policy, to: :networking_client @@ -100,6 +101,7 @@ module Gitlab # group client delegate :create_cilium_network_policy, :get_cilium_network_policies, + :get_cilium_network_policy, :update_cilium_network_policy, :delete_cilium_network_policy, to: :cilium_networking_client diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb index 28810dc4453..c8e9b987443 100644 --- a/lib/gitlab/kubernetes/network_policy.rb +++ b/lib/gitlab/kubernetes/network_policy.rb @@ -6,6 +6,8 @@ module Gitlab include NetworkPolicyCommon extend ::Gitlab::Utils::Override + KIND = 'NetworkPolicy' + def initialize(name:, namespace:, selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil) @name = name @namespace = namespace @@ -17,13 +19,6 @@ module Gitlab @egress = egress end - def generate - ::Kubeclient::Resource.new.tap do |resource| - resource.metadata = metadata - resource.spec = spec - end - end - def self.from_yaml(manifest) return unless manifest @@ -63,6 +58,15 @@ module Gitlab ) end + override :resource + def resource + { + kind: KIND, + metadata: metadata, + spec: spec + } + end + private attr_reader :name, :namespace, :labels, :creation_timestamp, :policy_types, :ingress, :egress @@ -71,7 +75,12 @@ module Gitlab @selector ||= {} end - override :spec + def metadata + meta = { name: name, namespace: namespace } + meta[:labels] = labels if labels + meta + end + def spec { podSelector: selector, diff --git a/lib/gitlab/kubernetes/network_policy_common.rb b/lib/gitlab/kubernetes/network_policy_common.rb index 3b6e46d21ef..99517454508 100644 --- a/lib/gitlab/kubernetes/network_policy_common.rb +++ b/lib/gitlab/kubernetes/network_policy_common.rb @@ -5,6 +5,10 @@ module Gitlab module NetworkPolicyCommon DISABLED_BY_LABEL = :'network-policy.gitlab.com/disabled_by' + def generate + ::Kubeclient::Resource.new(resource) + end + def as_json(opts = nil) { name: name, @@ -46,19 +50,12 @@ module Gitlab private - def metadata - meta = { name: name, namespace: namespace } - meta[:labels] = labels if labels - meta[:resourceVersion] = resource_version if defined?(resource_version) - meta - end - - def spec + def resource raise NotImplementedError end def manifest - YAML.dump({ metadata: metadata, spec: spec }.deep_stringify_keys) + YAML.dump(resource.deep_stringify_keys) end end end diff --git a/lib/gitlab/lfs/client.rb b/lib/gitlab/lfs/client.rb new file mode 100644 index 00000000000..e4d600694c2 --- /dev/null +++ b/lib/gitlab/lfs/client.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true +module Gitlab + module Lfs + # Gitlab::Lfs::Client implements a simple LFS client, designed to talk to + # LFS servers as described in these documents: + # * https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md + # * https://github.com/git-lfs/git-lfs/blob/master/docs/api/basic-transfers.md + class Client + attr_reader :base_url + + def initialize(base_url, credentials:) + @base_url = base_url + @credentials = credentials + end + + def batch(operation, objects) + body = { + operation: operation, + transfers: ['basic'], + # We don't know `ref`, so can't send it + objects: objects.map { |object| { oid: object.oid, size: object.size } } + } + + rsp = Gitlab::HTTP.post( + batch_url, + basic_auth: basic_auth, + body: body.to_json, + headers: { 'Content-Type' => 'application/vnd.git-lfs+json' } + ) + + raise BatchSubmitError unless rsp.success? + + # HTTParty provides rsp.parsed_response, but it only kicks in for the + # application/json content type in the response, which we can't rely on + body = Gitlab::Json.parse(rsp.body) + transfer = body.fetch('transfer', 'basic') + + raise UnsupportedTransferError.new(transfer.inspect) unless transfer == 'basic' + + body + end + + def upload(object, upload_action, authenticated:) + file = object.file.open + + params = { + body_stream: file, + headers: { + 'Content-Length' => object.size.to_s, + 'Content-Type' => 'application/octet-stream' + }.merge(upload_action['header'] || {}) + } + + params[:basic_auth] = basic_auth unless authenticated + + rsp = Gitlab::HTTP.put(upload_action['href'], params) + + raise ObjectUploadError unless rsp.success? + ensure + file&.close + end + + private + + attr_reader :credentials + + def batch_url + base_url + '/info/lfs/objects/batch' + end + + def basic_auth + return unless credentials[:auth_method] == "password" + + { username: credentials[:user], password: credentials[:password] } + end + + class BatchSubmitError < StandardError + def message + "Failed to submit batch" + end + end + + class UnsupportedTransferError < StandardError + def initialize(transfer = nil) + super + @transfer = transfer + end + + def message + "Unsupported transfer: #{@transfer}" + end + end + + class ObjectUploadError < StandardError + def message + "Failed to upload object" + end + end + end + end +end diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index 2ec8c268d09..89a4e36a232 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -32,7 +32,8 @@ module Gitlab end def self.build - Gitlab::SafeRequestStore[self.cache_key] ||= new(self.full_log_path) + Gitlab::SafeRequestStore[self.cache_key] ||= + new(self.full_log_path, level: ::Logger::DEBUG) end def self.full_log_path diff --git a/lib/gitlab/marginalia/comment.rb b/lib/gitlab/marginalia/comment.rb index d5dae5ef4b3..7b4e4b06f00 100644 --- a/lib/gitlab/marginalia/comment.rb +++ b/lib/gitlab/marginalia/comment.rb @@ -28,7 +28,7 @@ module Gitlab # We are using 'Marginalia::SidekiqInstrumentation' which does not support 'ActiveJob::Base'. # Gitlab also uses 'ActionMailer::MailDeliveryJob' which inherits from ActiveJob::Base. # So below condition is used to return metadata for such jobs. - if job.is_a?(ActionMailer::MailDeliveryJob) || job.is_a?(ActionMailer::DeliveryJob) + if job.is_a?(ActionMailer::MailDeliveryJob) { "class" => job.arguments.first, "jid" => job.job_id diff --git a/lib/gitlab/metrics/dashboard/importer.rb b/lib/gitlab/metrics/dashboard/importer.rb new file mode 100644 index 00000000000..ca835650648 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/importer.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + class Importer + def initialize(dashboard_path, project) + @dashboard_path = dashboard_path.to_s + @project = project + end + + def execute + return false unless Dashboard::Validator.validate(dashboard_hash, project: project, dashboard_path: dashboard_path) + + Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute + rescue Gitlab::Config::Loader::FormatError + false + end + + def execute! + Dashboard::Validator.validate!(dashboard_hash, project: project, dashboard_path: dashboard_path) + + Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute! + end + + private + + attr_accessor :dashboard_path, :project + + def dashboard_hash + @dashboard_hash ||= begin + raw_dashboard = Dashboard::RepoDashboardFinder.read_dashboard(project, dashboard_path) + return unless raw_dashboard.present? + + ::Gitlab::Config::Loader::Yaml.new(raw_dashboard).load_raw! + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb new file mode 100644 index 00000000000..d1490d5d9b6 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Importers + class PrometheusMetrics + ALLOWED_ATTRIBUTES = %i(title query y_label unit legend group dashboard_path).freeze + + # Takes a JSON schema validated dashboard hash and + # imports metrics to database + def initialize(dashboard_hash, project:, dashboard_path:) + @dashboard_hash = dashboard_hash + @project = project + @dashboard_path = dashboard_path + end + + def execute + import + rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError + false + end + + def execute! + import + end + + private + + attr_reader :dashboard_hash, :project, :dashboard_path + + def import + delete_stale_metrics + create_or_update_metrics + end + + # rubocop: disable CodeReuse/ActiveRecord + def create_or_update_metrics + # TODO: use upsert and worker for callbacks? + prometheus_metrics_attributes.each do |attributes| + prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project)) + prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES)) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def delete_stale_metrics + identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } + + stale_metrics = PrometheusMetric.for_project(project) + .for_dashboard_path(dashboard_path) + .for_group(Enums::PrometheusMetric.groups[:custom]) + .not_identifier(identifiers) + + # TODO: use destroy_all and worker for callbacks? + stale_metrics.each(&:destroy) + end + + def prometheus_metrics_attributes + @prometheus_metrics_attributes ||= begin + Dashboard::Transformers::Yml::V1::PrometheusMetrics.new( + dashboard_hash, + project: project, + dashboard_path: dashboard_path + ).execute + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb index cfec027155e..06cfa5cc58e 100644 --- a/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb @@ -27,7 +27,7 @@ module Gitlab private def custom_group_titles - @custom_group_titles ||= PrometheusMetricEnums.custom_group_details.values.map { |group_details| group_details[:group_title] } + @custom_group_titles ||= Enums::PrometheusMetric.custom_group_details.values.map { |group_details| group_details[:group_title] } end def edit_path(metric) diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb new file mode 100644 index 00000000000..4d94ab098ae --- /dev/null +++ b/lib/gitlab/metrics/dashboard/transformers/errors.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Transformers + TransformerError = Class.new(StandardError) + + module Errors + class MissingAttribute < TransformerError + def initialize(attribute_name) + super("Missing attribute: '#{attribute_name}'") + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb new file mode 100644 index 00000000000..4e46eec17d6 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Transformers + module Yml + module V1 + # Takes a JSON schema validated dashboard hash and + # maps it to PrometheusMetric model attributes + class PrometheusMetrics + def initialize(dashboard_hash, project: nil, dashboard_path: nil) + @dashboard_hash = dashboard_hash.with_indifferent_access + @project = project + @dashboard_path = dashboard_path + + @dashboard_hash.default_proc = -> (h, k) { raise Transformers::Errors::MissingAttribute, k.to_s } + end + + def execute + prometheus_metrics = [] + + dashboard_hash[:panel_groups].each do |panel_group| + panel_group[:panels].each do |panel| + panel[:metrics].each do |metric| + prometheus_metrics << { + project: project, + title: panel[:title], + y_label: panel[:y_label], + query: metric[:query_range] || metric[:query], + unit: metric[:unit], + legend: metric[:label], + identifier: metric[:id], + group: Enums::PrometheusMetric.groups[:custom], + common: false, + dashboard_path: dashboard_path + }.compact + end + end + end + + prometheus_metrics + end + + private + + attr_reader :dashboard_hash, :project, :dashboard_path + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index 160ecfb85c9..6dcc73c0f6a 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -10,20 +10,23 @@ module Gitlab QUERY_PATTERN = '(?<query>\?[a-zA-Z0-9%.()+_=-]+(&[a-zA-Z0-9%.()+_=-]+)*)?' ANCHOR_PATTERN = '(?<anchor>\#[a-z0-9_-]+)?' - OPTIONAL_DASH_PATTERN = '(?:/-)?' + DASH_PATTERN = '(?:/-)' - # Matches urls for a metrics dashboard. This could be - # either the /metrics endpoint or the /metrics_dashboard - # endpoint. + # Matches urls for a metrics dashboard. + # This regex needs to match the old metrics URL, the new metrics URL, + # and the dashboard URL (inline_metrics_redactor_filter.rb + # uses this regex to match against the dashboard URL.) # - # EX - https://<host>/<namespace>/<project>/environments/<env_id>/metrics + # EX - Old URL: https://<host>/<namespace>/<project>/environments/<env_id>/metrics + # OR + # New URL: https://<host>/<namespace>/<project>/-/metrics?environment=<env_id> + # OR + # dashboard URL: https://<host>/<namespace>/<project>/environments/<env_id>/metrics_dashboard def metrics_regex strong_memoize(:metrics_regex) do regex_for_project_metrics( %r{ - /environments - /(?<environment>\d+) - /(metrics_dashboard|metrics) + ( #{environment_metrics_regex} ) | ( #{non_environment_metrics_regex} ) }x ) end @@ -36,6 +39,7 @@ module Gitlab strong_memoize(:grafana_regex) do regex_for_project_metrics( %r{ + #{DASH_PATTERN}? /grafana /metrics_dashboard }x @@ -44,16 +48,22 @@ module Gitlab end # Matches dashboard urls for a metric chart embed - # for cluster metrics + # for cluster metrics. + # This regex needs to match the dashboard URL as well, not just the trigger URL. + # The inline_metrics_redactor_filter.rb uses this regex to match against + # the dashboard URL. # # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB) + # dashboard URL - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/metrics_dashboard?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB) def clusters_regex strong_memoize(:clusters_regex) do regex_for_project_metrics( %r{ + #{DASH_PATTERN}? /clusters /(?<cluster_id>\d+) /? + ( (/metrics) | ( /metrics_dashboard\.json ) )? }x ) end @@ -67,10 +77,11 @@ module Gitlab strong_memoize(:alert_regex) do regex_for_project_metrics( %r{ + #{DASH_PATTERN}? /prometheus /alerts /(?<alert>\d+) - /metrics_dashboard + /metrics_dashboard(\.json)? }x ) end @@ -95,16 +106,37 @@ module Gitlab private + def environment_metrics_regex + %r{ + #{DASH_PATTERN}? + /environments + /(?<environment>\d+) + /(metrics_dashboard|metrics) + }x + end + + def non_environment_metrics_regex + %r{ + #{DASH_PATTERN} + /metrics + (?= # Lookahead to ensure there is an environment query param + \? + .* + environment=(?<environment>\d+) + .* + ) + }x + end + def regex_for_project_metrics(path_suffix_pattern) %r{ - (?<url> + ^(?<url> #{gitlab_host_pattern} #{project_path_pattern} - #{OPTIONAL_DASH_PATTERN} #{path_suffix_pattern} #{QUERY_PATTERN} #{ANCHOR_PATTERN} - ) + )$ }x end diff --git a/lib/gitlab/metrics/dashboard/validator.rb b/lib/gitlab/metrics/dashboard/validator.rb index 8edd9c397f9..1e8dc059968 100644 --- a/lib/gitlab/metrics/dashboard/validator.rb +++ b/lib/gitlab/metrics/dashboard/validator.rb @@ -4,24 +4,22 @@ module Gitlab module Metrics module Dashboard module Validator - DASHBOARD_SCHEMA_PATH = 'lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json'.freeze + DASHBOARD_SCHEMA_PATH = Rails.root.join(*%w[lib gitlab metrics dashboard validator schemas dashboard.json]).freeze class << self def validate(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) - errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project) - errors.empty? + errors(content, schema_path, dashboard_path: dashboard_path, project: project).empty? end def validate!(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) - errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project) + errors = errors(content, schema_path, dashboard_path: dashboard_path, project: project) errors.empty? || raise(errors.first) end - private - - def _validate(content, schema_path, dashboard_path: nil, project: nil) - client = Validator::Client.new(content, schema_path, dashboard_path: dashboard_path, project: project) - client.execute + def errors(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) + Validator::Client + .new(content, schema_path, dashboard_path: dashboard_path, project: project) + .execute end end end diff --git a/lib/gitlab/metrics/dashboard/validator/client.rb b/lib/gitlab/metrics/dashboard/validator/client.rb index c63415abcfc..588c677ca28 100644 --- a/lib/gitlab/metrics/dashboard/validator/client.rb +++ b/lib/gitlab/metrics/dashboard/validator/client.rb @@ -46,7 +46,7 @@ module Gitlab def validate_against_schema schemer.validate(content).map do |error| - Errors::SchemaValidationError.new(error) + ::Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError.new(error) end end end diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json index 011eef53e40..2ae9608036e 100644 --- a/lib/gitlab/metrics/dashboard/validator/schemas/panel.json +++ b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json @@ -4,7 +4,7 @@ "properties": { "type": { "type": "string", - "enum": ["area-chart", "anomaly-chart", "bar", "column", "stacked-column", "single-stat", "heatmap"], + "enum": ["area-chart", "line-chart", "anomaly-chart", "bar", "column", "stacked-column", "single-stat", "heatmap", "gauge"], "default": "area-chart" }, "title": { "type": "string" }, diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb index 054b4949dd6..36262b09b2d 100644 --- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -12,7 +12,11 @@ module Gitlab end def log_filename - File.join(Rails.root, 'log', 'sidekiq_exporter.log') + if settings['log_enabled'] + File.join(Rails.root, 'log', 'sidekiq_exporter.log') + else + File::NULL + end end private diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index ff3fffe7b95..66361529546 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -120,9 +120,6 @@ module Gitlab def self.instrument(type, mod, name) return unless ::Gitlab::Metrics.enabled? - name = name.to_sym - target = type == :instance ? mod : mod.singleton_class - if type == :instance target = mod method_name = "##{name}" @@ -154,6 +151,8 @@ module Gitlab '*args' end + method_visibility = method_visibility_for(target, name) + proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) if trans = Gitlab::Metrics::Instrumentation.transaction @@ -163,11 +162,23 @@ module Gitlab super end end + #{method_visibility} :#{name} EOF target.prepend(proxy_module) end + def self.method_visibility_for(mod, name) + if mod.private_method_defined?(name) + :private + elsif mod.protected_method_defined?(name) + :protected + else + :public + end + end + private_class_method :method_visibility_for + # Small layer of indirection to make it easier to stub out the current # transaction. def self.transaction diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index 2b5d1c710f6..3100450bc00 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -52,7 +52,7 @@ module Gitlab end def disabled_by_feature(options) - options.with_feature && !::Feature.enabled?(options.with_feature) + options.with_feature && !::Feature.enabled?(options.with_feature, type: :ops) end def build_metric!(type, name, options) diff --git a/lib/gitlab/metrics/samplers/action_cable_sampler.rb b/lib/gitlab/metrics/samplers/action_cable_sampler.rb new file mode 100644 index 00000000000..9f4979fa673 --- /dev/null +++ b/lib/gitlab/metrics/samplers/action_cable_sampler.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Samplers + class ActionCableSampler < BaseSampler + SAMPLING_INTERVAL_SECONDS = 5 + + def initialize(interval = SAMPLING_INTERVAL_SECONDS, action_cable: ::ActionCable.server) + super(interval) + @action_cable = action_cable + end + + def metrics + @metrics ||= { + active_connections: ::Gitlab::Metrics.gauge( + :action_cable_active_connections, 'Number of ActionCable WS clients currently connected' + ), + pool_min_size: ::Gitlab::Metrics.gauge( + :action_cable_pool_min_size, 'Minimum number of worker threads in ActionCable thread pool' + ), + pool_max_size: ::Gitlab::Metrics.gauge( + :action_cable_pool_max_size, 'Maximum number of worker threads in ActionCable thread pool' + ), + pool_current_size: ::Gitlab::Metrics.gauge( + :action_cable_pool_current_size, 'Current number of worker threads in ActionCable thread pool' + ), + pool_largest_size: ::Gitlab::Metrics.gauge( + :action_cable_pool_largest_size, 'Largest number of worker threads observed so far in ActionCable thread pool' + ), + pool_completed_tasks: ::Gitlab::Metrics.gauge( + :action_cable_pool_tasks_total, 'Total number of tasks executed in ActionCable thread pool' + ), + pool_pending_tasks: ::Gitlab::Metrics.gauge( + :action_cable_pool_pending_tasks, 'Number of tasks waiting to be executed in ActionCable thread pool' + ) + } + end + + def sample + pool = @action_cable.worker_pool.executor + labels = { + server_mode: server_mode + } + + metrics[:active_connections].set(labels, @action_cable.connections.size) + metrics[:pool_min_size].set(labels, pool.min_length) + metrics[:pool_max_size].set(labels, pool.max_length) + metrics[:pool_current_size].set(labels, pool.length) + metrics[:pool_largest_size].set(labels, pool.largest_length) + metrics[:pool_completed_tasks].set(labels, pool.completed_task_count) + metrics[:pool_pending_tasks].set(labels, pool.queue_length) + end + + private + + def server_mode + Gitlab::ActionCable::Config.in_app? ? 'in-app' : 'standalone' + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index ff3e7be567f..39a49187e45 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -21,7 +21,7 @@ module Gitlab def safe_sample sample rescue => e - Rails.logger.warn("#{self.class}: #{e}, stopping") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping") stop end diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index b5343d5e66a..d295beb59f1 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -42,7 +42,7 @@ module Gitlab def puma_stats Puma.stats rescue NoMethodError - Rails.logger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" nil end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index c0b671abd44..8e6ac7610f2 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -29,6 +29,8 @@ module Gitlab module Middleware class Multipart RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS' + JWT_PARAM_SUFFIX = '.gitlab-workhorse-upload' + JWT_PARAM_FIXED_KEY = 'upload' class Handler def initialize(env, message) @@ -57,7 +59,8 @@ module Gitlab yield ensure - @open_files.each(&:close) + @open_files.compact + .each(&:close) end # This function calls itself recursively @@ -122,15 +125,89 @@ module Gitlab def allowed_paths [ + Dir.tmpdir, ::FileUploader.root, - Gitlab.config.uploads.storage_path, - JobArtifactUploader.workhorse_upload_path, - LfsObjectUploader.workhorse_upload_path, + ::Gitlab.config.uploads.storage_path, + ::JobArtifactUploader.workhorse_upload_path, + ::LfsObjectUploader.workhorse_upload_path, File.join(Rails.root, 'public/uploads/tmp') ] + package_allowed_paths end end + # TODO this class is meant to replace Handler when the feature flag + # upload_middleware_jwt_params_handler is removed + class HandlerForJWTParams < Handler + def with_open_files + @rewritten_fields.keys.each do |field| + parsed_field = Rack::Utils.parse_nested_query(field) + raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1 + + key, value = parsed_field.first + if value.nil? # we have a top level param, eg. field = 'foo' and not 'foo[bar]' + raise "invalid field: #{field.inspect}" if field != key + + value = open_file(extract_upload_params_from(@request.params, with_prefix: key)) + @open_files << value + else + value = decorate_params_value(value, @request.params[key]) + end + + update_param(key, value) + end + + yield + ensure + @open_files.compact + .each(&:close) + end + + # This function calls itself recursively + def decorate_params_value(hash_path, value_hash) + unless hash_path.is_a?(Hash) && hash_path.count == 1 + raise "invalid path: #{hash_path.inspect}" + end + + path_key, path_value = hash_path.first + + unless value_hash.is_a?(Hash) && value_hash[path_key] + raise "invalid value hash: #{value_hash.inspect}" + end + + case path_value + when nil + value_hash[path_key] = open_file(extract_upload_params_from(value_hash[path_key])) + @open_files << value_hash[path_key] + value_hash + when Hash + decorate_params_value(path_value, value_hash[path_key]) + value_hash + else + raise "unexpected path value: #{path_value.inspect}" + end + end + + def open_file(params) + ::UploadedFile.from_params_without_field(params, allowed_paths) + end + + private + + def extract_upload_params_from(params, with_prefix: '') + param_key = "#{with_prefix}#{JWT_PARAM_SUFFIX}" + jwt_token = params[param_key] + raise "Empty JWT param: #{param_key}" if jwt_token.blank? + + payload = Gitlab::Workhorse.decode_jwt(jwt_token).first + raise "Invalid JWT payload: not a Hash" unless payload.is_a?(Hash) + + upload_params = payload.fetch(JWT_PARAM_FIXED_KEY, {}) + raise "Empty params for: #{param_key}" if upload_params.empty? + + upload_params + end + end + def initialize(app) @app = app end @@ -139,14 +216,24 @@ module Gitlab encoded_message = env.delete(RACK_ENV_KEY) return @app.call(env) if encoded_message.blank? - message = Gitlab::Workhorse.decode_jwt(encoded_message)[0] + message = ::Gitlab::Workhorse.decode_jwt(encoded_message)[0] - Handler.new(env, message).with_open_files do + handler_class.new(env, message).with_open_files do @app.call(env) end rescue UploadedFile::InvalidPathError => e [400, { 'Content-Type' => 'text/plain' }, e.message] end + + private + + def handler_class + if Feature.enabled?(:upload_middleware_jwt_params_handler) + ::Gitlab::Middleware::Multipart::HandlerForJWTParams + else + ::Gitlab::Middleware::Multipart::Handler + end + end end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 6573506e926..cfea4aaddf3 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -36,7 +36,7 @@ module Gitlab def call if disallowed_request? && Gitlab::Database.read_only? - Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.debug('GitLab ReadOnly: preventing possible non read-only operation') if json_request? return [403, { 'Content-Type' => APPLICATION_JSON }, [{ 'message' => ERROR_MESSAGE }.to_json]] diff --git a/lib/gitlab/middleware/same_site_cookies.rb b/lib/gitlab/middleware/same_site_cookies.rb index 45968035e79..37ccc5abb10 100644 --- a/lib/gitlab/middleware/same_site_cookies.rb +++ b/lib/gitlab/middleware/same_site_cookies.rb @@ -30,6 +30,7 @@ module Gitlab set_cookie = headers['Set-Cookie']&.strip return result if set_cookie.blank? || !ssl? + return result if same_site_none_incompatible?(env['HTTP_USER_AGENT']) cookies = set_cookie.split(COOKIE_SEPARATOR) @@ -39,11 +40,11 @@ module Gitlab # Chrome will drop SameSite=None cookies without the Secure # flag. If we remove this middleware, we may need to ensure # that all cookies set this flag. - if ssl? && !(cookie =~ /;\s*secure/i) + unless SECURE_REGEX.match?(cookie) cookie << '; Secure' end - unless cookie =~ /;\s*samesite=/i + unless SAME_SITE_REGEX.match?(cookie) cookie << '; SameSite=None' end end @@ -55,9 +56,93 @@ module Gitlab private + # Taken from https://www.chromium.org/updates/same-site/incompatible-clients + # We use RE2 instead of the browser gem for performance. + IOS_REGEX = RE2('\(iP.+; CPU .*OS (\d+)[_\d]*.*\) AppleWebKit\/') + MACOS_REGEX = RE2('\(Macintosh;.*Mac OS X (\d+)_(\d+)[_\d]*.*\) AppleWebKit\/') + SAFARI_REGEX = RE2('Version\/.* Safari\/') + CHROMIUM_REGEX = RE2('Chrom(e|ium)') + CHROMIUM_VERSION_REGEX = RE2('Chrom[^ \/]+\/(\d+)') + UC_BROWSER_REGEX = RE2('UCBrowser\/') + UC_BROWSER_VERSION_REGEX = RE2('UCBrowser\/(\d+)\.(\d+)\.(\d+)') + + SECURE_REGEX = RE2(';\s*secure', case_sensitive: false) + SAME_SITE_REGEX = RE2(';\s*samesite=', case_sensitive: false) + def ssl? Gitlab.config.gitlab.https end + + def same_site_none_incompatible?(user_agent) + return false if user_agent.blank? + + has_webkit_same_site_bug?(user_agent) || drops_unrecognized_same_site_cookies?(user_agent) + end + + def has_webkit_same_site_bug?(user_agent) + ios_version?(12, user_agent) || + (macos_version?(10, 14, user_agent) && safari?(user_agent)) + end + + def drops_unrecognized_same_site_cookies?(user_agent) + if uc_browser?(user_agent) + return !uc_browser_version_at_least?(12, 13, 2, user_agent) + end + + chromium_based?(user_agent) && chromium_version_between?(51, 66, user_agent) + end + + def ios_version?(major, user_agent) + m = IOS_REGEX.match(user_agent) + + return false if m.nil? + + m[1].to_i == major + end + + def macos_version?(major, minor, user_agent) + m = MACOS_REGEX.match(user_agent) + + return false if m.nil? + + m[1].to_i == major && m[2].to_i == minor + end + + def safari?(user_agent) + SAFARI_REGEX.match?(user_agent) + end + + def chromium_based?(user_agent) + CHROMIUM_REGEX.match?(user_agent) + end + + def chromium_version_between?(from_major, to_major, user_agent) + m = CHROMIUM_VERSION_REGEX.match(user_agent) + + return false if m.nil? + + version = m[1].to_i + version >= from_major && version <= to_major + end + + def uc_browser?(user_agent) + UC_BROWSER_REGEX.match?(user_agent) + end + + def uc_browser_version_at_least?(major, minor, build, user_agent) + m = UC_BROWSER_VERSION_REGEX.match(user_agent) + + return false if m.nil? + + major_version = m[1].to_i + minor_version = m[2].to_i + build_version = m[3].to_i + + return major_version > major if major_version != major + return minor_version > minor if minor_version != minor + + build_version >= build + end end end end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 41d80fe9aa6..e18b6d003e0 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -7,18 +7,19 @@ module Gitlab class ObjectHierarchy DEPTH_COLUMN = :depth - attr_reader :ancestors_base, :descendants_base, :model + attr_reader :ancestors_base, :descendants_base, :model, :options # ancestors_base - An instance of ActiveRecord::Relation for which to # get parent objects. # descendants_base - An instance of ActiveRecord::Relation for which to # get child objects. If omitted, ancestors_base is used. - def initialize(ancestors_base, descendants_base = ancestors_base) + def initialize(ancestors_base, descendants_base = ancestors_base, options: {}) raise ArgumentError.new("Model of ancestors_base does not match model of descendants_base") if ancestors_base.model != descendants_base.model @ancestors_base = ancestors_base @descendants_base = descendants_base @model = ancestors_base.model + @options = options end # Returns the set of descendants of a given relation, but excluding the given @@ -133,8 +134,8 @@ module Gitlab # Recursively get all the ancestors of the base set. parent_query = model - .from([objects_table, cte.table]) - .where(objects_table[:id].eq(cte.table[:parent_id])) + .from(from_tables(cte)) + .where(ancestor_conditions(cte)) .except(:order) if hierarchy_order @@ -148,7 +149,7 @@ module Gitlab ).where(cte.table[:tree_cycle].eq(false)) end - parent_query = parent_query.where(cte.table[:parent_id].not_eq(stop_id)) if stop_id + parent_query = parent_query.where(parent_id_column(cte).not_eq(stop_id)) if stop_id cte << parent_query cte @@ -166,8 +167,8 @@ module Gitlab # Recursively get all the descendants of the base set. descendants_query = model - .from([objects_table, cte.table]) - .where(objects_table[:parent_id].eq(cte.table[:id])) + .from(from_tables(cte)) + .where(descendant_conditions(cte)) .except(:order) if with_depth @@ -190,6 +191,22 @@ module Gitlab model.arel_table end + def parent_id_column(cte) + cte.table[:parent_id] + end + + def from_tables(cte) + [objects_table, cte.table] + end + + def ancestor_conditions(cte) + objects_table[:id].eq(cte.table[:parent_id]) + end + + def descendant_conditions(cte) + objects_table[:parent_id].eq(cte.table[:id]) + end + def read_only(relation) # relations using a CTE are not safe to use with update_all as it will # throw away the CTE, hence we mark them as read-only. diff --git a/lib/gitlab/pages/settings.rb b/lib/gitlab/pages/settings.rb index e3dbeee7b13..8650a80a85e 100644 --- a/lib/gitlab/pages/settings.rb +++ b/lib/gitlab/pages/settings.rb @@ -6,12 +6,8 @@ module Gitlab DiskAccessDenied = Class.new(StandardError) def path - if ::Gitlab::Runtime.web_server? && ENV['GITLAB_PAGES_DENY_DISK_ACCESS'] == '1' - begin - raise DiskAccessDenied - rescue DiskAccessDenied => ex - ::Gitlab::ErrorTracking.track_exception(ex) - end + if ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite? + raise DiskAccessDenied end super diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb index a70dc826f97..8ae0ec5a78a 100644 --- a/lib/gitlab/pages_transfer.rb +++ b/lib/gitlab/pages_transfer.rb @@ -1,7 +1,26 @@ # frozen_string_literal: true +# To make a call happen in a new Sidekiq job, add `.async` before the call. For +# instance: +# +# PagesTransfer.new.async.move_namespace(...) +# module Gitlab class PagesTransfer < ProjectTransfer + class Async + METHODS = %w[move_namespace move_project rename_project rename_namespace].freeze + + METHODS.each do |meth| + define_method meth do |*args| + PagesTransferWorker.perform_async(meth, args) + end + end + end + + def async + @async ||= Async.new + end + def root_dir Gitlab.config.pages.path end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb index ff90a009b2e..23e380b3cf1 100644 --- a/lib/gitlab/project_authorizations.rb +++ b/lib/gitlab/project_authorizations.rb @@ -99,6 +99,7 @@ module Gitlab .and(members[:source_type].eq('Namespace')) .and(members[:requested_at].eq(nil)) .and(members[:user_id].eq(user.id)) + .and(members[:access_level].gt(Gitlab::Access::MINIMAL_ACCESS)) Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) end @@ -119,6 +120,7 @@ module Gitlab .and(members[:source_type].eq('Namespace')) .and(members[:requested_at].eq(nil)) .and(members[:user_id].eq(user.id)) + .and(members[:access_level].gt(Gitlab::Access::MINIMAL_ACCESS)) Arel::Nodes::InnerJoin.new(members, Arel::Nodes::On.new(cond)) end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index f8141278e48..ded8d4ade3f 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -4,11 +4,11 @@ module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref - def initialize(current_user, project, query, repository_ref = nil) - @current_user = current_user + def initialize(current_user, query, project:, repository_ref: nil, filters: {}) @project = project @repository_ref = repository_ref.presence - @query = query + + super(current_user, query, [project], filters: filters) end def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil) diff --git a/lib/gitlab/prometheus/internal.rb b/lib/gitlab/prometheus/internal.rb index d59352119ba..c2f4035821e 100644 --- a/lib/gitlab/prometheus/internal.rb +++ b/lib/gitlab/prometheus/internal.rb @@ -25,6 +25,10 @@ module Gitlab end end + def self.server_address + uri&.strip&.sub(/^http[s]?:\/\//, '') + end + def self.listen_address Gitlab.config.prometheus.listen_address.to_s if Gitlab.config.prometheus rescue Settingslogic::MissingSetting diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 56e1154a672..965349ad711 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -42,6 +42,15 @@ module Gitlab response_body == HEALTHY_RESPONSE end + def ready? + response = get(ready_url, {}) + + # From Prometheus docs: This endpoint returns 200 when Prometheus is ready to serve traffic (i.e. respond to queries). + response.code == 200 + rescue => e + raise PrometheusClient::UnexpectedResponseError, "#{e.message}" + end + def proxy(type, args) path = api_path(type) get(path, args) @@ -103,7 +112,11 @@ module Gitlab end def health_url - [api_url, '-/healthy'].join('/') + "#{api_url}/-/healthy" + end + + def ready_url + "#{api_url}/-/ready" end private diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index cd07122ffd9..dd7a27ead01 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -9,6 +9,62 @@ module Gitlab # extractor = Gitlab::QuickActions::Extractor.new([:open, :assign, :labels]) # ``` class Extractor + CODE_REGEX = %r{ + (?<code> + # Code blocks: + # ``` + # Anything, including `/cmd arg` which are ignored by this filter + # ``` + + ^``` + .+? + \n```$ + ) + }mix.freeze + + INLINE_CODE_REGEX = %r{ + (?<inline_code> + # Inline code on separate rows: + # ` + # Anything, including `/cmd arg` which are ignored by this filter + # ` + + ^.*`\n* + .+? + \n*`$ + ) + }mix.freeze + + HTML_BLOCK_REGEX = %r{ + (?<html> + # HTML block: + # <tag> + # Anything, including `/cmd arg` which are ignored by this filter + # </tag> + + ^<[^>]+?>\n + .+? + \n<\/[^>]+?>$ + ) + }mix.freeze + + QUOTE_BLOCK_REGEX = %r{ + (?<html> + # Quote block: + # >>> + # Anything, including `/cmd arg` which are ignored by this filter + # >>> + + ^>>> + .+? + \n>>>$ + ) + }mix.freeze + + EXCLUSION_REGEX = %r{ + #{CODE_REGEX} | #{INLINE_CODE_REGEX} | #{HTML_BLOCK_REGEX} | #{QUOTE_BLOCK_REGEX} + }mix.freeze + attr_reader :command_definitions def initialize(command_definitions) @@ -35,9 +91,7 @@ module Gitlab def extract_commands(content, only: nil) return [content, []] unless content - content, commands = perform_regex(content, only: only) - - perform_substitutions(content, commands) + perform_regex(content, only: only) end # Encloses quick action commands into code span markdown @@ -55,13 +109,19 @@ module Gitlab private def perform_regex(content, only: nil, redact: false) - commands = [] - content = content.dup + names = command_names(limit_to_commands: only).map(&:to_s) + sub_names = substitution_names.map(&:to_s) + commands = [] + content = content.dup content.delete!("\r") - names = command_names(limit_to_commands: only).map(&:to_s) - content.gsub!(commands_regex(names: names)) do - command, output = process_commands($~, redact) + content.gsub!(commands_regex(names: names, sub_names: sub_names)) do + command, output = if $~[:substitution] + process_substitutions($~) + else + process_commands($~, redact) + end + commands << command output end @@ -86,6 +146,21 @@ module Gitlab [command, output] end + def process_substitutions(matched_text) + output = matched_text[0] + command = [] + + if matched_text[:substitution] + cmd = matched_text[:substitution].downcase + command = [cmd, matched_text[:arg]].reject(&:blank?) + + substitution = substitution_definitions.find { |definition| definition.all_names.include?(cmd.to_sym) } + output = substitution.perform_substitution(self, output) if substitution + end + + [command, output] + end + # Builds a regular expression to match known commands. # First match group captures the command name and # second match group captures its arguments. @@ -93,51 +168,9 @@ module Gitlab # It looks something like: # # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ - def commands_regex(names:) + def commands_regex(names:, sub_names:) @commands_regex[names] ||= %r{ - (?<code> - # Code blocks: - # ``` - # Anything, including `/cmd arg` which are ignored by this filter - # ``` - - ^``` - .+? - \n```$ - ) - | - (?<inline_code> - # Inline code on separate rows: - # ` - # Anything, including `/cmd arg` which are ignored by this filter - # ` - - ^.*`\n* - .+? - \n*`$ - ) - | - (?<html> - # HTML block: - # <tag> - # Anything, including `/cmd arg` which are ignored by this filter - # </tag> - - ^<[^>]+?>\n - .+? - \n<\/[^>]+?>$ - ) - | - (?<html> - # Quote block: - # >>> - # Anything, including `/cmd arg` which are ignored by this filter - # >>> - - ^>>> - .+? - \n>>>$ - ) + #{EXCLUSION_REGEX} | (?: # Command not in a blockquote, blockcode, or HTML tag: @@ -151,32 +184,19 @@ module Gitlab )? (?:\s*\n|$) ) - }mix - end - - def perform_substitutions(content, commands) - return unless content - - substitution_definitions = self.command_definitions.select do |definition| - definition.is_a?(Gitlab::QuickActions::SubstitutionDefinition) - end - - substitution_definitions.each do |substitution| - regex = commands_regex(names: substitution.all_names) - content = content.gsub(regex) do |text| - if $~[:cmd] - command = [substitution.name.to_s] - command << $~[:arg] if $~[:arg].present? - commands << command - - substitution.perform_substitution(self, text) - else - text - end - end - end + | + (?: + # Substitution not in a blockquote, blockcode, or HTML tag: - [content, commands] + ^\/ + (?<substitution>#{Regexp.new(Regexp.union(sub_names).source, Regexp::IGNORECASE)}) + (?: + [ ] + (?<arg>[^\n]*) + )? + (?:\s*\n|$) + ) + }mix end def command_names(limit_to_commands:) @@ -190,6 +210,17 @@ module Gitlab command.all_names end.compact end + + def substitution_names + substitution_definitions.flat_map { |command| command.all_names } + .compact + end + + def substitution_definitions + @substition_definitions ||= command_definitions.select do |command| + command.is_a?(Gitlab::QuickActions::SubstitutionDefinition) + end + end end end end diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index aff3ed53734..6607c73a5c3 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -91,6 +91,7 @@ module Gitlab params '%"milestone"' types Issue, MergeRequest condition do + quick_action_target.supports_milestone? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) && find_milestones(project, state: 'active').any? end @@ -113,6 +114,7 @@ module Gitlab condition do quick_action_target.persisted? && quick_action_target.milestone_id? && + quick_action_target.supports_milestone? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end command :remove_milestone do diff --git a/lib/gitlab/quick_actions/relate_actions.rb b/lib/gitlab/quick_actions/relate_actions.rb new file mode 100644 index 00000000000..95f71214667 --- /dev/null +++ b/lib/gitlab/quick_actions/relate_actions.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + module RelateActions + extend ActiveSupport::Concern + include ::Gitlab::QuickActions::Dsl + + included do + desc _('Mark this issue as related to another issue') + explanation do |related_reference| + _('Marks this issue as related to %{issue_ref}.') % { issue_ref: related_reference } + end + execution_message do |related_reference| + _('Marked this issue as related to %{issue_ref}.') % { issue_ref: related_reference } + end + params '#issue' + types Issue + condition do + quick_action_target.persisted? && + current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) + end + command :relate do |related_param| + IssueLinks::CreateService.new(quick_action_target, current_user, { issuable_references: [related_param] }).execute + end + end + end + end +end diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb index cd4d202e8d0..24b4e3c62b3 100644 --- a/lib/gitlab/quick_actions/substitution_definition.rb +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -9,10 +9,6 @@ module Gitlab true end - def match(content) - content.match %r{^/#{all_names.join('|')}(?![\S]) ?(.*)$} - end - def perform_substitution(context, content) return unless content diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 8ab53700932..2848c9f0b59 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -9,7 +9,7 @@ module Gitlab SESSION_NAMESPACE = 'session:gitlab' USER_SESSIONS_NAMESPACE = 'session:user:gitlab' USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab' - IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab' + IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382' REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE' diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb index 5fdfa5e75ed..c2fa2e1330a 100644 --- a/lib/gitlab/reference_counter.rb +++ b/lib/gitlab/reference_counter.rb @@ -51,10 +51,8 @@ module Gitlab redis_cmd do |redis| current_value = redis.decr(key) if current_value < 0 - # rubocop:disable Gitlab/RailsLogger - Rails.logger.warn("Reference counter for #{gl_repository} decreased" \ + Gitlab::AppLogger.warn("Reference counter for #{gl_repository} decreased" \ " when its value was less than 1. Resetting the counter.") - # rubocop:enable Gitlab/RailsLogger redis.del(key) end end @@ -87,7 +85,7 @@ module Gitlab true rescue => e - Rails.logger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") false end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 2d625737e05..8e23ac6aca5 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -46,6 +46,21 @@ module Gitlab maven_app_name_regex end + def pypi_version_regex + # See the official regex: https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L159 + + @pypi_version_regex ||= %r{ + \A(?: + v? + (?:([0-9]+)!)? (?# epoch) + ([0-9]+(?:\.[0-9]+)*) (?# release segment) + ([-_\.]?((a|b|c|rc|alpha|beta|pre|preview))[-_\.]?([0-9]+)?)? (?# pre-release) + ((?:-([0-9]+))|(?:[-_\.]?(post|rev|r)[-_\.]?([0-9]+)?))? (?# post release) + ([-_\.]?(dev)[-_\.]?([0-9]+)?)? (?# dev release) + (?:\+([a-z0-9]+(?:[-_\.][a-z0-9]+)*))? (?# local version) + )\z}xi.freeze + end + def unbounded_semver_regex # See the official regex: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string @@ -84,6 +99,10 @@ module Gitlab \b (?# word boundary) /ix.freeze end + + def generic_package_version_regex + /\A\d+\.\d+\.\d+\z/ + end end extend self @@ -102,7 +121,11 @@ module Gitlab end def group_name_regex - @group_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9ff}_()\. ]*\z/.freeze + @group_name_regex ||= /\A#{group_name_regex_chars}\z/.freeze + end + + def group_name_regex_chars + @group_name_regex_chars ||= /[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9ff}_()\. ]*/.freeze end def group_name_regex_message @@ -269,7 +292,14 @@ module Gitlab def base64_regex @base64_regex ||= /(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?/.freeze end + + def feature_flag_regex + /\A[a-z]([-_a-z0-9]*[a-z0-9])?\z/ + end + + def feature_flag_regex_message + "can contain only lowercase letters, digits, '_' and '-'. " \ + "Must start with a letter, and cannot end with '-' or '_'" + end end end - -Gitlab::Regex.prepend_if_ee('EE::Gitlab::Regex') diff --git a/lib/gitlab/relative_positioning.rb b/lib/gitlab/relative_positioning.rb new file mode 100644 index 00000000000..b5a923f0824 --- /dev/null +++ b/lib/gitlab/relative_positioning.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + STEPS = 10 + IDEAL_DISTANCE = 2**(STEPS - 1) + 1 + + MIN_POSITION = Gitlab::Database::MIN_INT_VALUE + START_POSITION = 0 + MAX_POSITION = Gitlab::Database::MAX_INT_VALUE + + MAX_GAP = IDEAL_DISTANCE * 2 + MIN_GAP = 2 + + NoSpaceLeft = Class.new(StandardError) + end +end diff --git a/lib/gitlab/relative_positioning/gap.rb b/lib/gitlab/relative_positioning/gap.rb new file mode 100644 index 00000000000..ab894141a60 --- /dev/null +++ b/lib/gitlab/relative_positioning/gap.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# +module Gitlab + module RelativePositioning + class Gap + attr_reader :start_pos, :end_pos + + def initialize(start_pos, end_pos) + @start_pos, @end_pos = start_pos, end_pos + end + + def ==(other) + other.is_a?(self.class) && other.start_pos == start_pos && other.end_pos == end_pos + end + + def delta + ((start_pos - end_pos) / 2.0).abs.ceil.clamp(0, RelativePositioning::IDEAL_DISTANCE) + end + end + end +end diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb new file mode 100644 index 00000000000..cd03a347355 --- /dev/null +++ b/lib/gitlab/relative_positioning/item_context.rb @@ -0,0 +1,259 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + # This class is API private - it should not be explicitly instantiated + # outside of tests + # rubocop: disable CodeReuse/ActiveRecord + class ItemContext + include Gitlab::Utils::StrongMemoize + + attr_reader :object, :model_class, :range + attr_accessor :ignoring + + def initialize(object, range, ignoring: nil) + @object = object + @range = range + @model_class = object.class + @ignoring = ignoring + end + + def ==(other) + other.is_a?(self.class) && other.object == object && other.range == range && other.ignoring == ignoring + end + + def positioned? + relative_position.present? + end + + def min_relative_position + strong_memoize(:min_relative_position) { calculate_relative_position('MIN') } + end + + def max_relative_position + strong_memoize(:max_relative_position) { calculate_relative_position('MAX') } + end + + def prev_relative_position + calculate_relative_position('MAX') { |r| nextify(r, false) } if object.relative_position + end + + def next_relative_position + calculate_relative_position('MIN') { |r| nextify(r) } if object.relative_position + end + + def nextify(relation, gt = true) + if gt + relation.where("relative_position > ?", relative_position) + else + relation.where("relative_position < ?", relative_position) + end + end + + def relative_siblings(relation = scoped_items) + object.exclude_self(relation) + end + + # Handles the possibility that the position is already occupied by a sibling + def place_at_position(position, lhs) + current_occupant = relative_siblings.find_by(relative_position: position) + + if current_occupant.present? + Mover.new(position, range).move(object, lhs.object, current_occupant) + else + object.relative_position = position + end + end + + def lhs_neighbour + scoped_items + .where('relative_position < ?', relative_position) + .reorder(relative_position: :desc) + .first + .then { |x| neighbour(x) } + end + + def rhs_neighbour + scoped_items + .where('relative_position > ?', relative_position) + .reorder(relative_position: :asc) + .first + .then { |x| neighbour(x) } + end + + def neighbour(item) + return unless item.present? + + self.class.new(item, range, ignoring: ignoring) + end + + def scoped_items + r = model_class.relative_positioning_query_base(object) + r = object.exclude_self(r, excluded: ignoring) if ignoring.present? + r + end + + def calculate_relative_position(calculation) + # When calculating across projects, this is much more efficient than + # MAX(relative_position) without the GROUP BY, due to index usage: + # https://gitlab.com/gitlab-org/gitlab-foss/issues/54276#note_119340977 + relation = scoped_items + .order(Gitlab::Database.nulls_last_order('position', 'DESC')) + .group(grouping_column) + .limit(1) + + relation = yield relation if block_given? + + relation + .pluck(grouping_column, Arel.sql("#{calculation}(relative_position) AS position")) + .first&.last + end + + def grouping_column + model_class.relative_positioning_parent_column + end + + def max_sibling + sib = relative_siblings + .order(Gitlab::Database.nulls_last_order('relative_position', 'DESC')) + .first + + neighbour(sib) + end + + def min_sibling + sib = relative_siblings + .order(Gitlab::Database.nulls_last_order('relative_position', 'ASC')) + .first + + neighbour(sib) + end + + def shift_left + move_sequence_before(true) + object.reset + end + + def shift_right + move_sequence_after(true) + object.reset + end + + def create_space_left + find_next_gap_before.tap { |gap| move_sequence_before(false, next_gap: gap) } + end + + def create_space_right + find_next_gap_after.tap { |gap| move_sequence_after(false, next_gap: gap) } + end + + def find_next_gap_before + items_with_next_pos = scoped_items + .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position DESC) AS next_pos') + .where('relative_position <= ?', relative_position) + .order(relative_position: :desc) + + find_next_gap(items_with_next_pos, range.first) + end + + def find_next_gap_after + items_with_next_pos = scoped_items + .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position ASC) AS next_pos') + .where('relative_position >= ?', relative_position) + .order(:relative_position) + + find_next_gap(items_with_next_pos, range.last) + end + + def find_next_gap(items_with_next_pos, default_end) + gap = model_class + .from(items_with_next_pos, :items) + .where('next_pos IS NULL OR ABS(pos::bigint - next_pos::bigint) >= ?', MIN_GAP) + .limit(1) + .pluck(:pos, :next_pos) + .first + + return if gap.nil? || gap.first == default_end + + Gap.new(gap.first, gap.second || default_end) + end + + def relative_position + object.relative_position + end + + private + + # Moves the sequence before the current item to the middle of the next gap + # For example, we have + # + # 5 . . . . . 11 12 13 14 [15] 16 . 17 + # ----------- + # + # This moves the sequence [11 12 13 14] to [8 9 10 11], so we have: + # + # 5 . . 8 9 10 11 . . . [15] 16 . 17 + # --------- + # + # Creating a gap to the left of the current item. We can understand this as + # dividing the 5 spaces between 5 and 11 into two smaller gaps of 2 and 3. + # + # If `include_self` is true, the current item will also be moved, creating a + # gap to the right of the current item: + # + # 5 . . 8 9 10 11 [14] . . . 16 . 17 + # -------------- + # + # As an optimization, the gap can be precalculated and passed to this method. + # + # @api private + # @raises NoSpaceLeft if the sequence cannot be moved + def move_sequence_before(include_self = false, next_gap: find_next_gap_before) + raise NoSpaceLeft unless next_gap.present? + + delta = next_gap.delta + + move_sequence(next_gap.start_pos, relative_position, -delta, include_self) + end + + # Moves the sequence after the current item to the middle of the next gap + # For example, we have: + # + # 8 . 10 [11] 12 13 14 15 . . . . . 21 + # ----------- + # + # This moves the sequence [12 13 14 15] to [15 16 17 18], so we have: + # + # 8 . 10 [11] . . . 15 16 17 18 . . 21 + # ----------- + # + # Creating a gap to the right of the current item. We can understand this as + # dividing the 5 spaces between 15 and 21 into two smaller gaps of 3 and 2. + # + # If `include_self` is true, the current item will also be moved, creating a + # gap to the left of the current item: + # + # 8 . 10 . . . [14] 15 16 17 18 . . 21 + # ---------------- + # + # As an optimization, the gap can be precalculated and passed to this method. + # + # @api private + # @raises NoSpaceLeft if the sequence cannot be moved + def move_sequence_after(include_self = false, next_gap: find_next_gap_after) + raise NoSpaceLeft unless next_gap.present? + + delta = next_gap.delta + + move_sequence(relative_position, next_gap.start_pos, delta, include_self) + end + + def move_sequence(start_pos, end_pos, delta, include_self = false) + relation = include_self ? scoped_items : relative_siblings + + object.update_relative_siblings(relation, (start_pos..end_pos), delta) + end + end + # rubocop: enable CodeReuse/ActiveRecord + end +end diff --git a/lib/gitlab/relative_positioning/mover.rb b/lib/gitlab/relative_positioning/mover.rb new file mode 100644 index 00000000000..9d891bfbe3b --- /dev/null +++ b/lib/gitlab/relative_positioning/mover.rb @@ -0,0 +1,155 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + class Mover + attr_reader :range, :start_position + + def initialize(start, range) + @range = range + @start_position = start + end + + def move_to_end(object) + focus = context(object, ignoring: object) + max_pos = focus.max_relative_position + + move_to_range_end(focus, max_pos) + end + + def move_to_start(object) + focus = context(object, ignoring: object) + min_pos = focus.min_relative_position + + move_to_range_start(focus, min_pos) + end + + def move(object, first, last) + raise ArgumentError, 'object is required' unless object + + lhs = context(first, ignoring: object) + rhs = context(last, ignoring: object) + focus = context(object) + range = RelativePositioning.range(lhs, rhs) + + if range.cover?(focus) + # Moving a object already within a range is a no-op + elsif range.open_on_left? + move_to_range_start(focus, range.rhs.relative_position) + elsif range.open_on_right? + move_to_range_end(focus, range.lhs.relative_position) + else + pos_left, pos_right = create_space_between(range) + desired_position = position_between(pos_left, pos_right) + focus.place_at_position(desired_position, range.lhs) + end + end + + def context(object, ignoring: nil) + return unless object + + ItemContext.new(object, range, ignoring: ignoring) + end + + private + + def gap_too_small?(pos_a, pos_b) + return false unless pos_a && pos_b + + (pos_a - pos_b).abs < MIN_GAP + end + + def move_to_range_end(context, max_pos) + range_end = range.last + 1 + + new_pos = if max_pos.nil? + start_position + elsif gap_too_small?(max_pos, range_end) + max = context.max_sibling + max.ignoring = context.object + max.shift_left + position_between(max.relative_position, range_end) + else + position_between(max_pos, range_end) + end + + context.object.relative_position = new_pos + end + + def move_to_range_start(context, min_pos) + range_end = range.first - 1 + + new_pos = if min_pos.nil? + start_position + elsif gap_too_small?(min_pos, range_end) + sib = context.min_sibling + sib.ignoring = context.object + sib.shift_right + position_between(sib.relative_position, range_end) + else + position_between(min_pos, range_end) + end + + context.object.relative_position = new_pos + end + + def create_space_between(range) + pos_left = range.lhs&.relative_position + pos_right = range.rhs&.relative_position + + return [pos_left, pos_right] unless gap_too_small?(pos_left, pos_right) + + gap = range.rhs.create_space_left + [pos_left - gap.delta, pos_right] + rescue NoSpaceLeft + gap = range.lhs.create_space_right + [pos_left, pos_right + gap.delta] + end + + # This method takes two integer values (positions) and + # calculates the position between them. The range is huge as + # the maximum integer value is 2147483647. + # + # We avoid open ranges by clamping the range to [MIN_POSITION, MAX_POSITION]. + # + # Then we handle one of three cases: + # - If the gap is too small, we raise NoSpaceLeft + # - If the gap is larger than MAX_GAP, we place the new position at most + # IDEAL_DISTANCE from the edge of the gap. + # - otherwise we place the new position at the midpoint. + # + # The new position will always satisfy: pos_before <= midpoint <= pos_after + # + # As a precondition, the gap between pos_before and pos_after MUST be >= 2. + # If the gap is too small, NoSpaceLeft is raised. + # + # @raises NoSpaceLeft + def position_between(pos_before, pos_after) + pos_before ||= range.first + pos_after ||= range.last + + pos_before, pos_after = [pos_before, pos_after].sort + + gap_width = pos_after - pos_before + + if gap_too_small?(pos_before, pos_after) + raise NoSpaceLeft + elsif gap_width > MAX_GAP + if pos_before <= range.first + pos_after - IDEAL_DISTANCE + elsif pos_after >= range.last + pos_before + IDEAL_DISTANCE + else + midpoint(pos_before, pos_after) + end + else + midpoint(pos_before, pos_after) + end + end + + def midpoint(lower_bound, upper_bound) + ((lower_bound + upper_bound) / 2.0).ceil.clamp(lower_bound, upper_bound - 1) + end + end + end +end diff --git a/lib/gitlab/relative_positioning/range.rb b/lib/gitlab/relative_positioning/range.rb new file mode 100644 index 00000000000..174d5ef4b35 --- /dev/null +++ b/lib/gitlab/relative_positioning/range.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + IllegalRange = Class.new(ArgumentError) + + class Range + attr_reader :lhs, :rhs + + def open_on_left? + lhs.nil? + end + + def open_on_right? + rhs.nil? + end + + def cover?(item_context) + return false unless item_context + return false unless item_context.positioned? + return true if item_context.object == lhs&.object + return true if item_context.object == rhs&.object + + pos = item_context.relative_position + + return lhs.relative_position < pos if open_on_right? + return pos < rhs.relative_position if open_on_left? + + lhs.relative_position < pos && pos < rhs.relative_position + end + + def ==(other) + other.is_a?(RelativePositioning::Range) && lhs == other.lhs && rhs == other.rhs + end + end + + def self.range(lhs, rhs) + if lhs && rhs + ClosedRange.new(lhs, rhs) + elsif lhs + StartingFrom.new(lhs) + elsif rhs + EndingAt.new(rhs) + else + raise IllegalRange, 'One of rhs or lhs must be provided' unless lhs && rhs + end + end + + class ClosedRange < RelativePositioning::Range + def initialize(lhs, rhs) + @lhs, @rhs = lhs, rhs + raise IllegalRange, 'Either lhs or rhs is missing' unless lhs && rhs + raise IllegalRange, 'lhs and rhs cannot be the same object' if lhs == rhs + end + end + + class StartingFrom < RelativePositioning::Range + include Gitlab::Utils::StrongMemoize + + def initialize(lhs) + @lhs = lhs + raise IllegalRange, 'lhs is required' unless lhs + end + + def rhs + strong_memoize(:rhs) { lhs.rhs_neighbour } + end + end + + class EndingAt < RelativePositioning::Range + include Gitlab::Utils::StrongMemoize + + def initialize(rhs) + @rhs = rhs + raise IllegalRange, 'rhs is required' unless rhs + end + + def lhs + strong_memoize(:lhs) { rhs.lhs_neighbour } + end + end + end +end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index f6a5c6ed754..eb7c9bccf96 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -218,7 +218,7 @@ module Gitlab def expire_method_caches(methods) methods.each do |name| unless cached_methods.include?(name.to_sym) - Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error "Requested to expire non-existent method '#{name}' for Repository" next end diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index 1e2d86b7ad2..69c1688767c 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -22,7 +22,7 @@ module Gitlab with do |redis| redis.multi do - redis.del(full_key) + redis.unlink(full_key) # Splitting into groups of 1000 prevents us from creating a too-long # Redis command diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb index dd1482da40d..541d505e735 100644 --- a/lib/gitlab/request_profiler.rb +++ b/lib/gitlab/request_profiler.rb @@ -11,7 +11,7 @@ module Gitlab Profile.new(File.basename(path)) end.select(&:valid?) end - module_function :all + module_function :all # rubocop: disable Style/AccessModifierDeclarations def find(name) file_path = File.join(PROFILES_DIR, name) @@ -19,18 +19,18 @@ module Gitlab Profile.new(name) end - module_function :find + module_function :find # rubocop: disable Style/AccessModifierDeclarations def profile_token Rails.cache.fetch('profile-token') do Devise.friendly_token end end - module_function :profile_token + module_function :profile_token # rubocop: disable Style/AccessModifierDeclarations def remove_all_profiles FileUtils.rm_rf(PROFILES_DIR) end - module_function :remove_all_profiles + module_function :remove_all_profiles # rubocop: disable Style/AccessModifierDeclarations end end diff --git a/lib/gitlab/robots_txt.rb b/lib/gitlab/robots_txt.rb new file mode 100644 index 00000000000..2f395548770 --- /dev/null +++ b/lib/gitlab/robots_txt.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module RobotsTxt + def self.disallowed?(path) + parsed_robots_txt.disallowed?(path) + end + + def self.parsed_robots_txt + @parsed_robots_txt ||= Parser.new(robots_txt) + end + + def self.robots_txt + File.read(Rails.root.join('public', 'robots.txt')) + end + end +end diff --git a/lib/gitlab/robots_txt/parser.rb b/lib/gitlab/robots_txt/parser.rb new file mode 100644 index 00000000000..b9a3837e468 --- /dev/null +++ b/lib/gitlab/robots_txt/parser.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module RobotsTxt + class Parser + attr_reader :disallow_rules + + def initialize(content) + @raw_content = content + + @disallow_rules = parse_raw_content! + end + + def disallowed?(path) + disallow_rules.any? { |rule| path =~ rule } + end + + private + + # This parser is very basic as it only knows about `Disallow:` lines, + # and simply ignores all other lines. + # + # Order of predecence, 'Allow:`, etc are ignored for now. + def parse_raw_content! + @raw_content.each_line.map do |line| + if line.start_with?('Disallow:') + value = line.sub('Disallow:', '').strip + value = Regexp.escape(value).gsub('\*', '.*') + Regexp.new("^#{value}") + else + nil + end + end.compact + end + end + end +end diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index 7e22bf4d7df..78c517c49d8 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -48,7 +48,7 @@ module Gitlab attr_reader :logger - def initialize(logger: Rails.logger) # rubocop:disable Gitlab/RailsLogger + def initialize(logger: Gitlab::AppLogger) @logger = logger end diff --git a/lib/gitlab/search/recent_issues.rb b/lib/gitlab/search/recent_issues.rb new file mode 100644 index 00000000000..413218da64d --- /dev/null +++ b/lib/gitlab/search/recent_issues.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Search + class RecentIssues < RecentItems + private + + def type + Issue + end + + def finder + IssuesFinder + end + end + end +end diff --git a/lib/gitlab/search/recent_items.rb b/lib/gitlab/search/recent_items.rb new file mode 100644 index 00000000000..40d96ded275 --- /dev/null +++ b/lib/gitlab/search/recent_items.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Search + # This is an abstract class used for storing/searching recently viewed + # items. The #type and #finder methods are the only ones needed to be + # implemented by classes inheriting from this. + class RecentItems + ITEMS_LIMIT = 100 + EXPIRES_AFTER = 7.days + + def initialize(user:, items_limit: ITEMS_LIMIT, expires_after: EXPIRES_AFTER) + @user = user + @items_limit = items_limit + @expires_after = expires_after + end + + def log_view(item) + with_redis do |redis| + redis.zadd(key, Time.now.to_f, item.id) + redis.expire(key, @expires_after) + + # There is a race condition here where we could end up removing an + # item from 2 places concurrently but this is fine since worst case + # scenario we remove an extra item from the end of the list. + if redis.zcard(key) > @items_limit + redis.zremrangebyrank(key, 0, 0) # Remove least recent + end + end + end + + def search(term) + ids = with_redis do |redis| + redis.zrevrange(key, 0, @items_limit - 1) + end.map(&:to_i) + + finder.new(@user, search: term, in: 'title').execute.reorder(nil).id_in_ordered(ids) # rubocop: disable CodeReuse/ActiveRecord + end + + private + + def with_redis(&blk) + Gitlab::Redis::SharedState.with(&blk) # rubocop: disable CodeReuse/ActiveRecord + end + + def key + "recent_items:#{type.name.downcase}:#{@user.id}" + end + + def type + raise NotImplementedError + end + + def finder + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/search/recent_merge_requests.rb b/lib/gitlab/search/recent_merge_requests.rb new file mode 100644 index 00000000000..7b14e3b33e5 --- /dev/null +++ b/lib/gitlab/search/recent_merge_requests.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Search + class RecentMergeRequests < RecentItems + private + + def type + MergeRequest + end + + def finder + MergeRequestsFinder + end + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 3d5f64ce05b..06d8dca2f70 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -7,7 +7,7 @@ module Gitlab DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 20 - attr_reader :current_user, :query + attr_reader :current_user, :query, :filters # Limit search results by passed projects # It allows us to search only for projects user has access to @@ -19,11 +19,12 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, limit_projects, query, default_project_filter: false) + def initialize(current_user, query, limit_projects = nil, default_project_filter: false, filters: {}) @current_user = current_user - @limit_projects = limit_projects || Project.all @query = query + @limit_projects = limit_projects || Project.all @default_project_filter = default_project_filter + @filters = filters end def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, without_count: true, preload_method: nil) @@ -186,10 +187,12 @@ module Gitlab params[:sort] = 'updated_desc' if query =~ /#(\d+)\z/ - params[:iids] = $1 + params[:iids] = Regexp.last_match(1) else params[:search] = query end + + params[:state] = filters[:state] if filters.key?(:state) end end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 64a30fbe16c..4df6a50c8dd 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -28,6 +28,25 @@ module Gitlab end # rubocop:enable Rails/Output + module Workhorse + extend Gitlab::SetupHelper + class << self + def configuration_toml(dir, _) + config = { redis: { URL: redis_url } } + + TomlRB.dump(config) + end + + def redis_url + Gitlab::Redis::SharedState.url + end + + def get_config_path(dir) + File.join(dir, 'config.toml') + end + end + end + module Gitaly extend Gitlab::SetupHelper class << self diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 24f49f6b943..3419989c110 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -116,7 +116,7 @@ module Gitlab true rescue => e - Rails.logger.warn("Repository does not exist: #{e} at: #{disk_path}.git") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.warn("Repository does not exist: #{e} at: #{disk_path}.git") Gitlab::ErrorTracking.track_exception(e, path: disk_path, storage: storage) false diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index cbd89b7629f..6f1d2ad23c1 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -105,7 +105,7 @@ module Gitlab query.each_line .map { |line| line.strip } .join("\n") - .gsub(PREFIX_NEWLINE) { "\n#{$1} " } + .gsub(PREFIX_NEWLINE) { "\n#{Regexp.last_match(1)} " } end end end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index b8a4eedd620..e1a87a77f04 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -230,8 +230,10 @@ module Gitlab end def rss_increase_by_jobs - Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord - rss_increase_by_job(job) + Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do + Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord + rss_increase_by_job(job) + end end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb index bb0c18735bb..6417ec20960 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb @@ -5,7 +5,7 @@ module Gitlab module DuplicateJobs class Client def call(worker_class, job, queue, _redis_pool, &block) - DuplicateJob.new(job, queue).schedule(&block) + ::Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(job, queue).schedule(&block) end end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 0dc53c61e84..5efd1b34d32 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -71,7 +71,7 @@ module Gitlab end def droppable? - idempotent? && ::Feature.disabled?("disable_#{queue_name}_deduplication") + idempotent? && ::Feature.disabled?("disable_#{queue_name}_deduplication", type: :ops) end def scheduled_at diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb index a08310a58ff..6fdef4c354e 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb @@ -7,7 +7,8 @@ module Gitlab UnknownStrategyError = Class.new(StandardError) STRATEGIES = { - until_executing: UntilExecuting + until_executing: UntilExecuting, + none: None }.freeze def self.for(name) diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb new file mode 100644 index 00000000000..cd101cd16b6 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + module Strategies + # This strategy will never deduplicate a job + class None + def initialize(_duplicate_job) + end + + def schedule(_job) + yield + end + + def perform(_job) + yield + end + end + end + end + end +end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 1d253ca90f3..41ec19f0da8 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -4,11 +4,8 @@ module Gitlab class SnippetSearchResults < SearchResults include SnippetsHelper - attr_reader :current_user - def initialize(current_user, query) - @current_user = current_user - @query = query + super(current_user, query) end def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil) diff --git a/lib/gitlab/sourcegraph.rb b/lib/gitlab/sourcegraph.rb index ec404ebd309..231d5aea129 100644 --- a/lib/gitlab/sourcegraph.rb +++ b/lib/gitlab/sourcegraph.rb @@ -12,8 +12,8 @@ module Gitlab !feature.off? end - def feature_enabled?(thing = nil) - feature.enabled?(thing) + def feature_enabled?(actor = nil) + feature.enabled?(actor) end private diff --git a/lib/gitlab/sql/except.rb b/lib/gitlab/sql/except.rb new file mode 100644 index 00000000000..82cbfa8d4ab --- /dev/null +++ b/lib/gitlab/sql/except.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module SQL + # Class for building SQL EXCEPT statements. + # + # ORDER BYs are dropped from the relations as the final sort order is not + # guaranteed any way. + # + # Example usage: + # + # except = Gitlab::SQL::Except.new([user.projects, user.personal_projects]) + # sql = except.to_sql + # + # Project.where("id IN (#{sql})") + class Except < SetOperator + def self.operator_keyword + 'EXCEPT' + end + end + end +end diff --git a/lib/gitlab/sql/intersect.rb b/lib/gitlab/sql/intersect.rb new file mode 100644 index 00000000000..c661db3d4c5 --- /dev/null +++ b/lib/gitlab/sql/intersect.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module SQL + # Class for building SQL INTERSECT statements. + # + # ORDER BYs are dropped from the relations as the final sort order is not + # guaranteed any way. + # + # Example usage: + # + # hierarchies = [group1.self_and_hierarchy, group2.self_and_hierarchy] + # intersect = Gitlab::SQL::Intersect.new(hierarchies) + # sql = intersect.to_sql + # + # Project.where("id IN (#{sql})") + class Intersect < SetOperator + def self.operator_keyword + 'INTERSECT' + end + end + end +end diff --git a/lib/gitlab/sql/set_operator.rb b/lib/gitlab/sql/set_operator.rb new file mode 100644 index 00000000000..d58a1415493 --- /dev/null +++ b/lib/gitlab/sql/set_operator.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module SQL + # Class for building SQL set operator statements (UNION, INTERSECT, and + # EXCEPT). + # + # ORDER BYs are dropped from the relations as the final sort order is not + # guaranteed any way. + # + # Example usage: + # + # union = Gitlab::SQL::Union.new([user.personal_projects, user.projects]) + # sql = union.to_sql + # + # Project.where("id IN (#{sql})") + class SetOperator + def initialize(relations, remove_duplicates: true) + @relations = relations + @remove_duplicates = remove_duplicates + end + + def self.operator_keyword + raise NotImplementedError + end + + def to_sql + # Some relations may include placeholders for prepared statements, these + # aren't incremented properly when joining relations together this way. + # By using "unprepared_statements" we remove the usage of placeholders + # (thus fixing this problem), at a slight performance cost. + fragments = ActiveRecord::Base.connection.unprepared_statement do + relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) + end + + if fragments.any? + "(" + fragments.join(")\n#{operator_keyword_fragment}\n(") + ")" + else + 'NULL' + end + end + + # UNION [ALL] | INTERSECT [ALL] | EXCEPT [ALL] + def operator_keyword_fragment + remove_duplicates ? self.class.operator_keyword : "#{self.class.operator_keyword} ALL" + end + + private + + attr_reader :relations, :remove_duplicates + end + end +end diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index b15f2ca385a..7fb3487a5e5 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -13,30 +13,9 @@ module Gitlab # sql = union.to_sql # # Project.where("id IN (#{sql})") - class Union - def initialize(relations, remove_duplicates: true) - @relations = relations - @remove_duplicates = remove_duplicates - end - - def to_sql - # Some relations may include placeholders for prepared statements, these - # aren't incremented properly when joining relations together this way. - # By using "unprepared_statements" we remove the usage of placeholders - # (thus fixing this problem), at a slight performance cost. - fragments = ActiveRecord::Base.connection.unprepared_statement do - @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) - end - - if fragments.any? - "(" + fragments.join(")\n#{union_keyword}\n(") + ")" - else - 'NULL' - end - end - - def union_keyword - @remove_duplicates ? 'UNION' : 'UNION ALL' + class Union < SetOperator + def self.operator_keyword + 'UNION' end end end diff --git a/lib/gitlab/static_site_editor/config.rb b/lib/gitlab/static_site_editor/config.rb deleted file mode 100644 index d335a434335..00000000000 --- a/lib/gitlab/static_site_editor/config.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module StaticSiteEditor - class Config - SUPPORTED_EXTENSIONS = %w[.md].freeze - - def initialize(repository, ref, file_path, return_url) - @repository = repository - @ref = ref - @file_path = file_path - @return_url = return_url - @commit_id = repository.commit(ref)&.id if ref - end - - def payload - { - branch: ref, - path: file_path, - commit_id: commit_id, - project_id: project.id, - project: project.path, - namespace: project.namespace.full_path, - return_url: sanitize_url(return_url), - is_supported_content: supported_content?.to_s, - base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path) - } - end - - private - - attr_reader :repository, :ref, :file_path, :return_url, :commit_id - - delegate :project, to: :repository - - def supported_content? - master_branch? && extension_supported? && file_exists? - end - - def master_branch? - ref == 'master' - end - - def extension_supported? - return true if file_path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project) - - SUPPORTED_EXTENSIONS.any? { |ext| file_path.end_with?(ext) } - end - - def file_exists? - commit_id.present? && !repository.blob_at(commit_id, file_path).nil? - end - - def full_path - "#{ref}/#{file_path}" - end - - def sanitize_url(url) - url if Gitlab::UrlSanitizer.valid_web?(url) - end - end - end -end diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb new file mode 100644 index 00000000000..f647c85e1c8 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + def data + { + static_site_generator: 'middleman' + } + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb new file mode 100644 index 00000000000..f3dce74a32f --- /dev/null +++ b/lib/gitlab/static_site_editor/config/generated_config.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class GeneratedConfig + SUPPORTED_EXTENSIONS = %w[.md].freeze + + def initialize(repository, ref, path, return_url) + @repository = repository + @ref = ref + @path = path + @return_url = return_url + end + + def data + merge_requests_illustration_path = ActionController::Base.helpers.image_path('illustrations/merge_requests.svg') + { + branch: ref, + path: path, + commit_id: commit_id, + project_id: project.id, + project: project.path, + namespace: project.namespace.full_path, + return_url: sanitize_url(return_url), + is_supported_content: supported_content?.to_s, + base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path), + merge_requests_illustration_path: merge_requests_illustration_path + } + end + + private + + attr_reader :repository, :ref, :path, :return_url + + delegate :project, to: :repository + + def commit_id + repository.commit(ref)&.id if ref + end + + def supported_content? + master_branch? && extension_supported? && file_exists? + end + + def master_branch? + ref == 'master' + end + + def extension_supported? + return true if path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project) + + SUPPORTED_EXTENSIONS.any? { |ext| path.end_with?(ext) } + end + + def file_exists? + commit_id.present? && !repository.blob_at(commit_id, path).nil? + end + + def full_path + "#{ref}/#{path}" + end + + def sanitize_url(url) + url if Gitlab::UrlSanitizer.valid_web?(url) + end + end + end + end +end diff --git a/lib/gitlab/submodule_links.rb b/lib/gitlab/submodule_links.rb index b0ee0877f30..38b10c5892d 100644 --- a/lib/gitlab/submodule_links.rb +++ b/lib/gitlab/submodule_links.rb @@ -4,14 +4,18 @@ module Gitlab class SubmoduleLinks include Gitlab::Utils::StrongMemoize + Urls = Struct.new(:web, :tree, :compare) + def initialize(repository) @repository = repository @cache_store = {} end - def for(submodule, sha) + def for(submodule, sha, diff_file = nil) submodule_url = submodule_url_for(sha, submodule.path) - SubmoduleHelper.submodule_links_for_url(submodule.id, submodule_url, repository) + old_submodule_id = old_submodule_id(submodule_url, diff_file) + urls = SubmoduleHelper.submodule_links_for_url(submodule.id, submodule_url, repository, old_submodule_id) + Urls.new(*urls) if urls.any? end private @@ -29,5 +33,15 @@ module Gitlab urls = submodule_urls_for(sha) urls && urls[path] end + + def old_submodule_id(submodule_url, diff_file) + return unless diff_file&.old_blob && diff_file&.old_content_sha + + # if the submodule url has changed from old_sha to sha, a compare link does not make sense + # + old_submodule_url = submodule_url_for(diff_file.old_content_sha, diff_file.old_blob.path) + + diff_file.old_blob.id if old_submodule_url == submodule_url + end end end diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 73187d8dea8..c702c6f1add 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -24,6 +24,8 @@ module Gitlab # Returns "yes" the user chose to continue # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue def ask_to_continue + return if Gitlab::Utils.to_boolean(ENV['GITLAB_ASSUME_YES']) + answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) raise Gitlab::TaskAbortedByUserError unless answer == "yes" end diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 3669d652fd3..9b39d386674 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -5,10 +5,10 @@ module Gitlab module Template module Finders class GlobalTemplateFinder < BaseTemplateFinder - def initialize(base_dir, extension, categories = {}, exclusions: []) + def initialize(base_dir, extension, categories = {}, excluded_patterns: []) @categories = categories @extension = extension - @exclusions = exclusions + @excluded_patterns = excluded_patterns super(base_dir) end @@ -43,7 +43,7 @@ module Gitlab private def excluded?(file_name) - @exclusions.include?(file_name) + @excluded_patterns.any? { |pattern| pattern.match?(file_name) } end def select_directory(file_name) diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 26a9dc9fd38..bb1e9db55fa 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -3,12 +3,16 @@ module Gitlab module Template class GitlabCiYmlTemplate < BaseTemplate + BASE_EXCLUDED_PATTERNS = [%r{\.latest$}].freeze + def content explanation = "# This file is a template, and might need editing before it works on your project." [explanation, super].join("\n") end class << self + include Gitlab::Utils::StrongMemoize + def extension '.gitlab-ci.yml' end @@ -22,10 +26,14 @@ module Gitlab } end - def disabled_templates - %w[ - Verify/Browser-Performance - ] + def excluded_patterns + strong_memoize(:excluded_patterns) do + BASE_EXCLUDED_PATTERNS + additional_excluded_patterns + end + end + + def additional_excluded_patterns + [%r{Verify/Browser-Performance}] end def base_dir @@ -34,7 +42,7 @@ module Gitlab def finder(project = nil) Gitlab::Template::Finders::GlobalTemplateFinder.new( - self.base_dir, self.extension, self.categories, exclusions: self.disabled_templates + self.base_dir, self.extension, self.categories, excluded_patterns: self.excluded_patterns ) end end diff --git a/lib/gitlab/testing/robots_blocker_middleware.rb b/lib/gitlab/testing/robots_blocker_middleware.rb new file mode 100644 index 00000000000..034492122df --- /dev/null +++ b/lib/gitlab/testing/robots_blocker_middleware.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# rubocop:disable Style/ClassVars +module Gitlab + module Testing + class RobotsBlockerMiddleware + @@block_requests = Concurrent::AtomicBoolean.new(false) + + # Block requests according to robots.txt. + # Any new requests disallowed by robots.txt will return an HTTP 503 status. + def self.block_requests! + @@block_requests.value = true + end + + # Allows the server to accept requests again. + def self.allow_requests! + @@block_requests.value = false + end + + def initialize(app) + @app = app + end + + def call(env) + request = Rack::Request.new(env) + + if block_requests? && Gitlab::RobotsTxt.disallowed?(request.path_info) + block_request(env) + else + @app.call(env) + end + end + + private + + def block_requests? + @@block_requests.true? + end + + def block_request(env) + [503, {}, []] + end + end + end +end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 37688d6e0e7..02d354ec43a 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -47,8 +47,7 @@ module Gitlab cookie_domain: Gitlab::CurrentSettings.snowplow_cookie_domain, app_id: Gitlab::CurrentSettings.snowplow_app_id, form_tracking: additional_features, - link_click_tracking: additional_features, - iglu_registry_url: Gitlab::CurrentSettings.snowplow_iglu_registry_url + link_click_tracking: additional_features }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym } end @@ -56,12 +55,19 @@ module Gitlab def snowplow @snowplow ||= SnowplowTracker::Tracker.new( - SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'https'), + emitter, SnowplowTracker::Subject.new, SNOWPLOW_NAMESPACE, Gitlab::CurrentSettings.snowplow_app_id ) end + + def emitter + SnowplowTracker::AsyncEmitter.new( + Gitlab::CurrentSettings.snowplow_collector_hostname, + protocol: 'https' + ) + end end end end diff --git a/lib/gitlab/tracking/incident_management.rb b/lib/gitlab/tracking/incident_management.rb index 5fa819b3696..df2a0658b36 100644 --- a/lib/gitlab/tracking/incident_management.rb +++ b/lib/gitlab/tracking/incident_management.rb @@ -35,6 +35,9 @@ module Gitlab }, pagerduty_active: { name: 'pagerduty_webhook' + }, + auto_close_incident: { + name: 'auto_close_incident' } }.with_indifferent_access.freeze end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 70efe86143e..89605ce5d07 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -38,6 +38,8 @@ module Gitlab .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, last_28_days_time_period)) .merge(analytics_unique_visits_data) .merge(compliance_unique_visits_data) + .merge(search_unique_visits_data) + .merge(redis_hll_counters) end end @@ -110,6 +112,8 @@ module Gitlab clusters_applications_jupyter: count(::Clusters::Applications::Jupyter.available), clusters_applications_cilium: count(::Clusters::Applications::Cilium.available), clusters_management_project: count(::Clusters::Cluster.with_management_project), + kubernetes_agents: count(::Clusters::Agent), + kubernetes_agents_with_token: distinct_count(::Clusters::AgentToken, :agent_id), in_review_folder: count(::Environment.in_review_folder), grafana_integrated_projects: count(GrafanaIntegration.enabled), groups: count(Group), @@ -129,6 +133,8 @@ module Gitlab lfs_objects: count(LfsObject), milestone_lists: count(List.milestone), milestones: count(Milestone), + projects_with_packages: distinct_count(::Packages::Package, :project_id), + packages: count(::Packages::Package), pages_domains: count(PagesDomain), pool_repositories: count(PoolRepository), projects: count(Project), @@ -160,7 +166,8 @@ module Gitlab user_preferences_usage, ingress_modsecurity_usage, container_expiration_policies_usage, - service_desk_counts + service_desk_counts, + snowplow_event_counts ).tap do |data| data[:snippets] = data[:personal_snippets] + data[:project_snippets] end @@ -168,6 +175,19 @@ module Gitlab end # rubocop: enable Metrics/AbcSize + def snowplow_event_counts(time_period: {}) + return {} unless report_snowplow_events? + + { + promoted_issues: count( + self_monitoring_project + .product_analytics_events + .by_category_and_action('epics', 'promote') + .where(time_period) + ) + } + end + def system_usage_data_monthly { counts_monthly: { @@ -176,9 +196,12 @@ module Gitlab successful_deployments: deployment_count(Deployment.success.where(last_28_days_time_period)), failed_deployments: deployment_count(Deployment.failed.where(last_28_days_time_period)), # rubocop: enable UsageData/LargeTable: + packages: count(::Packages::Package.where(last_28_days_time_period)), personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)), project_snippets: count(ProjectSnippet.where(last_28_days_time_period)) - }.tap do |data| + }.merge( + snowplow_event_counts(time_period: last_28_days_time_period(column: :collector_tstamp)) + ).tap do |data| data[:snippets] = data[:personal_snippets] + data[:project_snippets] end } @@ -240,7 +263,8 @@ module Gitlab Gitlab::UsageDataCounters::ProductivityAnalyticsCounter, Gitlab::UsageDataCounters::SourceCodeCounter, Gitlab::UsageDataCounters::MergeRequestCounter, - Gitlab::UsageDataCounters::DesignsCounter + Gitlab::UsageDataCounters::DesignsCounter, + Gitlab::UsageDataCounters::KubernetesAgentCounter ] end @@ -264,6 +288,9 @@ module Gitlab database: { adapter: alt_usage_data { Gitlab::Database.adapter_name }, version: alt_usage_data { Gitlab::Database.version } + }, + mail: { + smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] } } } end @@ -371,7 +398,9 @@ module Gitlab # so we can just check for subdomains of atlassian.net results = { projects_jira_server_active: 0, - projects_jira_cloud_active: 0 + projects_jira_cloud_active: 0, + projects_jira_dvcs_cloud_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled), + projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) } # rubocop: disable UsageData/LargeTable: @@ -399,7 +428,7 @@ module Gitlab { jira_imports_total_imported_count: count(finished_jira_imports), jira_imports_projects_count: distinct_count(finished_jira_imports, :project_id), - jira_imports_total_imported_issues_count: alt_usage_data { JiraImportState.finished_imports_count } + jira_imports_total_imported_issues_count: sum(JiraImportState.finished, :imported_issues_count) } # rubocop: enable UsageData/LargeTable end @@ -409,7 +438,7 @@ module Gitlab def successful_deployments_with_cluster(scope) scope .joins(cluster: :deployments) - .merge(Clusters::Cluster.enabled) + .merge(::Clusters::Cluster.enabled) .merge(Deployment.success) end # rubocop: enable UsageData/LargeTable @@ -419,16 +448,17 @@ module Gitlab {} # augmented in EE end - # rubocop: disable CodeReuse/ActiveRecord def merge_requests_users(time_period) - distinct_count( - Event.where(target_type: Event::TARGET_TYPES[:merge_request].to_s).where(time_period), - :author_id, - start: user_minimum_id, - finish: user_maximum_id - ) + counter = Gitlab::UsageDataCounters::TrackUniqueEvents + + redis_usage_data do + counter.count_unique_events( + event_action: Gitlab::UsageDataCounters::TrackUniqueEvents::MERGE_REQUEST_ACTION, + date_from: time_period[:created_at].first, + date_to: time_period[:created_at].last + ) + end end - # rubocop: enable CodeReuse/ActiveRecord def installation_type if Rails.env.production? @@ -438,8 +468,8 @@ module Gitlab end end - def last_28_days_time_period - { created_at: 28.days.ago..Time.current } + def last_28_days_time_period(column: :created_at) + { column => 28.days.ago..Time.current } end # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv @@ -510,7 +540,22 @@ module Gitlab events: distinct_count(::Event.where(time_period), :author_id), groups: distinct_count(::GroupMember.where(time_period), :user_id), users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), - omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' } + omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, + projects_imported: { + gitlab_project: projects_imported_count('gitlab_project', time_period), + gitlab: projects_imported_count('gitlab', time_period), + github: projects_imported_count('github', time_period), + bitbucket: projects_imported_count('bitbucket', time_period), + bitbucket_server: projects_imported_count('bitbucket_server', time_period), + gitea: projects_imported_count('gitea', time_period), + git: projects_imported_count('git', time_period), + manifest: projects_imported_count('manifest', time_period) + }, + issues_imported: { + jira: distinct_count(::JiraImportState.where(time_period), :user_id), + fogbugz: projects_imported_count('fogbugz', time_period), + phabricator: projects_imported_count('phabricator', time_period) + } } end # rubocop: enable CodeReuse/ActiveRecord @@ -527,9 +572,13 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def usage_activity_by_stage_package(time_period) - {} + { + projects_with_packages: distinct_count(::Project.with_packages.where(time_period), :creator_id) + } end + # rubocop: enable CodeReuse/ActiveRecord # Omitted because no user, creator or author associated: `boards`, `labels`, `milestones`, `uploads` # Omitted because too expensive: `epics_deepest_relationship_level` @@ -542,7 +591,10 @@ module Gitlab projects: distinct_count(::Project.where(time_period), :creator_id), todos: distinct_count(::Todo.where(time_period), :author_id), service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period), - service_desk_issues: count(::Issue.service_desk.where(time_period)) + service_desk_issues: count(::Issue.service_desk.where(time_period)), + projects_jira_active: distinct_count(::Project.with_active_jira_services.where(time_period), :creator_id), + projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_jira_services.with_jira_dvcs_cloud.where(time_period), :creator_id), + projects_jira_dvcs_server_active: distinct_count(::Project.with_active_jira_services.with_jira_dvcs_server.where(time_period), :creator_id) } end # rubocop: enable CodeReuse/ActiveRecord @@ -583,9 +635,13 @@ module Gitlab {} end + def redis_hll_counters + { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } + end + def analytics_unique_visits_data - results = ::Gitlab::Analytics::UniqueVisits.analytics_ids.each_with_object({}) do |target_id, hash| - hash[target_id] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target_id) } + results = ::Gitlab::Analytics::UniqueVisits.analytics_events.each_with_object({}) do |target, hash| + hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } end results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics) } results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, start_date: 4.weeks.ago.to_date, end_date: Date.current) } @@ -594,8 +650,8 @@ module Gitlab end def compliance_unique_visits_data - results = ::Gitlab::Analytics::UniqueVisits.compliance_ids.each_with_object({}) do |target_id, hash| - hash[target_id] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target_id) } + results = ::Gitlab::Analytics::UniqueVisits.compliance_events.each_with_object({}) do |target, hash| + hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } end results['compliance_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance) } results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, start_date: 4.weeks.ago.to_date, end_date: Date.current) } @@ -603,41 +659,53 @@ module Gitlab { compliance_unique_visits: results } end + def search_unique_visits_data + events = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('search') + results = events.each_with_object({}) do |event, hash| + hash[event] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) } + end + + results['search_unique_visits_for_any_target_weekly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 7.days.ago.to_date, end_date: Date.current) } + results['search_unique_visits_for_any_target_monthly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: 4.weeks.ago.to_date, end_date: Date.current) } + + { search_unique_visits: results } + end + def action_monthly_active_users(time_period) - counter = Gitlab::UsageDataCounters::TrackUniqueActions + date_range = { date_from: time_period[:created_at].first, date_to: time_period[:created_at].last } - project_count = redis_usage_data do - counter.count_unique( - event_action: Gitlab::UsageDataCounters::TrackUniqueActions::PUSH_ACTION, - date_from: time_period[:created_at].first, - date_to: time_period[:created_at].last - ) - end + event_monthly_active_users(date_range) + .merge!(ide_monthly_active_users(date_range)) + end - design_count = redis_usage_data do - counter.count_unique( - event_action: Gitlab::UsageDataCounters::TrackUniqueActions::DESIGN_ACTION, - date_from: time_period[:created_at].first, - date_to: time_period[:created_at].last - ) - end + private - wiki_count = redis_usage_data do - counter.count_unique( - event_action: Gitlab::UsageDataCounters::TrackUniqueActions::WIKI_ACTION, - date_from: time_period[:created_at].first, - date_to: time_period[:created_at].last - ) + def event_monthly_active_users(date_range) + data = { + action_monthly_active_users_project_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::PUSH_ACTION, + action_monthly_active_users_design_management: Gitlab::UsageDataCounters::TrackUniqueEvents::DESIGN_ACTION, + action_monthly_active_users_wiki_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::WIKI_ACTION + } + + data.each do |key, event| + data[key] = redis_usage_data { Gitlab::UsageDataCounters::TrackUniqueEvents.count_unique_events(event_action: event, **date_range) } end + end + + def ide_monthly_active_users(date_range) + counter = Gitlab::UsageDataCounters::EditorUniqueCounter { - action_monthly_active_users_project_repo: project_count, - action_monthly_active_users_design_management: design_count, - action_monthly_active_users_wiki_repo: wiki_count + action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(date_range) }, + action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(date_range) }, + action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(date_range) }, + action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(date_range) } } end - private + def report_snowplow_events? + self_monitoring_project && Feature.enabled?(:product_analytics, self_monitoring_project) + end def distinct_count_service_desk_enabled_projects(time_period) project_creator_id_start = user_minimum_id @@ -716,6 +784,22 @@ module Gitlab end end + def project_minimum_id + strong_memoize(:project_minimum_id) do + ::Project.minimum(:id) + end + end + + def project_maximum_id + strong_memoize(:project_maximum_id) do + ::Project.maximum(:id) + end + end + + def self_monitoring_project + Gitlab::CurrentSettings.self_monitoring_project + end + def clear_memoized clear_memoization(:issue_minimum_id) clear_memoization(:issue_maximum_id) @@ -726,14 +810,14 @@ module Gitlab clear_memoization(:deployment_maximum_id) clear_memoization(:approval_merge_request_rule_minimum_id) clear_memoization(:approval_merge_request_rule_maximum_id) + clear_memoization(:project_minimum_id) + clear_memoization(:project_maximum_id) end # rubocop: disable CodeReuse/ActiveRecord - # rubocop: disable UsageData/DistinctCountByLargeForeignKey def cluster_applications_user_distinct_count(applications, time_period) distinct_count(applications.where(time_period).available.joins(:cluster), 'clusters.user_id') end - # rubocop: enable UsageData/DistinctCountByLargeForeignKey def clusters_user_distinct_count(clusters, time_period) distinct_count(clusters.where(time_period), :user_id) @@ -755,6 +839,10 @@ module Gitlab def deployment_count(relation) count relation, start: deployment_minimum_id, finish: deployment_maximum_id end + + def projects_imported_count(from, time_period) + distinct_count(::Project.imported_from(from).where(time_period), :creator_id) # rubocop: disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/usage_data/topology.rb b/lib/gitlab/usage_data/topology.rb index edc4dc75750..7f7854c3eb1 100644 --- a/lib/gitlab/usage_data/topology.rb +++ b/lib/gitlab/usage_data/topology.rb @@ -40,9 +40,10 @@ module Gitlab private def topology_fetch_all_data - with_prometheus_client(fallback: {}) do |client| + with_prometheus_client(fallback: {}, verify: false) do |client| { application_requests_per_hour: topology_app_requests_per_hour(client), + query_apdex_weekly_average: topology_query_apdex_weekly_average(client), nodes: topology_node_data(client) }.compact end @@ -63,6 +64,16 @@ module Gitlab (result['value'].last.to_f * 1.hour).to_i end + def topology_query_apdex_weekly_average(client) + result = query_safely('gitlab_usage_ping:sql_duration_apdex:ratio_rate5m', 'query_apdex', fallback: nil) do |query| + client.query(aggregate_one_week(query)).first + end + + return unless result + + result['value'].last.to_f + end + def topology_node_data(client) # node-level data by_instance_mem = topology_node_memory(client) diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb new file mode 100644 index 00000000000..b68d50ee419 --- /dev/null +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module EditorUniqueCounter + EDIT_BY_SNIPPET_EDITOR = 'g_edit_by_snippet_ide' + EDIT_BY_SFE = 'g_edit_by_sfe' + EDIT_BY_WEB_IDE = 'g_edit_by_web_ide' + EDIT_CATEGORY = 'ide_edit' + + class << self + def track_web_ide_edit_action(author:, time: Time.zone.now) + track_unique_action(EDIT_BY_WEB_IDE, author, time) + end + + def count_web_ide_edit_actions(date_from:, date_to:) + count_unique(EDIT_BY_WEB_IDE, date_from, date_to) + end + + def track_sfe_edit_action(author:, time: Time.zone.now) + track_unique_action(EDIT_BY_SFE, author, time) + end + + def count_sfe_edit_actions(date_from:, date_to:) + count_unique(EDIT_BY_SFE, date_from, date_to) + end + + def track_snippet_editor_edit_action(author:, time: Time.zone.now) + track_unique_action(EDIT_BY_SNIPPET_EDITOR, author, time) + end + + def count_snippet_editor_edit_actions(date_from:, date_to:) + count_unique(EDIT_BY_SNIPPET_EDITOR, date_from, date_to) + end + + def count_edit_using_editor(date_from:, date_to:) + events = Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(EDIT_CATEGORY) + count_unique(events, date_from, date_to) + end + + private + + def track_unique_action(action, author, time) + return unless Feature.enabled?(:track_editor_edit_actions, default_enabled: true) + return unless author + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time) + end + + def count_unique(actions, date_from, date_to) + Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: actions, start_date: date_from, end_date: date_to) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index c9c39225068..53bf6daea4c 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -31,7 +31,11 @@ module Gitlab # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event(user_id, 'g_compliance_dashboard') # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current) class << self + include Gitlab::Utils::UsageData + def track_event(entity_id, event_name, time = Time.zone.now) + return unless Gitlab::CurrentSettings.usage_ping_enabled? + event = event_for(event_name) raise UnknownEvent.new("Unknown event #{event_name}") unless event.present? @@ -50,15 +54,51 @@ module Gitlab keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date) - Gitlab::Redis::HLL.count(keys: keys) + redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } + end + + def categories + @categories ||= known_events.map { |event| event[:category] }.uniq end + # @param category [String] the category name + # @return [Array<String>] list of event names for given category def events_for_category(category) - known_events.select { |event| event[:category] == category }.map { |event| event[:name] } + known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] } + end + + def unique_events_data + categories.each_with_object({}) do |category, category_results| + events_names = events_for_category(category) + + event_results = events_names.each_with_object({}) do |event, hash| + hash[event] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) + end + + if eligible_for_totals?(events_names) + event_results["#{category}_total_unique_counts_weekly"] = unique_events(event_names: events_names, start_date: 7.days.ago.to_date, end_date: Date.current) + event_results["#{category}_total_unique_counts_monthly"] = unique_events(event_names: events_names, start_date: 4.weeks.ago.to_date, end_date: Date.current) + end + + category_results["#{category}"] = event_results + end + end + + def known_event?(event_name) + event_for(event_name).present? end private + # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level + # and if there are more than 1 event + def eligible_for_totals?(events_names) + return false if events_names.size <= 1 + + events = events_for(events_names) + events_in_same_slot?(events) && events_in_same_category?(events) && events_same_aggregation?(events) + end + def keys_for_aggregation(aggregation, events:, start_date:, end_date:) if aggregation.to_sym == :daily daily_redis_keys(events: events, start_date: start_date, end_date: end_date) @@ -76,8 +116,11 @@ module Gitlab end def events_in_same_slot?(events) + # if we check one event then redis_slot is only one to check + return true if events.size == 1 + slot = events.first[:redis_slot] - events.all? { |event| event[:redis_slot] == slot } + events.all? { |event| event[:redis_slot].present? && event[:redis_slot] == slot } end def events_in_same_category?(events) @@ -91,7 +134,7 @@ module Gitlab end def expiry(event) - return event[:expiry] if event[:expiry].present? + return event[:expiry].days if event[:expiry].present? event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH end diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb new file mode 100644 index 00000000000..fc1b5a59487 --- /dev/null +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module IssueActivityUniqueCounter + ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed' + ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed' + ISSUE_ASSIGNEE_CHANGED = 'g_project_management_issue_assignee_changed' + ISSUE_MADE_CONFIDENTIAL = 'g_project_management_issue_made_confidential' + ISSUE_MADE_VISIBLE = 'g_project_management_issue_made_visible' + ISSUE_CATEGORY = 'issues_edit' + + class << self + def track_issue_title_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_TITLE_CHANGED, author, time) + end + + def track_issue_description_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_DESCRIPTION_CHANGED, author, time) + end + + def track_issue_assignee_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_ASSIGNEE_CHANGED, author, time) + end + + def track_issue_made_confidential_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_MADE_CONFIDENTIAL, author, time) + end + + def track_issue_made_visible_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_MADE_VISIBLE, author, time) + end + + private + + def track_unique_action(action, author, time) + return unless Feature.enabled?(:track_issue_activity_actions) + return unless author + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml index b7e516fa8b1..25e7f858bb1 100644 --- a/lib/gitlab/usage_data_counters/known_events.yml +++ b/lib/gitlab/usage_data_counters/known_events.yml @@ -3,86 +3,206 @@ - name: g_compliance_dashboard redis_slot: compliance category: compliance - expiry: 84 # expiration time in days, equivalent to 12 weeks aggregation: weekly - name: g_compliance_audit_events category: compliance redis_slot: compliance - expiry: 84 aggregation: weekly - name: i_compliance_audit_events category: compliance redis_slot: compliance - expiry: 84 aggregation: weekly - name: i_compliance_credential_inventory category: compliance redis_slot: compliance - expiry: 84 + aggregation: weekly +- name: a_compliance_audit_events_api + category: compliance + redis_slot: compliance aggregation: weekly # Analytics category - name: g_analytics_contribution category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: g_analytics_insights category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: g_analytics_issues category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: g_analytics_productivity category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: g_analytics_valuestream category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_pipelines category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_code_reviews category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_valuestream category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_insights category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_issues category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_repo category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: i_analytics_cohorts category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: i_analytics_dev_ops_score category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly +- name: g_analytics_merge_request + category: analytics + redis_slot: analytics + aggregation: weekly +- name: p_analytics_merge_request + category: analytics + redis_slot: analytics + aggregation: weekly +- name: i_analytics_instance_statistics + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_edit_by_web_ide + category: ide_edit + redis_slot: edit + expiry: 29 + aggregation: daily +- name: g_edit_by_sfe + category: ide_edit + redis_slot: edit + expiry: 29 + aggregation: daily +- name: g_edit_by_snippet_ide + category: ide_edit + redis_slot: edit + expiry: 29 + aggregation: daily +- name: i_search_total + category: search + redis_slot: search + aggregation: weekly +- name: i_search_advanced + category: search + redis_slot: search + aggregation: weekly +- name: i_search_paid + category: search + redis_slot: search + aggregation: weekly +- name: wiki_action + category: source_code + aggregation: daily +- name: design_action + category: source_code + aggregation: daily +- name: project_action + category: source_code + aggregation: daily +- name: merge_request_action + category: source_code + aggregation: daily +- name: i_source_code_code_intelligence + redis_slot: source_code + category: source_code + aggregation: daily +# Incident management +- name: incident_management_alert_status_changed + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_alert_assigned + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_alert_todo + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_created + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_reopened + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_closed + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_assigned + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_todo + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_comment + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_zoom_meeting + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_published + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_relate + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_unrelate + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_change_confidential + redis_slot: incident_management + category: incident_management + aggregation: weekly +# Project Management group +- name: g_project_management_issue_title_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_description_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_assignee_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_made_confidential + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_made_visible + category: issues_edit + redis_slot: project_management + aggregation: daily diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb new file mode 100644 index 00000000000..eae42bdc4a1 --- /dev/null +++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class KubernetesAgentCounter < BaseCounter + PREFIX = 'kubernetes_agent' + KNOWN_EVENTS = %w[gitops_sync].freeze + + class << self + def increment_gitops_sync(incr) + raise ArgumentError, 'must be greater than or equal to zero' if incr < 0 + + # rather then hitting redis for this no-op, we return early + # note: redis returns the increment, so we mimic this here + return 0 if incr == 0 + + increment_by(redis_key(:gitops_sync), incr) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb index 75d5a75e3a4..2406f771fd8 100644 --- a/lib/gitlab/usage_data_counters/redis_counter.rb +++ b/lib/gitlab/usage_data_counters/redis_counter.rb @@ -9,6 +9,12 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } end + def increment_by(redis_counter_key, incr) + return unless Gitlab::CurrentSettings.usage_ping_enabled + + Gitlab::Redis::SharedState.with { |redis| redis.incrby(redis_counter_key, incr) } + end + def total_count(redis_counter_key) Gitlab::Redis::SharedState.with { |redis| redis.get(redis_counter_key).to_i } end diff --git a/lib/gitlab/usage_data_counters/track_unique_actions.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb index 0df982572a4..7053744b665 100644 --- a/lib/gitlab/usage_data_counters/track_unique_actions.rb +++ b/lib/gitlab/usage_data_counters/track_unique_events.rb @@ -2,12 +2,11 @@ module Gitlab module UsageDataCounters - module TrackUniqueActions - KEY_EXPIRY_LENGTH = 29.days - + module TrackUniqueEvents WIKI_ACTION = :wiki_action DESIGN_ACTION = :design_action PUSH_ACTION = :project_action + MERGE_REQUEST_ACTION = :merge_request_action ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({ wiki: { @@ -22,26 +21,30 @@ module Gitlab }, project: { pushed: PUSH_ACTION + }, + merge_request: { + closed: MERGE_REQUEST_ACTION, + merged: MERGE_REQUEST_ACTION, + created: MERGE_REQUEST_ACTION, + commented: MERGE_REQUEST_ACTION } }).freeze class << self def track_event(event_action:, event_target:, author_id:, time: Time.zone.now) - return unless Gitlab::CurrentSettings.usage_ping_enabled return unless valid_target?(event_target) return unless valid_action?(event_action) transformed_target = transform_target(event_target) transformed_action = transform_action(event_action, transformed_target) - target_key = key(transformed_action, time) - Gitlab::Redis::HLL.add(key: target_key, value: author_id, expiry: KEY_EXPIRY_LENGTH) - end + return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s) - def count_unique(event_action:, date_from:, date_to:) - keys = (date_from.to_date..date_to.to_date).map { |date| key(event_action, date) } + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, transformed_action.to_s, time) + end - Gitlab::Redis::HLL.count(keys: keys) + def count_unique_events(event_action:, date_from:, date_to:) + Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event_action.to_s, start_date: date_from, end_date: date_to) end private @@ -61,11 +64,6 @@ module Gitlab def valid_action?(action) Event.actions.key?(action) end - - def key(event_action, date) - year_day = date.strftime('%G-%j') - "#{year_day}-{#{event_action}}" - end end end end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb new file mode 100644 index 00000000000..bacd63ab282 --- /dev/null +++ b/lib/gitlab/usage_data_queries.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + # This class is used by the `gitlab:usage_data:dump_sql` rake tasks to output SQL instead of running it. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091 + class UsageDataQueries < UsageData + class << self + def count(relation, column = nil, *rest) + raw_sql(relation, column) + end + + def distinct_count(relation, column = nil, *rest) + raw_sql(relation, column, :distinct) + end + + def redis_usage_data(counter = nil, &block) + if block_given? + { redis_usage_data_block: block.to_s } + elsif counter.present? + { redis_usage_data_counter: counter } + end + end + + def sum(relation, column, *rest) + relation.select(relation.all.table[column].sum).to_sql # rubocop:disable CodeReuse/ActiveRecord + end + + private + + def raw_sql(relation, column, distinct = nil) + column ||= relation.primary_key + relation.select(relation.all.table[column].count(distinct)).to_sql + end + end + end +end diff --git a/lib/gitlab/utils/gzip.rb b/lib/gitlab/utils/gzip.rb new file mode 100644 index 00000000000..898be651554 --- /dev/null +++ b/lib/gitlab/utils/gzip.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + module Gzip + def gzip_compress(data) + # .compress returns ASCII-8BIT, so we need to force the encoding to + # UTF-8 before caching it in redis, else we risk encoding mismatch + # errors. + # + ActiveSupport::Gzip.compress(data).force_encoding("UTF-8") + rescue Zlib::GzipFile::Error + data + end + + def gzip_decompress(data) + # Since we could be dealing with an already populated cache full of data + # that isn't gzipped, we want to also check to see if the data is + # gzipped before we attempt to .decompress it, thus we check the first + # 2 bytes for "\x1F\x8B" to confirm it is a gzipped string. While a + # non-gzipped string will raise a Zlib::GzipFile::Error, which we're + # rescuing, we don't want to count on rescue for control flow. + # + data[0..1] == "\x1F\x8B" ? ActiveSupport::Gzip.decompress(data) : data + rescue Zlib::GzipFile::Error + data + end + end + end +end diff --git a/lib/gitlab/utils/markdown.rb b/lib/gitlab/utils/markdown.rb index 82c4a0e3b23..e783ac785cc 100644 --- a/lib/gitlab/utils/markdown.rb +++ b/lib/gitlab/utils/markdown.rb @@ -4,11 +4,13 @@ module Gitlab module Utils module Markdown PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze + PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate)(\s+only)?\)\**/.freeze def string_to_anchor(string) string .strip .downcase + .gsub(PRODUCT_SUFFIX, '') .gsub(PUNCTUATION_REGEXP, '') # remove punctuation .tr(' ', '-') # replace spaces with dash .squeeze('-') # replace multiple dashes with one diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 36046ca14bf..ca6a36c9cea 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -59,6 +59,12 @@ module Gitlab FALLBACK end + def sum(relation, column, batch_size: nil, start: nil, finish: nil) + Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish) + rescue ActiveRecord::StatementInvalid + FALLBACK + end + def alt_usage_data(value = nil, fallback: FALLBACK, &block) if block_given? yield @@ -77,11 +83,11 @@ module Gitlab end end - def with_prometheus_client(fallback: nil) - return fallback unless Gitlab::Prometheus::Internal.prometheus_enabled? + def with_prometheus_client(fallback: nil, verify: true) + client = prometheus_client(verify: verify) + return fallback unless client - prometheus_address = Gitlab::Prometheus::Internal.uri - yield Gitlab::PrometheusClient.new(prometheus_address, allow_local_requests: true) + yield client end def measure_duration @@ -96,8 +102,41 @@ module Gitlab yield.merge(key => Time.current) end + # @param event_name [String] the event name + # @param values [Array|String] the values counted + def track_usage_event(event_name, values) + return unless Feature.enabled?(:"usage_data_#{event_name}", default_enabled: true) + return unless Gitlab::CurrentSettings.usage_ping_enabled? + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name.to_s) + end + private + def prometheus_client(verify:) + server_address = prometheus_server_address + + return unless server_address + + # There really is not a way to discover whether a Prometheus connection is using TLS or not + # Try TLS first because HTTPS will return fast if failed. + %w[https http].find do |scheme| + api_url = "#{scheme}://#{server_address}" + client = Gitlab::PrometheusClient.new(api_url, allow_local_requests: true, verify: verify) + break client if client.ready? + rescue + nil + end + end + + def prometheus_server_address + if Gitlab::Prometheus::Internal.prometheus_enabled? + Gitlab::Prometheus::Internal.server_address + elsif Gitlab::Consul::Internal.api_url + Gitlab::Consul::Internal.discover_prometheus_server_address + end + end + def redis_usage_counter yield rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent diff --git a/lib/gitlab/web_ide/config.rb b/lib/gitlab/web_ide/config.rb index 3b1fa162b53..b2ab5c0b6e3 100644 --- a/lib/gitlab/web_ide/config.rb +++ b/lib/gitlab/web_ide/config.rb @@ -34,6 +34,10 @@ module Gitlab @global.terminal_value end + def schemas_value + @global.schemas_value + end + private def build_config(config, opts = {}) diff --git a/lib/gitlab/web_ide/config/entry/global.rb b/lib/gitlab/web_ide/config/entry/global.rb index 50c3f2d294f..2c67c7d02d4 100644 --- a/lib/gitlab/web_ide/config/entry/global.rb +++ b/lib/gitlab/web_ide/config/entry/global.rb @@ -12,18 +12,22 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[terminal].freeze + def self.allowed_keys + %i[terminal].freeze + end validations do - validates :config, allowed_keys: ALLOWED_KEYS + validates :config, allowed_keys: Global.allowed_keys end + attributes allowed_keys + entry :terminal, Entry::Terminal, description: 'Configuration of the webide terminal.' - - attributes :terminal end end end end end + +::Gitlab::WebIde::Config::Entry::Global.prepend_if_ee('EE::Gitlab::WebIde::Config::Entry::Global') diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e3b1cb3d016..8a5acd242d9 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -156,10 +156,11 @@ module Gitlab ] end - def send_scaled_image(location, width) + def send_scaled_image(location, width, content_type) params = { 'Location' => location, - 'Width' => width + 'Width' => width, + 'ContentType' => content_type } [ |