diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-12-20 09:07:57 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-12-20 09:07:57 +0000 |
| commit | 7881eb30eaa8b01dbcfe87faa09927c75c7d6e45 (patch) | |
| tree | 298bc8d2c62b2f2c29cb8ecbcf3de3eaaa6466d9 /lib/gitlab | |
| parent | 64b66e0cb6d1bfd27abf24e06653f00bddb60597 (diff) | |
| download | gitlab-ce-7881eb30eaa8b01dbcfe87faa09927c75c7d6e45.tar.gz | |
Add latest changes from gitlab-org/gitlab@12-6-stable-ee
Diffstat (limited to 'lib/gitlab')
168 files changed, 2915 insertions, 1004 deletions
diff --git a/lib/gitlab/action_rate_limiter.rb b/lib/gitlab/action_rate_limiter.rb deleted file mode 100644 index 0e8707af631..00000000000 --- a/lib/gitlab/action_rate_limiter.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - # This class implements a simple rate limiter that can be used to throttle - # certain actions. Unlike Rack Attack and Rack::Throttle, which operate at - # the middleware level, this can be used at the controller level. - class ActionRateLimiter - TIME_TO_EXPIRE = 60 # 1 min - - attr_accessor :action, :expiry_time - - def initialize(action:, expiry_time: TIME_TO_EXPIRE) - @action = action - @expiry_time = expiry_time - end - - # Increments the given cache key and increments the value by 1 with the - # given expiration time. Returns the incremented value. - # - # key - An array of ActiveRecord instances - def increment(key) - value = 0 - - Gitlab::Redis::Cache.with do |redis| - cache_key = action_key(key) - value = redis.incr(cache_key) - redis.expire(cache_key, expiry_time) if value == 1 - end - - value - end - - # Increments the given key and returns true if the action should - # be throttled. - # - # key - An array of ActiveRecord instances or strings - # threshold_value - The maximum number of times this action should occur in the given time interval. If number is zero is considered disabled. - def throttled?(key, threshold_value) - threshold_value > 0 && - self.increment(key) > threshold_value - end - - # Logs request into auth.log - # - # request - Web request to be logged - # type - A symbol key that represents the request. - # current_user - Current user of the request, it can be nil. - def log_request(request, type, current_user) - request_information = { - message: 'Action_Rate_Limiter_Request', - env: type, - remote_ip: request.ip, - request_method: request.request_method, - path: request.fullpath - } - - if current_user - request_information.merge!({ - user_id: current_user.id, - username: current_user.username - }) - end - - Gitlab::AuthLogger.error(request_information) - end - - private - - def action_key(key) - serialized = key.map do |obj| - if obj.is_a?(String) - "#{obj}" - else - "#{obj.class.model_name.to_s.underscore}:#{obj.id}" - end - end.join(":") - - "action_rate_limiter:#{action}:#{serialized}" - end - end -end diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb index 05b16672912..5eca364a697 100644 --- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -42,3 +42,5 @@ module Gitlab end end end + +Gitlab::Analytics::CycleAnalytics::DataCollector.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::DataCollector') diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index 2662aa38d6b..e8e269a88f0 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -21,7 +21,7 @@ module Gitlab finder_class: MergeRequestsFinder, serializer_class: AnalyticsMergeRequestSerializer, includes_for_query: { target_project: [:namespace], author: [] }, - columns_for_select: %I[title iid id created_at author_id state target_project_id] + columns_for_select: %I[title iid id created_at author_id state_id target_project_id] } }.freeze diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index 667d6def414..0c75a141c3c 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -8,6 +8,8 @@ module Gitlab class StageEvent include Gitlab::CycleAnalytics::MetricsTables + delegate :label_based?, to: :class + def initialize(params) @params = params end @@ -35,7 +37,7 @@ module Gitlab query end - def label_based? + def self.label_based? false end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb index 34c726b2254..29a2d55df1a 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb @@ -9,11 +9,11 @@ module Gitlab end def zero_interval - Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) + Arel::Nodes::NamedFunction.new('CAST', [Arel.sql("'0' AS INTERVAL")]) end def round_duration_to_seconds - Arel::Nodes::Extract.new(duration, :epoch) + Arel::Nodes::NamedFunction.new('ROUND', [Arel::Nodes::Extract.new(duration, :epoch)]) end def duration diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb new file mode 100644 index 00000000000..629632b744b --- /dev/null +++ b/lib/gitlab/application_rate_limiter.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +module Gitlab + # This class implements a simple rate limiter that can be used to throttle + # certain actions. Unlike Rack Attack and Rack::Throttle, which operate at + # the middleware level, this can be used at the controller or API level. + # + # @example + # if Gitlab::ApplicationRateLimiter.throttled?(:project_export, scope: [@project, @current_user]) + # flash[:alert] = 'error!' + # redirect_to(edit_project_path(@project), status: :too_many_requests) + # end + class ApplicationRateLimiter + class << self + # Application rate limits + # + # Threshold value can be either an Integer or a Proc + # in order to not evaluate it's value every time this method is called + # and only do that when it's needed. + def rate_limits + { + project_export: { threshold: 1, interval: 5.minutes }, + project_download_export: { threshold: 10, interval: 10.minutes }, + project_generate_new_export: { threshold: 1, interval: 5.minutes }, + play_pipeline_schedule: { threshold: 1, interval: 1.minute }, + show_raw_controller: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute } + }.freeze + end + + # Increments the given key and returns true if the action should + # be throttled. + # + # @param key [Symbol] Key attribute registered in `.rate_limits` + # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) + # @option threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` + # @option interval [Integer] Optional interval value to override default one registered in `.rate_limits` + # + # @return [Boolean] Whether or not a request should be throttled + def throttled?(key, scope: nil, interval: nil, threshold: nil) + return unless rate_limits[key] + + threshold_value = threshold || threshold(key) + + threshold_value > 0 && + increment(key, scope, interval) > threshold_value + end + + # Increments the given cache key and increments the value by 1 with the + # expiration interval defined in `.rate_limits`. + # + # @param key [Symbol] Key attribute registered in `.rate_limits` + # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) + # @option interval [Integer] Optional interval value to override default one registered in `.rate_limits` + # + # @return [Integer] incremented value + def increment(key, scope, interval = nil) + value = 0 + interval_value = interval || interval(key) + + Gitlab::Redis::Cache.with do |redis| + cache_key = action_key(key, scope) + value = redis.incr(cache_key) + redis.expire(cache_key, interval_value) if value == 1 + end + + value + end + + # Logs request using provided logger + # + # @param request [Http::Request] - Web request to be logged + # @param type [Symbol] A symbol key that represents the request + # @param current_user [User] Current user of the request, it can be nil + # @param logger [Logger] Logger to log request to a specific log file. Defaults to Gitlab::AuthLogger + def log_request(request, type, current_user, logger = Gitlab::AuthLogger) + request_information = { + message: 'Application_Rate_Limiter_Request', + env: type, + remote_ip: request.ip, + request_method: request.request_method, + path: request.fullpath + } + + if current_user + request_information.merge!({ + user_id: current_user.id, + username: current_user.username + }) + end + + logger.error(request_information) + end + + private + + def threshold(key) + value = rate_limit_value_by_key(key, :threshold) + + return value.call if value.is_a?(Proc) + + value.to_i + end + + def interval(key) + rate_limit_value_by_key(key, :interval).to_i + end + + def rate_limit_value_by_key(key, setting) + action = rate_limits[key] + + action[setting] if action + end + + def action_key(key, scope) + composed_key = [key, scope].flatten.compact + + serialized = composed_key.map do |obj| + if obj.is_a?(String) || obj.is_a?(Symbol) + "#{obj}" + else + "#{obj.class.model_name.to_s.underscore}:#{obj.id}" + end + end.join(":") + + "application_rate_limiter:#{serialized}" + end + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 4217859f9fb..dfdba617cb6 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -3,6 +3,7 @@ module Gitlab module Auth MissingPersonalAccessTokenError = Class.new(StandardError) + IpBlacklisted = Class.new(StandardError) # Scopes used for GitLab API access API_SCOPES = [:api, :read_user].freeze @@ -35,6 +36,10 @@ module Gitlab def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? + rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip) + + raise IpBlacklisted if !skip_rate_limit?(login: login) && rate_limiter.banned? + # `user_with_password_for_git` should be the last check # because it's the most expensive, especially when LDAP # is enabled. @@ -48,7 +53,7 @@ module Gitlab user_with_password_for_git(login, password) || Gitlab::Auth::Result.new - rate_limit!(ip, success: result.success?, login: login) unless skip_rate_limit?(login: login) + rate_limit!(rate_limiter, success: result.success?, login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) return result if result.success? || authenticate_using_internal_or_ldap_password? @@ -96,10 +101,11 @@ module Gitlab end end + private + # rubocop:disable Gitlab/RailsLogger - def rate_limit!(ip, success:, login:) - rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip) - return unless rate_limiter.enabled? + def rate_limit!(rate_limiter, success:, login:) + return if skip_rate_limit?(login: login) if success # Repeated login 'failures' are normal behavior for some Git clients so @@ -109,18 +115,16 @@ module Gitlab else # Register a login failure so that Rack::Attack can block the next # request from this IP if needed. - rate_limiter.register_fail! - - if rate_limiter.banned? - Rails.logger.info "IP #{ip} failed to login " \ + # This returns true when the failures are over the threshold and the IP + # is banned. + if rate_limiter.register_fail! + Rails.logger.info "IP #{rate_limiter.ip} failed to login " \ "as #{login} but has been temporarily banned from Git auth" end end end # rubocop:enable Gitlab/RailsLogger - private - def skip_rate_limit?(login:) ::Ci::Build::CI_REGISTRY_USER == login end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index e2f562c0843..33cbb070c2f 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -17,13 +17,17 @@ module Gitlab end end - module UserAuthFinders - prepend_if_ee('::EE::Gitlab::Auth::UserAuthFinders') # rubocop: disable Cop/InjectEnterpriseEditionModule + module AuthFinders + prepend_if_ee('::EE::Gitlab::Auth::AuthFinders') # rubocop: disable Cop/InjectEnterpriseEditionModule include Gitlab::Utils::StrongMemoize + include ActionController::HttpAuthentication::Basic PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN' PRIVATE_TOKEN_PARAM = :private_token + JOB_TOKEN_HEADER = "HTTP_JOB_TOKEN".freeze + JOB_TOKEN_PARAM = :job_token + RUNNER_TOKEN_PARAM = :token # Check the Rails session for valid authentication details def find_user_from_warden @@ -50,6 +54,33 @@ module Gitlab User.find_by_feed_token(token) || raise(UnauthorizedError) end + def find_user_from_job_token + return unless route_authentication_setting[:job_token_allowed] + + token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s + return unless token.present? + + job = ::Ci::Build.find_by_token(token) + raise ::Gitlab::Auth::UnauthorizedError unless job + + @current_authenticated_job = job # rubocop:disable Gitlab/ModuleWithInstanceVariables + + job.user + end + + def find_user_from_basic_auth_job + return unless has_basic_credentials?(current_request) + + login, password = user_name_and_password(current_request) + return unless login.present? && password.present? + return unless ::Ci::Build::CI_REGISTRY_USER == login + + job = ::Ci::Build.find_by_token(password) + raise UnauthorizedError unless job + + job.user + end + # We only allow Private Access Tokens with `api` scope to be used by web # requests on RSS feeds or ICS files for backwards compatibility. # It is also used by GraphQL/API requests. @@ -69,6 +100,15 @@ module Gitlab access_token.user || raise(UnauthorizedError) end + def find_runner_from_token + return unless api_request? + + token = current_request.params[RUNNER_TOKEN_PARAM].presence + return unless token + + ::Ci::Runner.find_by_token(token) || raise(UnauthorizedError) + end + def validate_access_token!(scopes: []) return unless access_token @@ -169,6 +209,8 @@ module Gitlab case request_format when :archive archive_request? + when :blob + blob_request? else false end @@ -183,12 +225,16 @@ module Gitlab end def api_request? - current_request.path.starts_with?("/api/") + current_request.path.starts_with?('/api/') end def archive_request? current_request.path.include?('/-/archive/') end + + def blob_request? + current_request.path.include?('/raw/') + end end end end diff --git a/lib/gitlab/auth/current_user_mode.rb b/lib/gitlab/auth/current_user_mode.rb index df5039f50c1..cb39baaa6cc 100644 --- a/lib/gitlab/auth/current_user_mode.rb +++ b/lib/gitlab/auth/current_user_mode.rb @@ -8,9 +8,13 @@ module Gitlab # an administrator must have explicitly enabled admin-mode # e.g. on web access require re-authentication class CurrentUserMode + NotRequestedError = Class.new(StandardError) + SESSION_STORE_KEY = :current_user_mode ADMIN_MODE_START_TIME_KEY = 'admin_mode' + ADMIN_MODE_REQUESTED_TIME_KEY = 'admin_mode_requested' MAX_ADMIN_MODE_TIME = 6.hours + ADMIN_MODE_REQUESTED_GRACE_PERIOD = 5.minutes def initialize(user) @user = user @@ -19,8 +23,16 @@ module Gitlab def admin_mode? return false unless user - Gitlab::SafeRequestStore.fetch(request_store_key) do - user&.admin? && any_session_with_admin_mode? + Gitlab::SafeRequestStore.fetch(admin_mode_rs_key) do + user.admin? && any_session_with_admin_mode? + end + end + + def admin_mode_requested? + return false unless user + + Gitlab::SafeRequestStore.fetch(admin_mode_requested_rs_key) do + user.admin? && admin_mode_requested_in_grace_period? end end @@ -28,20 +40,45 @@ module Gitlab return unless user&.admin? return unless skip_password_validation || user&.valid_password?(password) + raise NotRequestedError unless admin_mode_requested? + + reset_request_store + + current_session_data[ADMIN_MODE_REQUESTED_TIME_KEY] = nil current_session_data[ADMIN_MODE_START_TIME_KEY] = Time.now end + def enable_sessionless_admin_mode! + request_admin_mode! && enable_admin_mode!(skip_password_validation: true) + end + def disable_admin_mode! + return unless user&.admin? + + reset_request_store + + current_session_data[ADMIN_MODE_REQUESTED_TIME_KEY] = nil current_session_data[ADMIN_MODE_START_TIME_KEY] = nil - Gitlab::SafeRequestStore.delete(request_store_key) + end + + def request_admin_mode! + return unless user&.admin? + + reset_request_store + + current_session_data[ADMIN_MODE_REQUESTED_TIME_KEY] = Time.now end private attr_reader :user - def request_store_key - @request_store_key ||= { res: :current_user_mode, user: user.id } + def admin_mode_rs_key + @admin_mode_rs_key ||= { res: :current_user_mode, user: user.id, method: :admin_mode? } + end + + def admin_mode_requested_rs_key + @admin_mode_requested_rs_key ||= { res: :current_user_mode, user: user.id, method: :admin_mode_requested? } end def current_session_data @@ -61,6 +98,15 @@ module Gitlab Gitlab::NamespacedSessionStore.new(SESSION_STORE_KEY, session.with_indifferent_access ) end end + + def admin_mode_requested_in_grace_period? + current_session_data[ADMIN_MODE_REQUESTED_TIME_KEY].to_i > ADMIN_MODE_REQUESTED_GRACE_PERIOD.ago.to_i + end + + def reset_request_store + Gitlab::SafeRequestStore.delete(admin_mode_rs_key) + Gitlab::SafeRequestStore.delete(admin_mode_requested_rs_key) + end end end end diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb index acb46abb6f3..f301a2ec2e8 100644 --- a/lib/gitlab/auth/ip_rate_limiter.rb +++ b/lib/gitlab/auth/ip_rate_limiter.rb @@ -9,41 +9,48 @@ module Gitlab def initialize(ip) @ip = ip - @banned = false - end - - def enabled? - config.enabled end def reset! + return if skip_rate_limit? + Rack::Attack::Allow2Ban.reset(ip, config) end def register_fail! - return false if trusted_ip? + return false if skip_rate_limit? # Allow2Ban.filter will return false if this IP has not failed too often yet - @banned = Rack::Attack::Allow2Ban.filter(ip, config) do + Rack::Attack::Allow2Ban.filter(ip, config) do # We return true to increment the count for this IP true end end def banned? - @banned - end + return false if skip_rate_limit? - def trusted_ip? - trusted_ips.any? { |netmask| netmask.include?(ip) } + Rack::Attack::Allow2Ban.banned?(ip) end private + def skip_rate_limit? + !enabled? || trusted_ip? + end + + def enabled? + config.enabled + end + def config Gitlab.config.rack_attack.git_basic_auth end + def trusted_ip? + trusted_ips.any? { |netmask| netmask.include?(ip) } + end + def trusted_ips strong_memoize(:trusted_ips) do config.ip_whitelist.map do |proxy| diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index aca8804b04c..34ccff588f4 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -5,7 +5,7 @@ module Gitlab module Auth class RequestAuthenticator - include UserAuthFinders + include AuthFinders attr_reader :request @@ -23,10 +23,17 @@ module Gitlab find_user_from_warden end + def runner + find_runner_from_token + rescue Gitlab::Auth::AuthenticationError + nil + end + def find_sessionless_user(request_format) find_user_from_web_access_token(request_format) || find_user_from_feed_token(request_format) || - find_user_from_static_object_token(request_format) + find_user_from_static_object_token(request_format) || + find_user_from_basic_auth_job rescue Gitlab::Auth::AuthenticationError nil end diff --git a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb index 4377ec2987c..23d99274232 100644 --- a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb +++ b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb @@ -123,8 +123,6 @@ module Gitlab end def add_missing_db_timezone - return '' unless Gitlab::Database.postgresql? - 'at time zone \'UTC\'' end end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index e01ffb631ba..67118aed549 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -11,7 +11,6 @@ module Gitlab { title: 'task', color: '#7F8C8D' }].freeze attr_reader :project, :client, :errors, :users - attr_accessor :logger def initialize(project) @project = project @@ -20,7 +19,6 @@ module Gitlab @labels = {} @errors = [] @users = {} - @logger = Gitlab::Import::Logger.build end def execute @@ -47,7 +45,8 @@ module Gitlab backtrace = Gitlab::Profiler.clean_backtrace(ex.backtrace) error = { type: :pull_request, iid: pull_request.iid, errors: ex.message, trace: backtrace, raw_response: pull_request.raw } - log_error(error) + Gitlab::ErrorTracking.log_exception(ex, error) + # Omit the details from the database to avoid blowing up usage in the error column error.delete(:trace) error.delete(:raw_response) @@ -275,10 +274,6 @@ module Gitlab author.to_s + comment.note.to_s end - def log_error(details) - logger.error(log_base_data.merge(details)) - end - def log_base_data { class: self.class.name, diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 93c6fdcf69c..b7b2fe115c1 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -133,7 +133,10 @@ module Gitlab log_info(stage: 'import_repository', message: 'finished import') rescue Gitlab::Shell::Error => e - log_error(stage: 'import_repository', message: 'failed import', error: e.message) + Gitlab::ErrorTracking.log_exception( + e, + stage: 'import_repository', message: 'failed import', error: e.message + ) # Expire cache to prevent scenarios such as: # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true @@ -164,8 +167,10 @@ module Gitlab batch.each do |pull_request| import_bitbucket_pull_request(pull_request) rescue StandardError => e - backtrace = Gitlab::Profiler.clean_backtrace(e.backtrace) - log_error(stage: 'import_pull_requests', iid: pull_request.iid, error: e.message, backtrace: backtrace) + Gitlab::ErrorTracking.log_exception( + e, + stage: 'import_pull_requests', iid: pull_request.iid, error: e.message + ) errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, backtrace: backtrace.join("\n"), raw_response: pull_request.raw } end @@ -177,7 +182,11 @@ module Gitlab client.delete_branch(project_key, repository_slug, branch.name, branch.sha) project.repository.delete_branch(branch.name) rescue BitbucketServer::Connection::ConnectionError => e - log_error(stage: 'delete_temp_branches', branch: branch.name, error: e.message) + Gitlab::ErrorTracking.log_exception( + e, + stage: 'delete_temp_branches', branch: branch.name, error: e.message + ) + @errors << { type: :delete_temp_branches, branch_name: branch.name, errors: e.message } end end @@ -200,7 +209,6 @@ module Gitlab target_project_id: project.id, target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name), target_branch_sha: pull_request.target_branch_sha, - state: pull_request.state, state_id: MergeRequest.available_states[pull_request.state], author_id: author_id, assignee_id: nil, @@ -289,7 +297,11 @@ module Gitlab # a regular note. create_fallback_diff_note(merge_request, comment, position) rescue StandardError => e - log_error(stage: 'create_diff_note', comment_id: comment.id, error: e.message) + Gitlab::ErrorTracking.log_exception( + e, + stage: 'create_diff_note', comment_id: comment.id, error: e.message + ) + errors << { type: :pull_request, id: comment.id, errors: e.message } nil end @@ -326,7 +338,11 @@ module Gitlab merge_request.notes.create!(pull_request_comment_attributes(replies)) end rescue StandardError => e - log_error(stage: 'import_standalone_pr_comments', merge_request_id: merge_request.id, comment_id: comment.id, error: e.message) + Gitlab::ErrorTracking.log_exception( + e, + stage: 'import_standalone_pr_comments', merge_request_id: merge_request.id, comment_id: comment.id, error: e.message + ) + errors << { type: :pull_request, comment_id: comment.id, errors: e.message } end end @@ -361,10 +377,6 @@ module Gitlab logger.info(log_base_data.merge(details)) end - def log_error(details) - logger.error(log_base_data.merge(details)) - end - def log_warn(details) logger.warn(log_base_data.merge(details)) end diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb index 4f47cdef971..911f2993b8a 100644 --- a/lib/gitlab/chaos.rb +++ b/lib/gitlab/chaos.rb @@ -19,9 +19,11 @@ module Gitlab # cpu_spin will consume all CPU on a single core for the specified duration def self.cpu_spin(duration_s) - expected_end_time = Time.now + duration_s + return unless Gitlab::Metrics::System.thread_cpu_time + + expected_end_time = Gitlab::Metrics::System.thread_cpu_time + duration_s - rand while Time.now < expected_end_time + rand while Gitlab::Metrics::System.thread_cpu_time < expected_end_time end # db_spin will query the database in a tight loop for the specified duration diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index cbda3808b86..0373a12ab69 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -37,16 +37,13 @@ module Gitlab flush_current_line - # TODO: replace OpenStruct with a better type - # https://gitlab.com/gitlab-org/gitlab/issues/34305 - OpenStruct.new( + Gitlab::Ci::Ansi2json::Result.new( lines: @lines, state: @state.encode, append: append, truncated: truncated, offset: start_offset, - size: stream.tell - start_offset, - total: stream.size + stream: stream ) end diff --git a/lib/gitlab/ci/ansi2json/parser.rb b/lib/gitlab/ci/ansi2json/parser.rb index d428680fb2a..79b42a5f5bf 100644 --- a/lib/gitlab/ci/ansi2json/parser.rb +++ b/lib/gitlab/ci/ansi2json/parser.rb @@ -94,7 +94,7 @@ module Gitlab def on_38(stack) { fg: fg_color_256(stack) } end - def on_39(_) { fg: fg_color(9) } end + def on_39(_) { fg: nil } end def on_40(_) { bg: bg_color(0) } end @@ -114,8 +114,7 @@ module Gitlab def on_48(stack) { bg: bg_color_256(stack) } end - # TODO: all the x9 never get called? - def on_49(_) { fg: fg_color(9) } end + def on_49(_) { bg: nil } end def on_90(_) { fg: fg_color(0, 'l') } end diff --git a/lib/gitlab/ci/ansi2json/result.rb b/lib/gitlab/ci/ansi2json/result.rb new file mode 100644 index 00000000000..9b573882a52 --- /dev/null +++ b/lib/gitlab/ci/ansi2json/result.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# Convertion result object class +module Gitlab + module Ci + module Ansi2json + class Result + attr_reader :lines, :state, :append, :truncated, :offset, :size, :total + + def initialize(lines:, state:, append:, truncated:, offset:, stream:) + @lines = lines + @state = state + @append = append + @truncated = truncated + @offset = offset + @size = stream.tell - offset + @total = stream.size + end + end + end + end +end diff --git a/lib/gitlab/ci/ansi2json/style.rb b/lib/gitlab/ci/ansi2json/style.rb index 77f61178b37..4d38ea55866 100644 --- a/lib/gitlab/ci/ansi2json/style.rb +++ b/lib/gitlab/ci/ansi2json/style.rb @@ -61,9 +61,9 @@ module Gitlab case when changes[:reset] reset! - when changes[:fg] + when changes.key?(:fg) @fg = changes[:fg] - when changes[:bg] + when changes.key?(:bg) @bg = changes[:bg] when changes[:enable] @mask |= changes[:enable] diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb index 9950e1dec55..465877871ea 100644 --- a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb +++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb @@ -8,7 +8,7 @@ module Gitlab def unmet? deployment_cluster.present? && deployment_cluster.managed? && - missing_namespace? + (missing_namespace? || need_knative_version_role_binding?) end def complete! @@ -23,6 +23,10 @@ module Gitlab kubernetes_namespace.nil? || kubernetes_namespace.service_account_token.blank? end + def need_knative_version_role_binding? + !knative_serving_namespace.nil? && knative_version_role_binding.nil? + end + def deployment_cluster build.deployment&.cluster end @@ -31,6 +35,22 @@ module Gitlab build.deployment.environment end + def knative_serving_namespace + strong_memoize(:knative_serving_namespace) do + Clusters::KnativeServingNamespaceFinder.new( + deployment_cluster + ).execute + end + end + + def knative_version_role_binding + strong_memoize(:knative_version_role_binding) do + Clusters::KnativeVersionRoleBindingFinder.new( + deployment_cluster + ).execute + end + end + def kubernetes_namespace strong_memoize(:kubernetes_namespace) do Clusters::KubernetesNamespaceFinder.new( @@ -43,12 +63,33 @@ module Gitlab end def create_namespace + namespace = kubernetes_namespace || build_namespace_record + + return if conflicting_ci_namespace_requested?(namespace) + Clusters::Kubernetes::CreateOrUpdateNamespaceService.new( cluster: deployment_cluster, - kubernetes_namespace: kubernetes_namespace || build_namespace_record + kubernetes_namespace: namespace ).execute end + ## + # A namespace can only be specified via gitlab-ci.yml + # for unmanaged clusters, as we currently have no way + # of preventing a job requesting a namespace it + # shouldn't have access to. + # + # To make this clear, we fail the build instead of + # silently using a namespace other than the one + # explicitly specified. + # + # Support for managed clusters will be added in + # https://gitlab.com/gitlab-org/gitlab/issues/38054 + def conflicting_ci_namespace_requested?(namespace_record) + build.expanded_kubernetes_namespace.present? && + namespace_record.namespace != build.expanded_kubernetes_namespace + end + def build_namespace_record Clusters::BuildKubernetesNamespaceService.new( deployment_cluster, diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 9c1e6277e95..38ab3475d01 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -67,11 +67,11 @@ module Gitlab build_config(config) rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e - track_exception(e) + track_and_raise_for_dev_exception(e) raise Config::ConfigError, e.message rescue Gitlab::Ci::Config::External::Context::TimeoutError => e - track_exception(e) + track_and_raise_for_dev_exception(e) raise Config::ConfigError, TIMEOUT_MESSAGE end @@ -94,8 +94,8 @@ module Gitlab user: user) end - def track_exception(error) - Gitlab::Sentry.track_exception(error, extra: @context.sentry_payload) + def track_and_raise_for_dev_exception(error) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, @context.sentry_payload) end # Overriden in EE diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index 83127bde6e4..88db17a75da 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -14,7 +14,8 @@ module Gitlab include ::Gitlab::Config::Entry::Inheritable ALLOWED_KEYS = %i[before_script image services - after_script cache interruptible].freeze + after_script cache interruptible + timeout retry tags artifacts].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -40,11 +41,27 @@ module Gitlab description: 'Configure caching between build jobs.', inherit: true - entry :interruptible, Entry::Boolean, + entry :interruptible, ::Gitlab::Config::Entry::Boolean, description: 'Set jobs interruptible default value.', inherit: false - helpers :before_script, :image, :services, :after_script, :cache, :interruptible + entry :timeout, Entry::Timeout, + description: 'Set jobs default timeout.', + inherit: false + + entry :retry, Entry::Retry, + description: 'Set retry default value.', + inherit: false + + entry :tags, ::Gitlab::Config::Entry::ArrayOfStrings, + description: 'Set the default tags.', + inherit: false + + entry :artifacts, Entry::Artifacts, + description: 'Default artifacts.', + inherit: false + + helpers :before_script, :image, :services, :after_script, :cache private diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 5a13fd18504..fc62cca58ff 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -8,9 +8,11 @@ module Gitlab # Entry that represents an environment. # class Environment < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name url action on_stop].freeze + ALLOWED_KEYS = %i[name url action on_stop auto_stop_in kubernetes].freeze + + entry :kubernetes, Entry::Kubernetes, description: 'Kubernetes deployment configuration.' validations do validate do @@ -46,6 +48,8 @@ module Gitlab allow_nil: true validates :on_stop, type: String, allow_nil: true + validates :kubernetes, type: Hash, allow_nil: true + validates :auto_stop_in, duration: true, allow_nil: true end end @@ -73,6 +77,14 @@ module Gitlab value[:on_stop] end + def kubernetes + value[:kubernetes] + end + + def auto_stop_in + value[:auto_stop_in] + end + def value case @config when String then { name: @config, action: 'start' } @@ -80,6 +92,10 @@ module Gitlab else {} end end + + def skip_config_hash_validation? + true + end end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index c75ae87a985..6a55b8cda57 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -36,7 +36,6 @@ module Gitlab if: :has_rules? with_options allow_nil: true do - validates :tags, array_of_strings: true validates :allow_failure, boolean: true validates :parallel, numericality: { only_integer: true, greater_than_or_equal_to: 2, @@ -46,14 +45,12 @@ module Gitlab message: "should be one of: #{ALLOWED_WHEN.join(', ')}" } - validates :timeout, duration: { limit: ChronicDuration.output(Project::MAX_BUILD_TIMEOUT) } - validates :dependencies, array_of_strings: true validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true end - validates :start_in, duration: { limit: '1 day' }, if: :delayed? + validates :start_in, duration: { limit: '1 week' }, if: :delayed? validates :start_in, absence: true, if: -> { has_rules? || !delayed? } validate do @@ -99,13 +96,29 @@ module Gitlab description: 'Services that will be used to execute this job.', inherit: true - entry :interruptible, Entry::Boolean, + entry :interruptible, ::Gitlab::Config::Entry::Boolean, description: 'Set jobs interruptible value.', inherit: true + entry :timeout, Entry::Timeout, + description: 'Timeout duration of this job.', + inherit: true + + entry :retry, Entry::Retry, + description: 'Retry configuration for this job.', + inherit: true + + entry :tags, ::Gitlab::Config::Entry::ArrayOfStrings, + description: 'Set the tags.', + inherit: true + + entry :artifacts, Entry::Artifacts, + description: 'Artifacts configuration for this job.', + inherit: true + entry :only, Entry::Policy, description: 'Refs policy this job will be executed for.', - default: Entry::Policy::DEFAULT_ONLY, + default: ::Gitlab::Ci::Config::Entry::Policy::DEFAULT_ONLY, inherit: false entry :except, Entry::Policy, @@ -121,17 +134,13 @@ module Gitlab entry :needs, Entry::Needs, description: 'Needs configuration for this job.', - metadata: { allowed_needs: %i[job] }, + metadata: { allowed_needs: %i[job cross_dependency] }, inherit: false entry :variables, Entry::Variables, description: 'Environment variables available for this job.', inherit: false - entry :artifacts, Entry::Artifacts, - description: 'Artifacts configuration for this job.', - inherit: false - entry :environment, Entry::Environment, description: 'Environment configuration for this job.', inherit: false @@ -140,10 +149,6 @@ module Gitlab description: 'Coverage configuration for this job.', inherit: false - entry :retry, Entry::Retry, - description: 'Retry configuration for this job.', - inherit: false - helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, :artifacts, :environment, :coverage, :retry, :rules, @@ -170,11 +175,18 @@ module Gitlab @entries.delete(:type) - # This is something of a hack, see issue for details: - # https://gitlab.com/gitlab-org/gitlab-foss/issues/67150 - if !only_defined? && has_rules? - @entries.delete(:only) - @entries.delete(:except) + has_workflow_rules = deps&.workflow&.has_rules? + + # If workflow:rules: or rules: are used + # they are considered not compatible + # with `only/except` defaults + # + # Context: https://gitlab.com/gitlab-org/gitlab/merge_requests/21742 + if has_rules? || has_workflow_rules + # Remove only/except defaults + # defaults are not considered as defined + @entries.delete(:only) unless only_defined? + @entries.delete(:except) unless except_defined? end end end diff --git a/lib/gitlab/ci/config/entry/kubernetes.rb b/lib/gitlab/ci/config/entry/kubernetes.rb new file mode 100644 index 00000000000..2f1595d4437 --- /dev/null +++ b/lib/gitlab/ci/config/entry/kubernetes.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Kubernetes < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[namespace].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash + validates :config, allowed_keys: ALLOWED_KEYS + + validates :namespace, type: String, presence: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index b6db546d8ff..abfffb7a5ed 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -5,9 +5,12 @@ module Gitlab class Config module Entry class Need < ::Gitlab::Config::Entry::Simplifiable - strategy :Job, if: -> (config) { config.is_a?(String) } + strategy :JobString, if: -> (config) { config.is_a?(String) } - class Job < ::Gitlab::Config::Entry::Node + strategy :JobHash, + if: -> (config) { config.is_a?(Hash) && config.key?(:job) && !(config.key?(:project) || config.key?(:ref)) } + + class JobString < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable validations do @@ -20,7 +23,30 @@ module Gitlab end def value - { name: @config } + { name: @config, artifacts: true } + end + end + + class JobHash < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[job artifacts].freeze + attributes :job, :artifacts + + validations do + validates :config, presence: true + validates :config, allowed_keys: ALLOWED_KEYS + validates :job, type: String, presence: true + validates :artifacts, boolean: true, allow_nil: true + end + + def type + :job + end + + def value + { name: job, artifacts: artifacts || artifacts.nil? } end end diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb index 28452aaaa16..5301c453ed4 100644 --- a/lib/gitlab/ci/config/entry/needs.rb +++ b/lib/gitlab/ci/config/entry/needs.rb @@ -53,3 +53,5 @@ module Gitlab end end end + +::Gitlab::Ci::Config::Entry::Needs.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Needs') diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 25fb278d9b8..12dd942fc1c 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -67,7 +67,7 @@ module Gitlab entry :workflow, Entry::Workflow, description: 'List of evaluable rules to determine Pipeline status' - helpers :default, :jobs, :stages, :types, :variables + helpers :default, :jobs, :stages, :types, :variables, :workflow delegate :before_script_value, :image_value, @@ -106,6 +106,10 @@ module Gitlab self[:default] end + def workflow + self[:workflow] if workflow_defined? + end + private # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config/entry/boolean.rb b/lib/gitlab/ci/config/entry/timeout.rb index 10619ef9f8d..0bffa9340de 100644 --- a/lib/gitlab/ci/config/entry/boolean.rb +++ b/lib/gitlab/ci/config/entry/timeout.rb @@ -7,11 +7,11 @@ module Gitlab ## # Entry that represents the interrutible value. # - class Boolean < ::Gitlab::Config::Entry::Node + class Timeout < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable validations do - validates :config, boolean: true + validates :config, duration: { limit: ChronicDuration.output(Project::MAX_BUILD_TIMEOUT) } end end end diff --git a/lib/gitlab/ci/config/entry/workflow.rb b/lib/gitlab/ci/config/entry/workflow.rb index a51a3fbdcd2..1d9007c9b9b 100644 --- a/lib/gitlab/ci/config/entry/workflow.rb +++ b/lib/gitlab/ci/config/entry/workflow.rb @@ -18,6 +18,10 @@ module Gitlab entry :rules, Entry::Rules, description: 'List of evaluable Rules to determine Pipeline status.', metadata: { allowed_when: %w[always never] } + + def has_rules? + @config.try(:key?, :rules) + end end end end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index e714ef225f5..1139efee9e8 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -44,7 +44,7 @@ module Gitlab if all_job_names = parallelized_jobs[job_need_name] all_job_names.map do |job_name| - { name: job_name } + job_need.merge(name: job_name) end else job_need diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index a8cd99b8e92..d4b7444005e 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -8,21 +8,28 @@ module Gitlab class Content < Chain::Base include Chain::Helpers - def perform! - return if @command.config_content - - if content = content_from_repo - @command.config_content = content - @pipeline.config_source = :repository_source - # TODO: we should persist ci_config_path - # @pipeline.config_path = ci_config_path - elsif content = content_from_auto_devops - @command.config_content = content - @pipeline.config_source = :auto_devops_source - end + SOURCES = [ + Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime, + Gitlab::Ci::Pipeline::Chain::Config::Content::Repository, + Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject, + Gitlab::Ci::Pipeline::Chain::Config::Content::Remote, + Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops + ].freeze + + LEGACY_SOURCES = [ + Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime, + Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyRepository, + Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyAutoDevops + ].freeze - unless @command.config_content - return error("Missing #{ci_config_path} file") + def perform! + if config = find_config + # TODO: we should persist config_content + # @pipeline.config_content = config.content + @command.config_content = config.content + @pipeline.config_source = config.source + else + error('Missing CI config file') end end @@ -32,24 +39,21 @@ module Gitlab private - def content_from_repo - return unless project - return unless @pipeline.sha - return unless ci_config_path + def find_config + sources.each do |source| + config = source.new(@pipeline, @command) + return config if config.exists? + end - project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path) - rescue GRPC::NotFound, GRPC::Internal nil end - def content_from_auto_devops - return unless project&.auto_devops_enabled? - - Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content - end - - def ci_config_path - project.ci_config_path.presence || '.gitlab-ci.yml' + def sources + if Feature.enabled?(:ci_root_config_content, @command.project, default_enabled: true) + SOURCES + else + LEGACY_SOURCES + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb new file mode 100644 index 00000000000..e9bcc67de9c --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class AutoDevops < Source + def content + strong_memoize(:content) do + next unless project&.auto_devops_enabled? + + template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps') + YAML.dump('include' => [{ 'template' => template.full_name }]) + end + end + + def source + :auto_devops_source + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb new file mode 100644 index 00000000000..8a19e433483 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class ExternalProject < Source + def content + strong_memoize(:content) do + next unless external_project_path? + + path_file, path_project = ci_config_path.split('@', 2) + YAML.dump('include' => [{ 'project' => path_project, 'file' => path_file }]) + end + end + + def source + :external_project_source + end + + private + + # Example: path/to/.gitlab-ci.yml@another-group/another-project + def external_project_path? + ci_config_path =~ /\A.+(yml|yaml)@.+\z/ + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb new file mode 100644 index 00000000000..c4cef356628 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class LegacyAutoDevops < Source + def content + strong_memoize(:content) do + next unless project&.auto_devops_enabled? + + template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps') + template.content + end + end + + def source + :auto_devops_source + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/legacy_repository.rb b/lib/gitlab/ci/pipeline/chain/config/content/legacy_repository.rb new file mode 100644 index 00000000000..fa4a97c6880 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/legacy_repository.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class LegacyRepository < Source + def content + strong_memoize(:content) do + next unless project + next unless @pipeline.sha + next unless ci_config_path + + project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path) + rescue GRPC::NotFound, GRPC::Internal + nil + end + end + + def source + :repository_source + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb new file mode 100644 index 00000000000..dcc336b8929 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class Remote < Source + def content + strong_memoize(:content) do + next unless ci_config_path =~ URI.regexp(%w[http https]) + + YAML.dump('include' => [{ 'remote' => ci_config_path }]) + end + end + + def source + :remote_source + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/repository.rb b/lib/gitlab/ci/pipeline/chain/config/content/repository.rb new file mode 100644 index 00000000000..0752b099d3d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/repository.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class Repository < Source + def content + strong_memoize(:content) do + next unless file_in_repository? + + YAML.dump('include' => [{ 'local' => ci_config_path }]) + end + end + + def source + :repository_source + end + + private + + def file_in_repository? + return unless project + return unless @pipeline.sha + + project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path).present? + rescue GRPC::NotFound, GRPC::Internal + nil + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/runtime.rb b/lib/gitlab/ci/pipeline/chain/config/content/runtime.rb new file mode 100644 index 00000000000..4811d3d913d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/runtime.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class Runtime < Source + def content + @command.config_content + end + + def source + # The only case when this source is used is when the config content + # is passed in as parameter to Ci::CreatePipelineService. + # This would only occur with parent/child pipelines which is being + # implemented. + # TODO: change source to return :runtime_source + # https://gitlab.com/gitlab-org/gitlab/merge_requests/21041 + + nil + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/source.rb b/lib/gitlab/ci/pipeline/chain/config/content/source.rb new file mode 100644 index 00000000000..3389187473b --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/source.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class Source + include Gitlab::Utils::StrongMemoize + + DEFAULT_YAML_FILE = '.gitlab-ci.yml' + + def initialize(pipeline, command) + @pipeline = pipeline + @command = command + end + + def exists? + strong_memoize(:exists) do + content.present? + end + end + + def content + raise NotImplementedError + end + + def source + raise NotImplementedError + end + + def project + @project ||= @pipeline.project + end + + def ci_config_path + @ci_config_path ||= project.ci_config_path.presence || DEFAULT_YAML_FILE + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 731b0fdb286..09d1b0edc93 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -21,10 +21,10 @@ module Gitlab rescue Gitlab::Ci::YamlProcessor::ValidationError => ex error(ex.message, config_error: true) rescue => ex - Gitlab::Sentry.track_acceptable_exception(ex, extra: { + Gitlab::ErrorTracking.track_exception(ex, project_id: project.id, sha: @pipeline.sha - }) + ) error("Undefined error (#{Labkit::Correlation::CorrelationId.current_id})", config_error: true) diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb index 0ee9485eebc..81f5733b279 100644 --- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -9,7 +9,13 @@ module Gitlab include Chain::Helpers def perform! - return unless Feature.enabled?(:workflow_rules, @pipeline.project) + unless feature_enabled? + if has_workflow_rules? + error("Workflow rules are disabled", config_error: true) + end + + return + end unless workflow_passed? error('Pipeline filtered out by workflow rules.') @@ -17,13 +23,15 @@ module Gitlab end def break? - return false unless Feature.enabled?(:workflow_rules, @pipeline.project) - - !workflow_passed? + @pipeline.errors.any? || @pipeline.persisted? end private + def feature_enabled? + Feature.enabled?(:workflow_rules, @pipeline.project, default_enabled: true) + end + def workflow_passed? strong_memoize(:workflow_passed) do workflow_rules.evaluate(@pipeline, global_context).pass? @@ -40,6 +48,10 @@ module Gitlab @pipeline, yaml_variables: workflow_config[:yaml_variables]) end + def has_workflow_rules? + workflow_config[:rules].present? + end + def workflow_config @command.config_processor.workflow_attributes || {} end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index 8ccb1066575..982ecc0ff51 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -5,12 +5,13 @@ module Gitlab module Pipeline module Chain module Helpers - def error(message, config_error: false) + def error(message, config_error: false, drop_reason: nil) if config_error && command.save_incompleted + drop_reason = :config_error pipeline.yaml_errors = message - pipeline.drop!(:config_error) end + pipeline.drop!(drop_reason) if drop_reason pipeline.errors.add(:base, message) end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb new file mode 100644 index 00000000000..44dc333a6a1 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class External < Chain::Base + include Chain::Helpers + + InvalidResponseCode = Class.new(StandardError) + + VALIDATION_REQUEST_TIMEOUT = 5 + + def perform! + error('External validation failed', drop_reason: :external_validation_failure) unless validate_external + end + + def break? + @pipeline.errors.any? + end + + private + + def validate_external + return true unless validation_service_url + + # 200 - accepted + # 4xx - not accepted + # everything else - accepted and logged + response_code = validate_service_request.code + case response_code + when 200 + true + when 400..499 + false + else + raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" + end + rescue => ex + Gitlab::ErrorTracking.track_exception(ex) + + true + end + + def validate_service_request + Gitlab::HTTP.post( + validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT, + body: validation_service_payload(@pipeline, @command.config_processor.stages_attributes) + ) + end + + def validation_service_url + ENV['EXTERNAL_VALIDATION_SERVICE_URL'] + end + + def validation_service_payload(pipeline, stages_attributes) + { + project: { + id: pipeline.project.id, + path: pipeline.project.full_path + }, + user: { + id: pipeline.user.id, + username: pipeline.user.username, + email: pipeline.user.email + }, + pipeline: { + sha: pipeline.sha, + ref: pipeline.ref, + type: pipeline.source + }, + builds: builds_validation_payload(stages_attributes) + }.to_json + end + + def builds_validation_payload(stages_attributes) + stages_attributes.map { |stage| stage[:builds] }.flatten + .map(&method(:build_validation_payload)) + end + + def build_validation_payload(build) + { + name: build[:name], + stage: build[:stage], + image: build.dig(:options, :image, :name), + services: build.dig(:options, :services)&.map { |service| service[:name] }, + script: [ + build.dig(:options, :before_script), + build.dig(:options, :script), + build.dig(:options, :after_script) + ].flatten.compact + } + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index dce56b22666..590c7f4d1dd 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -10,7 +10,7 @@ module Gitlab delegate :dig, to: :@seed_attributes # When the `ci_dag_limit_needs` is enabled it uses the lower limit - LOW_NEEDS_LIMIT = 5 + LOW_NEEDS_LIMIT = 10 HARD_NEEDS_LIMIT = 50 def initialize(pipeline, attributes, previous_stages) diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index b0a79950667..426f0238f9d 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -15,15 +15,15 @@ performance: fi - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - mkdir gitlab-exporter - - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js + - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.0.0/index.js - mkdir sitespeed-results - | if [ -f .gitlab-urls.txt ] then sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:11.2.0 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt else - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:11.2.0 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" fi - mv sitespeed-results/data/performance.json performance.json artifacts: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index a60b00b2ee8..1708984c1cb 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,6 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.5" script: - | if ! docker info &>/dev/null; then @@ -14,11 +15,12 @@ code_quality: export DOCKER_HOST='tcp://localhost:2375' fi fi + - docker pull --quiet "$CODE_QUALITY_IMAGE" - docker run --env SOURCE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock - "registry.gitlab.com/gitlab-org/security-products/codequality:12-5-stable" /code + "$CODE_QUALITY_IMAGE" /code artifacts: reports: codequality: gl-code-quality-report.json diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 738be44d5f4..d20d04425f6 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.7.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.8.3" review: extends: .auto-deploy diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml new file mode 100644 index 00000000000..9a5b0f79ecf --- /dev/null +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -0,0 +1,16 @@ +apply: + stage: deploy + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.3.0" + environment: + name: production + variables: + TILLER_NAMESPACE: gitlab-managed-apps + GITLAB_MANAGED_APPS_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/config.yaml + INGRESS_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/ingress/values.yaml + SENTRY_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/sentry/values.yaml + script: + - kubectl get namespace "$TILLER_NAMESPACE" || kubectl create namespace "$TILLER_NAMESPACE" + - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml + only: + refs: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index 9a3ecd1c34f..975cb3b7698 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -1,5 +1,16 @@ -# Full project: https://gitlab.com/pages/hugo -image: dettmering/hugo-build +--- +# All available Hugo versions are listed here: +# https://gitlab.com/pages/hugo/container_registry +image: registry.gitlab.com/pages/hugo:latest + +variables: + GIT_SUBMODULE_STRATEGY: recursive + +test: + script: + - hugo + except: + - master pages: script: @@ -9,9 +20,3 @@ pages: - public only: - master - -test: - script: - - hugo - except: - - master diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index ef2fc561201..f708e95c2cf 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/ variables: - CS_MAJOR_VERSION: 1 + CS_MAJOR_VERSION: 2 container_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 4993d22d400..d73f6ccdb3f 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -6,7 +6,7 @@ variables: DS_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - DS_DEFAULT_ANALYZERS: "gemnasium, retire.js, gemnasium-python, gemnasium-maven, bundler-audit" + DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_MAJOR_VERSION: 2 DS_DISABLE_DIND: "false" @@ -43,15 +43,17 @@ dependency_scanning: DS_ANALYZER_IMAGE_TAG \ DS_DEFAULT_ANALYZERS \ DS_EXCLUDED_PATHS \ - DEP_SCAN_DISABLE_REMOTE_CHECKS \ DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ DS_PULL_ANALYZER_IMAGE_TIMEOUT \ DS_RUN_ANALYZER_TIMEOUT \ DS_PYTHON_VERSION \ + DS_PIP_VERSION \ DS_PIP_DEPENDENCY_PATH \ PIP_INDEX_URL \ PIP_EXTRA_INDEX_URL \ + PIP_REQUIREMENTS_FILE \ MAVEN_CLI_OPTS \ + BUNDLER_AUDIT_UPDATE_DISABLED \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ @@ -70,7 +72,7 @@ dependency_scanning: - $DEPENDENCY_SCANNING_DISABLED - $DS_DISABLE_DIND == 'true' -.analyzer: +.ds-analyzer: extends: dependency_scanning services: [] except: @@ -80,7 +82,7 @@ dependency_scanning: - /analyzer run gemnasium-dependency_scanning: - extends: .analyzer + extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium:$DS_MAJOR_VERSION" only: @@ -90,7 +92,7 @@ gemnasium-dependency_scanning: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby|javascript|php/ gemnasium-maven-dependency_scanning: - extends: .analyzer + extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" only: @@ -100,7 +102,7 @@ gemnasium-maven-dependency_scanning: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bjava\b/ gemnasium-python-dependency_scanning: - extends: .analyzer + extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" only: @@ -110,7 +112,7 @@ gemnasium-python-dependency_scanning: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/ bundler-audit-dependency_scanning: - extends: .analyzer + extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/bundler-audit:$DS_MAJOR_VERSION" only: @@ -120,7 +122,7 @@ bundler-audit-dependency_scanning: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/ retire-js-dependency_scanning: - extends: .analyzer + extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/retire.js:$DS_MAJOR_VERSION" only: diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index c81b4efddbc..34d84138a8b 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -6,9 +6,10 @@ variables: SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec" SAST_ANALYZER_IMAGE_TAG: 2 SAST_DISABLE_DIND: "false" + SCAN_KUBERNETES_MANIFESTS: "false" sast: stage: test @@ -49,7 +50,7 @@ sast: - $SAST_DISABLED - $SAST_DISABLE_DIND == 'true' -.analyzer: +.sast-analyzer: extends: sast services: [] except: @@ -59,7 +60,7 @@ sast: - /analyzer run bandit-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" only: @@ -69,7 +70,7 @@ bandit-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/ brakeman-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" only: @@ -79,7 +80,7 @@ brakeman-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/ eslint-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" only: @@ -89,7 +90,7 @@ eslint-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/ flawfinder-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" only: @@ -98,8 +99,18 @@ flawfinder-sast: $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ && $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\+\+|c)\b/ +kubesec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" + only: + variables: + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && + $SCAN_KUBERNETES_MANIFESTS == 'true' + gosec-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" only: @@ -109,7 +120,7 @@ gosec-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bgo\b/ nodejs-scan-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" only: @@ -119,7 +130,7 @@ nodejs-scan-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/ phpcs-security-audit-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" only: @@ -129,7 +140,7 @@ phpcs-security-audit-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /php/ pmd-apex-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" only: @@ -139,7 +150,7 @@ pmd-apex-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /apex/ secrets-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG" only: @@ -148,7 +159,7 @@ secrets-sast: $SAST_DEFAULT_ANALYZERS =~ /secrets/ security-code-scan-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" only: @@ -158,7 +169,7 @@ security-code-scan-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\#|visual basic\b)/ sobelow-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" only: @@ -168,7 +179,7 @@ sobelow-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /elixir/ spotbugs-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" only: @@ -178,7 +189,7 @@ spotbugs-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /java\b/ tslint-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint:$SAST_ANALYZER_IMAGE_TAG" only: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index eced181e966..e6097ae322e 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -11,7 +11,7 @@ performance: image: docker:git variables: URL: https://example.com - SITESPEED_VERSION: 6.3.1 + SITESPEED_VERSION: 11.2.0 SITESPEED_OPTIONS: '' services: - docker:stable-dind diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 833c545fc5b..27cd4f5fd6b 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -9,6 +9,12 @@ module Gitlab attr_reader :stages, :jobs + ResultWithErrors = Struct.new(:content, :errors) do + def valid? + errors.empty? + end + end + def initialize(config, opts = {}) @ci_config = Gitlab::Ci::Config.new(config, **opts) @config = @ci_config.to_hash @@ -22,6 +28,18 @@ module Gitlab raise ValidationError, e.message end + def self.new_with_validation_errors(content, opts = {}) + return ResultWithErrors.new('', ['Please provide content of .gitlab-ci.yml']) if content.blank? + + config = Gitlab::Ci::Config.new(content, **opts) + return ResultWithErrors.new("", config.errors) unless config.valid? + + config = Gitlab::Ci::YamlProcessor.new(content, opts) + ResultWithErrors.new(config, []) + rescue ValidationError, Gitlab::Ci::Config::ConfigError => e + ResultWithErrors.new('', [e.message]) + end + def builds @jobs.map do |name, _| build_attributes(name) @@ -42,6 +60,8 @@ module Gitlab yaml_variables: transform_to_yaml_variables(job_variables(name)), needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], + only: job[:only], + except: job[:except], rules: job[:rules], cache: job[:cache], options: { @@ -49,6 +69,7 @@ module Gitlab services: job[:services], artifacts: job[:artifacts], dependencies: job[:dependencies], + cross_dependencies: job.dig(:needs, :cross_dependency), job_timeout: job[:timeout], before_script: job[:before_script], script: job[:script], @@ -71,13 +92,7 @@ module Gitlab def stages_attributes @stages.uniq.map do |stage| - seeds = stage_builds_attributes(stage).map do |attributes| - job = @jobs.fetch(attributes[:name].to_sym) - - attributes - .merge(only: job.fetch(:only, {})) - .merge(except: job.fetch(:except, {})) - end + seeds = stage_builds_attributes(stage) { name: stage, index: @stages.index(stage), builds: seeds } end diff --git a/lib/gitlab/config/entry/array_of_strings.rb b/lib/gitlab/config/entry/array_of_strings.rb new file mode 100644 index 00000000000..403b15e8f32 --- /dev/null +++ b/lib/gitlab/config/entry/array_of_strings.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # Entry that represents a array of strings value. + # + class ArrayOfStrings < Node + include Validatable + + validations do + validates :config, array_of_strings: true + end + end + end + end +end diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb index bda84dc2cff..d5a093a469a 100644 --- a/lib/gitlab/config/entry/configurable.rb +++ b/lib/gitlab/config/entry/configurable.rb @@ -25,7 +25,6 @@ module Gitlab end end - # rubocop: disable CodeReuse/ActiveRecord def compose!(deps = nil) return unless valid? @@ -35,11 +34,7 @@ module Gitlab # we can end with different config types like String next unless config.is_a?(Hash) - factory - .value(config[key]) - .with(key: key, parent: self) - - entries[key] = factory.create! + entry_create!(key, config[key]) end yield if block_given? @@ -49,6 +44,16 @@ module Gitlab end end end + + # rubocop: disable CodeReuse/ActiveRecord + def entry_create!(key, value) + factory = self.class + .nodes[key] + .value(value) + .with(key: key, parent: self) + + entries[key] = factory.create! + end # rubocop: enable CodeReuse/ActiveRecord def skip_config_hash_validation? diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb index fcc282bf7a6..d75da76415a 100644 --- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -10,7 +10,7 @@ module Gitlab mr_table[:iid], mr_table[:id], mr_table[:created_at], - mr_table[:state], + mr_table[:state_id], mr_table[:author_id]] @order = mr_table[:created_at] diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb index 4b5d79097b7..f5f8c19683d 100644 --- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -10,7 +10,7 @@ module Gitlab mr_table[:iid], mr_table[:id], mr_table[:created_at], - mr_table[:state], + mr_table[:state_id], mr_table[:author_id]] super(*args) diff --git a/lib/gitlab/danger/changelog.rb b/lib/gitlab/danger/changelog.rb new file mode 100644 index 00000000000..b53516081be --- /dev/null +++ b/lib/gitlab/danger/changelog.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Danger + module Changelog + NO_CHANGELOG_LABELS = %w[backstage ci-build meta].freeze + NO_CHANGELOG_CATEGORIES = %i[docs none].freeze + + def needed? + categories_need_changelog? && (gitlab.mr_labels & NO_CHANGELOG_LABELS).empty? + end + + def found + git.added_files.find { |path| path =~ %r{\A(ee/)?(changelogs/unreleased)(-ee)?/} } + end + + def presented_no_changelog_labels + NO_CHANGELOG_LABELS.map { |label| "~#{label}" }.join(', ') + end + + def sanitized_mr_title + gitlab.mr_json["title"].gsub(/^WIP: */, '').gsub(/`/, '\\\`') + end + + def ee_changelog?(changelog_path) + changelog_path =~ /unreleased-ee/ + end + + def ce_port_changelog?(changelog_path) + helper.ee? && !ee_changelog?(changelog_path) + end + + private + + def categories_need_changelog? + (helper.changes_by_category.keys - NO_CHANGELOG_CATEGORIES).any? + end + end + end +end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 0e7e0c40a8a..cd7d617509b 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -153,7 +153,10 @@ module Gitlab # Fallbacks in case the above patterns miss anything %r{\.rb\z} => :backend, - %r{\.(md|txt)\z} => :none, # To reinstate roulette for documentation, set to `:docs`. + %r{( + \.(md|txt)\z | + \.markdownlint\.json + )}x => :none, # To reinstate roulette for documentation, set to `:docs`. %r{\.js\z} => :frontend }.freeze diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index e96f5177195..55476cd9789 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -63,7 +63,7 @@ module Gitlab def has_capability?(project, category, kind, labels) case category when :test - area = role[/Test Automation Engineer(?:.*?, (\w+))/, 1] + area = role[/Software Engineer in Test(?:.*?, (\w+))/, 1] area && labels.any?("devops::#{area.downcase}") if kind == :reviewer when :engineering_productivity diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 3407380127e..f7b7db50b2f 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -31,6 +31,8 @@ module Gitlab build_duration: build.duration, build_allow_failure: build.allow_failure, build_failure_reason: build.failure_reason, + pipeline_id: commit.id, + runner: build_runner(build.runner), # TODO: do we still need it? project_id: project.id, @@ -43,6 +45,7 @@ module Gitlab }, commit: { + # note: commit.id is actually the pipeline id id: commit.id, sha: commit.sha, message: commit.git_commit_message, @@ -75,6 +78,17 @@ module Gitlab author = commit.try(:author) author ? Gitlab::Routing.url_helpers.user_url(author) : "mailto:#{pipeline.git_author_email}" end + + def build_runner(runner) + return unless runner + + { + id: runner.id, + description: runner.description, + active: runner.active?, + is_shared: runner.instance_type? + } + end end end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index da3d6c47431..8e699de8164 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -61,6 +61,7 @@ module Gitlab finished_at: build.finished_at, when: build.when, manual: build.action?, + allow_failure: build.allow_failure, user: build.user.try(:hook_attrs), runner: build.runner && runner_hook_attrs(build.runner), artifacts_file: { diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 50e23681de0..ceab9322857 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -95,6 +95,10 @@ module Gitlab version.to_f >= 9.6 end + def self.upsert_supported? + version.to_f >= 9.5 + end + # map some of the function names that changed between PostgreSQL 9 and 10 # https://wiki.postgresql.org/wiki/New_in_postgres_10 def self.pg_wal_lsn_diff @@ -158,7 +162,9 @@ module Gitlab # disable_quote - A key or an Array of keys to exclude from quoting (You # become responsible for protection from SQL injection for # these keys!) - def self.bulk_insert(table, rows, return_ids: false, disable_quote: []) + # on_conflict - Defines an upsert. Values can be: :disabled (default) or + # :do_nothing + def self.bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil) return if rows.empty? keys = rows.first.keys @@ -176,10 +182,12 @@ module Gitlab VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} EOF - if return_ids - sql = "#{sql}RETURNING id" + if upsert_supported? && on_conflict == :do_nothing + sql = "#{sql} ON CONFLICT DO NOTHING" end + sql = "#{sql} RETURNING id" if return_ids + result = connection.execute(sql) if return_ids diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 7ea7565f758..f9340b262e5 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -155,6 +155,7 @@ module Gitlab # column - The name of the column to create the foreign key on. # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". + # name - The name of the foreign key. # # rubocop:disable Gitlab/RailsLogger def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, name: nil) @@ -164,25 +165,31 @@ module Gitlab raise 'add_concurrent_foreign_key can not be run inside a transaction' end - on_delete = 'SET NULL' if on_delete == :nullify + options = { + column: column, + on_delete: on_delete, + name: name.presence || concurrent_foreign_key_name(source, column) + } - key_name = name || concurrent_foreign_key_name(source, column) - - unless foreign_key_exists?(source, target, column: column) - Rails.logger.warn "Foreign key not created because it exists already " \ + if foreign_key_exists?(source, target, options) + warning_message = "Foreign key not created because it exists already " \ "(this may be due to an aborted migration or similar): " \ - "source: #{source}, target: #{target}, column: #{column}" + "source: #{source}, target: #{target}, column: #{options[:column]}, "\ + "name: #{options[:name]}, on_delete: #{options[:on_delete]}" + Rails.logger.warn warning_message + else # Using NOT VALID allows us to create a key without immediately # validating it. This means we keep the ALTER TABLE lock only for a # short period of time. The key _is_ enforced for any newly created # data. + execute <<-EOF.strip_heredoc ALTER TABLE #{source} - ADD CONSTRAINT #{key_name} - FOREIGN KEY (#{column}) + ADD CONSTRAINT #{options[:name]} + FOREIGN KEY (#{options[:column]}) REFERENCES #{target} (id) - #{on_delete ? "ON DELETE #{on_delete.upcase}" : ''} + #{on_delete_statement(options[:on_delete])} NOT VALID; EOF end @@ -193,18 +200,15 @@ module Gitlab # # Note this is a no-op in case the constraint is VALID already disable_statement_timeout do - execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};") + execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{options[:name]};") end end # rubocop:enable Gitlab/RailsLogger - def foreign_key_exists?(source, target = nil, column: nil) - foreign_keys(source).any? do |key| - if column - key.options[:column].to_s == column.to_s - else - key.to_table.to_s == target.to_s - end + def foreign_key_exists?(source, target = nil, **options) + foreign_keys(source).any? do |foreign_key| + tables_match?(target.to_s, foreign_key.to_table.to_s) && + options_match?(foreign_key.options, options) end end @@ -1048,8 +1052,32 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(index_sql).to_i > 0 end + def create_or_update_plan_limit(limit_name, plan_name, limit_value) + execute <<~SQL + INSERT INTO plan_limits (plan_id, #{quote_column_name(limit_name)}) + VALUES + ((SELECT id FROM plans WHERE name = #{quote(plan_name)} LIMIT 1), #{quote(limit_value)}) + ON CONFLICT (plan_id) DO UPDATE SET #{quote_column_name(limit_name)} = EXCLUDED.#{quote_column_name(limit_name)}; + SQL + end + private + def tables_match?(target_table, foreign_key_table) + target_table.blank? || foreign_key_table == target_table + end + + def options_match?(foreign_key_options, options) + options.all? { |k, v| foreign_key_options[k].to_s == v.to_s } + end + + def on_delete_statement(on_delete) + return '' if on_delete.blank? + return 'ON DELETE SET NULL' if on_delete == :nullify + + "ON DELETE #{on_delete.upcase}" + end + def create_column_from(table, old, new, type: nil) old_col = column_for(table, old) new_type = type || old_col.type diff --git a/lib/gitlab/database/obsolete_ignored_columns.rb b/lib/gitlab/database/obsolete_ignored_columns.rb index 6266b6a4b65..ad5473f1b74 100644 --- a/lib/gitlab/database/obsolete_ignored_columns.rb +++ b/lib/gitlab/database/obsolete_ignored_columns.rb @@ -23,8 +23,15 @@ module Gitlab private def ignored_columns_safe_to_remove_for(klass) - ignored = klass.ignored_columns.map(&:to_s) + ignores = ignored_and_not_present(klass).each_with_object({}) do |col, h| + h[col] = klass.ignored_columns_details[col.to_sym] + end + + ignores.select { |_, i| i&.safe_to_remove? } + end + def ignored_and_not_present(klass) + ignored = klass.ignored_columns.map(&:to_s) return [] if ignored.empty? schema = klass.connection.schema_cache.columns_hash(klass.table_name) diff --git a/lib/gitlab/database/sha256_attribute.rb b/lib/gitlab/database/sha256_attribute.rb new file mode 100644 index 00000000000..adf3f7fb5a6 --- /dev/null +++ b/lib/gitlab/database/sha256_attribute.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # Class for casting binary data to hexadecimal SHA256 hashes (and vice-versa). + # + # Using Sha256Attribute allows you to store SHA256 values as binary while still + # using them as if they were stored as string values. This gives you the + # ease of use of string values, but without the storage overhead. + class Sha256Attribute < ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea + # Casts binary data to a SHA256 and remove trailing = and newline from encode64 + def deserialize(value) + value = super(value) + if value.present? + Base64.encode64(value).delete("=").chomp("\n") + else + nil + end + end + + # Casts a SHA256 in a proper binary format. which is 32 bytes long + def serialize(value) + arg = if value.present? + Base64.decode64(value) + else + nil + end + + super(arg) + end + end + end +end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index 8cd9694b741..fbf252b7ec3 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -29,10 +29,11 @@ module Gitlab def execute! result = execute_steps - if result[:status] == :success + ::Gitlab::Tracking.event("self_monitoring", "project_created") result elsif STEPS_ALLOWED_TO_FAIL.include?(result[:last_step]) + ::Gitlab::Tracking.event("self_monitoring", "project_created") success else raise StandardError, result[:message] diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index ffad00fa7d7..dd7ab92c6ae 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -62,7 +62,7 @@ module Gitlab end def link_tag(name, url) - %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>} + %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}.html_safe end # Links package names based on regex. diff --git a/lib/gitlab/diff/deprecated_highlight_cache.rb b/lib/gitlab/diff/deprecated_highlight_cache.rb new file mode 100644 index 00000000000..47347686973 --- /dev/null +++ b/lib/gitlab/diff/deprecated_highlight_cache.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true +# +module Gitlab + module Diff + class DeprecatedHighlightCache + delegate :diffable, to: :@diff_collection + delegate :diff_options, to: :@diff_collection + + def initialize(diff_collection, backend: Rails.cache) + @backend = backend + @diff_collection = diff_collection + end + + # - Reads from cache + # - Assigns DiffFile#highlighted_diff_lines for cached files + def decorate(diff_file) + if content = read_file(diff_file) + diff_file.highlighted_diff_lines = content.map do |line| + Gitlab::Diff::Line.init_from_hash(line) + end + end + end + + # It populates a Hash in order to submit a single write to the memory + # cache. This avoids excessive IO generated by N+1's (1 writing for + # each highlighted line or file). + def write_if_empty + return if cached_content.present? + + @diff_collection.diff_files.each do |diff_file| + next unless cacheable?(diff_file) + + diff_file_id = diff_file.file_identifier + + cached_content[diff_file_id] = diff_file.highlighted_diff_lines.map(&:to_hash) + end + + cache.write(key, cached_content, expires_in: 1.week) + end + + def clear + cache.delete(key) + end + + def key + [diffable, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] + end + + private + + def read_file(diff_file) + cached_content[diff_file.file_identifier] + end + + def cache + @backend + end + + def cached_content + @cached_content ||= cache.read(key) || {} + end + + def cacheable?(diff_file) + diffable.present? && diff_file.text? && diff_file.diffable? + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index c5bbf522f7c..38b636e4e5a 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs, :diffable - delegate :count, :size, :real_size, to: :diff_files + delegate :count, :size, :real_size, to: :raw_diff_files def self.default_options ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true) @@ -31,7 +31,23 @@ module Gitlab end def diff_files - @diff_files ||= diffs.decorate! { |diff| decorate_diff!(diff) } + raw_diff_files + end + + def raw_diff_files + @raw_diff_files ||= diffs.decorate! { |diff| decorate_diff!(diff) } + end + + def diff_file_paths + diff_files.map(&:file_path) + end + + def pagination_data + { + current_page: nil, + next_page: nil, + total_pages: nil + } end # This mutates `diff_files` lines. diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index c4288ca6408..fe7df1062c0 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -4,37 +4,6 @@ module Gitlab module Diff module FileCollection class MergeRequestDiff < MergeRequestDiffBase - def diff_files - diff_files = super - - diff_files.each { |diff_file| cache.decorate(diff_file) } - - diff_files - end - - override :write_cache - def write_cache - cache.write_if_empty - end - - override :clear_cache - def clear_cache - cache.clear - end - - def cache_key - cache.key - end - - def real_size - @merge_request_diff.real_size - end - - private - - def cache - @cache ||= Gitlab::Diff::HighlightCache.new(self) - end end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index a747a6ed475..06cf3d4d168 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -15,6 +15,44 @@ module Gitlab diff_refs: merge_request_diff.diff_refs, fallback_diff_refs: merge_request_diff.fallback_diff_refs) end + + def diff_files + strong_memoize(:diff_files) do + diff_files = super + + diff_files.each { |diff_file| cache.decorate(diff_file) } + + diff_files + end + end + + override :write_cache + def write_cache + cache.write_if_empty + end + + override :clear_cache + def clear_cache + cache.clear + end + + def cache_key + cache.key + end + + def real_size + @merge_request_diff.real_size + end + + private + + def cache + @cache ||= if Feature.enabled?(:hset_redis_diff_caching, project) + Gitlab::Diff::HighlightCache.new(self) + else + Gitlab::Diff::DeprecatedHighlightCache.new(self) + end + end end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb index 663326e01d5..c6d1e0b93a7 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb @@ -29,10 +29,6 @@ module Gitlab } end - def diff_file_paths - diff_files.map(&:file_path) - end - override :diffs def diffs strong_memoize(:diffs) do diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index ca7974930af..0d027809ba8 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -35,7 +35,7 @@ module Gitlab # match the blob, which is a bug. But we shouldn't fail to render # completely in that case, even though we want to report the error. rescue RangeError => e - Gitlab::Sentry.track_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/45441') + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/45441') end end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index e4390771db2..403effbb0c6 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -3,66 +3,154 @@ module Gitlab module Diff class HighlightCache - delegate :diffable, to: :@diff_collection + include Gitlab::Utils::StrongMemoize + + EXPIRATION = 1.week + VERSION = 1 + + delegate :diffable, to: :@diff_collection delegate :diff_options, to: :@diff_collection - def initialize(diff_collection, backend: Rails.cache) - @backend = backend + def initialize(diff_collection) @diff_collection = diff_collection end # - Reads from cache # - Assigns DiffFile#highlighted_diff_lines for cached files + # def decorate(diff_file) if content = read_file(diff_file) diff_file.highlighted_diff_lines = content.map do |line| - Gitlab::Diff::Line.init_from_hash(line) + Gitlab::Diff::Line.safe_init_from_hash(line) end end end - # It populates a Hash in order to submit a single write to the memory - # cache. This avoids excessive IO generated by N+1's (1 writing for - # each highlighted line or file). + # For every file that isn't already contained in the redis hash, store the + # result of #highlighted_diff_lines, then submit the uncached content + # to #write_to_redis_hash to submit a single write. This avoids excessive + # IO generated by N+1's (1 writing for each highlighted line or file). + # def write_if_empty - return if cached_content.present? + return if cacheable_files.empty? - @diff_collection.diff_files.each do |diff_file| - next unless cacheable?(diff_file) + new_cache_content = {} - diff_file_id = diff_file.file_identifier - - cached_content[diff_file_id] = diff_file.highlighted_diff_lines.map(&:to_hash) + cacheable_files.each do |diff_file| + new_cache_content[diff_file.file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) end - cache.write(key, cached_content, expires_in: 1.week) + write_to_redis_hash(new_cache_content) end def clear - cache.delete(key) + Gitlab::Redis::Cache.with do |redis| + redis.del(key) + end end def key - [diffable, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] + strong_memoize(:redis_key) do + ['highlighted-diff-files', diffable.cache_key, VERSION, diff_options].join(":") + end end private - def read_file(diff_file) - cached_content[diff_file.file_identifier] + # We create a Gitlab::Diff::DeprecatedHighlightCache here in order to + # expire deprecated cache entries while we make the transition. This can + # be removed when :hset_redis_diff_caching is fully launched. + # See https://gitlab.com/gitlab-org/gitlab/issues/38008 + # + def deprecated_cache + strong_memoize(:deprecated_cache) do + Gitlab::Diff::DeprecatedHighlightCache.new(@diff_collection) + end + end + + def cacheable_files + strong_memoize(:cacheable_files) do + diff_files.select { |file| cacheable?(file) && read_file(file).nil? } + end end - def cache - @backend + # Given a hash of: + # { "file/to/cache" => + # [ { line_code: "a5cc2925ca8258af241be7e5b0381edf30266302_19_19", + # rich_text: " <span id=\"LC19\" class=\"line\" lang=\"plaintext\">config/initializers/secret_token.rb</span>\n", + # text: " config/initializers/secret_token.rb", + # type: nil, + # index: 3, + # old_pos: 19, + # new_pos: 19 } + # ] } + # + # ...it will write/update a Gitlab::Redis hash (HSET) + # + def write_to_redis_hash(hash) + Gitlab::Redis::Cache.with do |redis| + redis.pipelined do + hash.each do |diff_file_id, highlighted_diff_lines_hash| + redis.hset(key, diff_file_id, highlighted_diff_lines_hash.to_json) + end + + # HSETs have to have their expiration date manually updated + # + redis.expire(key, EXPIRATION) + end + end + + # Subsequent read_file calls would need the latest cache. + # + clear_memoization(:cached_content) + clear_memoization(:cacheable_files) + + # Clean up any deprecated hash entries + # + deprecated_cache.clear + end + + def file_paths + strong_memoize(:file_paths) do + diff_files.collect(&:file_path) + end + end + + def read_file(diff_file) + cached_content[diff_file.file_path] end def cached_content - @cached_content ||= cache.read(key) || {} + strong_memoize(:cached_content) { read_cache } + end + + def read_cache + return {} unless file_paths.any? + + results = [] + + Gitlab::Redis::Cache.with do |redis| + results = redis.hmget(key, file_paths) + end + + results.map! do |result| + JSON.parse(result, symbolize_names: true) unless result.nil? + end + + file_paths.zip(results).to_h end def cacheable?(diff_file) diffable.present? && diff_file.text? && diff_file.diffable? end + + def diff_files + # We access raw_diff_files here, as diff_files will attempt to apply the + # highlighting code found in this class, leading to a circular + # reference. + # + @diff_collection.raw_diff_files + end end end end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 1bbde1ffd2a..29dff699ba5 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -9,7 +9,7 @@ module Gitlab def mark(line_inline_diffs, mode: nil) super(line_inline_diffs) do |text, left:, right:| - %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>} + %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>}.html_safe end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 001748afb41..379fc6af875 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -3,6 +3,9 @@ module Gitlab module Diff class Line + # When SERIALIZE_KEYS is updated, to reset the redis cache entries you'll + # need to bump the VERSION constant on Gitlab::Diff::HighlightCache + # SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze attr_reader :line_code, :type, :old_pos, :new_pos @@ -31,6 +34,14 @@ module Gitlab rich_text: hash[:rich_text]) end + def self.safe_init_from_hash(hash) + line = hash.with_indifferent_access + rich_text = line[:rich_text] + line[:rich_text] = rich_text&.html_safe + + init_from_hash(line) + end + def to_hash hash = {} SERIALIZE_KEYS.each { |key| hash[key] = send(key) } # rubocop:disable GitlabSecurity/PublicSend diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index 369c6b87fb4..1f64883cb69 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -43,11 +43,7 @@ module Gitlab next unless lines JSON.parse(lines).map! do |line| - line = line.with_indifferent_access - rich_text = line[:rich_text] - line[:rich_text] = rich_text&.html_safe - - Gitlab::Diff::Line.init_from_hash(line) + Gitlab::Diff::Line.safe_init_from_hash(line) end end end diff --git a/lib/gitlab/elasticsearch/logger.rb b/lib/gitlab/elasticsearch/logger.rb new file mode 100644 index 00000000000..86cd1d942f2 --- /dev/null +++ b/lib/gitlab/elasticsearch/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Elasticsearch + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'elasticsearch' + end + end + end +end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb new file mode 100644 index 00000000000..6df9bfad657 --- /dev/null +++ b/lib/gitlab/error_tracking.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class << self + def configure + Raven.configure do |config| + config.dsn = sentry_dsn + config.release = Gitlab.revision + config.current_environment = Gitlab.config.sentry.environment + + # Sanitize fields based on those sanitized from Rails. + config.sanitize_fields = Rails.application.config.filter_parameters.map(&:to_s) + # Sanitize authentication headers + config.sanitize_http_headers = %w[Authorization Private-Token] + config.tags = { program: Gitlab.process_name } + # Debugging for https://gitlab.com/gitlab-org/gitlab-foss/issues/57727 + config.before_send = method(:add_context_from_exception_type) + end + end + + def with_context(current_user = nil) + last_user_context = Raven.context.user + + user_context = { + id: current_user&.id, + email: current_user&.email, + username: current_user&.username + }.compact + + Raven.tags_context(default_tags) + Raven.user_context(user_context) + + yield + ensure + Raven.user_context(last_user_context) + end + + # This should be used when you want to passthrough exception handling: + # rescue and raise to be catched in upper layers of the application. + # + # If the exception implements the method `sentry_extra_data` and that method + # returns a Hash, then the return value of that method will be merged into + # `extra`. Exceptions can use this mechanism to provide structured data + # to sentry in addition to their message and back-trace. + def track_and_raise_exception(exception, extra = {}) + process_exception(exception, sentry: true, extra: extra) + + raise exception + end + + # This can be used for investigating exceptions that can be recovered from in + # code. The exception will still be raised in development and test + # environments. + # + # That way we can track down these exceptions with as much information as we + # need to resolve them. + # + # If the exception implements the method `sentry_extra_data` and that method + # returns a Hash, then the return value of that method will be merged into + # `extra`. Exceptions can use this mechanism to provide structured data + # to sentry in addition to their message and back-trace. + # + # Provide an issue URL for follow up. + # as `issue_url: 'http://gitlab.com/gitlab-org/gitlab/issues/111'` + def track_and_raise_for_dev_exception(exception, extra = {}) + process_exception(exception, sentry: true, extra: extra) + + raise exception if should_raise_for_dev? + end + + # This should be used when you only want to track the exception. + # + # If the exception implements the method `sentry_extra_data` and that method + # returns a Hash, then the return value of that method will be merged into + # `extra`. Exceptions can use this mechanism to provide structured data + # to sentry in addition to their message and back-trace. + def track_exception(exception, extra = {}) + process_exception(exception, sentry: true, extra: extra) + end + + # This should be used when you only want to log the exception, + # but not send it to Sentry. + # + # If the exception implements the method `sentry_extra_data` and that method + # returns a Hash, then the return value of that method will be merged into + # `extra`. Exceptions can use this mechanism to provide structured data + # to sentry in addition to their message and back-trace. + def log_exception(exception, extra = {}) + process_exception(exception, extra: extra) + end + + private + + def process_exception(exception, sentry: false, logging: true, extra:) + exception.try(:sentry_extra_data)&.tap do |data| + extra = extra.merge(data) if data.is_a?(Hash) + end + + if sentry && Raven.configuration.server + Raven.capture_exception(exception, tags: default_tags, extra: extra) + end + + if logging + # TODO: this logic could migrate into `Gitlab::ExceptionLogFormatter` + # and we could also flatten deep nested hashes if required for search + # (e.g. if `extra` includes hash of hashes). + # In the current implementation, we don't flatten multi-level folded hashes. + log_hash = {} + Raven.context.tags.each { |name, value| log_hash["tags.#{name}"] = value } + Raven.context.user.each { |name, value| log_hash["user.#{name}"] = value } + Raven.context.extra.merge(extra).each { |name, value| log_hash["extra.#{name}"] = value } + + Gitlab::ExceptionLogFormatter.format!(exception, log_hash) + + Gitlab::ErrorTracking::Logger.error(log_hash) + end + end + + def sentry_dsn + return unless Rails.env.production? || Rails.env.development? + return unless Gitlab.config.sentry.enabled + + Gitlab.config.sentry.dsn + end + + def should_raise_for_dev? + Rails.env.development? || Rails.env.test? + end + + def default_tags + { + Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id, + locale: I18n.locale + } + end + + def add_context_from_exception_type(event, hint) + if ActiveModel::MissingAttributeError === hint[:exception] + columns_hash = ActiveRecord::Base + .connection + .schema_cache + .instance_variable_get(:@columns_hash) + .map { |k, v| [k, v.map(&:first)] } + .to_h + + event.extra.merge!(columns_hash) + end + + event + end + end + end +end diff --git a/lib/gitlab/error_tracking/detailed_error.rb b/lib/gitlab/error_tracking/detailed_error.rb index 225280a42f4..169d6c03f12 100644 --- a/lib/gitlab/error_tracking/detailed_error.rb +++ b/lib/gitlab/error_tracking/detailed_error.rb @@ -4,6 +4,7 @@ module Gitlab module ErrorTracking class DetailedError include ActiveModel::Model + include GlobalID::Identification attr_accessor :count, :culprit, @@ -13,6 +14,8 @@ module Gitlab :first_release_short_version, :first_seen, :frequency, + :gitlab_project, + :gitlab_issue, :id, :last_release_last_commit, :last_release_short_version, @@ -26,6 +29,10 @@ module Gitlab :title, :type, :user_count + + def self.declarative_policy_class + 'ErrorTracking::DetailedErrorPolicy' + end end end end diff --git a/lib/gitlab/error_tracking/logger.rb b/lib/gitlab/error_tracking/logger.rb new file mode 100644 index 00000000000..1b081f943aa --- /dev/null +++ b/lib/gitlab/error_tracking/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'exceptions_json' + end + end + end +end diff --git a/lib/gitlab/error_tracking/stack_trace_highlight_decorator.rb b/lib/gitlab/error_tracking/stack_trace_highlight_decorator.rb new file mode 100644 index 00000000000..a403275fd4e --- /dev/null +++ b/lib/gitlab/error_tracking/stack_trace_highlight_decorator.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + module StackTraceHighlightDecorator + extend self + + def decorate(error_event) + ::Gitlab::ErrorTracking::ErrorEvent.new( + issue_id: error_event.issue_id, + date_received: error_event.date_received, + stack_trace_entries: highlight_stack_trace(error_event.stack_trace_entries) + ) + end + + private + + def highlight_stack_trace(stack_trace) + stack_trace.map do |entry| + highlight_stack_trace_entry(entry) + end + end + + def highlight_stack_trace_entry(entry) + return entry unless entry['context'] + + entry.merge('context' => highlight_entry_context(entry['filename'], entry['context'])) + end + + def highlight_entry_context(filename, context) + language = Rouge::Lexer.guess_by_filename(filename).tag + + context.map do |line_number, line_of_code| + [ + line_number, + # Passing nil for the blob name allows skipping linking dependencies for the line_of_code + Gitlab::Highlight.highlight(nil, line_of_code, language: language) + ] + end + end + end + end +end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index efddda0ec65..17d9cf08367 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -23,6 +23,10 @@ module Gitlab 'issue_notes' ), Gitlab::EtagCaching::Router::Route.new( + %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z), + 'merge_request_notes' + ), + Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z), 'issue_title' ), diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 948f720b01b..4fbf15d521a 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -14,7 +14,7 @@ module Gitlab signup_flow: { feature_toggle: :experimental_separate_sign_up_flow, environment: ::Gitlab.dev_env_or_com?, - enabled_ratio: 0.1, + enabled_ratio: 1, tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow' } }.freeze diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 9fc2217ad43..a386c21983d 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -36,7 +36,10 @@ module Gitlab podspec_json: %r{\A[^/]*\.podspec\.json\z}, podspec: %r{\A[^/]*\.podspec\z}, requirements_txt: %r{\A[^/]*requirements\.txt\z}, - yarn_lock: 'yarn.lock' + yarn_lock: 'yarn.lock', + + # OpenAPI Specification files + openapi: %r{.*(openapi|swagger).*\.(yaml|yml|json)\z}i }.freeze # Returns an Array of file types based on the given paths. diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb index ca78d49f99b..e052792675a 100644 --- a/lib/gitlab/file_type_detection.rb +++ b/lib/gitlab/file_type_detection.rb @@ -20,6 +20,7 @@ module Gitlab module FileTypeDetection SAFE_IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze + PDF_EXT = 'pdf' # We recommend using the .mp4 format over .mov. Videos in .mov format can # still be used but you really need to make sure they are served with the # proper MIME type video/mp4 and not video/quicktime or your videos won't play @@ -46,6 +47,10 @@ module Gitlab extension_match?(SAFE_AUDIO_EXT) end + def pdf? + extension_match?([PDF_EXT]) + end + def embeddable? image? || video? || audio? end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 6210223917b..b2dc9a8a3c8 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -370,15 +370,26 @@ module Gitlab # subject from the message to make it clearer when there's one # available but not the other. @message = message_from_gitaly_body - @authored_date = Time.at(commit.author.date.seconds).utc + @authored_date = init_date_from_gitaly(commit.author) @author_name = commit.author.name.dup @author_email = commit.author.email.dup - @committed_date = Time.at(commit.committer.date.seconds).utc + + @committed_date = init_date_from_gitaly(commit.committer) @committer_name = commit.committer.name.dup @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) end + # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone + # offset in author.timezone. If the latter isn't present, assume UTC. + def init_date_from_gitaly(author) + if author.timezone.present? + Time.strptime("#{author.date.seconds} #{author.timezone}", '%s %z') + else + Time.at(author.date.seconds).utc + end + end + def serialize_keys SERIALIZE_KEYS end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 23d989ff258..0218f6e6232 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -62,6 +62,10 @@ module Gitlab encode! @message end + def tagger + @raw_tag.tagger + end + private def message_from_gitaly_tag diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 5b47853b9c1..9e033c705bd 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -67,8 +67,7 @@ module Gitlab File.read(cert_file).scan(PEM_REGEX).map do |cert| OpenSSL::X509::Certificate.new(cert).to_pem rescue OpenSSL::OpenSSLError => e - Rails.logger.error "Could not load certificate #{cert_file} #{e}" # rubocop:disable Gitlab/RailsLogger - Gitlab::Sentry.track_exception(e, extra: { cert_file: cert_file }) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, cert_file: cert_file) nil end.compact end.uniq.join("\n") diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index a468f6d8821..8648cbaec9d 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -52,7 +52,6 @@ module Gitlab project_id: project.id, description: description, milestone_id: milestone_finder.id_for(issue), - state: issue.state, state_id: ::Issue.available_states[issue.state], created_at: issue.created_at, updated_at: issue.updated_at diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index 377e873d24d..6d2aff63a47 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -54,7 +54,6 @@ module Gitlab target_project_id: project.id, source_branch: pull_request.formatted_source_branch, target_branch: pull_request.target_branch, - state: pull_request.state, state_id: ::MergeRequest.available_states[pull_request.state], milestone_id: milestone_finder.id_for(pull_request), author_id: author_id, @@ -92,12 +91,10 @@ module Gitlab project.repository.add_branch(project.creator, source_branch, pull_request.source_branch_sha) rescue Gitlab::Git::CommandError => e - Gitlab::Sentry.track_acceptable_exception(e, - extra: { - source_branch: source_branch, - project_id: merge_request.project.id, - merge_request_id: merge_request.id - }) + Gitlab::ErrorTracking.track_exception(e, + source_branch: source_branch, + project_id: merge_request.project.id, + merge_request_id: merge_request.id) end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2616a19fdaa..f22c69c531a 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -41,7 +41,7 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend - push_frontend_feature_flag(:suppress_ajax_navigation_errors, default_enabled: true) + push_frontend_feature_flag(:snippets_vue, default_enabled: false) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 1dce26efc65..e3c474bc0fe 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -5,7 +5,7 @@ module Gitlab extend self CleanupError = Class.new(StandardError) - BG_CLEANUP_RUNTIME_S = 2 + BG_CLEANUP_RUNTIME_S = 10 FG_CLEANUP_RUNTIME_S = 0.5 MUTEX = Mutex.new @@ -107,19 +107,18 @@ module Gitlab begin cleanup_tmp_dir(tmp_dir) rescue CleanupError => e + folder_contents = Dir.children(tmp_dir) # This means we left a GPG-agent process hanging. Logging the problem in # sentry will make this more visible. - Gitlab::Sentry.track_exception(e, + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab/issues/20918', - extra: { tmp_dir: tmp_dir }) + tmp_dir: tmp_dir, contents: folder_contents) end tmp_keychains_removed.increment unless File.exist?(tmp_dir) end def cleanup_tmp_dir(tmp_dir) - return FileUtils.remove_entry(tmp_dir, true) if Feature.disabled?(:gpg_cleanup_retries) - # Retry when removing the tmp directory failed, as we may run into a # race condition: # The `gpg-agent` agent process may clean up some files as well while diff --git a/lib/gitlab/grafana_embed_usage_data.rb b/lib/gitlab/grafana_embed_usage_data.rb new file mode 100644 index 00000000000..78a87623e1f --- /dev/null +++ b/lib/gitlab/grafana_embed_usage_data.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + class GrafanaEmbedUsageData + class << self + def issue_count + # rubocop:disable CodeReuse/ActiveRecord + Issue.joins('JOIN grafana_integrations USING (project_id)') + .where("issues.description LIKE '%' || grafana_integrations.grafana_url || '%'") + .where(grafana_integrations: { enabled: true }) + .count + # rubocop:enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index df60b9d8346..26e8c53032f 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -6,6 +6,8 @@ module Gitlab module AuthorizeResource extend ActiveSupport::Concern + RESOURCE_ACCESS_ERROR = "The resource that you are attempting to access does not exist or you don't have permission to perform this action" + class_methods do def required_permissions # If the `#authorize` call is used on multiple classes, we add the @@ -38,8 +40,7 @@ module Gitlab def authorize!(object) unless authorized_resource?(object) - raise Gitlab::Graphql::Errors::ResourceNotAvailable, - "The resource that you are attempting to access does not exist or you don't have permission to perform this action" + raise_resource_not_avaiable_error! end end @@ -61,6 +62,10 @@ module Gitlab Ability.allowed?(current_user, ability, object, scope: :user) end end + + def raise_resource_not_avaiable_error! + raise Gitlab::Graphql::Errors::ResourceNotAvailable, RESOURCE_ACCESS_ERROR + end end end end diff --git a/lib/gitlab/graphql/calls_gitaly/instrumentation.rb b/lib/gitlab/graphql/calls_gitaly/instrumentation.rb index fbd5e348c7d..11d3c50e093 100644 --- a/lib/gitlab/graphql/calls_gitaly/instrumentation.rb +++ b/lib/gitlab/graphql/calls_gitaly/instrumentation.rb @@ -32,7 +32,7 @@ module Gitlab # Will inform you if there needs to be `calls_gitaly: true` as a kwarg in the field declaration # if there is at least 1 Gitaly call involved with the field resolution. error = RuntimeError.new("Gitaly is called for field '#{type_object.name}' on #{type_object.owner.try(:name)} - please either specify a constant complexity or add `calls_gitaly: true` to the field declaration") - Gitlab::Sentry.track_exception(error) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) end end end diff --git a/lib/gitlab/graphql/connections/keyset/connection.rb b/lib/gitlab/graphql/connections/keyset/connection.rb index c75ea206edb..5de075f2f7a 100644 --- a/lib/gitlab/graphql/connections/keyset/connection.rb +++ b/lib/gitlab/graphql/connections/keyset/connection.rb @@ -32,18 +32,11 @@ module Gitlab class Connection < GraphQL::Relay::BaseConnection include Gitlab::Utils::StrongMemoize - # TODO https://gitlab.com/gitlab-org/gitlab/issues/35104 - include Gitlab::Graphql::Connections::Keyset::LegacyKeysetConnection - def cursor_from_node(node) - return legacy_cursor_from_node(node) if use_legacy_pagination? - encoded_json_from_ordering(node) end def sliced_nodes - return legacy_sliced_nodes if use_legacy_pagination? - @sliced_nodes ||= begin OrderInfo.validate_ordering(ordered_nodes, order_list) @@ -137,14 +130,7 @@ module Gitlab def ordering_from_encoded_json(cursor) JSON.parse(decode(cursor)) rescue JSON::ParserError - # for the transition period where a client might request using an - # old style cursor. Once removed, make it an error: - # raise Gitlab::Graphql::Errors::ArgumentError, "Please provide a valid cursor" - # TODO can be removed in next release - # https://gitlab.com/gitlab-org/gitlab/issues/32933 - field_name = order_list.first.attribute_name - - { field_name => decode(cursor) } + raise Gitlab::Graphql::Errors::ArgumentError, "Please provide a valid cursor" end end end diff --git a/lib/gitlab/graphql/connections/keyset/legacy_keyset_connection.rb b/lib/gitlab/graphql/connections/keyset/legacy_keyset_connection.rb deleted file mode 100644 index baf900d1048..00000000000 --- a/lib/gitlab/graphql/connections/keyset/legacy_keyset_connection.rb +++ /dev/null @@ -1,66 +0,0 @@ -# frozen_string_literal: true - -# TODO https://gitlab.com/gitlab-org/gitlab/issues/35104 -module Gitlab - module Graphql - module Connections - module Keyset - module LegacyKeysetConnection - def legacy_cursor_from_node(node) - encode(node[legacy_order_field].to_s) - end - - # rubocop: disable CodeReuse/ActiveRecord - def legacy_sliced_nodes - @sliced_nodes ||= - begin - sliced = nodes - - sliced = sliced.where(legacy_before_slice) if before.present? - sliced = sliced.where(legacy_after_slice) if after.present? - - sliced - end - end - # rubocop: enable CodeReuse/ActiveRecord - - private - - def use_legacy_pagination? - strong_memoize(:feature_disabled) do - Feature.disabled?(:graphql_keyset_pagination, default_enabled: true) - end - end - - def legacy_before_slice - if legacy_sort_direction == :asc - arel_table[legacy_order_field].lt(decode(before)) - else - arel_table[legacy_order_field].gt(decode(before)) - end - end - - def legacy_after_slice - if legacy_sort_direction == :asc - arel_table[legacy_order_field].gt(decode(after)) - else - arel_table[legacy_order_field].lt(decode(after)) - end - end - - def legacy_order_info - @legacy_order_info ||= nodes.order_values.first - end - - def legacy_order_field - @legacy_order_field ||= legacy_order_info&.expr&.name || nodes.primary_key - end - - def legacy_sort_direction - @legacy_order_direction ||= legacy_order_info&.direction || :desc - end - end - end - end - end -end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb index 01b55a1667f..327a9c549d5 100644 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -18,7 +18,7 @@ module Gitlab variables: variables }) rescue => e - Gitlab::Sentry.track_exception(e) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) default_initial_values(query) end @@ -38,7 +38,7 @@ module Gitlab GraphqlLogger.info(memo.except!(:time_started, :query)) rescue => e - Gitlab::Sentry.track_exception(e) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end private diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 14727b03ce9..7965f165683 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -47,23 +47,13 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def self.legacy_attachments_relation - Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) - JOIN projects - ON (uploads.model_type='Project' AND uploads.model_id=projects.id) - SQL + Upload.inner_join_local_uploads_projects.merge(Project.without_storage_feature(:attachments)) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def self.hashed_attachments_relation - Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments]) - JOIN projects - ON (uploads.model_type='Project' AND uploads.model_id=projects.id) - SQL + Upload.inner_join_local_uploads_projects.merge(Project.with_storage_feature(:attachments)) end - # rubocop: enable CodeReuse/ActiveRecord def self.relation_summary(relation_name, relation) relation_count = relation.count diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 381f1dd4e55..2c243a0d0ae 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -61,7 +61,7 @@ module Gitlab tokens = lexer.lex(text, continue: continue) Timeout.timeout(timeout_time) { @formatter.format(tokens, tag: tag).html_safe } rescue Timeout::Error => e - Gitlab::Sentry.track_exception(e) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) highlight_plain(text) rescue highlight_plain(text) diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 0678799b64b..ae2ec424ce5 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -23,7 +23,7 @@ module Gitlab milestone_id source_branch source_project_id - state + state_id target_branch target_project_id time_estimate @@ -53,7 +53,8 @@ module Gitlab human_total_time_spent: merge_request.human_total_time_spent, human_time_estimate: merge_request.human_time_estimate, assignee_ids: merge_request.assignee_ids, - assignee_id: merge_request.assignee_ids.first # This key is deprecated + assignee_id: merge_request.assignee_ids.first, # This key is deprecated + state: merge_request.state # This key is deprecated } merge_request.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index 50fec9f3eb9..c8dbec7bcba 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -3,7 +3,7 @@ module Gitlab module ImportExport class AttributeCleaner - ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + %w[group_id commit_id] + ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + %w[group_id commit_id discussion_id] PROHIBITED_REFERENCES = Regexp.union(/\Acached_markdown_version\Z/, /_id\Z/, /_ids\Z/, /_html\Z/).freeze def self.clean(*args) diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 2fd12e3aa78..9d04d55770d 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -5,6 +5,8 @@ module Gitlab class FileImporter include Gitlab::ImportExport::CommandLineUtil + ImporterError = Class.new(StandardError) + MAX_RETRIES = 8 IGNORED_FILENAMES = %w(. ..).freeze @@ -12,8 +14,8 @@ module Gitlab new(*args).import end - def initialize(project:, archive_file:, shared:) - @project = project + def initialize(importable:, archive_file:, shared:) + @importable = importable @archive_file = archive_file @shared = shared end @@ -52,7 +54,7 @@ module Gitlab def decompress_archive result = untar_zxf(archive: @archive_file, dir: @shared.export_path) - raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + raise ImporterError.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result result end @@ -60,9 +62,9 @@ module Gitlab def copy_archive return if @archive_file - @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project)) + @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @importable)) - download_or_copy_upload(@project.import_export_upload.import_file, @archive_file) + download_or_copy_upload(@importable.import_export_upload.import_file, @archive_file) end def remove_symlinks diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group_import_export.yml index c1900350c86..049d81f96a4 100644 --- a/lib/gitlab/import_export/group_import_export.yml +++ b/lib/gitlab/import_export/group_import_export.yml @@ -14,9 +14,16 @@ tree: - :user included_attributes: + user: + - :id + - :email + - :username + author: + - :name excluded_attributes: group: + - :id - :runners_token - :runners_token_encrypted @@ -25,6 +32,8 @@ methods: - :type badges: - :type + notes: + - :type preloads: @@ -33,4 +42,11 @@ preloads: ee: tree: group: - - :epics + - epics: + - :parent + - notes: + - :author + - boards: + - :board_assignee + - labels: + - :priorities diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 1aafe5804c0..4f4b4c02eb9 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -29,6 +29,7 @@ tree: - :priorities - :issue_assignees - :zoom_meetings + - :sentry_issue - snippets: - :award_emoji - notes: @@ -72,6 +73,7 @@ tree: - :auto_devops - :triggers - :pipeline_schedules + - :container_expiration_policy - :services - protected_branches: - :merge_access_levels @@ -163,28 +165,43 @@ excluded_attributes: - :identifier snippets: - :expired_at + - :secret + - :encrypted_secret_token + - :encrypted_secret_token_iv merge_request_diff: - :external_diff - :stored_externally - :external_diff_store + - :merge_request_id + merge_request_diff_commits: + - :merge_request_diff_id merge_request_diff_files: - :diff - :external_diff_offset - :external_diff_size + - :merge_request_diff_id issues: - :milestone_id + - :moved_to_id + - :state_id + - :duplicated_to_id + - :promoted_to_epic_id merge_request: - :milestone_id - :ref_fetched - :merge_jid - :rebase_jid - :latest_merge_request_diff_id + - :head_pipeline_id + - :state_id merge_requests: - :milestone_id - :ref_fetched - :merge_jid - :rebase_jid - :latest_merge_request_diff_id + - :head_pipeline_id + - :state_id award_emoji: - :awardable_id statuses: @@ -198,6 +215,16 @@ excluded_attributes: - :artifacts_metadata_store - :artifacts_size - :commands + - :runner_id + - :trigger_request_id + - :erased_by_id + - :auto_canceled_by_id + - :stage_id + - :upstream_pipeline_id + - :resource_group_id + - :waiting_for_resource_at + sentry_issue: + - :issue_id push_event_payload: - :event_id project_badges: @@ -206,6 +233,9 @@ excluded_attributes: - :reference - :reference_html - :epic_id + - :issue_id + - :merge_request_id + - :label_id runners: - :token - :token_encrypted @@ -215,7 +245,66 @@ excluded_attributes: - :encrypted_token - :encrypted_token_iv - :enabled - + service_desk_setting: + - :outgoing_name + priorities: + - :label_id + events: + - :target_id + timelogs: + - :issue_id + - :merge_request_id + notes: + - :noteable_id + - :review_id + label_links: + - :label_id + - :target_id + issue_assignees: + - :issue_id + zoom_meetings: + - :issue_id + design: + - :issue_id + designs: + - :issue_id + design_versions: + - :issue_id + actions: + - :design_id + - :version_id + links: + - :release_id + project_members: + - :source_id + metrics: + - :merge_request_id + - :pipeline_id + suggestions: + - :note_id + ci_pipelines: + - :auto_canceled_by_id + - :pipeline_schedule_id + - :merge_request_id + - :external_pull_request_id + stages: + - :pipeline_id + merge_access_levels: + - :protected_branch_id + push_access_levels: + - :protected_branch_id + unprotect_access_levels: + - :protected_branch_id + create_access_levels: + - :protected_tag_id + deploy_access_levels: + - :protected_environment_id + boards: + - :milestone_id + lists: + - :board_id + - :label_id + - :milestone_id methods: notes: - :type @@ -267,8 +356,9 @@ ee: - :push_event_payload - design_versions: - actions: - - :design # Duplicate export of issues.designs in order to link the record to both Issue and DesignVersion + - :design # Duplicate export of issues.designs in order to link the record to both Issue and Action - protected_branches: - :unprotect_access_levels - protected_environments: - :deploy_access_levels + - :service_desk_setting diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 62cf6c86906..a6463ed678c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -39,7 +39,7 @@ module Gitlab end def import_file - Gitlab::ImportExport::FileImporter.import(project: project, + Gitlab::ImportExport::FileImporter.import(importable: project, archive_file: archive_file, shared: shared) end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 4e976cfca3a..d2e27388b51 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -3,10 +3,10 @@ module Gitlab module ImportExport class MembersMapper - def initialize(exported_members:, user:, project:) + def initialize(exported_members:, user:, importable:) @exported_members = user.admin? ? exported_members : [] @user = user - @project = project + @importable = importable # This needs to run first, as second call would be from #map # which means project members already exist. @@ -19,7 +19,7 @@ module Gitlab @exported_members.inject(missing_keys_tracking_hash) do |hash, member| if member['user'] old_user_id = member['user']['id'] - existing_user = User.where(find_project_user_query(member)).first + existing_user = User.where(find_user_query(member)).first hash[old_user_id] = existing_user.id if existing_user && add_team_member(member, existing_user) else add_team_member(member) @@ -47,39 +47,48 @@ module Gitlab end def ensure_default_member! - @project.project_members.destroy_all # rubocop: disable DestroyAll + @importable.members.destroy_all # rubocop: disable DestroyAll - ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) + relation_class.create!(user: @user, access_level: relation_class::MAINTAINER, source_id: @importable.id, importing: true) rescue => e - raise e, "Error adding importer user to project members. #{e.message}" + raise e, "Error adding importer user to #{@importable.class} members. #{e.message}" end def add_team_member(member, existing_user = nil) member['user'] = existing_user - ProjectMember.create(member_hash(member)).persisted? + relation_class.create(member_hash(member)).persisted? end def member_hash(member) parsed_hash(member).merge( - 'source_id' => @project.id, + 'source_id' => @importable.id, 'importing' => true, - 'access_level' => [member['access_level'], ProjectMember::MAINTAINER].min + 'access_level' => [member['access_level'], relation_class::MAINTAINER].min ).except('user_id') end def parsed_hash(member) - Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, - relation_class: ProjectMember) + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, + relation_class: relation_class) end - def find_project_user_query(member) + def find_user_query(member) user_arel[:email].eq(member['user']['email']).or(user_arel[:username].eq(member['user']['username'])) end def user_arel @user_arel ||= User.arel_table end + + def relation_class + case @importable + when Project + ProjectMember + when Group + GroupMember + end + end end end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c401f96b5c1..e274b68a94f 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -3,9 +3,6 @@ module Gitlab module ImportExport class ProjectTreeRestorer - # Relations which cannot be saved at project level (and have a group assigned) - GROUP_MODELS = [GroupLabel, Milestone].freeze - attr_reader :user attr_reader :shared attr_reader :project @@ -13,34 +10,23 @@ module Gitlab def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @user = user - @shared = shared + @shared = shared @project = project end def restore - begin - @tree_hash = read_tree_hash - rescue => e - Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') - end - + @tree_hash = read_tree_hash @project_members = @tree_hash.delete('project_members') RelationRenameService.rename(@tree_hash) - ActiveRecord::Base.uncached do - ActiveRecord::Base.no_touching do - update_project_params! - create_project_relations! - post_import! - end - end - - # ensure that we have latest version of the restore - @project.reload # rubocop:disable Cop/ActiveRecordAssociationReload + if relation_tree_restorer.restore + @project.merge_requests.set_latest_merge_request_diff_ids! - true + true + else + false + end rescue => e @shared.error(e) false @@ -51,173 +37,36 @@ module Gitlab def read_tree_hash json = IO.read(@path) ActiveSupport::JSON.decode(json) + rescue => e + Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') end - def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, - user: @user, - project: @project) - end - - # A Hash of the imported merge request ID -> imported ID. - def merge_requests_mapping - @merge_requests_mapping ||= {} - end - - # Loops through the tree of models defined in import_export.yml and - # finds them in the imported JSON so they can be instantiated and saved - # in the DB. The structure and relationships between models are guessed from - # the configuration yaml file too. - # Finally, it updates each attribute in the newly imported project. - def create_project_relations! - project_relations.each(&method( - :process_project_relation!)) - end - - def post_import! - @project.merge_requests.set_latest_merge_request_diff_ids! - end - - def process_project_relation!(relation_key, relation_definition) - data_hashes = @tree_hash.delete(relation_key) - return unless data_hashes - - # we do not care if we process array or hash - data_hashes = [data_hashes] unless data_hashes.is_a?(Array) - - # consume and remove objects from memory - while data_hash = data_hashes.shift - process_project_relation_item!(relation_key, relation_definition, data_hash) - end - end - - def process_project_relation_item!(relation_key, relation_definition, data_hash) - relation_object = build_relation(relation_key, relation_definition, data_hash) - return unless relation_object - return if group_model?(relation_object) - - relation_object.project = @project - relation_object.save! - - save_id_mapping(relation_key, data_hash, relation_object) - end - - # Older, serialized CI pipeline exports may only have a - # merge_request_id and not the full hash of the merge request. To - # import these pipelines, we need to preserve the mapping between - # the old and new the merge request ID. - def save_id_mapping(relation_key, data_hash, relation_object) - return unless relation_key == 'merge_requests' - - merge_requests_mapping[data_hash['id']] = relation_object.id - end - - def project_relations - @project_relations ||= - reader - .attributes_finder - .find_relations_tree(:project) - .deep_stringify_keys - end - - def update_project_params! - project_params = @tree_hash.reject do |key, value| - project_relations.include?(key) - end - - project_params = project_params.merge( - present_project_override_params) - - # Cleaning all imported and overridden params - project_params = Gitlab::ImportExport::AttributeCleaner.clean( - relation_hash: project_params, - relation_class: Project, - excluded_keys: excluded_keys_for_relation(:project)) - - @project.assign_attributes(project_params) - @project.drop_visibility_level! - - Gitlab::Timeless.timeless(@project) do - @project.save! - end - end - - def present_project_override_params - # we filter out the empty strings from the overrides - # keeping the default values configured - project_override_params.transform_values do |value| - value.is_a?(String) ? value.presence : value - end.compact - end - - def project_override_params - @project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {} - end - - def build_relations(relation_key, relation_definition, data_hashes) - data_hashes.map do |data_hash| - build_relation(relation_key, relation_definition, data_hash) - end.compact - end - - def build_relation(relation_key, relation_definition, data_hash) - # TODO: This is hack to not create relation for the author - # Rather make `RelationFactory#set_note_author` to take care of that - return data_hash if relation_key == 'author' - - # create relation objects recursively for all sub-objects - relation_definition.each do |sub_relation_key, sub_relation_definition| - transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) - end - - Gitlab::ImportExport::RelationFactory.create( - relation_sym: relation_key.to_sym, - relation_hash: data_hash, - members_mapper: members_mapper, - merge_requests_mapping: merge_requests_mapping, + def relation_tree_restorer + @relation_tree_restorer ||= RelationTreeRestorer.new( user: @user, - project: @project, - excluded_keys: excluded_keys_for_relation(relation_key)) + shared: @shared, + importable: @project, + tree_hash: @tree_hash, + members_mapper: members_mapper, + relation_factory: relation_factory, + reader: reader + ) end - def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) - sub_data_hash = data_hash[sub_relation_key] - return unless sub_data_hash - - # if object is a hash we can create simple object - # as it means that this is 1-to-1 vs 1-to-many - sub_data_hash = - if sub_data_hash.is_a?(Array) - build_relations( - sub_relation_key, - sub_relation_definition, - sub_data_hash).presence - else - build_relation( - sub_relation_key, - sub_relation_definition, - sub_data_hash) - end - - # persist object(s) or delete from relation - if sub_data_hash - data_hash[sub_relation_key] = sub_data_hash - else - data_hash.delete(sub_relation_key) - end + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + importable: @project) end - def group_model?(relation_object) - GROUP_MODELS.include?(relation_object.class) && relation_object.group_id + def relation_factory + Gitlab::ImportExport::RelationFactory end def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end - - def excluded_keys_for_relation(relation) - reader.attributes_finder.find_excluded_keys(relation) - end end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 5d907300d68..1438a7db001 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -38,12 +38,12 @@ module Gitlab IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request ProjectCiCdSetting].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request ProjectCiCdSetting container_expiration_policy].freeze TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze # This represents all relations that have unique key on `project_id` - UNIQUE_RELATIONS = %i[project_feature ProjectCiCdSetting].freeze + UNIQUE_RELATIONS = %i[project_feature ProjectCiCdSetting container_expiration_policy].freeze def self.create(*args) new(*args).create diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb new file mode 100644 index 00000000000..d9c253788b4 --- /dev/null +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -0,0 +1,240 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class RelationTreeRestorer + # Relations which cannot be saved at project level (and have a group assigned) + GROUP_MODELS = [GroupLabel, Milestone].freeze + + attr_reader :user + attr_reader :shared + attr_reader :importable + attr_reader :tree_hash + + def initialize(user:, shared:, importable:, tree_hash:, members_mapper:, relation_factory:, reader:) + @user = user + @shared = shared + @importable = importable + @tree_hash = tree_hash + @members_mapper = members_mapper + @relation_factory = relation_factory + @reader = reader + end + + def restore + ActiveRecord::Base.uncached do + ActiveRecord::Base.no_touching do + update_params! + create_relations! + end + end + + # ensure that we have latest version of the restore + @importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload + + true + rescue => e + @shared.error(e) + false + end + + private + + # Loops through the tree of models defined in import_export.yml and + # finds them in the imported JSON so they can be instantiated and saved + # in the DB. The structure and relationships between models are guessed from + # the configuration yaml file too. + # Finally, it updates each attribute in the newly imported project/group. + def create_relations! + relations.each(&method(:process_relation!)) + end + + def process_relation!(relation_key, relation_definition) + data_hashes = @tree_hash.delete(relation_key) + return unless data_hashes + + # we do not care if we process array or hash + data_hashes = [data_hashes] unless data_hashes.is_a?(Array) + + relation_index = 0 + + # consume and remove objects from memory + while data_hash = data_hashes.shift + process_relation_item!(relation_key, relation_definition, relation_index, data_hash) + relation_index += 1 + end + end + + def process_relation_item!(relation_key, relation_definition, relation_index, data_hash) + relation_object = build_relation(relation_key, relation_definition, data_hash) + return unless relation_object + return if importable_class == Project && group_model?(relation_object) + + relation_object.assign_attributes(importable_class_sym => @importable) + relation_object.save! + + save_id_mapping(relation_key, data_hash, relation_object) + rescue => e + # re-raise if not enabled + raise e unless Feature.enabled?(:import_graceful_failures, @importable.group, default_enabled: true) + + log_import_failure(relation_key, relation_index, e) + end + + def log_import_failure(relation_key, relation_index, exception) + Gitlab::ErrorTracking.track_exception(exception, + project_id: @importable.id, relation_key: relation_key, relation_index: relation_index) + + ImportFailure.create( + project: @importable, + relation_key: relation_key, + relation_index: relation_index, + exception_class: exception.class.to_s, + exception_message: exception.message.truncate(255), + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id + ) + end + + # Older, serialized CI pipeline exports may only have a + # merge_request_id and not the full hash of the merge request. To + # import these pipelines, we need to preserve the mapping between + # the old and new the merge request ID. + def save_id_mapping(relation_key, data_hash, relation_object) + return unless importable_class == Project + return unless relation_key == 'merge_requests' + + merge_requests_mapping[data_hash['id']] = relation_object.id + end + + def relations + @relations ||= + @reader + .attributes_finder + .find_relations_tree(importable_class_sym) + .deep_stringify_keys + end + + def update_params! + params = @tree_hash.reject do |key, _| + relations.include?(key) + end + + params = params.merge(present_override_params) + + # Cleaning all imported and overridden params + params = Gitlab::ImportExport::AttributeCleaner.clean( + relation_hash: params, + relation_class: importable_class, + excluded_keys: excluded_keys_for_relation(importable_class_sym)) + + @importable.assign_attributes(params) + @importable.drop_visibility_level! if importable_class == Project + + Gitlab::Timeless.timeless(@importable) do + @importable.save! + end + end + + def present_override_params + # we filter out the empty strings from the overrides + # keeping the default values configured + override_params&.transform_values do |value| + value.is_a?(String) ? value.presence : value + end&.compact + end + + def override_params + @importable_override_params ||= importable_override_params + end + + def importable_override_params + if @importable.respond_to?(:import_data) + @importable.import_data&.data&.fetch('override_params', nil) || {} + else + {} + end + end + + def build_relations(relation_key, relation_definition, data_hashes) + data_hashes.map do |data_hash| + build_relation(relation_key, relation_definition, data_hash) + end.compact + end + + def build_relation(relation_key, relation_definition, data_hash) + # TODO: This is hack to not create relation for the author + # Rather make `RelationFactory#set_note_author` to take care of that + return data_hash if relation_key == 'author' + + # create relation objects recursively for all sub-objects + relation_definition.each do |sub_relation_key, sub_relation_definition| + transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) + end + + @relation_factory.create(relation_factory_params(relation_key, data_hash)) + end + + def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) + sub_data_hash = data_hash[sub_relation_key] + return unless sub_data_hash + + # if object is a hash we can create simple object + # as it means that this is 1-to-1 vs 1-to-many + sub_data_hash = + if sub_data_hash.is_a?(Array) + build_relations( + sub_relation_key, + sub_relation_definition, + sub_data_hash).presence + else + build_relation( + sub_relation_key, + sub_relation_definition, + sub_data_hash) + end + + # persist object(s) or delete from relation + if sub_data_hash + data_hash[sub_relation_key] = sub_data_hash + else + data_hash.delete(sub_relation_key) + end + end + + def group_model?(relation_object) + GROUP_MODELS.include?(relation_object.class) && relation_object.group_id + end + + def excluded_keys_for_relation(relation) + @reader.attributes_finder.find_excluded_keys(relation) + end + + def importable_class + @importable.class + end + + def importable_class_sym + importable_class.to_s.downcase.to_sym + end + + # A Hash of the imported merge request ID -> imported ID. + def merge_requests_mapping + @merge_requests_mapping ||= {} + end + + def relation_factory_params(relation_key, data_hash) + base_params = { + relation_sym: relation_key.to_sym, + relation_hash: data_hash, + members_mapper: @members_mapper, + user: @user, + excluded_keys: excluded_keys_for_relation(relation_key) + } + + base_params[:merge_requests_mapping] = merge_requests_mapping if importable_class == Project + base_params[importable_class_sym] = @importable + base_params + end + end + end +end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 2539a6828c3..8d81b2af065 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -56,11 +56,7 @@ module Gitlab end def error(error) - error_payload = { message: error.message } - error_payload[:error_backtrace] = Gitlab::Profiler.clean_backtrace(error.backtrace) if error.backtrace - log_error(error_payload) - - Gitlab::Sentry.track_acceptable_exception(error, extra: log_base_data) + Gitlab::ErrorTracking.track_exception(error, log_base_data) add_error_message(error.message) end diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb index e4f0e9d2c73..7b1cf5e7931 100644 --- a/lib/gitlab/insecure_key_fingerprint.rb +++ b/lib/gitlab/insecure_key_fingerprint.rb @@ -10,6 +10,7 @@ module Gitlab # class InsecureKeyFingerprint attr_accessor :key + alias_attribute :fingerprint_md5, :fingerprint # # Gets the base64 encoded string representing a rsa or dsa key @@ -21,5 +22,9 @@ module Gitlab def fingerprint OpenSSL::Digest::MD5.hexdigest(Base64.decode64(@key)).scan(/../).join(':') end + + def fingerprint_sha256 + Digest::SHA256.base64digest(Base64.decode64(@key)).scan(/../).join('').delete("=") + end end end diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index 9aa71db6b18..4314c131ada 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -80,15 +80,10 @@ module Gitlab # when the new_record? method incorrectly returns false. # # See https://gitlab.com/gitlab-org/gitlab/issues/9903#note_145329964 - klass - .allocate - .init_with( - "attributes" => attributes_for(klass, raw), - "new_record" => new_record?(raw, klass) - ) + klass.allocate.init_with(encode_for(klass, raw)) end - def attributes_for(klass, raw) + def encode_for(klass, raw) # We have models that leave out some fields from the JSON export for # security reasons, e.g. models that include the CacheMarkdownField. # The ActiveRecord::AttributeSet we build from raw does know about @@ -96,7 +91,10 @@ module Gitlab missing_attributes = (klass.columns.map(&:name) - raw.keys) missing_attributes.each { |column| raw[column] = nil } - klass.attributes_builder.build_from_database(raw, {}) + coder = {} + klass.new(raw).encode_with(coder) + coder["new_record"] = new_record?(raw, klass) + coder end def new_record?(raw, klass) diff --git a/lib/gitlab/kubernetes/cluster_role.rb b/lib/gitlab/kubernetes/cluster_role.rb new file mode 100644 index 00000000000..4d40736a0b5 --- /dev/null +++ b/lib/gitlab/kubernetes/cluster_role.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class ClusterRole + attr_reader :name, :rules + + def initialize(name:, rules:) + @name = name + @rules = rules + end + + def generate + ::Kubeclient::Resource.new( + metadata: metadata, + rules: rules + ) + end + + private + + def metadata + { + name: name + } + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb index a3f732e1283..b953ce24c4a 100644 --- a/lib/gitlab/kubernetes/helm/client_command.rb +++ b/lib/gitlab/kubernetes/helm/client_command.rb @@ -5,14 +5,24 @@ module Gitlab module Helm module ClientCommand def init_command - # Here we are always upgrading to the latest version of Tiller when - # installing an app. We ensure the helm version stored in the - # database is correct by also updating this after transition to - # :installed,:updated in Clusters::Concerns::ApplicationStatus - 'helm init --upgrade' + if local_tiller_enabled? + <<~HEREDOC.chomp + export HELM_HOST="localhost:44134" + tiller -listen ${HELM_HOST} -alsologtostderr & + helm init --client-only + HEREDOC + else + # Here we are always upgrading to the latest version of Tiller when + # installing an app. We ensure the helm version stored in the + # database is correct by also updating this after transition to + # :installed,:updated in Clusters::Concerns::ApplicationStatus + 'helm init --upgrade' + end end def wait_for_tiller_command + return if local_tiller_enabled? + helm_check = ['helm', 'version', *optional_tls_flags].shelljoin # This is necessary to give Tiller time to restart after upgrade. # Ideally we'd be able to use --wait but cannot because of @@ -25,6 +35,18 @@ module Gitlab ['helm', 'repo', 'add', name, repository].shelljoin if repository end + private + + def tls_flags_if_remote_tiller + return [] if local_tiller_enabled? + + optional_tls_flags + end + + def repository_update_command + 'helm repo update' + end + def optional_tls_flags return [] unless files.key?(:'ca.pem') @@ -35,6 +57,10 @@ module Gitlab '--tls-key', "#{files_dir}/key.pem" ] end + + def local_tiller_enabled? + Feature.enabled?(:managed_apps_local_tiller) + end end end end diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb index dcf22e7abb6..9d0fd30ba8f 100644 --- a/lib/gitlab/kubernetes/helm/delete_command.rb +++ b/lib/gitlab/kubernetes/helm/delete_command.rb @@ -39,7 +39,7 @@ module Gitlab private def delete_command - command = ['helm', 'delete', '--purge', name] + optional_tls_flags + command = ['helm', 'delete', '--purge', name] + tls_flags_if_remote_tiller command.shelljoin end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index ccb053f507d..8e24cb4c24f 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -39,17 +39,13 @@ module Gitlab private - def repository_update_command - 'helm repo update' - end - # Uses `helm upgrade --install` which means we can use this for both # installation and uprade of applications def install_command command = ['helm', 'upgrade', name, chart] + install_flag + reset_values_flag + - optional_tls_flags + + tls_flags_if_remote_tiller + optional_version_flag + rbac_create_flag + namespace_flag + diff --git a/lib/gitlab/kubernetes/helm/patch_command.rb b/lib/gitlab/kubernetes/helm/patch_command.rb new file mode 100644 index 00000000000..ed7a5c2b2d6 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/patch_command.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +# PatchCommand is for updating values in installed charts without overwriting +# existing values. +module Gitlab + module Kubernetes + module Helm + class PatchCommand + include BaseCommand + include ClientCommand + + attr_reader :name, :files, :chart, :repository + attr_accessor :version + + def initialize(name:, chart:, files:, rbac:, version:, repository: nil) + # version is mandatory to prevent chart mismatches + # we do not want our values interpreted in the context of the wrong version + raise ArgumentError, 'version is required' if version.blank? + + @name = name + @chart = chart + @version = version + @rbac = rbac + @files = files + @repository = repository + end + + def generate_script + super + [ + init_command, + wait_for_tiller_command, + repository_command, + repository_update_command, + upgrade_command + ].compact.join("\n") + end + + def rbac? + @rbac + end + + private + + def upgrade_command + command = ['helm', 'upgrade', name, chart] + + reuse_values_flag + + tls_flags_if_remote_tiller + + version_flag + + namespace_flag + + value_flag + + command.shelljoin + end + + def reuse_values_flag + ['--reuse-values'] + end + + def value_flag + ['-f', "/data/helm/#{name}/config/values.yaml"] + end + + def namespace_flag + ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] + end + + def version_flag + ['--version', version] + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 66c28a9b702..b23ca095414 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -56,6 +56,7 @@ module Gitlab # group client delegate :create_cluster_role_binding, :get_cluster_role_binding, + :get_cluster_role_bindings, :update_cluster_role_binding, to: :rbac_client @@ -68,6 +69,13 @@ module Gitlab # RBAC methods delegates to the apis/rbac.authorization.k8s.io api # group client + delegate :create_cluster_role, + :get_cluster_role, + :update_cluster_role, + to: :rbac_client + + # RBAC methods delegates to the apis/rbac.authorization.k8s.io api + # group client delegate :create_role_binding, :get_role_binding, :update_role_binding, diff --git a/lib/gitlab/kubernetes/kubectl_cmd.rb b/lib/gitlab/kubernetes/kubectl_cmd.rb index 981eb5681dc..e8fde28b44d 100644 --- a/lib/gitlab/kubernetes/kubectl_cmd.rb +++ b/lib/gitlab/kubernetes/kubectl_cmd.rb @@ -13,6 +13,16 @@ module Gitlab %w(kubectl apply -f).concat([filename], args).shelljoin end + + def delete_crds_from_group(group) + api_resources_args = %w(-o name --api-group).push(group) + + api_resources(*api_resources_args) + " | xargs " + delete('--ignore-not-found', 'crd') + end + + def api_resources(*args) + %w(kubectl api-resources).concat(args).shelljoin + end end end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 78f2d83c1af..f7699ef1718 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -4,15 +4,21 @@ require 'yaml' require 'json' require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) +# This service is run independently of the main Rails process, +# therefore the `Rails` class and its methods are unavailable. + module Gitlab module MailRoom + RAILS_ROOT_DIR = Pathname.new('../..').expand_path(__dir__).freeze + DEFAULT_CONFIG = { enabled: false, port: 143, ssl: false, start_tls: false, mailbox: 'inbox', - idle_timeout: 60 + idle_timeout: 60, + log_path: RAILS_ROOT_DIR.join('log', 'mail_room_json.log') }.freeze class << self @@ -33,7 +39,7 @@ module Gitlab def fetch_config return {} unless File.exist?(config_file) - config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys[:incoming_email] || {} + config = load_from_yaml || {} config = DEFAULT_CONFIG.merge(config) do |_key, oldval, newval| newval.nil? ? oldval : newval end @@ -47,6 +53,7 @@ module Gitlab end end + config[:log_path] = File.expand_path(config[:log_path], RAILS_ROOT_DIR) config end @@ -57,6 +64,10 @@ module Gitlab def config_file ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../config/gitlab.yml', __dir__) end + + def load_from_yaml + YAML.load_file(config_file)[rails_env].deep_symbolize_keys[:incoming_email] + end end end end diff --git a/lib/gitlab/marginalia.rb b/lib/gitlab/marginalia.rb new file mode 100644 index 00000000000..2be96cecae3 --- /dev/null +++ b/lib/gitlab/marginalia.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Marginalia + MARGINALIA_FEATURE_FLAG = :marginalia + + def self.set_application_name + ::Marginalia.application_name = Gitlab.process_name + end + + def self.enable_sidekiq_instrumentation + if Sidekiq.server? + ::Marginalia::SidekiqInstrumentation.enable! + end + end + + def self.cached_feature_enabled? + !!@enabled + end + + def self.set_feature_cache + # During db:create and db:bootstrap skip feature query as DB is not available yet. + return false unless ActiveRecord::Base.connected? && Gitlab::Database.cached_table_exists?('features') + + @enabled = Feature.enabled?(MARGINALIA_FEATURE_FLAG) + end + end +end diff --git a/lib/gitlab/marginalia/active_record_instrumentation.rb b/lib/gitlab/marginalia/active_record_instrumentation.rb new file mode 100644 index 00000000000..3266b9f8336 --- /dev/null +++ b/lib/gitlab/marginalia/active_record_instrumentation.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Patch to annotate sql only when the feature is enabled. +module Gitlab + module Marginalia + module ActiveRecordInstrumentation + def annotate_sql(sql) + Gitlab::Marginalia.cached_feature_enabled? ? super(sql) : sql + end + end + end +end diff --git a/lib/gitlab/marginalia/comment.rb b/lib/gitlab/marginalia/comment.rb new file mode 100644 index 00000000000..a0eee823763 --- /dev/null +++ b/lib/gitlab/marginalia/comment.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# Module to support correlation_id and additional job details. +module Gitlab + module Marginalia + module Comment + private + + def jid + bg_job["jid"] if bg_job.present? + end + + def job_class + bg_job["class"] if bg_job.present? + end + + def correlation_id + if bg_job.present? + bg_job["correlation_id"] + else + Labkit::Correlation::CorrelationId.current_id + end + end + + def bg_job + job = ::Marginalia::Comment.marginalia_job + + # We are using 'Marginalia::SidekiqInstrumentation' which does not support 'ActiveJob::Base'. + # Gitlab also uses 'ActionMailer::DeliveryJob' which inherits from ActiveJob::Base. + # So below condition is used to return metadata for such jobs. + if job && job.is_a?(ActionMailer::DeliveryJob) + { + "class" => job.arguments.first, + "jid" => job.job_id + } + else + job + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb index aee7f6685ad..5b6f25420e0 100644 --- a/lib/gitlab/metrics/dashboard/service_selector.rb +++ b/lib/gitlab/metrics/dashboard/service_selector.rb @@ -22,6 +22,7 @@ module Gitlab return SERVICES::DynamicEmbedService if dynamic_embed?(params) return SERVICES::DefaultEmbedService if params[:embedded] return SERVICES::SystemDashboardService if system_dashboard?(params[:dashboard_path]) + return SERVICES::PodDashboardService if pod_dashboard?(params[:dashboard_path]) return SERVICES::ProjectDashboardService if params[:dashboard_path] default_service @@ -34,7 +35,11 @@ module Gitlab end def system_dashboard?(filepath) - SERVICES::SystemDashboardService.system_dashboard?(filepath) + SERVICES::SystemDashboardService.matching_dashboard?(filepath) + end + + def pod_dashboard?(filepath) + SERVICES::PodDashboardService.matching_dashboard?(filepath) end def custom_metric_embed?(params) diff --git a/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb index c00ef208848..4f5e9a98799 100644 --- a/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb @@ -16,12 +16,20 @@ module Gitlab private def endpoint_for_metric(metric) - Gitlab::Routing.url_helpers.prometheus_api_project_environment_path( - project, - params[:environment], - proxy_path: query_type(metric), - query: query_for_metric(metric) - ) + if ENV['USE_SAMPLE_METRICS'] + Gitlab::Routing.url_helpers.sample_metrics_project_environment_path( + project, + params[:environment], + identifier: metric[:id] + ) + else + Gitlab::Routing.url_helpers.prometheus_api_project_environment_path( + project, + params[:environment], + proxy_path: query_type(metric), + query: query_for_metric(metric) + ) + end end def query_type(metric) diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 115368c8bc6..552eae639e6 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -164,7 +164,6 @@ module Gitlab docstring 'Transaction allocated memory bytes' base_labels BASE_LABELS buckets [100, 1000, 10000, 100000, 1000000, 10000000] - with_feature :prometheus_metrics_transaction_allocated_memory end def self.transaction_metric(name, type, prefix: nil, tags: {}) diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index f9efef38825..f207d91235f 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -25,13 +25,13 @@ module Gitlab def render_go_doc(request) return unless go_request?(request) - path = project_path(request) + path, branch = project_path(request) return unless path - body = go_body(path) + body, code = go_response(path, branch) return unless body - response = Rack::Response.new(body, 200, { 'Content-Type' => 'text/html' }) + response = Rack::Response.new(body, code, { 'Content-Type' => 'text/html' }) response.finish end @@ -39,8 +39,15 @@ module Gitlab request["go-get"].to_i == 1 && request.env["PATH_INFO"].present? end - def go_body(path) + def go_response(path, branch) config = Gitlab.config + body_tag = content_tag :body, "go get #{config.gitlab.url}/#{path}" + + unless branch + html_tag = content_tag :html, body_tag + return html_tag, 404 + end + project_url = Gitlab::Utils.append_path(config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) @@ -52,9 +59,11 @@ module Gitlab "#{project_url}.git" end - meta_tag = tag :meta, name: 'go-import', content: "#{import_prefix} git #{repository_url}" - head_tag = content_tag :head, meta_tag - content_tag :html, head_tag + meta_import_tag = tag :meta, name: 'go-import', content: "#{import_prefix} git #{repository_url}" + meta_source_tag = tag :meta, name: 'go-source', content: "#{import_prefix} #{project_url} #{project_url}/tree/#{branch}{/dir} #{project_url}/blob/#{branch}{/dir}/{file}#L{line}" + head_tag = content_tag :head, meta_import_tag + meta_source_tag + html_tag = content_tag :html, head_tag + body_tag + [html_tag, 200] end def strip_url(url) @@ -80,9 +89,6 @@ module Gitlab path_segments = path.split('/') simple_project_path = path_segments.first(2).join('/') - # If the path is at most 2 segments long, it is a simple `namespace/project` path and we're done - return simple_project_path if path_segments.length <= 2 - project_paths = [] begin project_paths << path_segments.join('/') @@ -94,7 +100,7 @@ module Gitlab if project # If a project is found and the user has access, we return the full project path - project.full_path + return project.full_path, project.default_branch else # If not, we return the first two components as if it were a simple `namespace/project` path, # so that we don't reveal the existence of a nested project the user doesn't have access to. @@ -105,7 +111,7 @@ module Gitlab # `go get gitlab.com/group/subgroup/project/subpackage` will not work for private projects. # `go get gitlab.com/group/subgroup/project.git/subpackage` will work, since Go is smart enough # to figure that out. `import 'gitlab.com/...'` behaves the same as `go get`. - simple_project_path + return simple_project_path, 'master' end end diff --git a/lib/gitlab/pagination/keyset.rb b/lib/gitlab/pagination/keyset.rb new file mode 100644 index 00000000000..5bd45fa9b56 --- /dev/null +++ b/lib/gitlab/pagination/keyset.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + def self.paginate(request_context, relation) + Gitlab::Pagination::Keyset::Pager.new(request_context).paginate(relation) + end + + def self.available?(request_context, relation) + order_by = request_context.page.order_by + + # This is only available for Project and order-by id (asc/desc) + return false unless relation.klass == Project + return false unless order_by.size == 1 && order_by[:id] + + true + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/page.rb b/lib/gitlab/pagination/keyset/page.rb new file mode 100644 index 00000000000..735f54faf0f --- /dev/null +++ b/lib/gitlab/pagination/keyset/page.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + # A Page models the pagination information for a particular page of the collection + class Page + # Default number of records for a page + DEFAULT_PAGE_SIZE = 20 + + # Maximum number of records for a page + MAXIMUM_PAGE_SIZE = 100 + + attr_accessor :lower_bounds, :end_reached + attr_reader :order_by + + def initialize(order_by: {}, lower_bounds: nil, per_page: DEFAULT_PAGE_SIZE, end_reached: false) + @order_by = order_by.symbolize_keys + @lower_bounds = lower_bounds&.symbolize_keys + @per_page = per_page + @end_reached = end_reached + end + + # Number of records to return per page + def per_page + return DEFAULT_PAGE_SIZE if @per_page <= 0 + + [@per_page, MAXIMUM_PAGE_SIZE].min + end + + # Determine whether this page indicates the end of the collection + def end_reached? + @end_reached + end + + # Construct a Page for the next page + # Uses identical order_by/per_page information for the next page + def next(lower_bounds, end_reached) + dup.tap do |next_page| + next_page.lower_bounds = lower_bounds&.symbolize_keys + next_page.end_reached = end_reached + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/pager.rb b/lib/gitlab/pagination/keyset/pager.rb new file mode 100644 index 00000000000..99b125cc2a0 --- /dev/null +++ b/lib/gitlab/pagination/keyset/pager.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class Pager + attr_reader :request + + def initialize(request) + @request = request + end + + def paginate(relation) + # Validate assumption: The last two columns must match the page order_by + validate_order!(relation) + + # This performs the database query and retrieves records + # We retrieve one record more to check if we have data beyond this page + all_records = relation.limit(page.per_page + 1).to_a # rubocop: disable CodeReuse/ActiveRecord + + records_for_page = all_records.first(page.per_page) + + # If we retrieved more records than belong on this page, + # we know there's a next page + there_is_more = all_records.size > records_for_page.size + apply_headers(records_for_page.last, there_is_more) + + records_for_page + end + + private + + def apply_headers(last_record_in_page, there_is_more) + end_reached = last_record_in_page.nil? || !there_is_more + lower_bounds = last_record_in_page&.slice(page.order_by.keys) + + next_page = page.next(lower_bounds, end_reached) + + request.apply_headers(next_page) + end + + def page + @page ||= request.page + end + + def validate_order!(rel) + present_order = rel.order_values.map { |val| [val.expr.name.to_sym, val.direction] }.last(2).to_h + + unless page.order_by == present_order + raise ArgumentError, "Page's order_by does not match the relation's order: #{present_order} vs #{page.order_by}" + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/request_context.rb b/lib/gitlab/pagination/keyset/request_context.rb new file mode 100644 index 00000000000..aeaed7587b3 --- /dev/null +++ b/lib/gitlab/pagination/keyset/request_context.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class RequestContext + attr_reader :request + + DEFAULT_SORT_DIRECTION = :desc + PRIMARY_KEY = :id + + # A tie breaker is added as an additional order-by column + # to establish a well-defined order. We use the primary key + # column here. + TIE_BREAKER = { PRIMARY_KEY => DEFAULT_SORT_DIRECTION }.freeze + + def initialize(request) + @request = request + end + + # extracts Paging information from request parameters + def page + @page ||= Page.new(order_by: order_by, per_page: params[:per_page]) + end + + def apply_headers(next_page) + request.header('Links', pagination_links(next_page)) + end + + private + + def order_by + return TIE_BREAKER.dup unless params[:order_by] + + order_by = { params[:order_by].to_sym => params[:sort]&.to_sym || DEFAULT_SORT_DIRECTION } + + # Order by an additional unique key, we use the primary key here + order_by = order_by.merge(TIE_BREAKER) unless order_by[PRIMARY_KEY] + + order_by + end + + def params + @params ||= request.params + end + + def lower_bounds_params(page) + page.lower_bounds.each_with_object({}) do |(column, value), params| + filter = filter_with_comparator(page, column) + params[filter] = value + end + end + + def filter_with_comparator(page, column) + direction = page.order_by[column] + + if direction&.to_sym == :desc + "#{column}_before" + else + "#{column}_after" + end + end + + def page_href(page) + base_request_uri.tap do |uri| + uri.query = query_params_for(page).to_query + end.to_s + end + + def pagination_links(next_page) + return if next_page.end_reached? + + %(<#{page_href(next_page)}>; rel="next") + end + + def base_request_uri + @base_request_uri ||= URI.parse(request.request.url).tap do |uri| + uri.host = Gitlab.config.gitlab.host + uri.port = Gitlab.config.gitlab.port + end + end + + def query_params_for(page) + request.params.merge(lower_bounds_params(page)) + end + end + end + end +end diff --git a/lib/gitlab/patch/draw_route.rb b/lib/gitlab/patch/draw_route.rb index 4c8ca015974..4d1b57fbbbb 100644 --- a/lib/gitlab/patch/draw_route.rb +++ b/lib/gitlab/patch/draw_route.rb @@ -10,7 +10,7 @@ module Gitlab RoutesNotFound = Class.new(StandardError) def draw(routes_name) - drawn_any = draw_ce(routes_name) | draw_ee(routes_name) + drawn_any = draw_ee(routes_name) | draw_ce(routes_name) drawn_any || raise(RoutesNotFound.new("Cannot find #{routes_name}")) end diff --git a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb index cddd4f18cc3..805283b0f93 100644 --- a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb +++ b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb @@ -5,7 +5,7 @@ module Gitlab module PerformanceBar module RedisAdapterWhenPeekEnabled def save(request_id) - super if ::Gitlab::PerformanceBar.enabled_for_request? && request_id.present? + super if ::Gitlab::PerformanceBar.enabled_for_request? end end end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 560618bb486..f2f6180c464 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -118,6 +118,8 @@ module Gitlab end def self.clean_backtrace(backtrace) + return unless backtrace + Array(Rails.backtrace_cleaner.clean(backtrace)).reject do |line| line.match(Regexp.union(IGNORE_BACKTRACES)) end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 279fc4aa375..b4ee8818925 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -54,6 +54,7 @@ module Gitlab ProjectTemplate.new('nfplainhtml', 'Netlify/Plain HTML', _('A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfplain-html', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfgitbook', 'Netlify/GitBook', _('A GitBook site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfgitbook', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('salesforcedx', 'SalesforceDX', _('A project boilerplate for Salesforce App development with Salesforce Developer tools.'), 'https://gitlab.com/gitlab-org/project-templates/salesforcedx'), ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg') ].freeze diff --git a/lib/gitlab/puma_logging/json_formatter.rb b/lib/gitlab/puma_logging/json_formatter.rb new file mode 100644 index 00000000000..9eeb980fc53 --- /dev/null +++ b/lib/gitlab/puma_logging/json_formatter.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'json' + +module Gitlab + module PumaLogging + class JSONFormatter + def call(str) + { timestamp: Time.now.utc.iso8601(3), pid: $$, message: str }.to_json + end + end + end +end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 93c0f3132d0..333f848df9b 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -16,11 +16,12 @@ module Gitlab ] }, ci: { - keys: [:skip] + keys: [:skip, :variable] } }).freeze MULTI_VALUE_OPTIONS = [ + %w[ci variable], %w[merge_request label], %w[merge_request unlabel] ].freeze diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 838aefb59f0..b0aae363749 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -183,6 +183,7 @@ module Gitlab command :zoom do |link| result = @zoom_service.add_link(link) @execution_message[:zoom] = result.message + @updates.merge!(result.payload) if result.payload end desc _('Remove Zoom meeting') diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index ea2b03b42c1..f095ac9ffd1 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -3,7 +3,7 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor - REFERABLES = %i(user issue label milestone + REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project merge_request snippet commit commit_range directly_addressed_user epic).freeze attr_accessor :project, :current_user, :author diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index e3a434dfe35..d9300da38a5 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -19,7 +19,7 @@ module Gitlab # See https://github.com/docker/distribution/blob/master/reference/regexp.go. # def container_repository_name_regex - @container_repository_regex ||= %r{\A[a-z0-9]+((?:[._/]|__|[-])[a-z0-9]+)*\Z} + @container_repository_regex ||= %r{\A[a-z0-9]+((?:[._/]|__|[-]{0,10})[a-z0-9]+)*\Z} end ## diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index f96346322db..53cbd5b21ea 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -16,6 +16,8 @@ module Gitlab class Seeder extend ActionView::Helpers::NumberHelper + MASS_INSERT_PROJECT_START = 'mass_insert_project_' + MASS_INSERT_USER_START = 'mass_insert_user_' ESTIMATED_INSERT_PER_MINUTE = 2_000_000 MASS_INSERT_ENV = 'MASS_INSERT' @@ -24,7 +26,7 @@ module Gitlab included do scope :not_mass_generated, -> do - where.not("path LIKE '#{Gitlab::Seeder::Projects::MASS_INSERT_NAME_START}%'") + where.not("path LIKE '#{MASS_INSERT_PROJECT_START}%'") end end end @@ -34,7 +36,7 @@ module Gitlab included do scope :not_mass_generated, -> do - where.not("username LIKE '#{Gitlab::Seeder::Users::MASS_INSERT_USERNAME_START}%'") + where.not("username LIKE '#{MASS_INSERT_USER_START}%'") end end end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb deleted file mode 100644 index 005cb3112b8..00000000000 --- a/lib/gitlab/sentry.rb +++ /dev/null @@ -1,77 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Sentry - def self.enabled? - (Rails.env.production? || Rails.env.development?) && - Gitlab.config.sentry.enabled - end - - def self.context(current_user = nil) - return unless enabled? - - Raven.tags_context(default_tags) - - if current_user - Raven.user_context( - id: current_user.id, - email: current_user.email, - username: current_user.username - ) - end - end - - # This can be used for investigating exceptions that can be recovered from in - # code. The exception will still be raised in development and test - # environments. - # - # That way we can track down these exceptions with as much information as we - # need to resolve them. - # - # Provide an issue URL for follow up. - def self.track_exception(exception, issue_url: nil, extra: {}) - track_acceptable_exception(exception, issue_url: issue_url, extra: extra) - - raise exception if should_raise_for_dev? - end - - # This should be used when you do not want to raise an exception in - # development and test. If you need development and test to behave - # just the same as production you can use this instead of - # track_exception. - # - # If the exception implements the method `sentry_extra_data` and that method - # returns a Hash, then the return value of that method will be merged into - # `extra`. Exceptions can use this mechanism to provide structured data - # to sentry in addition to their message and back-trace. - def self.track_acceptable_exception(exception, issue_url: nil, extra: {}) - if enabled? - extra = build_extra_data(exception, issue_url, extra) - context # Make sure we've set everything we know in the context - - Raven.capture_exception(exception, tags: default_tags, extra: extra) - end - end - - def self.should_raise_for_dev? - Rails.env.development? || Rails.env.test? - end - - def self.default_tags - { - Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id, - locale: I18n.locale - } - end - - def self.build_extra_data(exception, issue_url, extra) - exception.try(:sentry_extra_data)&.tap do |data| - extra.merge!(data) if data.is_a?(Hash) - end - - extra.merge({ issue_url: issue_url }.compact) - end - - private_class_method :build_extra_data - end -end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 28e5d0ba8f5..290c4cff329 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -126,7 +126,7 @@ module Gitlab true rescue => e - Gitlab::Sentry.track_acceptable_exception(e, extra: { path: path, new_path: new_path, storage: storage }) + Gitlab::ErrorTracking.track_exception(e, path: path, new_path: new_path, storage: storage) false end @@ -158,7 +158,7 @@ module Gitlab true rescue => e Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") # rubocop:disable Gitlab/RailsLogger - Gitlab::Sentry.track_acceptable_exception(e, extra: { path: name, storage: storage }) + Gitlab::ErrorTracking.track_exception(e, path: name, storage: storage) false end @@ -267,7 +267,7 @@ module Gitlab def mv_namespace(storage, old_name, new_name) Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name) rescue GRPC::InvalidArgument => e - Gitlab::Sentry.track_acceptable_exception(e, extra: { old_name: old_name, new_name: new_name, storage: storage }) + Gitlab::ErrorTracking.track_exception(e, old_name: old_name, new_name: new_name, storage: storage) false end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb new file mode 100644 index 00000000000..c6726dcfa67 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + # The SidekiqMiddleware class is responsible for configuring the + # middleware stacks used in the client and server middlewares + module SidekiqMiddleware + # The result of this method should be passed to + # Sidekiq's `config.server_middleware` method + # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)` + def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true, request_store: true) + lambda do |chain| + chain.add Gitlab::SidekiqMiddleware::Monitor + chain.add Gitlab::SidekiqMiddleware::Metrics if metrics + chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger + chain.add Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer + chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware if request_store + chain.add Gitlab::SidekiqMiddleware::BatchLoader + chain.add Gitlab::SidekiqMiddleware::CorrelationLogger + chain.add Gitlab::SidekiqMiddleware::InstrumentationLogger + chain.add Gitlab::SidekiqStatus::ServerMiddleware + end + end + + # The result of this method should be passed to + # Sidekiq's `config.client_middleware` method + # eg: `config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)` + def self.client_configurator + lambda do |chain| + chain.add Gitlab::SidekiqStatus::ClientMiddleware + chain.add Gitlab::SidekiqMiddleware::CorrelationInjector + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb index bd819843bd4..7bfb0d54d80 100644 --- a/lib/gitlab/sidekiq_middleware/metrics.rb +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -7,14 +7,17 @@ module Gitlab # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze + TRUE_LABEL = "yes" + FALSE_LABEL = "no" + def initialize @metrics = init_metrics @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) end - def call(_worker, job, queue) - labels = create_labels(queue) + def call(worker, job, queue) + labels = create_labels(worker.class, queue) queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration @@ -42,7 +45,7 @@ module Gitlab @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded # job_status: done, fail match the job_status attribute in structured logging - labels[:job_status] = job_succeeded ? :done : :fail + labels[:job_status] = job_succeeded ? "done" : "fail" @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) end @@ -62,10 +65,24 @@ module Gitlab } end - def create_labels(queue) - { - queue: queue - } + def create_labels(worker_class, queue) + labels = { queue: queue.to_s, latency_sensitive: FALSE_LABEL, external_dependencies: FALSE_LABEL, feature_category: "", boundary: "" } + return labels unless worker_class.include? WorkerAttributes + + labels[:latency_sensitive] = bool_as_label(worker_class.latency_sensitive_worker?) + labels[:external_dependencies] = bool_as_label(worker_class.worker_has_external_dependencies?) + + feature_category = worker_class.get_feature_category + labels[:feature_category] = feature_category.to_s + + resource_boundary = worker_class.get_worker_resource_boundary + labels[:boundary] = resource_boundary == :unknown ? "" : resource_boundary.to_s + + labels + end + + def bool_as_label(value) + value ? TRUE_LABEL : FALSE_LABEL end def get_thread_cputime diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb index fbc3cf2e049..c9c5c6da3bf 100644 --- a/lib/gitlab/slash_commands/presenters/access.rb +++ b/lib/gitlab/slash_commands/presenters/access.rb @@ -34,8 +34,8 @@ module Gitlab def authorize message = - if @resource - ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{@resource})." + if resource + ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{resource})." else ":sweat_smile: Couldn't identify you, nor can I authorize you!" end diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index 73814aa180f..54d74ed3998 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -18,6 +18,8 @@ module Gitlab private + attr_reader :resource + def header_with_list(header, items) message = [header] @@ -67,12 +69,51 @@ module Gitlab def resource_url url_for( [ - @resource.project.namespace.becomes(Namespace), - @resource.project, - @resource + resource.project.namespace.becomes(Namespace), + resource.project, + resource ] ) end + + def project_link + "[#{project.full_name}](#{project.web_url})" + end + + def author_profile_link + "[#{author.to_reference}](#{url_for(author)})" + end + + def response_message(custom_pretext: pretext) + { + attachments: [ + { + title: "#{issue.title} · #{issue.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url, + fallback: fallback_message, + pretext: custom_pretext, + text: text, + color: color(resource), + fields: fields, + mrkdwn_in: fields_with_markdown + } + ] + } + end + + def pretext + '' + end + + def text + '' + end + + def fields_with_markdown + %i(title pretext fields) + end end end end diff --git a/lib/gitlab/slash_commands/presenters/issue_base.rb b/lib/gitlab/slash_commands/presenters/issue_base.rb index 0be31e234b5..4bc05d1f318 100644 --- a/lib/gitlab/slash_commands/presenters/issue_base.rb +++ b/lib/gitlab/slash_commands/presenters/issue_base.rb @@ -42,17 +42,11 @@ module Gitlab ] end - def project_link - "[#{project.full_name}](#{project.web_url})" - end - - def author_profile_link - "[#{author.to_reference}](#{url_for(author)})" - end - private attr_reader :resource + + alias_method :issue, :resource end end end diff --git a/lib/gitlab/slash_commands/presenters/issue_close.rb b/lib/gitlab/slash_commands/presenters/issue_close.rb index b3f24f4296a..f8d9af2c3c6 100644 --- a/lib/gitlab/slash_commands/presenters/issue_close.rb +++ b/lib/gitlab/slash_commands/presenters/issue_close.rb @@ -7,43 +7,25 @@ module Gitlab include Presenters::IssueBase def present - if @resource.confidential? - ephemeral_response(close_issue) + if resource.confidential? + ephemeral_response(response_message) else - in_channel_response(close_issue) + in_channel_response(response_message) end end def already_closed - ephemeral_response(text: "Issue #{@resource.to_reference} is already closed.") + ephemeral_response(text: "Issue #{resource.to_reference} is already closed.") end private - def close_issue - { - attachments: [ - { - title: "#{@resource.title} · #{@resource.to_reference}", - title_link: resource_url, - author_name: author.name, - author_icon: author.avatar_url, - fallback: "Closed issue #{@resource.to_reference}: #{@resource.title}", - pretext: pretext, - color: color(@resource), - fields: fields, - mrkdwn_in: [ - :title, - :pretext, - :fields - ] - } - ] - } + def fallback_message + "Closed issue #{issue.to_reference}: #{issue.title}" end def pretext - "I closed an issue on #{author_profile_link}'s behalf: *#{@resource.to_reference}* in #{project_link}" + "I closed an issue on #{author_profile_link}'s behalf: *#{issue.to_reference}* in #{project_link}" end end end diff --git a/lib/gitlab/slash_commands/presenters/issue_comment.rb b/lib/gitlab/slash_commands/presenters/issue_comment.rb index cce71e23b21..6ad56dd3682 100644 --- a/lib/gitlab/slash_commands/presenters/issue_comment.rb +++ b/lib/gitlab/slash_commands/presenters/issue_comment.rb @@ -7,31 +7,13 @@ module Gitlab include Presenters::NoteBase def present - ephemeral_response(new_note) + ephemeral_response(response_message) end private - def new_note - { - attachments: [ - { - title: "#{issue.title} · #{issue.to_reference}", - title_link: resource_url, - author_name: author.name, - author_icon: author.avatar_url, - fallback: "New comment on #{issue.to_reference}: #{issue.title}", - pretext: pretext, - color: color, - fields: fields, - mrkdwn_in: [ - :title, - :pretext, - :fields - ] - } - ] - } + def fallback_message + "New comment on #{issue.to_reference}: #{issue.title}" end def pretext diff --git a/lib/gitlab/slash_commands/presenters/issue_move.rb b/lib/gitlab/slash_commands/presenters/issue_move.rb index 01f2025ee10..5b9ca89c063 100644 --- a/lib/gitlab/slash_commands/presenters/issue_move.rb +++ b/lib/gitlab/slash_commands/presenters/issue_move.rb @@ -19,30 +19,15 @@ module Gitlab private def moved_issue(old_issue) - { - attachments: [ - { - title: "#{@resource.title} · #{@resource.to_reference}", - title_link: resource_url, - author_name: author.name, - author_icon: author.avatar_url, - fallback: "Issue #{@resource.to_reference}: #{@resource.title}", - pretext: pretext(old_issue), - color: color(@resource), - fields: fields, - mrkdwn_in: [ - :title, - :pretext, - :text, - :fields - ] - } - ] - } + response_message(custom_pretext: custom_pretext(old_issue)) end - def pretext(old_issue) - "Moved issue *#{issue_link(old_issue)}* to *#{issue_link(@resource)}*" + def fallback_message + "Issue #{issue.to_reference}: #{issue.title}" + end + + def custom_pretext(old_issue) + "Moved issue *#{issue_link(old_issue)}* to *#{issue_link(issue)}*" end def issue_link(issue) diff --git a/lib/gitlab/slash_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb index 1424a4ac381..552456f5836 100644 --- a/lib/gitlab/slash_commands/presenters/issue_new.rb +++ b/lib/gitlab/slash_commands/presenters/issue_new.rb @@ -7,36 +7,21 @@ module Gitlab include Presenters::IssueBase def present - in_channel_response(new_issue) + in_channel_response(response_message) end private - def new_issue - { - attachments: [ - { - title: "#{@resource.title} · #{@resource.to_reference}", - title_link: resource_url, - author_name: author.name, - author_icon: author.avatar_url, - fallback: "New issue #{@resource.to_reference}: #{@resource.title}", - pretext: pretext, - color: color(@resource), - fields: fields, - mrkdwn_in: [ - :title, - :pretext, - :text, - :fields - ] - } - ] - } + def fallback_message + "New issue #{issue.to_reference}: #{issue.title}" + end + + def fields_with_markdown + %i(title pretext text fields) end def pretext - "I created an issue on #{author_profile_link}'s behalf: *#{@resource.to_reference}* in #{project_link}" + "I created an issue on #{author_profile_link}'s behalf: *#{issue.to_reference}* in #{project_link}" end end end diff --git a/lib/gitlab/slash_commands/presenters/issue_search.rb b/lib/gitlab/slash_commands/presenters/issue_search.rb index 0d497efec0e..fffa082baac 100644 --- a/lib/gitlab/slash_commands/presenters/issue_search.rb +++ b/lib/gitlab/slash_commands/presenters/issue_search.rb @@ -7,12 +7,12 @@ module Gitlab include Presenters::IssueBase def present - text = if @resource.count >= 5 + text = if resource.count >= 5 "Here are the first 5 issues I found:" - elsif @resource.one? + elsif resource.one? "Here is the only issue I found:" else - "Here are the #{@resource.count} issues I found:" + "Here are the #{resource.count} issues I found:" end ephemeral_response(text: text, attachments: attachments) @@ -21,7 +21,7 @@ module Gitlab private def attachments - @resource.map do |issue| + resource.map do |issue| url = "[#{issue.to_reference}](#{url_for([namespace, project, issue])})" { @@ -37,7 +37,7 @@ module Gitlab end def project - @project ||= @resource.first.project + @project ||= resource.first.project end def namespace diff --git a/lib/gitlab/slash_commands/presenters/issue_show.rb b/lib/gitlab/slash_commands/presenters/issue_show.rb index 5a2c79a928e..448381b64ed 100644 --- a/lib/gitlab/slash_commands/presenters/issue_show.rb +++ b/lib/gitlab/slash_commands/presenters/issue_show.rb @@ -7,55 +7,36 @@ module Gitlab include Presenters::IssueBase def present - if @resource.confidential? - ephemeral_response(show_issue) + if resource.confidential? + ephemeral_response(response_message) else - in_channel_response(show_issue) + in_channel_response(response_message) end end private - def show_issue - { - attachments: [ - { - title: "#{@resource.title} · #{@resource.to_reference}", - title_link: resource_url, - author_name: author.name, - author_icon: author.avatar_url, - fallback: "Issue #{@resource.to_reference}: #{@resource.title}", - pretext: pretext, - text: text, - color: color(@resource), - fields: fields, - mrkdwn_in: [ - :pretext, - :text, - :fields - ] - } - ] - } + def fallback_message + "Issue #{resource.to_reference}: #{resource.title}" end def text - message = ["**#{status_text(@resource)}**"] + message = ["**#{status_text(resource)}**"] - if @resource.upvotes.zero? && @resource.downvotes.zero? && @resource.user_notes_count.zero? + if resource.upvotes.zero? && resource.downvotes.zero? && resource.user_notes_count.zero? return message.join end message << " · " - message << ":+1: #{@resource.upvotes} " unless @resource.upvotes.zero? - message << ":-1: #{@resource.downvotes} " unless @resource.downvotes.zero? - message << ":speech_balloon: #{@resource.user_notes_count}" unless @resource.user_notes_count.zero? + message << ":+1: #{resource.upvotes} " unless resource.upvotes.zero? + message << ":-1: #{resource.downvotes} " unless resource.downvotes.zero? + message << ":speech_balloon: #{resource.user_notes_count}" unless resource.user_notes_count.zero? message.join end def pretext - "Issue *#{@resource.to_reference}* from #{project.full_name}" + "Issue *#{resource.to_reference}* from #{project.full_name}" end end end diff --git a/lib/gitlab/slash_commands/presenters/note_base.rb b/lib/gitlab/slash_commands/presenters/note_base.rb index 7758fc740de..71a9b99d0fd 100644 --- a/lib/gitlab/slash_commands/presenters/note_base.rb +++ b/lib/gitlab/slash_commands/presenters/note_base.rb @@ -6,7 +6,7 @@ module Gitlab module NoteBase GREEN = '#38ae67' - def color + def color(_) GREEN end @@ -18,18 +18,10 @@ module Gitlab issue.project end - def project_link - "[#{project.full_name}](#{project.web_url})" - end - def author resource.author end - def author_profile_link - "[#{author.to_reference}](#{url_for(author)})" - end - def fields [ { diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index f6edbfced7f..ca7ae429986 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -35,7 +35,7 @@ module Gitlab query.length >= min_chars_for_partial_matching end - # column - The column name to search in. + # column - The column name / Arel column to search in. # query - The text to search for. # lower_exact_match - When set to `true` we'll fall back to using # `LOWER(column) = query` instead of using `ILIKE`. @@ -43,19 +43,21 @@ module Gitlab query = query.squish return unless query.present? + arel_column = column.is_a?(Arel::Attributes::Attribute) ? column : arel_table[column] + words = select_fuzzy_words(query, use_minimum_char_limit: use_minimum_char_limit) if words.any? - words.map { |word| arel_table[column].matches(to_pattern(word, use_minimum_char_limit: use_minimum_char_limit)) }.reduce(:and) + words.map { |word| arel_column.matches(to_pattern(word, use_minimum_char_limit: use_minimum_char_limit)) }.reduce(:and) else # No words of at least 3 chars, but we can search for an exact # case insensitive match with the query as a whole if lower_exact_match Arel::Nodes::NamedFunction - .new('LOWER', [arel_table[column]]) + .new('LOWER', [arel_column]) .eq(query) else - arel_table[column].matches(sanitize_sql_like(query)) + arel_column.matches(sanitize_sql_like(query)) end end end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 91e2ff0b10d..37688d6e0e7 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -29,14 +29,14 @@ module Gitlab def event(category, action, label: nil, property: nil, value: nil, context: nil) return unless enabled? - snowplow.track_struct_event(category, action, label, property, value, context, Time.now.to_i) + snowplow.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) end def self_describing_event(schema_url, event_data_json, context: nil) return unless enabled? event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, event_data_json) - snowplow.track_self_describing_event(event_json, context, Time.now.to_i) + snowplow.track_self_describing_event(event_json, context, (Time.now.to_f * 1000).to_i) end def snowplow_options(group) diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 038067eeae4..4bedf7a301e 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -6,10 +6,10 @@ module Gitlab include GitlabRoutingHelper include ActionView::RecordIdentifier - attr_reader :object + attr_reader :object, :opts - def self.build(object) - new(object).url + def self.build(object, opts = {}) + new(object, opts).url end def url @@ -24,10 +24,8 @@ module Gitlab note_url when WikiPage wiki_page_url - when ProjectSnippet - project_snippet_url(object.project, object) when Snippet - snippet_url(object) + opts[:raw].present? ? gitlab_raw_snippet_url(object) : gitlab_snippet_url(object) when Milestone milestone_url(object) when ::Ci::Build @@ -41,8 +39,9 @@ module Gitlab private - def initialize(object) + def initialize(object, opts = {}) @object = object + @opts = opts end def commit_url(opts = {}) @@ -66,13 +65,7 @@ module Gitlab merge_request_url(object.noteable, anchor: dom_id(object)) elsif object.for_snippet? - snippet = object.noteable - - if snippet.is_a?(PersonalSnippet) - snippet_url(snippet, anchor: dom_id(object)) - else - project_snippet_url(snippet.project, snippet, anchor: dom_id(object)) - end + gitlab_snippet_url(object.noteable, anchor: dom_id(object)) end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index b6effac25c6..ec2243345e1 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -82,8 +82,10 @@ module Gitlab grafana_integrated_projects: count(GrafanaIntegration.enabled), groups: count(Group), issues: count(Issue), + issues_created_from_gitlab_error_tracking_ui: count(SentryIssue), issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue), issues_using_zoom_quick_actions: count(ZoomMeeting.select(:issue_id).distinct), + issues_with_embedded_grafana_charts_approx: ::Gitlab::GrafanaEmbedUsageData.issue_count, keys: count(Key), label_lists: count(List.label), lfs_objects: count(LfsObject), @@ -107,7 +109,8 @@ module Gitlab services_usage, approximate_counts, usage_counters, - user_preferences_usage + user_preferences_usage, + ingress_modsecurity_usage ) } end @@ -169,6 +172,10 @@ module Gitlab } end + def ingress_modsecurity_usage + ::Clusters::Applications::IngressModsecurityUsageService.new.execute + end + # rubocop: disable CodeReuse/ActiveRecord def services_usage types = { diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb index 2b52571c3cc..77fc216738f 100644 --- a/lib/gitlab/usage_data_counters/base_counter.rb +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -8,7 +8,7 @@ module Gitlab::UsageDataCounters class << self def redis_key(event) - Gitlab::Sentry.track_exception(UnknownEvent, extra: { event: event }) unless known_events.include?(event.to_s) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new, event: event) unless known_events.include?(event.to_s) "USAGE_#{prefix}_#{event}".upcase end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 35aea209cb9..784a6686962 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -146,7 +146,8 @@ module Gitlab def prepended(base = nil) super - queue_verification(base) if base + # prepend can override methods, thus we need to verify it like classes + queue_verification(base, verify: true) if base end def extended(mod = nil) @@ -155,11 +156,15 @@ module Gitlab queue_verification(mod.singleton_class) if mod end - def queue_verification(base) + def queue_verification(base, verify: false) return unless ENV['STATIC_VERIFICATION'] - if base.is_a?(Class) # We could check for Class in `override` - # This could be `nil` if `override` was never called + # We could check for Class in `override` + # This could be `nil` if `override` was never called. + # We also force verification for prepend because it can also override + # a method like a class, but not the cases for include or extend. + # This includes Rails helpers but not limited to. + if base.is_a?(Class) || verify Override.extensions[self]&.add_class(base) end end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index e2787744f09..082d93aa354 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -29,7 +29,7 @@ module Gitlab def levels_for_user(user = nil) return [PUBLIC] unless user - if user.full_private_access? + if user.can_read_all_resources? [PRIVATE, INTERNAL, PUBLIC] elsif user.external? [PUBLIC] @@ -115,6 +115,18 @@ module Gitlab end end + def visibility_level_decreased? + return false unless visibility_level_previous_changes + + before, after = visibility_level_previous_changes + + before && after && after < before + end + + def visibility_level_previous_changes + previous_changes[:visibility_level] + end + def private? visibility_level_value == PRIVATE end diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb index 1d2aff5e5b4..d2c01bbd55e 100644 --- a/lib/gitlab/webpack/manifest.rb +++ b/lib/gitlab/webpack/manifest.rb @@ -12,11 +12,12 @@ module Gitlab def entrypoint_paths(source) raise ::Webpack::Rails::Manifest::WebpackError, manifest["errors"] unless manifest_bundled? + dll_assets = manifest.fetch("dllAssets", []) entrypoint = manifest["entrypoints"][source] if entrypoint && entrypoint["assets"] # Can be either a string or an array of strings. # Do not include source maps as they are not javascript - [entrypoint["assets"]].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p| + [dll_assets, entrypoint["assets"]].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p| "/#{::Rails.configuration.webpack.public_path}/#{p}" end else |