diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-07-20 15:40:28 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-07-20 15:40:28 +0000 |
| commit | b595cb0c1dec83de5bdee18284abe86614bed33b (patch) | |
| tree | 8c3d4540f193c5ff98019352f554e921b3a41a72 /lib | |
| parent | 2f9104a328fc8a4bddeaa4627b595166d24671d0 (diff) | |
| download | gitlab-ce-b595cb0c1dec83de5bdee18284abe86614bed33b.tar.gz | |
Add latest changes from gitlab-org/gitlab@15-2-stable-eev15.2.0-rc42
Diffstat (limited to 'lib')
295 files changed, 4966 insertions, 1503 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 8cafde4fedb..8827371546c 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -28,7 +28,8 @@ module API Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new, Gitlab::GrapeLogging::Loggers::ContextLogger.new, Gitlab::GrapeLogging::Loggers::ContentLogger.new, - Gitlab::GrapeLogging::Loggers::UrgencyLogger.new + Gitlab::GrapeLogging::Loggers::UrgencyLogger.new, + Gitlab::GrapeLogging::Loggers::ResponseLogger.new ] allow_access_with_scope :api @@ -242,6 +243,7 @@ module API mount ::API::MergeRequestApprovals mount ::API::MergeRequestDiffs mount ::API::MergeRequests + mount ::API::Metadata mount ::API::Metrics::Dashboard::Annotations mount ::API::Metrics::UserStarredDashboards mount ::API::Namespaces @@ -313,6 +315,7 @@ module API mount ::API::Internal::Lfs mount ::API::Internal::Pages mount ::API::Internal::Kubernetes + mount ::API::Internal::ErrorTracking mount ::API::Internal::MailRoom mount ::API::Internal::ContainerRegistry::Migration mount ::API::Internal::Workhorse diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index c8485054377..fd36b364d56 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -6,8 +6,6 @@ module API helpers ::API::Helpers::AwardEmoji - before { authenticate! } - Helpers::AwardEmoji.awardables.each do |awardable_params| resource awardable_params[:resource], requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do awardable_string = awardable_params[:type].pluralize @@ -82,7 +80,7 @@ module API delete "#{endpoint}/:award_id", feature_category: awardable_params[:feature_category] do award = awardable.award_emoji.find(params[:award_id]) - unauthorized! unless award.user == current_user || current_user.admin? + unauthorized! unless award.user == current_user || current_user&.admin? destroy_conditionally!(award) end diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index b5d68ca5de2..e818cad0d03 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -4,7 +4,7 @@ module API class BroadcastMessages < ::API::Base include PaginationParams - feature_category :navigation + feature_category :onboarding urgency :low resource :broadcast_messages do diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb index 72e36d95dc5..fe49074afed 100644 --- a/lib/api/ci/helpers/runner.rb +++ b/lib/api/ci/helpers/runner.rb @@ -12,6 +12,7 @@ module API JOB_TOKEN_PARAM = :token def authenticate_runner! + track_runner_authentication forbidden! unless current_runner current_runner @@ -42,6 +43,14 @@ module API end end + def track_runner_authentication + if current_runner + metrics.increment_runner_authentication_success_counter(runner_type: current_runner.runner_type) + else + metrics.increment_runner_authentication_failure_counter + end + end + # HTTP status codes to terminate the job on GitLab Runner: # - 403 def authenticate_job!(require_running: true, heartbeat_runner: false) @@ -149,6 +158,10 @@ module API def request_using_running_job_token? current_job.present? && current_authenticated_job.present? && current_job != current_authenticated_job end + + def metrics + strong_memoize(:metrics) { ::Gitlab::Ci::Runner::Metrics.new } + end end end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index dedda82091f..5fd9a8e3278 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -139,7 +139,7 @@ module API if find_user_from_warden Gitlab::UsageDataCounters::WebIdeCounter.increment_commits_count - Gitlab::UsageDataCounters::EditorUniqueCounter.track_web_ide_edit_action(author: current_user) + Gitlab::UsageDataCounters::EditorUniqueCounter.track_web_ide_edit_action(author: current_user, project: user_project) end present commit_detail, with: Entities::CommitDetail, stats: params[:stats] diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb index d1cc35b16d8..a90269b565c 100644 --- a/lib/api/concerns/packages/conan_endpoints.rb +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -43,6 +43,7 @@ module API end before do + not_found! if Gitlab::FIPS.enabled? require_packages_enabled! # Personal access token will be extracted from Bearer or Basic authorization diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index 1f640cc17d0..8bf4ac22802 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -6,6 +6,10 @@ module API project_id: %r{[0-9]+}.freeze ).freeze + before do + not_found! if Gitlab::FIPS.enabled? + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do def user_project diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 5fb11db8938..ca576254c3d 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -14,6 +14,10 @@ module API file_name: API::NO_SLASH_URL_PART_REGEX }.freeze + before do + not_found! if Gitlab::FIPS.enabled? + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do def project_or_group diff --git a/lib/api/entities/ci/job_request/service.rb b/lib/api/entities/ci/job_request/service.rb index d9da2c92ec7..7d494c7e516 100644 --- a/lib/api/entities/ci/job_request/service.rb +++ b/lib/api/entities/ci/job_request/service.rb @@ -8,6 +8,7 @@ module API expose :name, :entrypoint expose :ports, using: Entities::Ci::JobRequest::Port + expose :pull_policy, if: ->(_) { ::Feature.enabled?(:ci_docker_image_pull_policy) } expose :alias, :command expose :variables end diff --git a/lib/api/entities/deploy_key.rb b/lib/api/entities/deploy_key.rb index e8537c4c677..2c9c33549a1 100644 --- a/lib/api/entities/deploy_key.rb +++ b/lib/api/entities/deploy_key.rb @@ -4,7 +4,8 @@ module API module Entities class DeployKey < Entities::SSHKey expose :key - expose :fingerprint + expose :fingerprint, if: ->(key, _) { key.fingerprint.present? } + expose :fingerprint_sha256 expose :projects_with_write_access, using: Entities::ProjectIdentity, if: -> (_, options) { options[:include_projects_with_write_access] } end diff --git a/lib/api/entities/environment.rb b/lib/api/entities/environment.rb index b1a720ac6bb..3b6ed94c3f1 100644 --- a/lib/api/entities/environment.rb +++ b/lib/api/entities/environment.rb @@ -4,48 +4,11 @@ module API module Entities class Environment < Entities::EnvironmentBasic include RequestAwareEntity - include Gitlab::Utils::StrongMemoize expose :tier expose :project, using: Entities::BasicProjectDetails expose :last_deployment, using: Entities::Deployment, if: { last_deployment: true } expose :state - - expose :enable_advanced_logs_querying, if: -> (*) { can_read_pod_logs? } do |environment| - environment.elastic_stack_available? - end - - expose :logs_api_path, if: -> (*) { can_read_pod_logs? } do |environment| - if environment.elastic_stack_available? - elasticsearch_project_logs_path(environment.project, environment_name: environment.name, format: :json) - else - k8s_project_logs_path(environment.project, environment_name: environment.name, format: :json) - end - end - - expose :gitlab_managed_apps_logs_path, if: -> (*) { can_read_pod_logs? && cluster } do |environment| - ::Clusters::ClusterPresenter.new(cluster, current_user: current_user).gitlab_managed_apps_logs_path # rubocop: disable CodeReuse/Presenter - end - - private - - alias_method :environment, :object - - def can_read_pod_logs? - strong_memoize(:can_read_pod_logs) do - current_user&.can?(:read_pod_logs, environment.project) - end - end - - def cluster - strong_memoize(:cluster) do - environment&.last_deployment&.cluster - end - end - - def current_user - options[:current_user] - end end end end diff --git a/lib/api/entities/group_detail.rb b/lib/api/entities/group_detail.rb index e6872709432..e521de0d572 100644 --- a/lib/api/entities/group_detail.rb +++ b/lib/api/entities/group_detail.rb @@ -6,10 +6,12 @@ module API expose :shared_with_groups do |group, options| SharedGroupWithGroup.represent(group.shared_with_group_links_visible_to_user(options[:current_user])) end - expose :runners_token, if: lambda { |group, options| options[:user_can_admin_group] } + expose :runners_token, if: ->(_, options) { options[:user_can_admin_group] } expose :prevent_sharing_groups_outside_hierarchy, if: ->(group) { group.root? } - expose :projects, using: Entities::Project do |group, options| + expose :projects, + if: ->(_, options) { options[:with_projects] }, + using: Entities::Project do |group, options| projects = GroupProjectsFinder.new( group: group, current_user: options[:current_user], @@ -19,7 +21,9 @@ module API Entities::Project.prepare_relation(projects, options) end - expose :shared_projects, using: Entities::Project do |group, options| + expose :shared_projects, + if: ->(_, options) { options[:with_projects] }, + using: Entities::Project do |group, options| projects = GroupProjectsFinder.new( group: group, current_user: options[:current_user], diff --git a/lib/api/entities/hook.rb b/lib/api/entities/hook.rb index d176e76b321..95924321221 100644 --- a/lib/api/entities/hook.rb +++ b/lib/api/entities/hook.rb @@ -8,6 +8,11 @@ module API expose :alert_status expose :disabled_until + expose :url_variables + + def url_variables + object.url_variables.keys.map { { key: _1 } } + end end end end diff --git a/lib/api/entities/issue.rb b/lib/api/entities/issue.rb index 1060b2c517a..7630fd1e94e 100644 --- a/lib/api/entities/issue.rb +++ b/lib/api/entities/issue.rb @@ -31,9 +31,7 @@ module API end expose :closed_as_duplicate_of do |issue| - if ::Feature.enabled?(:closed_as_duplicate_of_issues_api, issue.project) && - issue.duplicated? && - options[:current_user]&.can?(:read_issue, issue.duplicated_to) + if issue.duplicated? && options[:current_user]&.can?(:read_issue, issue.duplicated_to) expose_url( api_v4_project_issue_path(id: issue.duplicated_to.project_id, issue_iid: issue.duplicated_to.iid) ) diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 9e216b0aed5..906c252d7f9 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -104,6 +104,7 @@ module API expose :ci_forward_deployment_enabled expose :ci_job_token_scope_enabled expose :ci_separated_caches + expose :ci_opt_in_jwt expose :public_builds, as: :public_jobs expose :build_git_strategy, if: lambda { |project, options| options[:user_can_admin_project] } do |project, options| project.build_allow_git_fetch ? 'fetch' : 'clone' diff --git a/lib/api/entities/terraform/module_version.rb b/lib/api/entities/terraform/module_version.rb new file mode 100644 index 00000000000..411fa09465e --- /dev/null +++ b/lib/api/entities/terraform/module_version.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module API + module Entities + module Terraform + class ModuleVersion < Grape::Entity + expose :name + expose :provider + expose :providers + expose :root + expose :source + expose :submodules + expose :version + expose :versions + end + end + end +end diff --git a/lib/api/entities/unleash/client_feature_flags.rb b/lib/api/entities/unleash/client_feature_flags.rb new file mode 100644 index 00000000000..8c96d0610a4 --- /dev/null +++ b/lib/api/entities/unleash/client_feature_flags.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module API + module Entities + module Unleash + class ClientFeatureFlags < Grape::Entity + expose :unleash_api_version, as: :version + expose :unleash_api_features, as: :features, using: ::API::Entities::UnleashFeature + end + end + end +end diff --git a/lib/api/entities/user.rb b/lib/api/entities/user.rb index 2366d137cc2..a86039b856a 100644 --- a/lib/api/entities/user.rb +++ b/lib/api/entities/user.rb @@ -19,7 +19,7 @@ module API user.followees.size end expose :is_followed, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) && opts[:current_user] } do |user, opts| - opts[:current_user].following?(user) + user.followed_by?(opts[:current_user]) end expose :local_time do |user| local_time(user.timezone) diff --git a/lib/api/feature_flags_user_lists.rb b/lib/api/feature_flags_user_lists.rb index 854719db4a1..f4771c07260 100644 --- a/lib/api/feature_flags_user_lists.rb +++ b/lib/api/feature_flags_user_lists.rb @@ -44,9 +44,13 @@ module API requires :user_xids, type: String, desc: 'A comma separated list of external user ids' end post do + # TODO: Move the business logic to a service class in app/services/feature_flags. + # https://gitlab.com/gitlab-org/gitlab/-/issues/367021 list = user_project.operations_feature_flags_user_lists.create(declared_params) if list.save + update_last_feature_flag_updated_at! + present list, with: ::API::Entities::FeatureFlag::UserList else render_api_error!(list.errors.full_messages, :bad_request) @@ -76,9 +80,13 @@ module API optional :user_xids, type: String, desc: 'A comma separated list of external user ids' end put do + # TODO: Move the business logic to a service class in app/services/feature_flags. + # https://gitlab.com/gitlab-org/gitlab/-/issues/367021 list = user_project.operations_feature_flags_user_lists.find_by_iid!(params[:iid]) if list.update(declared_params(include_missing: false)) + update_last_feature_flag_updated_at! + present list, with: ::API::Entities::FeatureFlag::UserList else render_api_error!(list.errors.full_messages, :bad_request) @@ -89,8 +97,14 @@ module API detail 'This feature was introduced in GitLab 12.10' end delete do + # TODO: Move the business logic to a service class in app/services/feature_flags. + # https://gitlab.com/gitlab-org/gitlab/-/issues/367021 list = user_project.operations_feature_flags_user_lists.find_by_iid!(params[:iid]) - unless list.destroy + if list.destroy + update_last_feature_flag_updated_at! + + nil + else render_api_error!(list.errors.full_messages, :conflict) end end @@ -101,6 +115,10 @@ module API def authorize_admin_feature_flags_user_lists! authorize! :admin_feature_flags_user_lists, user_project end + + def update_last_feature_flag_updated_at! + Operations::FeatureFlagsClient.update_last_feature_flag_updated_at!(user_project) + end end end end diff --git a/lib/api/geo.rb b/lib/api/geo.rb index 85f242cd135..cb04d2a4e1e 100644 --- a/lib/api/geo.rb +++ b/lib/api/geo.rb @@ -8,7 +8,7 @@ module API helpers do # Overridden in EE def geo_proxy_response - {} + { geo_enabled: false } end end diff --git a/lib/api/group_debian_distributions.rb b/lib/api/group_debian_distributions.rb index f0376fe2c9c..1f43bb0e2b3 100644 --- a/lib/api/group_debian_distributions.rb +++ b/lib/api/group_debian_distributions.rb @@ -6,6 +6,10 @@ module API requires :id, type: String, desc: 'The ID of a group' end + before do + not_found! if Gitlab::FIPS.enabled? + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do after_validation do require_packages_enabled! diff --git a/lib/api/groups.rb b/lib/api/groups.rb index c17bc432404..b63396ed073 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -109,6 +109,19 @@ module API present paginate(groups), options end + def present_group_details(params, group, with_projects: true) + options = { + with: Entities::GroupDetail, + with_projects: with_projects, + current_user: current_user, + user_can_admin_group: can?(current_user, :admin_group, group) + } + + group, options = with_custom_attributes(group, options) if params[:with_custom_attributes] + + present group, options + end + def present_groups_with_pagination_strategies(params, groups) return present_groups(params, groups) if current_user.present? @@ -236,7 +249,7 @@ module API authorize! :admin_group, group if update_group(group) - present group, with: Entities::GroupDetail, current_user: current_user + present_group_details(params, group, with_projects: true) else render_validation_error!(group) end @@ -254,15 +267,7 @@ module API group = find_group!(params[:id]) group.preload_shared_group_links - options = { - with: params[:with_projects] ? Entities::GroupDetail : Entities::Group, - current_user: current_user, - user_can_admin_group: can?(current_user, :admin_group, group) - } - - group, options = with_custom_attributes(group, options) - - present group, options + present_group_details(params, group, with_projects: params[:with_projects]) end desc 'Remove a group.' diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index fc1037131d8..e462ca19ba6 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -225,7 +225,11 @@ module API def find_project_issue(iid, project_id = nil) project = project_id ? find_project!(project_id) : user_project - ::IssuesFinder.new(current_user, project_id: project.id).find_by!(iid: iid) + ::IssuesFinder.new( + current_user, + project_id: project.id, + issue_types: WorkItems::Type.allowed_types_for_issues + ).find_by!(iid: iid) end # rubocop: enable CodeReuse/ActiveRecord @@ -476,9 +480,9 @@ module API render_api_error!('202 Accepted', 202) end - def render_validation_error!(model) + def render_validation_error!(model, status = 400) if model.errors.any? - render_api_error!(model_error_messages(model) || '400 Bad Request', 400) + render_api_error!(model_error_messages(model) || '400 Bad Request', status) end end @@ -637,6 +641,7 @@ module API :last_activity_after, :last_activity_before, :topic, + :topic_id, :repository_storage) .symbolize_keys .compact diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index 0fbd0e6be44..0b0100c7d7f 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -633,6 +633,12 @@ module API }, { required: false, + name: :notify_only_default_branch, + type: Boolean, + desc: 'Send notifications only for the default branch' + }, + { + required: false, name: :branches_to_be_notified, type: String, desc: 'Branches for which notifications are to be sent' diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb index 185a10a250c..47ea9c9fe2c 100644 --- a/lib/api/helpers/issues_helpers.rb +++ b/lib/api/helpers/issues_helpers.rb @@ -60,7 +60,7 @@ module API args[:not][:label_name] ||= args[:not].delete(:labels) args[:scope] = args[:scope].underscore if args[:scope] args[:sort] = "#{args[:order_by]}_#{args[:sort]}" - args[:issue_types] ||= args.delete(:issue_type) + args[:issue_types] ||= args.delete(:issue_type) || WorkItems::Type.allowed_types_for_issues IssuesFinder.new(current_user, args) end diff --git a/lib/api/helpers/pagination_strategies.rb b/lib/api/helpers/pagination_strategies.rb index 8c2186768ea..4e244ea589e 100644 --- a/lib/api/helpers/pagination_strategies.rb +++ b/lib/api/helpers/pagination_strategies.rb @@ -49,6 +49,7 @@ module API offset_limit = limit_for_scope(request_scope) if (Gitlab::Pagination::Keyset.available_for_type?(relation) || cursor_based_keyset_pagination_supported?(relation)) && + cursor_based_keyset_pagination_enforced?(relation) && offset_limit_exceeded?(offset_limit) return error!("Offset pagination has a maximum allowed offset of #{offset_limit} " \ @@ -63,6 +64,10 @@ module API Gitlab::Pagination::CursorBasedKeyset.available_for_type?(relation) end + def cursor_based_keyset_pagination_enforced?(relation) + Gitlab::Pagination::CursorBasedKeyset.enforced_for_type?(relation) + end + def keyset_pagination_enabled? params[:pagination] == 'keyset' end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 52cb398d6bf..3a518959b2c 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -96,6 +96,7 @@ module API params :optional_update_params_ce do optional :ci_forward_deployment_enabled, type: Boolean, desc: 'Skip older deployment jobs that are still pending' + optional :ci_separated_caches, type: Boolean, desc: 'Enable or disable separated caches based on branch protection.' optional :restrict_user_defined_variables, type: Boolean, desc: 'Restrict use of user-defined variables when triggering a pipeline' end @@ -130,6 +131,7 @@ module API :ci_config_path, :ci_default_git_depth, :ci_forward_deployment_enabled, + :ci_separated_caches, :container_registry_access_level, :container_expiration_policy_attributes, :default_branch, diff --git a/lib/api/helpers/protected_tags_helpers.rb b/lib/api/helpers/protected_tags_helpers.rb new file mode 100644 index 00000000000..cad4ec8d5bd --- /dev/null +++ b/lib/api/helpers/protected_tags_helpers.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module API + module Helpers + module ProtectedTagsHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_params_ee do + end + end + end +end + +API::Helpers::ProtectedTagsHelpers.prepend_mod_with('API::Helpers::ProtectedTagsHelpers') diff --git a/lib/api/helpers/web_hooks_helpers.rb b/lib/api/helpers/web_hooks_helpers.rb new file mode 100644 index 00000000000..a71e56af4c3 --- /dev/null +++ b/lib/api/helpers/web_hooks_helpers.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module API + module Helpers + module WebHooksHelpers + extend Grape::API::Helpers + + params :requires_url do + requires :url, type: String, desc: "The URL to send the request to" + end + + params :optional_url do + optional :url, type: String, desc: "The URL to send the request to" + end + + params :url_variables do + optional :url_variables, type: Array, desc: 'URL variables for interpolation' do + requires :key, type: String, desc: 'Name of the variable' + requires :value, type: String, desc: 'Value of the variable' + end + end + + def find_hook + hook_scope.find(params.delete(:hook_id)) + end + + def create_hook_params + hook_params = declared_params(include_missing: false) + url_variables = hook_params.delete(:url_variables) + + if url_variables.present? + hook_params[:url_variables] = url_variables.to_h { [_1[:key], _1[:value]] } + end + + hook_params + end + + def update_hook(entity:) + hook = find_hook + update_params = update_hook_params(hook) + + hook.assign_attributes(update_params) + + save_hook(hook, entity) + end + + def update_hook_params(hook) + update_params = declared_params(include_missing: false) + url_variables = update_params.delete(:url_variables) || [] + url_variables = url_variables.to_h { [_1[:key], _1[:value]] } + update_params[:url_variables] = hook.url_variables.merge(url_variables) if url_variables.present? + + error!('No parameters provided', :bad_request) if update_params.empty? + + update_params + end + + def save_hook(hook, entity) + if hook.save + present hook, with: entity + else + error!("Invalid url given", 422) if hook.errors[:url].present? + error!("Invalid branch filter given", 422) if hook.errors[:push_events_branch_filter].present? + + render_validation_error!(hook, 422) + end + end + end + end +end diff --git a/lib/api/hooks/test.rb b/lib/api/hooks/test.rb new file mode 100644 index 00000000000..4871955c6e0 --- /dev/null +++ b/lib/api/hooks/test.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module API + module Hooks + # It is important that this re-usable module is not a Grape Instance, + # since it will be re-mounted. + # rubocop: disable API/Base + class Test < ::Grape::API + params do + requires :hook_id, type: Integer, desc: 'The ID of the hook' + end + post ":hook_id" do + hook = find_hook + data = configuration[:data].dup + hook.execute(data, configuration[:kind]) + data + end + end + # rubocop: enable API/Base + end +end diff --git a/lib/api/hooks/url_variables.rb b/lib/api/hooks/url_variables.rb new file mode 100644 index 00000000000..708b78134e5 --- /dev/null +++ b/lib/api/hooks/url_variables.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module API + module Hooks + # It is important that this re-usable module is not a Grape Instance, + # since it will be re-mounted. + # rubocop: disable API/Base + class UrlVariables < ::Grape::API + params do + requires :hook_id, type: Integer, desc: 'The ID of the hook' + requires :key, type: String, desc: 'The key of the variable' + end + namespace ':hook_id/url_variables' do + desc 'Set a url variable' + params do + requires :value, type: String, desc: 'The value of the variable' + end + put ":key" do + hook = find_hook + key = params.delete(:key) + value = params.delete(:value) + vars = hook.url_variables.merge(key => value) + + error!('Illegal key or value', 422) unless hook.update(url_variables: vars) + + status :no_content + end + + desc 'Un-Set a url variable' + delete ":key" do + hook = find_hook + key = params.delete(:key) + not_found!('URL variable') unless hook.url_variables.key?(key) + + vars = hook.url_variables.reject { _1 == key } + + error!('Could not unset variable', 422) unless hook.update(url_variables: vars) + + status :no_content + end + end + end + # rubocop: enable API/Base + end +end diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 3edd38a0108..b53f855c3a2 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -164,18 +164,6 @@ module API check_allowed(params) end - post '/error_tracking_allowed', feature_category: :error_tracking do - public_key = params[:public_key] - project_id = params[:project_id] - - unprocessable_entity! if public_key.blank? || project_id.blank? - - enabled = ::ErrorTracking::ClientKey.enabled_key_for(project_id, public_key).exists? - - status 200 - { enabled: enabled } - end - post "/lfs_authenticate", feature_category: :source_code_management, urgency: :high do not_found! unless container&.lfs_enabled? diff --git a/lib/api/internal/error_tracking.rb b/lib/api/internal/error_tracking.rb new file mode 100644 index 00000000000..bad790b0e43 --- /dev/null +++ b/lib/api/internal/error_tracking.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module API + module Internal + class ErrorTracking < ::API::Base + GITLAB_ERROR_TRACKING_TOKEN_HEADER = "Gitlab-Error-Tracking-Token" + + feature_category :error_tracking + + helpers do + def verify_error_tracking_token! + input = params['error_tracking_token'] + + if headers.key?(GITLAB_ERROR_TRACKING_TOKEN_HEADER) + input ||= Base64.decode64(headers[GITLAB_ERROR_TRACKING_TOKEN_HEADER]) + end + + input&.chomp! + + unauthorized! unless Devise.secure_compare(error_tracking_token, input) + end + + def error_tracking_token + Gitlab::CurrentSettings.error_tracking_access_token + end + + def error_tracking_enabled? + Gitlab::CurrentSettings.error_tracking_enabled + end + end + + namespace 'internal' do + namespace 'error_tracking' do + before do + verify_error_tracking_token! + end + + post '/allowed', urgency: :high do + public_key = params[:public_key] + project_id = params[:project_id] + + unprocessable_entity! if public_key.blank? || project_id.blank? + + project = Project.find(project_id) + enabled = error_tracking_enabled? && + Feature.enabled?(:use_click_house_database_for_error_tracking, project) && + ::ErrorTracking::ClientKey.enabled_key_for(project_id, public_key).exists? + + status 200 + { enabled: enabled } + end + end + end + end + end +end diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 34acfac4cb1..f7c6e48e54f 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -38,7 +38,6 @@ module API def gitaly_repository(project) { - default_branch: project.default_branch_or_main, storage_name: project.repository_storage, relative_path: project.disk_path + '.git', gl_repository: repo_type.identifier_for_container(project), @@ -76,7 +75,8 @@ module API agent_id: agent.id, agent_name: agent.name, gitaly_info: gitaly_info(project), - gitaly_repository: gitaly_repository(project) + gitaly_repository: gitaly_repository(project), + default_branch: project.default_branch_or_main } end end diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb index 8eaeeae26c2..20ca7038471 100644 --- a/lib/api/internal/pages.rb +++ b/lib/api/internal/pages.rb @@ -54,7 +54,15 @@ module API virtual_domain = host.pages_virtual_domain no_content! unless virtual_domain - present virtual_domain, with: Entities::Internal::Pages::VirtualDomain + if virtual_domain.cache_key.present? + # Cache context is not added to make it easier to expire the cache with + # Gitlab::Pages::CacheControl + present_cached virtual_domain, + cache_context: nil, + with: Entities::Internal::Pages::VirtualDomain + else + present virtual_domain, with: Entities::Internal::Pages::VirtualDomain + end end end end diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index 2fed724f947..e2481dcb8c1 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -35,6 +35,8 @@ module API name, _, format = file_name.rpartition('.') if %w(md5 sha1).include?(format) + unprocessable_entity! if Gitlab::FIPS.enabled? && format == 'md5' + [name, format] else [file_name, format] @@ -109,6 +111,7 @@ module API route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get 'packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do # return a similar failure to authorize_read_package!(project) + forbidden! unless path_exists?(params[:path]) file_name, format = extract_format(params[:file_name]) @@ -241,6 +244,7 @@ module API end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true put ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do + unprocessable_entity! if Gitlab::FIPS.enabled? && params['file.md5'] authorize_upload! bad_request!('File is too large') if user_project.actual_limits.exceeded?(:maven_max_file_size, params[:file].size) diff --git a/lib/api/metadata.rb b/lib/api/metadata.rb new file mode 100644 index 00000000000..c4984f0e7f0 --- /dev/null +++ b/lib/api/metadata.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module API + class Metadata < ::API::Base + helpers ::API::Helpers::GraphqlHelpers + include APIGuard + + allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } + + before { authenticate! } + + feature_category :not_owned # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned + + METADATA_QUERY = <<~EOF + { + metadata { + version + revision + kas { + enabled + externalUrl + version + } + } + } + EOF + + desc 'Get the metadata information of the GitLab instance.' do + detail 'This feature was introduced in GitLab 15.2.' + end + get '/metadata' do + run_graphql!( + query: METADATA_QUERY, + context: { current_user: current_user }, + transform: ->(result) { result.dig('data', 'metadata') } + ) + end + end +end diff --git a/lib/api/project_debian_distributions.rb b/lib/api/project_debian_distributions.rb index 2ba1ff85adb..b8ca9428fa3 100644 --- a/lib/api/project_debian_distributions.rb +++ b/lib/api/project_debian_distributions.rb @@ -6,6 +6,10 @@ module API requires :id, type: String, desc: 'The ID of a project' end + before do + not_found! if Gitlab::FIPS.enabled? + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do after_validation do require_packages_enabled! diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 431ba199131..466e80d68c8 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -9,16 +9,21 @@ module API feature_category :integrations + helpers ::API::Helpers::WebHooksHelpers + helpers do - params :project_hook_properties do - requires :url, type: String, desc: "The URL to send the request to" + def hook_scope + user_project.hooks + end + + params :common_hook_parameters do optional :push_events, type: Boolean, desc: "Trigger hook on push events" optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" optional :confidential_issues_events, type: Boolean, desc: "Trigger hook on confidential issues events" optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" - optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" - optional :confidential_note_events, type: Boolean, desc: "Trigger hook on confidential note(comment) events" + optional :note_events, type: Boolean, desc: "Trigger hook on note (comment) events" + optional :confidential_note_events, type: Boolean, desc: "Trigger hook on confidential note (comment) events" optional :job_events, type: Boolean, desc: "Trigger hook on job events" optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" @@ -27,6 +32,7 @@ module API optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" optional :push_events_branch_filter, type: String, desc: "Trigger hook on specified branch only" + use :url_variables end end @@ -34,6 +40,10 @@ module API requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/hooks' do + mount ::API::Hooks::UrlVariables + end + desc 'Get project hooks' do success Entities::ProjectHook end @@ -59,43 +69,26 @@ module API success Entities::ProjectHook end params do - use :project_hook_properties + use :requires_url + use :common_hook_parameters end post ":id/hooks" do - hook_params = declared_params(include_missing: false) - + hook_params = create_hook_params hook = user_project.hooks.new(hook_params) - if hook.save - present hook, with: Entities::ProjectHook - else - error!("Invalid url given", 422) if hook.errors[:url].present? - error!("Invalid branch filter given", 422) if hook.errors[:push_events_branch_filter].present? - - not_found!("Project hook #{hook.errors.messages}") - end + save_hook(hook, Entities::ProjectHook) end - desc 'Update an existing project hook' do + desc 'Update an existing hook' do success Entities::ProjectHook end params do requires :hook_id, type: Integer, desc: "The ID of the hook to update" - use :project_hook_properties + use :optional_url + use :common_hook_parameters end put ":id/hooks/:hook_id" do - hook = user_project.hooks.find(params.delete(:hook_id)) - - update_params = declared_params(include_missing: false) - - if hook.update(update_params) - present hook, with: Entities::ProjectHook - else - error!("Invalid url given", 422) if hook.errors[:url].present? - error!("Invalid branch filter given", 422) if hook.errors[:push_events_branch_filter].present? - - not_found!("Project hook #{hook.errors.messages}") - end + update_hook(entity: Entities::ProjectHook) end desc 'Deletes project hook' do @@ -105,7 +98,7 @@ module API requires :hook_id, type: Integer, desc: 'The ID of the hook to delete' end delete ":id/hooks/:hook_id" do - hook = user_project.hooks.find(params.delete(:hook_id)) + hook = find_hook destroy_conditionally!(hook) do WebHooks::DestroyService.new(current_user).execute(hook) diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 44b1acaca88..6530887c1c3 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -134,6 +134,7 @@ module API optional :last_activity_before, type: DateTime, desc: 'Limit results to projects with last_activity before specified time. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins' optional :topic, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of topics. Limit results to projects having all topics' + optional :topic_id, type: Integer, desc: 'Limit results to projects with the assigned topic given by the topic ID' use :optional_filter_params_ee end diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb index b9385df1f8d..4611ee58479 100644 --- a/lib/api/protected_tags.rb +++ b/lib/api/protected_tags.rb @@ -10,6 +10,8 @@ module API feature_category :source_code_management + helpers Helpers::ProtectedTagsHelpers + params do requires :id, type: String, desc: 'The ID of a project' end @@ -50,14 +52,15 @@ module API end params do requires :name, type: String, desc: 'The name of the protected tag' - optional :create_access_level, type: Integer, default: Gitlab::Access::MAINTAINER, + optional :create_access_level, type: Integer, values: ProtectedTag::CreateAccessLevel.allowed_access_levels, desc: 'Access levels allowed to create (defaults: `40`, maintainer access level)' + use :optional_params_ee end post ':id/protected_tags' do protected_tags_params = { name: params[:name], - create_access_levels_attributes: [{ access_level: params[:create_access_level] }] + create_access_levels_attributes: ::ProtectedRefs::AccessLevelParams.new(:create, params).access_levels } protected_tag = ::ProtectedTags::CreateService.new(user_project, diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index 5bf3c3b8aac..ae53f08fb1d 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -217,6 +217,8 @@ module API track_package_event('push_package', :pypi, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace) + unprocessable_entity! if Gitlab::FIPS.enabled? && declared_params[:md5_digest].present? + ::Packages::Pypi::CreatePackageService .new(authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job)) .execute diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 2e21f591667..4c7cc6be8b6 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -238,6 +238,10 @@ module API end params do use :release_params + + optional :config_file, + type: String, + desc: "The file path to the configuration file as stored in the project's Git repository. Defaults to '.gitlab/changelog_config.yml'" end get ':id/repository/changelog' do service = ::Repositories::ChangelogService.new( @@ -262,6 +266,10 @@ module API type: String, desc: 'The branch to commit the changelog changes to' + optional :config_file, + type: String, + desc: "The file path to the configuration file as stored in the project's Git repository. Defaults to '.gitlab/changelog_config.yml'" + optional :file, type: String, desc: 'The file to commit the changelog changes to', diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 496532a15b2..4e70ebddf94 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -29,12 +29,16 @@ module API success Entities::Snippet end params do + optional :created_after, type: DateTime, desc: 'Return snippets created after the specified time' + optional :created_before, type: DateTime, desc: 'Return snippets created before the specified time' + use :pagination end get do authenticate! - present paginate(snippets_for_current_user), with: Entities::Snippet, current_user: current_user + filter_params = declared_params(include_missing: false).merge(author: current_user) + present paginate(SnippetsFinder.new(current_user, filter_params).execute), with: Entities::Snippet, current_user: current_user end desc 'List all public personal snippets current_user has access to' do @@ -42,12 +46,16 @@ module API success Entities::PersonalSnippet end params do + optional :created_after, type: DateTime, desc: 'Return snippets created after the specified time' + optional :created_before, type: DateTime, desc: 'Return snippets created before the specified time' + use :pagination end get 'public', urgency: :low do authenticate! - present paginate(public_snippets), with: Entities::PersonalSnippet, current_user: current_user + filter_params = declared_params(include_missing: false).merge(only_personal: true) + present paginate(SnippetsFinder.new(nil, filter_params).execute), with: Entities::PersonalSnippet, current_user: current_user end desc 'Get a single snippet' do diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 7c91fbd36d9..804cedfefe9 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -11,7 +11,27 @@ module API authenticated_as_admin! end + helpers ::API::Helpers::WebHooksHelpers + + helpers do + def hook_scope + SystemHook + end + + params :hook_parameters do + optional :token, type: String, desc: 'The token used to validate payloads' + optional :push_events, type: Boolean, desc: "Trigger hook on push events" + optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :merge_requests_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :repository_update_events, type: Boolean, desc: "Trigger hook on repository update events" + optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" + use :url_variables + end + end + resource :hooks do + mount ::API::Hooks::UrlVariables + desc 'Get the list of system hooks' do success Entities::Hook end @@ -26,70 +46,63 @@ module API success Entities::Hook end params do - requires :id, type: Integer, desc: 'The ID of the system hook' + requires :hook_id, type: Integer, desc: 'The ID of the system hook' end - get ":id" do - hook = SystemHook.find(params[:id]) - - present hook, with: Entities::Hook + get ":hook_id" do + present find_hook, with: Entities::Hook end desc 'Create a new system hook' do success Entities::Hook end params do - requires :url, type: String, desc: "The URL to send the request to" - optional :token, type: String, desc: 'The token used to validate payloads' - optional :push_events, type: Boolean, desc: "Trigger hook on push events" - optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" - optional :merge_requests_events, type: Boolean, desc: "Trigger hook on tag push events" - optional :repository_update_events, type: Boolean, desc: "Trigger hook on repository update events" - optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" + use :requires_url + use :hook_parameters end post do - hook = SystemHook.new(declared_params(include_missing: false)) + hook_params = create_hook_params + hook = SystemHook.new(hook_params) - if hook.save - present hook, with: Entities::Hook - else - render_validation_error!(hook) - end + save_hook(hook, Entities::Hook) end - desc 'Test a hook' + desc 'Update an existing system hook' do + success Entities::Hook + end params do - requires :id, type: Integer, desc: 'The ID of the system hook' + requires :hook_id, type: Integer, desc: "The ID of the hook to update" + use :optional_url + use :hook_parameters end - post ":id" do - hook = SystemHook.find(params[:id]) - data = { + put ":hook_id" do + update_hook(entity: Entities::Hook) + end + + mount ::API::Hooks::Test, with: { + data: { event_name: "project_create", name: "Ruby", path: "ruby", project_id: 1, owner_name: "Someone", owner_email: "example@gitlabhq.com" - } - hook.execute(data, 'system_hooks') - data - end + }, + kind: 'system_hooks' + } desc 'Delete a hook' do success Entities::Hook end params do - requires :id, type: Integer, desc: 'The ID of the system hook' + requires :hook_id, type: Integer, desc: 'The ID of the system hook' end - # rubocop: disable CodeReuse/ActiveRecord - delete ":id" do - hook = SystemHook.find_by(id: params[:id]) - not_found!('System hook') unless hook + delete ":hook_id" do + hook = find_hook destroy_conditionally!(hook) do WebHooks::DestroyService.new(current_user).execute(hook) end end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 0fa8c21f8d7..97a2aebf53b 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -6,7 +6,11 @@ module API TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX) - before { authorize! :download_code, user_project } + before do + authorize! :download_code, user_project + + not_found! unless user_project.repo_exists? + end params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb index f96cffb008c..267d41e5fb9 100644 --- a/lib/api/terraform/modules/v1/packages.rb +++ b/lib/api/terraform/modules/v1/packages.rb @@ -97,6 +97,33 @@ module API present presenter, with: ::API::Entities::Terraform::ModuleVersions end + get 'download' do + latest_version = packages.order_version.last&.version + + render_api_error!({ error: "No version found for #{params[:module_name]} module" }, :not_found) if latest_version.nil? + + download_path = api_v4_packages_terraform_modules_v1_module_version_download_path( + { + module_namespace: params[:module_namespace], + module_name: params[:module_name], + module_system: params[:module_system], + module_version: latest_version + }, + true + ) + + redirect(download_path) + end + + get do + latest_package = packages.order_version.last + + render_api_error!({ error: "No version found for #{params[:module_name]} module" }, :not_found) if latest_package&.version.nil? + + presenter = ::Terraform::ModuleVersionPresenter.new(latest_package, params[:module_system]) + present presenter, with: ::API::Entities::Terraform::ModuleVersion + end + params do includes :module_version end @@ -133,6 +160,16 @@ module API present_carrierwave_file!(package_file.file) end end + + # This endpoint has to be the last within namespace '*module_version' block + # due to how the route matching works in grape + # format: false is required, otherwise grape splits the semver version into 2 params: + # params[:module_version] and params[:format], + # thus leading to an invalid/not found module version + get format: false do + presenter = ::Terraform::ModuleVersionPresenter.new(package, params[:module_system]) + present presenter, with: ::API::Entities::Terraform::ModuleVersion + end end end diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb index b727fbd9f65..a19919b5e76 100644 --- a/lib/api/terraform/state.rb +++ b/lib/api/terraform/state.rb @@ -81,7 +81,7 @@ module API delete do authorize! :admin_terraform_state, user_project - remote_state_handler.handle_with_lock do |state| + remote_state_handler.find_with_lock do |state| ::Terraform::States::TriggerDestroyService.new(state, current_user: current_user).execute end diff --git a/lib/api/unleash.rb b/lib/api/unleash.rb index 37fe540cde1..2d528ad47a2 100644 --- a/lib/api/unleash.rb +++ b/lib/api/unleash.rb @@ -25,14 +25,22 @@ module API desc 'Get a list of features (deprecated, v2 client support)' get 'features' do - present :version, 1 - present :features, feature_flags, with: ::API::Entities::UnleashFeature + if ::Feature.enabled?(:cache_unleash_client_api, project) + present_feature_flags + else + present :version, 1 + present :features, feature_flags, with: ::API::Entities::UnleashFeature + end end desc 'Get a list of features' get 'client/features' do - present :version, 1 - present :features, feature_flags, with: ::API::Entities::UnleashFeature + if ::Feature.enabled?(:cache_unleash_client_api, project) + present_feature_flags + else + present :version, 1 + present :features, feature_flags, with: ::API::Entities::UnleashFeature + end end post 'client/register' do @@ -49,10 +57,24 @@ module API end helpers do + def present_feature_flags + present_cached feature_flags_client, + with: ::API::Entities::Unleash::ClientFeatureFlags, + cache_context: -> (client) { client.unleash_api_cache_key } + end + def project @project ||= find_project(params[:project_id]) end + def feature_flags_client + strong_memoize(:feature_flags_client) do + client = Operations::FeatureFlagsClient.find_for_project_and_token(project, unleash_instance_id) + client.unleash_app_name = unleash_app_name if client + client + end + end + def unleash_instance_id env['HTTP_UNLEASH_INSTANCEID'] || params[:instance_id] end @@ -62,8 +84,7 @@ module API end def authorize_by_unleash_instance_id! - unauthorized! unless Operations::FeatureFlagsClient - .find_for_project_and_token(project, unleash_instance_id) + unauthorized! unless feature_flags_client end def feature_flags diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index 6e81a578d4a..9e446aff605 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -29,7 +29,7 @@ module API params do requires :event, type: String, desc: 'The event name that should be tracked' end - post 'increment_unique_users' do + post 'increment_unique_users', urgency: :low do event_name = params[:event] increment_unique_values(event_name, current_user.id) diff --git a/lib/api/users.rb b/lib/api/users.rb index 93df9413119..d66d86a9055 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -125,7 +125,7 @@ module API entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic if entity == Entities::UserWithAdmin - users = users.preload(:identities, :u2f_registrations, :webauthn_registrations, :namespace) + users = users.preload(:identities, :u2f_registrations, :webauthn_registrations, :namespace, :followers, :followees, :user_preference) end users, options = with_custom_attributes(users, { with: entity, current_user: current_user }) @@ -325,6 +325,30 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + desc "Disable two factor authentication for a user. Available only for admins" do + detail 'This feature was added in GitLab 15.2' + success Entities::UserWithAdmin + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + patch ":id/disable_two_factor", feature_category: :authentication_and_authorization do + authenticated_as_admin! + + user = User.find_by_id(params[:id]) + not_found!('User') unless user + + forbidden!('Two-factor authentication for admins cannot be disabled via the API. Use the Rails console') if user.admin? + + result = TwoFactor::DestroyService.new(current_user, user: user).execute + + if result[:status] == :success + no_content! + else + bad_request!(result[:message]) + end + end + desc "Delete a user's identity. Available only for admins" do success Entities::UserWithAdmin end @@ -1260,3 +1284,5 @@ module API end end end + +API::Users.prepend_mod diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index 077eabdd131..a995f308c2b 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -96,8 +96,8 @@ module Backup def build_env { - 'SSL_CERT_FILE' => OpenSSL::X509::DEFAULT_CERT_FILE, - 'SSL_CERT_DIR' => OpenSSL::X509::DEFAULT_CERT_DIR + 'SSL_CERT_FILE' => Gitlab::X509::Certificate.default_cert_file, + 'SSL_CERT_DIR' => Gitlab::X509::Certificate.default_cert_dir }.merge(ENV) end diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb index 537b7c80d91..f5c4b788ad8 100644 --- a/lib/banzai/filter/footnote_filter.rb +++ b/lib/banzai/filter/footnote_filter.rb @@ -71,7 +71,12 @@ module Banzai private def random_number - @random_number ||= rand(10000) + # We allow overriding the randomness with a static value from GITLAB_TEST_FOOTNOTE_ID. + # This allows stable generation of example HTML during GLFM Snapshot Testing + # (https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/#markdown-snapshot-testing), + # and reduces the need for normalization of the example HTML + # (https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/#normalization) + @random_number ||= ENV.fetch('GITLAB_TEST_FOOTNOTE_ID', rand(10000)) end def fn_id(num) diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb index df27275b664..3067e0997c2 100644 --- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb +++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb @@ -7,10 +7,15 @@ module BulkImports def transform(context, data) import_entity = context.entity + if import_entity.destination_namespace.present? + namespace = Namespace.find_by_full_path(import_entity.destination_namespace) + end + data + .then { |data| transform_name(import_entity, namespace, data) } .then { |data| transform_path(import_entity, data) } .then { |data| transform_full_path(data) } - .then { |data| transform_parent(context, import_entity, data) } + .then { |data| transform_parent(context, import_entity, namespace, data) } .then { |data| transform_visibility_level(data) } .then { |data| transform_project_creation_level(data) } .then { |data| transform_subgroup_creation_level(data) } @@ -18,6 +23,20 @@ module BulkImports private + def transform_name(import_entity, namespace, data) + if namespace.present? + namespace_children_names = namespace.children.pluck(:name) # rubocop: disable CodeReuse/ActiveRecord + + if namespace_children_names.include?(data['name']) + data['name'] = Uniquify.new(1).string(-> (counter) { "#{data['name']}(#{counter})" }) do |base| + namespace_children_names.include?(base) + end + end + end + + data + end + def transform_path(import_entity, data) data['path'] = import_entity.destination_name.parameterize data @@ -28,11 +47,8 @@ module BulkImports data end - def transform_parent(context, import_entity, data) - unless import_entity.destination_namespace.blank? - namespace = Namespace.find_by_full_path(import_entity.destination_namespace) - data['parent_id'] = namespace.id - end + def transform_parent(context, import_entity, namespace, data) + data['parent_id'] = namespace.id if namespace.present? data end diff --git a/lib/bulk_imports/network_error.rb b/lib/bulk_imports/network_error.rb index d69b0172f6c..3514291a75d 100644 --- a/lib/bulk_imports/network_error.rb +++ b/lib/bulk_imports/network_error.rb @@ -7,9 +7,9 @@ module BulkImports RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS RETRIABLE_HTTP_CODES = [429].freeze - DEFAULT_RETRY_DELAY_SECONDS = 60 + DEFAULT_RETRY_DELAY_SECONDS = 30 - MAX_RETRIABLE_COUNT = 3 + MAX_RETRIABLE_COUNT = 10 def initialize(message = nil, response: nil) raise ArgumentError, 'message or response required' if message.blank? && response.blank? diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index 8f515b571a6..c03da7d8d01 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -56,12 +56,16 @@ module BulkImports pipeline_step: step, step_class: class_name ) + rescue BulkImports::NetworkError => e + if e.retriable?(context.tracker) + raise BulkImports::RetryPipelineError.new(e.message, e.retry_delay) + else + log_and_fail(e, step) + end + rescue BulkImports::RetryPipelineError + raise rescue StandardError => e - log_import_failure(e, step) - - mark_as_failed if abort_on_failure? - - nil + log_and_fail(e, step) end def extracted_data_from @@ -74,11 +78,17 @@ module BulkImports run if extracted_data.has_next_page? end - def mark_as_failed - warn(message: 'Pipeline failed') + def log_and_fail(exception, step) + log_import_failure(exception, step) - context.entity.fail_op! tracker.fail_op! + + if abort_on_failure? + warn(message: 'Aborting entity migration due to pipeline failure') + context.entity.fail_op! + end + + nil end def skip!(message, extra = {}) diff --git a/lib/bulk_imports/retry_pipeline_error.rb b/lib/bulk_imports/retry_pipeline_error.rb new file mode 100644 index 00000000000..a1b02addf45 --- /dev/null +++ b/lib/bulk_imports/retry_pipeline_error.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module BulkImports + class RetryPipelineError < Error + attr_reader :retry_delay + + def initialize(message, retry_delay) + super(message) + + @retry_delay = retry_delay + end + end +end diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb index 0cd8f8509f6..c68b222af97 100644 --- a/lib/container_registry/gitlab_api_client.rb +++ b/lib/container_registry/gitlab_api_client.rb @@ -28,7 +28,7 @@ module ContainerRegistry end def self.deduplicated_size(path) - with_dummy_client(token_config: { type: :nested_repositories_token, path: path }) do |client| + with_dummy_client(token_config: { type: :nested_repositories_token, path: path&.downcase }) do |client| client.repository_details(path, sizing: :self_with_descendants)['size_bytes'] end end diff --git a/lib/error_tracking/collector/dsn.rb b/lib/error_tracking/collector/dsn.rb deleted file mode 100644 index 665181328f3..00000000000 --- a/lib/error_tracking/collector/dsn.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module ErrorTracking - module Collector - class Dsn - # Build a sentry compatible DSN URL for GitLab collector. - # - # The expected URL looks like that: - # https://PUBLIC_KEY@gitlab.example.com/api/v4/error_tracking/collector/PROJECT_ID - # - def self.build_url(public_key, project_id) - gitlab = Settings.gitlab - - custom_port = Settings.gitlab_on_standard_port? ? nil : ":#{gitlab.port}" - - base_url = [ - gitlab.protocol, - "://", - public_key, - '@', - gitlab.host, - custom_port, - gitlab.relative_url_root - ].join('') - - "#{base_url}/api/v4/error_tracking/collector/#{project_id}" - end - end - end -end diff --git a/lib/error_tracking/stacktrace_builder.rb b/lib/error_tracking/stacktrace_builder.rb index 4f331bc4e06..024587e8683 100644 --- a/lib/error_tracking/stacktrace_builder.rb +++ b/lib/error_tracking/stacktrace_builder.rb @@ -29,6 +29,10 @@ module ErrorTracking exception_entry = payload['exception'] return unless exception_entry + # Some SDK send exception payload as Array. For exmple Go lang SDK. + # We need to convert it to hash format we expect. + exception_entry = { 'values' => exception_entry } if exception_entry.is_a?(Array) + exception_values = exception_entry['values'] stack_trace_entry = exception_values&.detect { |h| h['stacktrace'].present? } stack_trace_entry&.dig('stacktrace', 'frames') diff --git a/lib/feature.rb b/lib/feature.rb index 3bba4be7514..ca91d86c199 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -3,7 +3,7 @@ require 'flipper/adapters/active_record' require 'flipper/adapters/active_support_cache_store' -class Feature +module Feature # Classes to override flipper table names class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature include DatabaseReflection @@ -104,7 +104,14 @@ class Feature def enable(key, thing = true) log(key: key, action: __method__, thing: thing) - with_feature(key) { _1.enable(thing) } + return_value = with_feature(key) { _1.enable(thing) } + + # rubocop:disable Gitlab/RailsLogger + Rails.logger.warn('WARNING: Understand the stability and security risks of enabling in-development features with feature flags.') + Rails.logger.warn('See https://docs.gitlab.com/ee/administration/feature_flags.html#risks-when-enabling-features-still-in-development for more information.') + # rubocop:enable Gitlab/RailsLogger + + return_value end def disable(key, thing = false) diff --git a/lib/feature/definition.rb b/lib/feature/definition.rb index 1551af730db..270bf46221d 100644 --- a/lib/feature/definition.rb +++ b/lib/feature/definition.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Feature +module Feature class Definition include ::Feature::Shared diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index 04ed78b8a51..0c6b9dfde7a 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Feature +module Feature class Gitaly PREFIX = "gitaly_" diff --git a/lib/feature/logger.rb b/lib/feature/logger.rb index 784a619e182..95e160273b6 100644 --- a/lib/feature/logger.rb +++ b/lib/feature/logger.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Feature +module Feature class Logger < ::Gitlab::JsonLogger def self.file_name_noext 'features_json' diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index 40f21fc4f50..edfc39aea0c 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -4,7 +4,7 @@ # 1. `Pure Ruby`: `bin/feature-flag` # 2. `GitLab Rails`: `lib/feature/definition.rb` -class Feature +module Feature module Shared # optional: defines if a on-disk definition is required for this feature flag type # rollout_issue: defines if `bin/feature-flag` asks for rollout issue diff --git a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb index 58d3257d07e..8cd03978f27 100644 --- a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb +++ b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb @@ -8,15 +8,17 @@ module Gitlab desc 'Generates a metric definition .yml file with defaults for Redis HLL.' argument :category, type: :string, desc: "Category name" - argument :event, type: :string, desc: "Event name" + argument :events, type: :array, desc: "Unique event names", banner: 'event_one event_two event_three' class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if metric is for ee' def create_metrics - weekly_params = ["#{key_path}_weekly", '--dir', '7d', '--class_name', 'RedisHLLMetric'] + weekly_key_paths = key_paths.map { |key_path| "#{key_path}_weekly" } + weekly_params = [*weekly_key_paths, '--dir', '7d', '--class_name', 'RedisHLLMetric'] weekly_params << '--ee' if ee? Gitlab::UsageMetricDefinitionGenerator.start(weekly_params) - monthly_params = ["#{key_path}_monthly", '--dir', '28d', '--class_name', 'RedisHLLMetric'] + monthly_key_paths = key_paths.map { |key_path| "#{key_path}_monthly" } + monthly_params = [*monthly_key_paths, '--dir', '28d', '--class_name', 'RedisHLLMetric'] monthly_params << '--ee' if ee? Gitlab::UsageMetricDefinitionGenerator.start(monthly_params) end @@ -27,8 +29,8 @@ module Gitlab options[:ee] end - def key_path - "redis_hll_counters.#{category}.#{event}" + def key_paths + events.map { |event| "redis_hll_counters.#{category}.#{event}" } end end end diff --git a/lib/generators/model/model_generator.rb b/lib/generators/model/model_generator.rb new file mode 100644 index 00000000000..533b2ce679d --- /dev/null +++ b/lib/generators/model/model_generator.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'rails/generators' +require 'rails/generators/active_record/model/model_generator' + +module Model + class ModelGenerator < ActiveRecord::Generators::ModelGenerator + source_root File.expand_path('../../../generator_templates/active_record/migration/', __dir__) + + def create_migration_file + return if skip_migration_creation? + + if options[:indexes] == false + attributes.each { |a| a.attr_options.delete(:index) if a.reference? && !a.has_index? } + end + + migration_template "create_table_migration.rb", File.join(db_migrate_path, "create_#{table_name}.rb") + end + + # Override to find templates from superclass as well + def source_paths + super + [self.class.superclass.default_source_root] + end + end +end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 722ee061eba..0c52ce8aba4 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -37,6 +37,7 @@ module Gitlab users_get_by_id: { threshold: -> { application_settings.users_get_by_id_limit }, interval: 10.minutes }, username_exists: { threshold: 20, interval: 1.minute }, user_sign_up: { threshold: 20, interval: 1.minute }, + user_sign_in: { threshold: 5, interval: 10.minutes }, profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, profile_update_username: { threshold: 10, interval: 1.minute }, update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, @@ -45,7 +46,10 @@ module Gitlab search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, gitlab_shell_operation: { threshold: 600, interval: 1.minute }, pipelines_create: { threshold: -> { application_settings.pipeline_limit_per_project_user_sha }, interval: 1.minute }, - temporary_email_failure: { threshold: 50, interval: 1.day } + temporary_email_failure: { threshold: 50, interval: 1.day }, + project_testing_integration: { threshold: 5, interval: 1.minute }, + email_verification: { threshold: 10, interval: 10.minutes }, + email_verification_code_send: { threshold: 10, interval: 1.hour } }.freeze end @@ -53,15 +57,26 @@ module Gitlab # be throttled. # # @param key [Symbol] Key attribute registered in `.rate_limits` - # @param scope [Array<ActiveRecord>] Array of ActiveRecord models, Strings or Symbols to scope throttling to a specific request (e.g. per user per project) - # @param threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` - # @param users_allowlist [Array<String>] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. - # @param peek [Boolean] Optional. When true the key will not be incremented but the current throttled state will be returned. + # @param scope [Array<ActiveRecord>] Array of ActiveRecord models, Strings + # or Symbols to scope throttling to a specific request (e.g. per user + # per project) + # @param resource [ActiveRecord] An ActiveRecord model to count an action + # for (e.g. limit unique project (resource) downloads (action) to five + # per user (scope)) + # @param threshold [Integer] Optional threshold value to override default + # one registered in `.rate_limits` + # @param users_allowlist [Array<String>] Optional list of usernames to + # exclude from the limit. This param will only be functional if Scope + # includes a current user. + # @param peek [Boolean] Optional. When true the key will not be + # incremented but the current throttled state will be returned. # # @return [Boolean] Whether or not a request should be throttled - def throttled?(key, scope:, threshold: nil, users_allowlist: nil, peek: false) + def throttled?(key, scope:, resource: nil, threshold: nil, users_allowlist: nil, peek: false) raise InvalidKeyError unless rate_limits[key] + strategy = resource.present? ? IncrementPerActionedResource.new(resource.id) : IncrementPerAction.new + ::Gitlab::Instrumentation::RateLimitingGates.track(key) return false if scoped_user_in_allowlist?(scope, users_allowlist) @@ -71,6 +86,9 @@ module Gitlab return false if threshold_value == 0 interval_value = interval(key) + + return false if interval_value == 0 + # `period_key` is based on the current time and interval so when time passes to the next interval # the key changes and the rate limit count starts again from 0. # Based on https://github.com/rack/rack-attack/blob/886ba3a18d13c6484cd511a4dc9b76c0d14e5e96/lib/rack/attack/cache.rb#L63-L68 @@ -78,9 +96,12 @@ module Gitlab cache_key = cache_key(key, scope, period_key) value = if peek - read(cache_key) + strategy.read(cache_key) else - increment(cache_key, interval_value, time_elapsed_in_period) + # We add a 1 second buffer to avoid timing issues when we're at the end of a period + expiry = interval_value - time_elapsed_in_period + 1 + + strategy.increment(cache_key, expiry) end value > threshold_value @@ -128,40 +149,25 @@ module Gitlab def threshold(key) value = rate_limit_value_by_key(key, :threshold) - return value.call if value.is_a?(Proc) - - value.to_i + rate_limit_value(value) end def interval(key) - rate_limit_value_by_key(key, :interval).to_i - end - - def rate_limit_value_by_key(key, setting) - action = rate_limits[key] + value = rate_limit_value_by_key(key, :interval) - action[setting] if action + rate_limit_value(value) end - # Increments the rate limit count and returns the new count value. - def increment(cache_key, interval_value, time_elapsed_in_period) - # We add a 1 second buffer to avoid timing issues when we're at the end of a period - expiry = interval_value - time_elapsed_in_period + 1 + def rate_limit_value(value) + value = value.call if value.is_a?(Proc) - ::Gitlab::Redis::RateLimiting.with do |redis| - redis.pipelined do - redis.incr(cache_key) - redis.expire(cache_key, expiry) - end.first - end + value.to_i end - # Returns the rate limit count. - # Will be 0 if there is no data in the cache. - def read(cache_key) - ::Gitlab::Redis::RateLimiting.with do |redis| - redis.get(cache_key).to_i - end + def rate_limit_value_by_key(key, setting) + action = rate_limits[key] + + action[setting] if action end def cache_key(key, scope, period_key) diff --git a/lib/gitlab/application_rate_limiter/base_strategy.rb b/lib/gitlab/application_rate_limiter/base_strategy.rb new file mode 100644 index 00000000000..b97770c0524 --- /dev/null +++ b/lib/gitlab/application_rate_limiter/base_strategy.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module ApplicationRateLimiter + class BaseStrategy + # Increment the rate limit count and return the new count value + def increment(cache_key, expiry) + raise NotImplementedError + end + + # Return the rate limit count. + # Should be 0 if there is no data in the cache. + def read(cache_key) + raise NotImplementedError + end + + private + + def with_redis(&block) + ::Gitlab::Redis::RateLimiting.with(&block) # rubocop: disable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/application_rate_limiter/increment_per_action.rb b/lib/gitlab/application_rate_limiter/increment_per_action.rb new file mode 100644 index 00000000000..c99d03f1344 --- /dev/null +++ b/lib/gitlab/application_rate_limiter/increment_per_action.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module ApplicationRateLimiter + class IncrementPerAction < BaseStrategy + def increment(cache_key, expiry) + with_redis do |redis| + redis.pipelined do + redis.incr(cache_key) + redis.expire(cache_key, expiry) + end.first + end + end + + def read(cache_key) + with_redis do |redis| + redis.get(cache_key).to_i + end + end + end + end +end diff --git a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb new file mode 100644 index 00000000000..8b4197cfff9 --- /dev/null +++ b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module ApplicationRateLimiter + class IncrementPerActionedResource < BaseStrategy + def initialize(resource_key) + @resource_key = resource_key + end + + def increment(cache_key, expiry) + with_redis do |redis| + redis.pipelined do + redis.sadd(cache_key, resource_key) + redis.expire(cache_key, expiry) + redis.scard(cache_key) + end.last + end + end + + def read(cache_key) + with_redis do |redis| + redis.scard(cache_key) + end + end + + private + + attr_accessor :resource_key + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 5d5a431f206..6c3487c28ea 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -217,7 +217,7 @@ module Gitlab return unless valid_scoped_token?(token, all_available_scopes) if project && token.user.project_bot? - return unless token_bot_in_resource?(token.user, project) + return unless can_read_project?(token.user, project) end if token.user.can_log_in_with_non_expired_password? || token.user.project_bot? @@ -225,22 +225,8 @@ module Gitlab end end - def token_bot_in_project?(user, project) - project.bots.include?(user) - end - - # rubocop: disable CodeReuse/ActiveRecord - - # A workaround for adding group-level automation is to add the bot user of a project access token as a group member. - # In order to make project access tokens work this way during git authentication, we need to add an additional check for group membership. - # This is a temporary workaround until service accounts are implemented. - def token_bot_in_group?(user, project) - project.group && project.group.members_with_parents.where(user_id: user.id).exists? - end - # rubocop: enable CodeReuse/ActiveRecord - - def token_bot_in_resource?(user, project) - token_bot_in_project?(user, project) || token_bot_in_group?(user, project) + def can_read_project?(user, project) + user.can?(:read_project, project) end def valid_oauth_token?(token) @@ -323,7 +309,7 @@ module Gitlab return unless build.project.builds_enabled? if build.user - return unless build.user.can_log_in_with_non_expired_password? || (build.user.project_bot? && token_bot_in_resource?(build.user, build.project)) + return unless build.user.can_log_in_with_non_expired_password? || (build.user.project_bot? && can_read_project?(build.user, build.project)) # If user is assigned to build, use restricted credentials of user Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index 56c2af1910e..957ec5fa479 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -31,7 +31,11 @@ module Gitlab end def valid_sign_in? - allowed? && super + # The order is important here: we need to ensure the + # associated GitLab user entry is valid and persisted in the + # database. Otherwise, the LDAP access check will fail since + # the user doesn't have an associated LDAP identity. + super && allowed? end def ldap_config diff --git a/lib/gitlab/background_migration/backfill_ci_runner_semver.rb b/lib/gitlab/background_migration/backfill_ci_runner_semver.rb new file mode 100644 index 00000000000..0901649f789 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_ci_runner_semver.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to update semver column in ci_runners in batches based on existing version values + class BackfillCiRunnerSemver < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + each_sub_batch( + operation_name: :backfill_ci_runner_semver, + batching_scope: ->(relation) { relation.where('semver::cidr IS NULL') } + ) do |sub_batch| + ranged_query = sub_batch.select( + %q(id AS r_id, + substring(ci_runners.version FROM 'v?(\d+\.\d+\.\d+)') AS extracted_semver) + ) + + update_sql = <<~SQL + UPDATE + ci_runners + SET semver = extracted_semver + FROM (#{ranged_query.to_sql}) v + WHERE id = v.r_id + AND v.extracted_semver IS NOT NULL + SQL + + connection.execute(update_sql) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb new file mode 100644 index 00000000000..b2d38ce6aa4 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + # Rechedules the backfill for the `issue_search_data` table for issues imported prior + # to the fix for the imported issues search data bug: + + class BackfillImportedIssueSearchData < BatchedMigrationJob + SUB_BATCH_SIZE = 1_000 + + def perform + each_sub_batch( + operation_name: :update_search_data + ) do |sub_batch| + update_search_data(sub_batch) + rescue ActiveRecord::StatementInvalid => e + raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') + + update_search_data_individually(sub_batch) + end + end + + private + + def update_search_data(relation) + ApplicationRecord.connection.execute( + <<~SQL + INSERT INTO issue_search_data + SELECT + project_id, + id, + NOW(), + NOW(), + setweight(to_tsvector('english', LEFT(title, 255)), 'A') || setweight(to_tsvector('english', LEFT(REGEXP_REPLACE(description, '[A-Za-z0-9+/@]{50,}', ' ', 'g'), 1048576)), 'B') + FROM (#{relation.limit(SUB_BATCH_SIZE).to_sql}) issues + ON CONFLICT DO NOTHING + SQL + ) + end + + def update_search_data_individually(relation) + relation.pluck(:id).each do |issue_id| + update_search_data(relation.klass.where(id: issue_id)) + sleep(pause_ms * 0.001) + rescue ActiveRecord::StatementInvalid => e + raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') + + logger.error( + message: "Error updating search data: #{e.message}", + class: relation.klass.name, + model_id: issue_id + ) + end + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb b/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb new file mode 100644 index 00000000000..ec813022b8f --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fill container_registry_size for project_statistics + class BackfillProjectStatisticsContainerRepositorySize < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + # no-op + end + end + end +end + +Gitlab::BackgroundMigration::BackfillProjectStatisticsContainerRepositorySize.prepend_mod_with('Gitlab::BackgroundMigration::BackfillProjectStatisticsContainerRepositorySize') # rubocop:disable Layout/LineLength diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 442eab0673e..c47b1735ccf 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -44,7 +44,19 @@ module Gitlab end end - def parent_batch_relation(batching_scope) + def distinct_each_batch(operation_name: :default, batching_arguments: {}) + all_batching_arguments = { column: batch_column, of: sub_batch_size }.merge(batching_arguments) + + parent_batch_relation.distinct_each_batch(**all_batching_arguments) do |relation| + batch_metrics.instrument_operation(operation_name) do + yield relation + end + + sleep([pause_ms, 0].max * 0.001) + end + end + + def parent_batch_relation(batching_scope = nil) parent_relation = define_batchable_model(batch_table, connection: connection) .where(batch_column => start_id..end_id) diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb index 06036eebcb9..7d5fef67c25 100644 --- a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb @@ -8,7 +8,7 @@ module Gitlab # # If no more batches exist in the table, returns nil. class BackfillIssueWorkItemTypeBatchingStrategy < PrimaryKeyBatchingStrategy - def apply_additional_filters(relation, job_arguments:) + def apply_additional_filters(relation, job_arguments:, job_class: nil) issue_type = job_arguments.first relation.where(issue_type: issue_type) diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_project_namespace_per_group_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_project_namespace_per_group_batching_strategy.rb index f352c527b54..68be42dc0a0 100644 --- a/lib/gitlab/background_migration/batching_strategies/backfill_project_namespace_per_group_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/backfill_project_namespace_per_group_batching_strategy.rb @@ -16,7 +16,7 @@ module Gitlab # batch_min_value - The minimum value which the next batch will start at # batch_size - The size of the next batch # job_arguments - The migration job arguments - def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:) + def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:, job_class: nil) next_batch_bounds = nil model_class = ::Gitlab::BackgroundMigration::ProjectNamespaces::Models::Project quoted_column_name = model_class.connection.quote_column_name(column_name) diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb new file mode 100644 index 00000000000..9ad119310f7 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Batching class to use for back-filling project_statistic's container_registry_size. + # Batches will be scoped to records where the project_ids are migrated + # + # If no more batches exist in the table, returns nil. + class BackfillProjectStatisticsWithContainerRegistrySizeBatchingStrategy < PrimaryKeyBatchingStrategy + MIGRATION_PHASE_1_ENDED_AT = Date.new(2022, 01, 23).freeze + + def apply_additional_filters(relation, job_arguments: [], job_class: nil) + relation.where(created_at: MIGRATION_PHASE_1_ENDED_AT..).or( + relation.where(migration_state: 'import_done') + ).select(:project_id).distinct + end + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb new file mode 100644 index 00000000000..e1855b6cfee --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Batching class to use for setting state in vulnerabilitites table. + # Batches will be scoped to records where the dismissed_at is set. + # + # If no more batches exist in the table, returns nil. + class DismissedVulnerabilitiesStrategy < PrimaryKeyBatchingStrategy + def apply_additional_filters(relation, job_arguments: [], job_class: nil) + relation.where.not(dismissed_at: nil) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy.rb new file mode 100644 index 00000000000..5cad9d2e3c4 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # This strategy provides an efficient way to iterate over columns with non-distinct values. + # A common use case would be iterating over a foreign key columns, for example issues.project_id + class LooseIndexScanBatchingStrategy < BaseStrategy + include Gitlab::Database::DynamicModelHelpers + + # Finds and returns the next batch in the table. + # + # table_name - The table to batch over + # column_name - The column to batch over + # batch_min_value - The minimum value which the next batch will start at + # batch_size - The size of the next batch + # job_arguments - The migration job arguments + # job_class - The migration job class + def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:, job_class: nil) + model_class = define_batchable_model(table_name, connection: connection) + + quoted_column_name = model_class.connection.quote_column_name(column_name) + relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) + next_batch_bounds = nil + + relation.distinct_each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop + next_batch_bounds = batch.pluck(Arel.sql("MIN(#{quoted_column_name}), MAX(#{quoted_column_name})")).first + + break + end + + next_batch_bounds + end + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb index e7a68b183b8..c2f59bf9c76 100644 --- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -18,12 +18,13 @@ module Gitlab # batch_min_value - The minimum value which the next batch will start at # batch_size - The size of the next batch # job_arguments - The migration job arguments - def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:) + # job_class - The migration job class + def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:, job_class: nil) model_class = define_batchable_model(table_name, connection: connection) quoted_column_name = model_class.connection.quote_column_name(column_name) relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) - relation = apply_additional_filters(relation, job_arguments: job_arguments) + relation = apply_additional_filters(relation, job_arguments: job_arguments, job_class: job_class) next_batch_bounds = nil relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop @@ -35,19 +36,11 @@ module Gitlab next_batch_bounds end - # Strategies based on PrimaryKeyBatchingStrategy can use - # this method to easily apply additional filters. - # - # Example: - # - # class MatchingType < PrimaryKeyBatchingStrategy - # def apply_additional_filters(relation, job_arguments:) - # type = job_arguments.first - # - # relation.where(type: type) - # end - # end - def apply_additional_filters(relation, job_arguments: []) + def apply_additional_filters(relation, job_arguments: [], job_class: nil) + if job_class.respond_to?(:batching_scope) + return job_class.batching_scope(relation, job_arguments: job_arguments) + end + relation end end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb new file mode 100644 index 00000000000..e759d504f8d --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for inactive, public projects + class DisableLegacyOpenSourceLicenseForInactivePublicProjects < + ::Gitlab::BackgroundMigration::BatchedMigrationJob + PUBLIC = 20 + LAST_ACTIVITY_DATE = '2021-07-01' + + # Migration only version of `project_settings` table + class ProjectSetting < ApplicationRecord + self.table_name = 'project_settings' + end + + def perform + each_sub_batch( + operation_name: :disable_legacy_open_source_license_available, + batching_scope: ->(relation) { + relation.where(visibility_level: PUBLIC).where('last_activity_at < ?', LAST_ACTIVITY_DATE) + } + ) do |sub_batch| + ProjectSetting.where(project_id: sub_batch).update_all(legacy_open_source_license_available: false) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb index a34e923545c..914ababa5c2 100644 --- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb +++ b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb @@ -68,7 +68,7 @@ module Gitlab def valid_json?(metadata) Oj.load(metadata) true - rescue Oj::ParseError, Encoding::UndefinedConversionError + rescue Oj::ParseError, EncodingError, JSON::ParserError, Encoding::UndefinedConversionError false end diff --git a/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb new file mode 100644 index 00000000000..3f04e04fc4d --- /dev/null +++ b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates the value operations_access_level to the new colums + # monitor_access_level, deployments_access_level, infrastructure_access_level. + # The operations_access_level setting is being split into three seperate toggles. + class PopulateOperationVisibilityPermissionsFromOperations < BatchedMigrationJob + def perform + each_sub_batch(operation_name: :populate_operations_visibility) do |batch| + batch.update_all('monitor_access_level=operations_access_level,' \ + 'infrastructure_access_level=operations_access_level,' \ + ' feature_flags_access_level=operations_access_level,'\ + ' environments_access_level=operations_access_level') + end + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'PopulateOperationVisibilityPermissionsFromOperations', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index db7afd59f4d..72380af2c53 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -79,10 +79,6 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # r # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength def perform(start_id, end_id) - unless Feature.enabled?(:migrate_vulnerability_finding_uuids) - return log_info('Migration is disabled by the feature flag', start_id: start_id, end_id: end_id) - end - log_info('Migration started', start_id: start_id, end_id: end_id) VulnerabilitiesFinding diff --git a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb new file mode 100644 index 00000000000..fd6cbcb8d05 --- /dev/null +++ b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Change the vulnerability state to `dismissed` if `dismissed_at` field is not null + class SetCorrectVulnerabilityState < BatchedMigrationJob + DISMISSED_STATE = 2 + + def perform + each_sub_batch( + operation_name: :update_vulnerabilities_state, + batching_scope: -> (relation) { relation.where.not(dismissed_at: nil) } + ) do |sub_batch| + sub_batch.update_all(state: DISMISSED_STATE) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb new file mode 100644 index 00000000000..04a2ceebef8 --- /dev/null +++ b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is used to update the delayed_project_removal column + # for user namespaces of the namespace_settings table. + class UpdateDelayedProjectRemovalToNullForUserNamespaces < Gitlab::BackgroundMigration::BatchedMigrationJob + # Migration only version of `namespace_settings` table + class NamespaceSetting < ::ApplicationRecord + self.table_name = 'namespace_settings' + end + + def perform + each_sub_batch( + operation_name: :set_delayed_project_removal_to_null_for_user_namespace + ) do |sub_batch| + set_delayed_project_removal_to_null_for_user_namespace(sub_batch) + end + end + + private + + def set_delayed_project_removal_to_null_for_user_namespace(relation) + NamespaceSetting.connection.execute( + <<~SQL + UPDATE namespace_settings + SET delayed_project_removal = NULL + WHERE + namespace_settings.namespace_id IN ( + SELECT + namespace_settings.namespace_id + FROM + namespace_settings + INNER JOIN namespaces ON namespaces.id = namespace_settings.namespace_id + WHERE + namespaces.id IN (#{relation.select(:namespace_id).to_sql}) + AND namespaces.type = 'User' + AND namespace_settings.delayed_project_removal = FALSE) + SQL + ) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f79baadb8ea..d58de7eb211 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -86,13 +86,15 @@ module Gitlab create_labels + issue_type_id = WorkItems::Type.default_issue_type.id + client.issues(repo).each do |issue| - import_issue(issue) + import_issue(issue, issue_type_id) end end # rubocop: disable CodeReuse/ActiveRecord - def import_issue(issue) + def import_issue(issue, issue_type_id) description = '' description += @formatter.author_line(issue.author) unless find_user_id(issue.author) description += issue.description @@ -106,7 +108,9 @@ module Gitlab description: description, state_id: Issue.available_states[issue.state], author_id: gitlab_user_id(project, issue.author), + namespace_id: project.project_namespace_id, milestone: milestone, + work_item_type_id: issue_type_id, created_at: issue.created_at, updated_at: issue.updated_at ) diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb index 9cb3d71f5c3..8fcc03ec437 100644 --- a/lib/gitlab/changelog/config.rb +++ b/lib/gitlab/changelog/config.rb @@ -7,9 +7,9 @@ module Gitlab # When rendering changelog entries, authors are not included. AUTHORS_NONE = 'none' - # The path to the configuration file as stored in the project's Git + # The default path to the configuration file as stored in the project's Git # repository. - FILE_PATH = '.gitlab/changelog_config.yml' + DEFAULT_FILE_PATH = '.gitlab/changelog_config.yml' # The default date format to use for formatting release dates. DEFAULT_DATE_FORMAT = '%Y-%m-%d' @@ -36,8 +36,9 @@ module Gitlab attr_accessor :date_format, :categories, :template, :tag_regex, :always_credit_user_ids - def self.from_git(project, user = nil) - if (yaml = project.repository.changelog_config.presence) + def self.from_git(project, user = nil, path = nil) + yaml = project.repository.changelog_config('HEAD', path.presence || DEFAULT_FILE_PATH) + if yaml.present? from_hash(project, YAML.safe_load(yaml), user) else new(project) diff --git a/lib/gitlab/ci/build/artifacts/expire_in_parser.rb b/lib/gitlab/ci/build/artifacts/expire_in_parser.rb deleted file mode 100644 index 848208c5cdd..00000000000 --- a/lib/gitlab/ci/build/artifacts/expire_in_parser.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Build - module Artifacts - class ExpireInParser - def self.validate_duration(value) - new(value).validate_duration - end - - def initialize(value) - @value = value - end - - def validate_duration - return true if never? - - cached_parse - end - - def seconds_from_now - parse&.seconds&.from_now - end - - private - - attr_reader :value - - def cached_parse - return validation_cache[value] if validation_cache.key?(value) - - validation_cache[value] = safe_parse - end - - def safe_parse - parse - rescue ChronicDuration::DurationParseError - false - end - - def parse - return if never? - - ChronicDuration.parse(value) - end - - def validation_cache - Gitlab::SafeRequestStore[:ci_expire_in_parser_cache] ||= {} - end - - def never? - value.to_s.casecmp('never') == 0 - end - end - end - end - end -end diff --git a/lib/gitlab/ci/build/duration_parser.rb b/lib/gitlab/ci/build/duration_parser.rb new file mode 100644 index 00000000000..9385dccd5f3 --- /dev/null +++ b/lib/gitlab/ci/build/duration_parser.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class DurationParser + def self.validate_duration(value) + new(value).validate_duration + end + + def initialize(value) + @value = value + end + + def validate_duration + return true if never? + + cached_parse + end + + def seconds_from_now + parse&.seconds&.from_now + end + + private + + attr_reader :value + + def cached_parse + return validation_cache[value] if validation_cache.key?(value) + + validation_cache[value] = safe_parse + end + + def safe_parse + parse + rescue ChronicDuration::DurationParseError + false + end + + def parse + return if never? + + ChronicDuration.parse(value) + end + + def validation_cache + Gitlab::SafeRequestStore[:ci_expire_in_parser_cache] ||= {} + end + + def never? + value.to_s.casecmp('never') == 0 + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 4c5f02b4f7b..1bcd87c9d93 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -4,8 +4,10 @@ module Gitlab module Ci module Build class Rules::Rule::Clause::Changes < Rules::Rule::Clause + include Gitlab::Utils::StrongMemoize + def initialize(globs) - @globs = Array(globs) + @globs = globs end def satisfied_by?(pipeline, context) @@ -19,13 +21,25 @@ module Gitlab end end + private + def expand_globs(context) - return @globs unless context + return paths unless context - @globs.map do |glob| + paths.map do |glob| ExpandVariables.expand_existing(glob, -> { context.variables_hash }) end end + + def paths + strong_memoize(:paths) do + if @globs.is_a?(Array) + @globs + else + Array(@globs[:paths]) + end + end + end end end end diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 56eeb5eeb06..3b0cbc6b69e 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -42,7 +42,7 @@ module Gitlab inclusion: { in: %w[on_success on_failure always], message: 'should be on_success, on_failure ' \ 'or always' } - validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::Artifacts::ExpireInParser } + validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::DurationParser } end end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index bc39abfe977..96ba3553b46 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -54,7 +54,7 @@ module Gitlab validates :on_stop, type: String, allow_nil: true validates :kubernetes, type: Hash, allow_nil: true - validates :auto_stop_in, duration: true, allow_nil: true + validates :auto_stop_in, duration: { parser: ::Gitlab::Ci::Build::DurationParser }, allow_nil: true end end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 79443f69b03..96ac959a3f4 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -48,7 +48,7 @@ module Gitlab { name: @config[:name], entrypoint: @config[:entrypoint], - ports: ports_value, + ports: (ports_value if ports_defined?), pull_policy: (ci_docker_image_pull_policy_enabled? ? pull_policy_value : nil) }.compact else diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 46afedbcc3a..78794f524f4 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -71,9 +71,9 @@ module Gitlab end def compose!(deps = nil) - super do - has_workflow_rules = deps&.workflow_entry&.has_rules? + has_workflow_rules = deps&.workflow_entry&.has_rules? + super do # If workflow:rules: or rules: are used # they are considered not compatible # with `only/except` defaults @@ -86,12 +86,10 @@ module Gitlab @entries.delete(:except) unless except_defined? # rubocop:disable Gitlab/ModuleWithInstanceVariables end - unless has_workflow_rules - validate_against_warnings - end - yield if block_given? end + + validate_against_warnings unless has_workflow_rules end def validate_against_warnings diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb index 53e52981471..91be1bb3ee4 100644 --- a/lib/gitlab/ci/config/entry/rules.rb +++ b/lib/gitlab/ci/config/entry/rules.rb @@ -13,7 +13,7 @@ module Gitlab end def value - [@config].flatten + [super].flatten end def composable_class diff --git a/lib/gitlab/ci/config/entry/rules/rule/changes.rb b/lib/gitlab/ci/config/entry/rules/rule/changes.rb index be57e089f34..a56b928450a 100644 --- a/lib/gitlab/ci/config/entry/rules/rule/changes.rb +++ b/lib/gitlab/ci/config/entry/rules/rule/changes.rb @@ -6,13 +6,51 @@ module Gitlab module Entry class Rules class Rule - class Changes < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable + class Changes < ::Gitlab::Config::Entry::Simplifiable + strategy :SimpleChanges, if: -> (config) { config.is_a?(Array) } + strategy :ComplexChanges, if: -> (config) { config.is_a?(Hash) } - validations do - validates :config, - array_of_strings: true, - length: { maximum: 50, too_long: "has too many entries (maximum %{count})" } + class SimpleChanges < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, + array_of_strings: true, + length: { maximum: 50, too_long: "has too many entries (maximum %{count})" } + end + + def value + { + paths: config + }.compact + end + end + + class ComplexChanges < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[paths].freeze + REQUIRED_KEYS = %i[paths].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, required_keys: REQUIRED_KEYS + + with_options allow_nil: false do + validates :paths, + array_of_strings: true, + length: { maximum: 50, too_long: "has too many entries (maximum %{count})" } + end + end + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be an array or a hash"] + end end end end diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index f27dca4986e..1a35f7de6cf 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -15,11 +15,13 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name entrypoint command alias ports variables].freeze + ALLOWED_KEYS = %i[name entrypoint command alias ports variables pull_policy].freeze + LEGACY_ALLOWED_KEYS = %i[name entrypoint command alias ports variables].freeze validations do validates :config, hash_or_string: true - validates :config, allowed_keys: ALLOWED_KEYS + validates :config, allowed_keys: ALLOWED_KEYS, if: :ci_docker_image_pull_policy_enabled? + validates :config, allowed_keys: LEGACY_ALLOWED_KEYS, unless: :ci_docker_image_pull_policy_enabled? validates :config, disallowed_keys: %i[ports], unless: :with_image_ports? validates :name, type: String, presence: true validates :entrypoint, array_of_strings: true, allow_nil: true @@ -32,11 +34,14 @@ module Gitlab entry :ports, Entry::Ports, description: 'Ports used to expose the service' + entry :pull_policy, Entry::PullPolicy, + description: 'Pull policy for the service' + entry :variables, ::Gitlab::Ci::Config::Entry::Variables, description: 'Environment variables available for this service.', inherit: false - attributes :ports + attributes :ports, :pull_policy, :variables def alias value[:alias] @@ -55,16 +60,25 @@ module Gitlab end def value - return { name: @config } if string? - return @config if hash? - - {} + if string? + { name: @config } + elsif hash? + @config.merge( + pull_policy: (pull_policy_value if ci_docker_image_pull_policy_enabled?) + ).compact + else + {} + end end def with_image_ports? opt(:with_image_ports) end + def ci_docker_image_pull_policy_enabled? + ::Feature.enabled?(:ci_docker_image_pull_policy) + end + def skip_config_hash_validation? true end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 2def565bc19..ec628399785 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -9,14 +9,20 @@ module Gitlab TimeoutError = Class.new(StandardError) + MAX_INCLUDES = 100 + TRIAL_MAX_INCLUDES = 250 + include ::Gitlab::Utils::StrongMemoize attr_reader :project, :sha, :user, :parent_pipeline, :variables - attr_reader :expandset, :execution_deadline, :logger + attr_reader :expandset, :execution_deadline, :logger, :max_includes delegate :instrument, to: :logger - def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, logger: nil) + def initialize( + project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, + logger: nil + ) @project = project @sha = sha @user = user @@ -25,7 +31,7 @@ module Gitlab @expandset = Set.new @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) - + @max_includes = Feature.enabled?(:ci_increase_includes_to_250, project) ? TRIAL_MAX_INCLUDES : MAX_INCLUDES yield self if block_given? end @@ -52,6 +58,7 @@ module Gitlab ctx.expandset = expandset ctx.execution_deadline = execution_deadline ctx.logger = logger + ctx.max_includes = max_includes end end @@ -86,7 +93,7 @@ module Gitlab protected - attr_writer :expandset, :execution_deadline, :logger + attr_writer :expandset, :execution_deadline, :logger, :max_includes private diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index b7fef081269..89418bd6a21 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -13,7 +13,7 @@ module Gitlab def initialize(params, context) @location = params[:file] - @project_name = params[:project] + @project_name = get_project_name(params[:project]) @ref_name = params[:ref] || 'HEAD' super @@ -122,6 +122,16 @@ module Gitlab ) end end + + # TODO: To be removed after we deprecate usage of array in `project` keyword. + # https://gitlab.com/gitlab-org/gitlab/-/issues/365975 + def get_project_name(project_name) + if project_name.is_a?(Array) + project_name.first + else + project_name + end + end end end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index c1250c82750..2a1060a6059 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,8 +7,6 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize - MAX_INCLUDES = 100 - FILE_CLASSES = [ External::File::Remote, External::File::Template, @@ -134,8 +132,8 @@ module Gitlab end def verify_max_includes! - if expandset.count >= MAX_INCLUDES - raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" + if expandset.count >= context.max_includes + raise TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!" end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 19678def666..c294291e538 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -64,7 +64,8 @@ module Gitlab if environment.present? fields.merge!( environment: environment.name, - environment_protected: environment_protected?.to_s + environment_protected: environment_protected?.to_s, + deployment_tier: build.environment_deployment_tier || environment.tier ) end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 71dfc1a676c..207b4b5ff8b 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -11,10 +11,10 @@ module Gitlab def perform! logger.instrument_with_sql(:pipeline_save) do BulkInsertableAssociations.with_bulk_insert do - with_bulk_insert_tags do + ::Ci::BulkInsertableTags.with_bulk_insert_tags do pipeline.transaction do pipeline.save! - CommitStatus.bulk_insert_tags!(statuses) + Gitlab::Ci::Tags::BulkInsert.bulk_insert_tags!(statuses) end end end @@ -29,14 +29,6 @@ module Gitlab private - def with_bulk_insert_tags - previous = Thread.current['ci_bulk_insert_tags'] - Thread.current['ci_bulk_insert_tags'] = true - yield - ensure - Thread.current['ci_bulk_insert_tags'] = previous - end - def statuses strong_memoize(:statuses) do pipeline diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 33b9ac9b641..c3e0f043b44 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -42,6 +42,15 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end + def self.pipeline_age_histogram + name = :gitlab_ci_pipeline_age_minutes + comment = 'Pipeline age histogram' + buckets = [5, 30, 120, 720, 1440, 7200, 21600, 43200, 86400, 172800, 518400, 1036800] + # 5m 30m 2h 12h 24h 5d 15d 30d 60d 180d 360d 2y + + ::Gitlab::Metrics.histogram(name, comment, {}, buckets) + end + def self.active_jobs_histogram name = :gitlab_ci_active_jobs comment = 'Total amount of active jobs' diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 7d8303214a5..5cee73238ca 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -250,11 +250,7 @@ module Gitlab end def running_jobs_relation(job) - if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data) - ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) - else - job.project.builds.running.where(runner: ::Ci::Runner.instance_type) - end + ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/ci/reports/coverage_report_generator.rb b/lib/gitlab/ci/reports/coverage_report_generator.rb index fd73ed6fd25..76992a48b0a 100644 --- a/lib/gitlab/ci/reports/coverage_report_generator.rb +++ b/lib/gitlab/ci/reports/coverage_report_generator.rb @@ -35,17 +35,7 @@ module Gitlab private def report_builds - if child_pipeline_feature_enabled? - @pipeline.latest_report_builds_in_self_and_descendants(::Ci::JobArtifact.coverage_reports) - else - @pipeline.latest_report_builds(::Ci::JobArtifact.coverage_reports) - end - end - - def child_pipeline_feature_enabled? - strong_memoize(:feature_enabled) do - Feature.enabled?(:ci_child_pipeline_coverage_reports, @pipeline.project) - end + @pipeline.latest_report_builds_in_self_and_descendants(::Ci::JobArtifact.coverage_reports) end end end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_report.rb index a5a630642e5..4fc10dd736e 100644 --- a/lib/gitlab/ci/reports/test_reports.rb +++ b/lib/gitlab/ci/reports/test_report.rb @@ -3,7 +3,7 @@ module Gitlab module Ci module Reports - class TestReports + class TestReport attr_reader :test_suites def initialize diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb index c6f17f0764f..497831ae5a7 100644 --- a/lib/gitlab/ci/reports/test_reports_comparer.rb +++ b/lib/gitlab/ci/reports/test_reports_comparer.rb @@ -9,7 +9,7 @@ module Gitlab attr_reader :base_reports, :head_reports def initialize(base_reports, head_reports) - @base_reports = base_reports || TestReports.new + @base_reports = base_reports || TestReport.new @head_reports = head_reports end diff --git a/lib/gitlab/ci/runner/metrics.rb b/lib/gitlab/ci/runner/metrics.rb new file mode 100644 index 00000000000..8df126decff --- /dev/null +++ b/lib/gitlab/ci/runner/metrics.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Runner + class Metrics + extend Gitlab::Utils::StrongMemoize + + def increment_runner_authentication_success_counter(runner_type: 'unknown_type') + raise ArgumentError, "unknown runner type: #{runner_type}" unless + ::Ci::Runner.runner_types.include? runner_type + + self.class.runner_authentication_success_counter.increment(runner_type: runner_type) + end + + def increment_runner_authentication_failure_counter + self.class.runner_authentication_failure_counter.increment + end + + def self.runner_authentication_success_counter + strong_memoize(:runner_authentication_success) do + name = :gitlab_ci_runner_authentication_success_total + comment = 'Runner authentication success' + labels = { runner_type: nil } + + ::Gitlab::Metrics.counter(name, comment, labels) + end + end + + def self.runner_authentication_failure_counter + strong_memoize(:runner_authentication_failure) do + name = :gitlab_ci_runner_authentication_failure_total + comment = 'Runner authentication failure' + + ::Gitlab::Metrics.counter(name, comment) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/runner_releases.rb b/lib/gitlab/ci/runner_releases.rb index 944c24ca128..8773ecbf09e 100644 --- a/lib/gitlab/ci/runner_releases.rb +++ b/lib/gitlab/ci/runner_releases.rb @@ -6,48 +6,83 @@ module Gitlab include Singleton RELEASES_VALIDITY_PERIOD = 1.day - RELEASES_VALIDITY_AFTER_ERROR_PERIOD = 5.seconds INITIAL_BACKOFF = 5.seconds MAX_BACKOFF = 1.hour BACKOFF_GROWTH_FACTOR = 2.0 def initialize - reset! + reset_backoff! end # Returns a sorted list of the publicly available GitLab Runner releases # def releases - return @releases unless Time.now.utc >= @expire_time + return if backoff_active? + + Rails.cache.fetch( + cache_key, + skip_nil: true, + expires_in: RELEASES_VALIDITY_PERIOD, + race_condition_ttl: 10.seconds + ) do + response = Gitlab::HTTP.try_get(runner_releases_url) + @releases_by_minor = nil + + unless response&.success? + @backoff_expire_time = next_backoff.from_now + break nil + end + + reset_backoff! + extract_releases(response) + end + end + + # Returns a hash with the latest runner version per minor release + # + def releases_by_minor + return unless releases - @releases = fetch_new_releases + @releases_by_minor ||= releases.group_by(&:without_patch).transform_values(&:max) end - def reset! - @expire_time = Time.now.utc - @releases = nil + def reset_backoff! + @backoff_expire_time = nil @backoff_count = 0 end - public_class_method :instance - private - def fetch_new_releases - response = Gitlab::HTTP.try_get(::Gitlab::CurrentSettings.current_application_settings.public_runner_releases_url) + def runner_releases_url + @runner_releases_url ||= ::Gitlab::CurrentSettings.current_application_settings.public_runner_releases_url + end - releases = response.success? ? extract_releases(response) : nil - ensure - @expire_time = (releases ? RELEASES_VALIDITY_PERIOD : next_backoff).from_now + def cache_key + runner_releases_url + end + + def backoff_active? + return false unless @backoff_expire_time + + Time.now.utc < @backoff_expire_time end def extract_releases(response) - response.parsed_response.map { |release| parse_runner_release(release) }.sort! + return unless response.parsed_response.is_a?(Array) + + releases = response.parsed_response + .map { |release| parse_runner_release(release) } + .select(&:valid?) + .sort! + + return if releases.empty? && response.parsed_response.present? + + releases end def parse_runner_release(release) - ::Gitlab::VersionInfo.parse(release['name'].delete_prefix('v')) + ::Gitlab::VersionInfo.parse(release['name'], parse_suffix: true) end def next_backoff diff --git a/lib/gitlab/ci/runner_upgrade_check.rb b/lib/gitlab/ci/runner_upgrade_check.rb index 0808290fe5b..10a89bb15d4 100644 --- a/lib/gitlab/ci/runner_upgrade_check.rb +++ b/lib/gitlab/ci/runner_upgrade_check.rb @@ -5,76 +5,71 @@ module Gitlab class RunnerUpgradeCheck include Singleton - STATUSES = { - invalid: 'Runner version is not valid.', - not_available: 'Upgrade is not available for the runner.', - available: 'Upgrade is available for the runner.', - recommended: 'Upgrade is available and recommended for the runner.' - }.freeze - - def initialize - reset! - end - def check_runner_upgrade_status(runner_version) - return :invalid unless runner_version + runner_version = ::Gitlab::VersionInfo.parse(runner_version, parse_suffix: true) - releases = RunnerReleases.instance.releases - orig_runner_version = runner_version - runner_version = ::Gitlab::VersionInfo.parse(runner_version) unless runner_version.is_a?(::Gitlab::VersionInfo) + return { invalid_version: runner_version } unless runner_version.valid? + return { error: runner_version } unless runner_releases_store.releases - raise ArgumentError, "'#{orig_runner_version}' is not a valid version" unless runner_version.valid? + # Recommend update if outside of backport window + recommended_version = recommendation_if_outside_backport_window(runner_version) + return { recommended: recommended_version } if recommended_version - gitlab_minor_version = version_without_patch(@gitlab_version) + # Recommend patch update if there's a newer release in a same minor branch as runner + recommended_version = recommended_runner_release_update(runner_version) + return { recommended: recommended_version } if recommended_version - available_releases = releases - .reject { |release| release.major > @gitlab_version.major } - .reject do |release| - release_minor_version = version_without_patch(release) + # Consider update if there's a newer release within the currently deployed GitLab version + available_version = available_runner_release(runner_version) + return { available: available_version } if available_version - # Do not reject a patch update, even if the runner is ahead of the instance version - next false if version_without_patch(runner_version) == release_minor_version + { not_available: runner_version } + end - release_minor_version > gitlab_minor_version - end + private - return :recommended if available_releases.any? { |available_rel| patch_update?(available_rel, runner_version) } - return :recommended if outside_backport_window?(runner_version, releases) - return :available if available_releases.any? { |available_rel| available_rel > runner_version } + def recommended_runner_release_update(runner_version) + recommended_release = runner_releases_store.releases_by_minor[runner_version.without_patch] + return recommended_release if recommended_release && recommended_release > runner_version - :not_available + # Consider the edge case of pre-release runner versions that get registered, but are never published. + # In this case, suggest the latest compatible runner version + latest_release = runner_releases_store.releases_by_minor.values.select { |v| v < gitlab_version }.max + latest_release if latest_release && latest_release > runner_version end - def reset! - @gitlab_version = ::Gitlab::VersionInfo.parse(::Gitlab::VERSION) + def available_runner_release(runner_version) + available_release = runner_releases_store.releases_by_minor[gitlab_version.without_patch] + available_release if available_release && available_release > runner_version end - public_class_method :instance - - private - - def patch_update?(available_release, runner_version) - # https://docs.gitlab.com/ee/policy/maintenance.html#patch-releases - available_release.major == runner_version.major && - available_release.minor == runner_version.minor && - available_release.patch > runner_version.patch + def gitlab_version + @gitlab_version ||= ::Gitlab::VersionInfo.parse(::Gitlab::VERSION, parse_suffix: true) end - def outside_backport_window?(runner_version, releases) - return false if runner_version >= releases.last # return early if runner version is too new - - latest_minor_releases = releases.map { |r| version_without_patch(r) }.uniq { |v| v.to_s } - latest_version_position = latest_minor_releases.count - 1 - runner_version_position = latest_minor_releases.index(version_without_patch(runner_version)) - - return true if runner_version_position.nil? # consider outside if version is too old - - # https://docs.gitlab.com/ee/policy/maintenance.html#backporting-to-older-releases - latest_version_position - runner_version_position > 2 + def runner_releases_store + RunnerReleases.instance end - def version_without_patch(version) - ::Gitlab::VersionInfo.new(version.major, version.minor, 0) + def recommendation_if_outside_backport_window(runner_version) + return if runner_releases_store.releases.empty? + return if runner_version >= runner_releases_store.releases.last # return early if runner version is too new + + minor_releases_with_index = runner_releases_store.releases_by_minor.keys.each_with_index.to_h + runner_minor_version_index = minor_releases_with_index[runner_version.without_patch] + if runner_minor_version_index + # https://docs.gitlab.com/ee/policy/maintenance.html#backporting-to-older-releases + outside_window = minor_releases_with_index.count - runner_minor_version_index > 3 + + if outside_window + recommended_release = runner_releases_store.releases_by_minor[gitlab_version.without_patch] + + recommended_release if recommended_release && recommended_release > runner_version + end + else + # If unknown runner version, then recommend the latest version for the GitLab instance + recommended_runner_release_update(gitlab_version) + end end end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 1a074c1af53..5d60aa8f540 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -20,10 +20,13 @@ module Gitlab scheduler_failure: 'scheduler failure', data_integrity_failure: 'data integrity failure', forward_deployment_failure: 'forward deployment failure', + protected_environment_failure: 'protected environment failure', pipeline_loop_detected: 'job would create infinitely looping pipelines', invalid_bridge_trigger: 'downstream pipeline trigger definition is invalid', downstream_bridge_project_not_found: 'downstream project could not be found', + upstream_bridge_project_not_found: 'upstream project could not be found', insufficient_bridge_permissions: 'no permissions to trigger downstream pipeline', + insufficient_upstream_permissions: 'no permissions to read upstream project', bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline', downstream_pipeline_creation_failed: 'downstream pipeline can not be created', secrets_provider_not_found: 'secrets provider can not be found', @@ -75,5 +78,3 @@ module Gitlab end end end - -Gitlab::Ci::Status::Build::Failed.prepend_mod_with('Gitlab::Ci::Status::Build::Failed') diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 3b2da773102..e854164d377 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -7,10 +7,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize # This class accepts an array of arrays/hashes/or objects - # - # The parameter `project` is only used for the feature flag check, and will be removed with - # https://gitlab.com/gitlab-org/gitlab/-/issues/321972 - def initialize(all_statuses, with_allow_failure: true, dag: false, project: nil) + def initialize(all_statuses, with_allow_failure: true, dag: false) unless all_statuses.respond_to?(:pluck) raise ArgumentError, "all_statuses needs to respond to `.pluck`" end @@ -19,7 +16,6 @@ module Gitlab @status_key = 0 @allow_failure_key = 1 if with_allow_failure @dag = dag - @project = project consume_all_statuses(all_statuses) end diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb index 29f3731a9b4..2e56e47f5b8 100644 --- a/lib/gitlab/ci/tags/bulk_insert.rb +++ b/lib/gitlab/ci/tags/bulk_insert.rb @@ -9,40 +9,44 @@ module Gitlab TAGGINGS_BATCH_SIZE = 1000 TAGS_BATCH_SIZE = 500 - def initialize(statuses) - @statuses = statuses + def self.bulk_insert_tags!(taggables) + Gitlab::Ci::Tags::BulkInsert.new(taggables).insert! + end + + def initialize(taggables) + @taggables = taggables end def insert! - return false if tag_list_by_status.empty? + return false if tag_list_by_taggable.empty? persist_build_tags! end private - attr_reader :statuses + attr_reader :taggables - def tag_list_by_status - strong_memoize(:tag_list_by_status) do - statuses.each.with_object({}) do |status, acc| - tag_list = status.tag_list + def tag_list_by_taggable + strong_memoize(:tag_list_by_taggable) do + taggables.each.with_object({}) do |taggable, acc| + tag_list = taggable.tag_list next unless tag_list - acc[status] = tag_list + acc[taggable] = tag_list end end end def persist_build_tags! - all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?) + all_tags = tag_list_by_taggable.values.flatten.uniq.reject(&:blank?) tag_records_by_name = create_tags(all_tags).index_by(&:name) taggings = build_taggings_attributes(tag_records_by_name) return false if taggings.empty? taggings.each_slice(TAGGINGS_BATCH_SIZE) do |taggings_slice| - ActsAsTaggableOn::Tagging.insert_all!(taggings) + ActsAsTaggableOn::Tagging.insert_all!(taggings_slice) end true @@ -65,24 +69,24 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def build_taggings_attributes(tag_records_by_name) - taggings = statuses.flat_map do |status| - tag_list = tag_list_by_status[status] + taggings = taggables.flat_map do |taggable| + tag_list = tag_list_by_taggable[taggable] next unless tag_list tags = tag_records_by_name.values_at(*tag_list) - taggings_for(tags, status) + taggings_for(tags, taggable) end taggings.compact! taggings end - def taggings_for(tags, status) + def taggings_for(tags, taggable) tags.map do |tag| { tag_id: tag.id, - taggable_type: CommitStatus.name, - taggable_id: status.id, + taggable_type: taggable.class.base_class.name, + taggable_id: taggable.id, created_at: Time.current, context: 'tags' } diff --git a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml index 17e49440784..1ac9c319429 100644 --- a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml @@ -7,6 +7,7 @@ stages: - build - test - review + - dast - deploy - production - cleanup diff --git a/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml index 9f0e9bcc1f2..ee52bc91ab3 100644 --- a/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml @@ -30,24 +30,24 @@ before_script: - apt-get --quiet update --yes - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 - # Setup path as ANDROID_SDK_ROOT for moving/exporting the downloaded sdk into it - - export ANDROID_SDK_ROOT="${PWD}/android-home" + # Setup path as ANDROID_HOME for moving/exporting the downloaded sdk into it + - export ANDROID_HOME="${PWD}/android-home" # Create a new directory at specified location - - install -d $ANDROID_SDK_ROOT + - install -d $ANDROID_HOME # Here we are installing androidSDK tools from official source, # (the key thing here is the url from where you are downloading these sdk tool for command line, so please do note this url pattern there and here as well) # after that unzipping those tools and # then running a series of SDK manager commands to install necessary android SDK packages that'll allow the app to build - - wget --output-document=$ANDROID_SDK_ROOT/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip - # move to the archive at ANDROID_SDK_ROOT - - pushd $ANDROID_SDK_ROOT + - wget --output-document=$ANDROID_HOME/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip + # move to the archive at ANDROID_HOME + - pushd $ANDROID_HOME - unzip -d cmdline-tools cmdline-tools.zip - pushd cmdline-tools # since commandline tools version 7583922 the root folder is named "cmdline-tools" so we rename it if necessary - mv cmdline-tools tools || true - popd - popd - - export PATH=$PATH:${ANDROID_SDK_ROOT}/cmdline-tools/tools/bin/ + - export PATH=$PATH:${ANDROID_HOME}/cmdline-tools/tools/bin/ # Nothing fancy here, just checking sdkManager version - sdkmanager --version diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index f39a84bceec..004c2897b60 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index c078c99f352..3096af1b173 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index f166da9bdd6..a64f87193a9 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -1,14 +1,17 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Chef.gitlab-ci.yml - # This template uses Test Kitchen with the kitchen-dokken driver to # perform functional testing. Doing so requires that your runner be a # Docker runner configured for privileged mode. Please see # https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode # for help configuring your runner properly, or, if you want to switch # to a different driver, see http://kitchen.ci/docs/drivers +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Chef.gitlab-ci.yml image: "chef/chefdk" services: diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index 0f9e28c9a8e..4fe37ceaeaa 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml index 8886929646d..68b55b782cd 100644 --- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml index 6354db38f58..35401e62fe2 100644 --- a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml index 1eb920c7747..83ddce936e6 100644 --- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index a5c261e367a..021662ab416 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml index 21a599fc78d..464b81965f2 100644 --- a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml - # This is a sample GitLab CI/CD configuration file that should run without any modifications. # It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, # it uses echo commands to simulate the pipeline execution. @@ -11,6 +6,14 @@ # Stages run in sequential order, but jobs within stages run in parallel. # # For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml stages: # List of stages for jobs, and their order of execution - build diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index bd8e1020c4e..603aede4d46 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 7e59354c4a1..03c8941169f 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Grails.gitlab-ci.yml - # This template uses the java:8 docker image because there isn't any # official Grails image at this moment # @@ -12,6 +7,14 @@ # Feel free to change GRAILS_VERSION version with your project version (3.0.1, 3.1.1,...) # Feel free to change GRADLE_VERSION version with your gradle project version (2.13, 2.14,...) # If you use Angular profile, this yml it's prepared to work with it +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Grails.gitlab-ci.yml image: java:8 diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 6a95d042842..86e3ace84c5 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.26" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.29" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 8f1124373c4..b41e92e3a56 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -4,6 +4,14 @@ variables: .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" +.common_rules: &common_rules + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME + when: never + - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH + when: never + - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given + when: never + dast_environment_deploy: extends: .dast-auto-deploy stage: review @@ -23,12 +31,7 @@ dast_environment_deploy: artifacts: paths: [environment_url.txt] rules: - - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - when: never - - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - when: never - - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given - when: never + - *common_rules - if: $CI_COMMIT_BRANCH && ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ @@ -47,13 +50,53 @@ stop_dast_environment: action: stop needs: ["dast"] rules: - - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - when: never - - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - when: never - - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given - when: never + - *common_rules - if: $CI_COMMIT_BRANCH && ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ when: always + +.ecs_image: + image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + +.ecs_rules: &ecs_rules + - if: $AUTO_DEVOPS_PLATFORM_TARGET != "ECS" + when: never + - if: $CI_KUBERNETES_ACTIVE || $KUBECONFIG + when: never + +dast_ecs_environment_deploy: + extends: .ecs_image + stage: review + script: + - ecs update-task-definition + - echo "http://$(ecs get-task-hostname)" > environment_url.txt + environment: + name: dast-default + on_stop: stop_dast_ecs_environment + artifacts: + paths: + - environment_url.txt + rules: + - *common_rules + - *ecs_rules + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ + +stop_dast_ecs_environment: + extends: .ecs_image + stage: cleanup + variables: + GIT_STRATEGY: none + script: + - ecs stop-task + allow_failure: true + environment: + name: dast-default + action: stop + needs: + - dast + rules: + - *common_rules + - *ecs_rules + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ + when: always diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index b95b36fd555..a9d9c400a34 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -46,10 +46,10 @@ dependency_scanning: script: - /analyzer run -.cyclone-dx-reports: +.cyclonedx-reports: artifacts: paths: - - "**/cyclonedx-*.json" + - "**/gl-sbom-*.cdx.json" .gemnasium-shared-rule: exists: @@ -66,7 +66,7 @@ dependency_scanning: gemnasium-dependency_scanning: extends: - .ds-analyzer - - .cyclone-dx-reports + - .cyclonedx-reports variables: DS_ANALYZER_NAME: "gemnasium" GEMNASIUM_LIBRARY_SCAN_ENABLED: "true" @@ -81,6 +81,7 @@ gemnasium-dependency_scanning: exists: !reference [.gemnasium-shared-rule, exists] variables: DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ exists: !reference [.gemnasium-shared-rule, exists] @@ -95,7 +96,7 @@ gemnasium-dependency_scanning: gemnasium-maven-dependency_scanning: extends: - .ds-analyzer - - .cyclone-dx-reports + - .cyclonedx-reports variables: DS_ANALYZER_NAME: "gemnasium-maven" rules: @@ -125,7 +126,7 @@ gemnasium-maven-dependency_scanning: gemnasium-python-dependency_scanning: extends: - .ds-analyzer - - .cyclone-dx-reports + - .cyclonedx-reports variables: DS_ANALYZER_NAME: "gemnasium-python" rules: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index 9bb2ba69d84..c2d31fd9669 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -20,6 +20,11 @@ .review_ecs_base: stage: review extends: .deploy_to_ecs + after_script: + - echo "http://$(ecs get-task-hostname)" > environment_url.txt + artifacts: + paths: + - environment_url.txt .production_ecs_base: stage: production diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index 4687a07d05b..34084272b29 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Julia.gitlab-ci.yml - # This is an example .gitlab-ci.yml file to test (and optionally report the coverage # results of) your [Julia][1] packages. Please refer to the [documentation][2] # for more information about package development in Julia. @@ -12,6 +7,14 @@ # # [1]: http://julialang.org/ # [2]: https://docs.julialang.org/en/v1/manual/documentation/index.html +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Julia.gitlab-ci.yml # Below is the template to run your tests in Julia .test_template: &test_definition diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index 0ec67526234..3a490012f3d 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 2f214347ec3..65db649e22f 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Mono.gitlab-ci.yml - # This is a simple gitlab continuous integration template (compatible with the shared runner provided on gitlab.com) # using the official mono docker image to build a visual studio project. # @@ -15,6 +10,14 @@ # # Please find the full example project here: # https://gitlab.com/tobiaskoch/gitlab-ci-example-mono +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Mono.gitlab-ci.yml # see https://hub.docker.com/_/mono/ image: mono:latest diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index 44370f896a7..7a4f7ed628b 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 7c8bbe464af..0eb3b483067 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index 4edc003a638..12640d28d29 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 690a5a291e1..aab408aa830 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index 390f0bb8061..a83f84da818 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: @@ -26,3 +29,14 @@ test:cargo: script: - rustc --version && cargo --version # Print version info for debugging - cargo test --workspace --verbose + +# Optional: Use a third party library to generate gitlab junit reports +# test:junit-report: +# script: +# Should be specified in Cargo.toml +# - cargo install junitify +# - cargo test -- --format=json -Z unstable-options --report-time | junitify --out $CI_PROJECT_DIR/tests/ +# artifacts: +# when: always +# reports: +# junit: $CI_PROJECT_DIR/tests/*.xml diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index de54d64dc42..26efe7a8908 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index eedb3b7a310..3c4533d603e 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index 841f17767eb..50ce181095e 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -23,6 +23,9 @@ # You need to have the network drive mapped as Local System user for gitlab-runner service to see it # The best way to persist the mapping is via a scheduled task # running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. # place project specific paths in variables to make the rest of the script more generic variables: diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml index 0b75c298167..58e840da713 100644 --- a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml - # This is a very simple template that mainly relies on FastLane to build and distribute your app. # Read more about how to use this template on the blog post https://about.gitlab.com/2019/03/06/ios-publishing-with-gitlab-and-fastlane/ # You will also need fastlane and signing configuration for this to work, along with a MacOS runner. @@ -15,6 +10,14 @@ # https://docs.gitlab.com/runner/security/#usage-of-shell-executor for additional # detail on what to keep in mind in this scenario. +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. + +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml + stages: - build - test diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index a452cb197ae..95dff83506d 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -26,7 +26,6 @@ module Gitlab variables.concat(secret_instance_variables) variables.concat(secret_group_variables(environment: environment)) variables.concat(secret_project_variables(environment: environment)) - variables.concat(job.trigger_request.user_variables) if job.trigger_request variables.concat(pipeline.variables) variables.concat(pipeline_schedule_variables) end @@ -52,7 +51,7 @@ module Gitlab # https://gitlab.com/groups/gitlab-org/configure/-/epics/8 # Until then, we need to make both the old and the new KUBECONFIG contexts available collection.concat(deployment_variables(environment: environment, job: job)) - template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.token).execute + template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.try(:token)).execute kubeconfig_yaml = collection['KUBECONFIG']&.value template.merge_yaml(kubeconfig_yaml) if kubeconfig_yaml.present? diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 15ebd506055..59acfa80258 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -16,6 +16,14 @@ module Gitlab end def execute + Gitlab::Ci::YamlProcessor::FeatureFlags.with_actor(project) do + parse_config + end + end + + private + + def parse_config if @config_content.blank? return Result.new(errors: ['Please provide content of .gitlab-ci.yml']) end @@ -35,7 +43,9 @@ module Gitlab Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) end - private + def project + @opts[:project] + end def run_logical_validations! @stages = @ci_config.stages diff --git a/lib/gitlab/ci/yaml_processor/feature_flags.rb b/lib/gitlab/ci/yaml_processor/feature_flags.rb new file mode 100644 index 00000000000..f03db9d0e6b --- /dev/null +++ b/lib/gitlab/ci/yaml_processor/feature_flags.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class YamlProcessor + module FeatureFlags + ACTOR_KEY = 'ci_yaml_processor_feature_flag_actor' + NO_ACTOR_VALUE = :no_actor + + NoActorError = Class.new(StandardError) + NO_ACTOR_MESSAGE = "Actor not set. Ensure to call `enabled?` inside `with_actor` block" + + class << self + # Cache a feature flag actor as thread local variable so + # we can have it available later with #enabled? + def with_actor(actor) + previous = Thread.current[ACTOR_KEY] + + # When actor is `nil` the method `Thread.current[]=` does not + # create the ACTOR_KEY. Instead, we want to still save an explicit + # value to know that we are within the `with_actor` block. + Thread.current[ACTOR_KEY] = actor || NO_ACTOR_VALUE + + yield + ensure + Thread.current[ACTOR_KEY] = previous + end + + # Use this to check if a feature flag is enabled + def enabled?(feature_flag) + ::Feature.enabled?(feature_flag, current_actor) + end + + private + + def current_actor + value = Thread.current[ACTOR_KEY] || (raise NoActorError, NO_ACTOR_MESSAGE) + return if value == NO_ACTOR_VALUE + + value + rescue NoActorError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + + nil + end + end + end + end + end +end diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index e634291f894..5908de68687 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -5,10 +5,10 @@ module Gitlab class PumaWorkerKillerInitializer def self.start( puma_options, - puma_per_worker_max_memory_mb: 1024, - puma_master_max_memory_mb: 800, - additional_puma_dev_max_memory_mb: 200 - ) + puma_per_worker_max_memory_mb: 1200, + puma_master_max_memory_mb: 950, + additional_puma_dev_max_memory_mb: 200) + require 'puma_worker_killer' PumaWorkerKiller.config do |config| diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 574a7dceaa4..8648ffe5f49 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -7,6 +7,8 @@ module Gitlab form_action frame_ancestors frame_src img_src manifest_src media_src object_src report_uri script_src style_src worker_src).freeze + DEFAULT_FALLBACK_VALUE = '<default_value>' + def self.default_enabled Rails.env.development? || Rails.env.test? end @@ -62,8 +64,10 @@ module Gitlab end def initialize(csp_directives) + # Using <default_value> falls back to the default values. + directives = csp_directives.reject { |_, value| value == DEFAULT_FALLBACK_VALUE } @merged_csp_directives = - HashWithIndifferentAccess.new(csp_directives) + HashWithIndifferentAccess.new(directives) .reverse_merge(::Gitlab::ContentSecurityPolicy::ConfigLoader.default_directives) end @@ -134,9 +138,8 @@ module Gitlab def self.allow_sentry(directives) sentry_dsn = Gitlab.config.sentry.clientside_dsn sentry_uri = URI(sentry_dsn) - sentry_uri.user = nil - append_to_directive(directives, 'connect_src', sentry_uri.to_s) + append_to_directive(directives, 'connect_src', "#{sentry_uri.scheme}://#{sentry_uri.host}") end def self.allow_letter_opener(directives) diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index f48ba27888c..4d289a59a6a 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -12,11 +12,8 @@ module Gitlab @contributor = contributor @contributor_time_instance = local_timezone_instance(contributor.timezone).now @current_user = current_user - @projects = if @contributor.include_private_contributions? - ContributedProjectsFinder.new(@contributor).execute(@contributor) - else - ContributedProjectsFinder.new(contributor).execute(current_user) - end + @projects = ContributedProjectsFinder.new(contributor) + .execute(current_user, ignore_visibility: @contributor.include_private_contributions?) end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index 0e6841e10a7..a9c69e3f997 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -5,7 +5,8 @@ module Gitlab module Deployment extend self - def build(deployment, status_changed_at) + # NOTE: Time-sensitive attributes should be explicitly passed as argument instead of reading from database. + def build(deployment, status, status_changed_at) # Deployments will not have a deployable when created using the API. deployable_url = if deployment.deployable @@ -22,9 +23,13 @@ module Gitlab Gitlab::UrlBuilder.build(deployment.deployed_by) end + # `status` argument could be `nil` during the upgrade. We can remove `deployment.status` in GitLab 15.5. + # See https://docs.gitlab.com/ee/development/multi_version_compatibility.html for more info. + deployment_status = status || deployment.status + { object_kind: 'deployment', - status: deployment.status, + status: deployment_status, status_changed_at: status_changed_at, deployment_id: deployment.id, deployable_id: deployment.deployable_id, diff --git a/lib/gitlab/data_builder/issuable.rb b/lib/gitlab/data_builder/issuable.rb index 9a0b964915c..d12537c4874 100644 --- a/lib/gitlab/data_builder/issuable.rb +++ b/lib/gitlab/data_builder/issuable.rb @@ -18,7 +18,7 @@ module Gitlab user: user.hook_attrs, project: issuable.project.hook_attrs, object_attributes: issuable_builder.new(issuable).build, - labels: issuable.labels.map(&:hook_attrs), + labels: issuable.labels_hook_attrs, changes: final_changes(changes.slice(*safe_keys)), # DEPRECATED repository: issuable.project.hook_attrs.slice(:name, :url, :description, :homepage) diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index c13bb1d6a9a..2c124b07006 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -118,6 +118,7 @@ module Gitlab finished_at: build.finished_at, duration: build.duration, queued_duration: build.queued_duration, + failure_reason: (build.failure_reason if build.failed?), when: build.when, manual: build.action?, allow_failure: build.allow_failure, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index b42d164d9c4..8703365b678 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -63,13 +63,45 @@ module Gitlab }.compact.with_indifferent_access.freeze end + # This returns a list of databases that contains all the gitlab_shared schema + # tables. We can't reuse database_base_models because Geo does not support + # the gitlab_shared tables yet. + def self.database_base_models_with_gitlab_shared + @database_base_models_with_gitlab_shared ||= { + # Note that we use ActiveRecord::Base here and not ApplicationRecord. + # This is deliberate, as we also use these classes to apply load + # balancing to, and the load balancer must be enabled for _all_ models + # that inher from ActiveRecord::Base; not just our own models that + # inherit from ApplicationRecord. + main: ::ActiveRecord::Base, + ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil + }.compact.with_indifferent_access.freeze + end + + # This returns a list of databases whose connection supports database load + # balancing. We can't reuse the database_base_models method because the Geo + # database does not support load balancing yet. + # + # TODO: https://gitlab.com/gitlab-org/geo-team/discussions/-/issues/5032 + def self.database_base_models_using_load_balancing + @database_base_models_with_gitlab_shared ||= { + # Note that we use ActiveRecord::Base here and not ApplicationRecord. + # This is deliberate, as we also use these classes to apply load + # balancing to, and the load balancer must be enabled for _all_ models + # that inher from ActiveRecord::Base; not just our own models that + # inherit from ApplicationRecord. + main: ::ActiveRecord::Base, + ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil + }.compact.with_indifferent_access.freeze + end + # This returns a list of base models with connection associated for a given gitlab_schema def self.schemas_to_base_models @schemas_to_base_models ||= { gitlab_main: [self.database_base_models.fetch(:main)], gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main - gitlab_shared: self.database_base_models.values, # all models - gitlab_internal: self.database_base_models.values # all models + gitlab_shared: database_base_models_with_gitlab_shared.values, # all models + gitlab_internal: database_base_models.values # all models }.with_indifferent_access.freeze end @@ -168,7 +200,7 @@ module Gitlab # can potentially upgrade from read to read-write mode (using a different connection), we specify # up-front that we'll explicitly use the primary for the duration of the operation. Gitlab::Database::LoadBalancing::Session.current.use_primary do - base_models = database_base_models.values + base_models = database_base_models_using_load_balancing.values base_models.reduce(block) { |blk, model| -> { model.uncached(&blk) } }.call end end @@ -286,7 +318,7 @@ module Gitlab extend ActiveSupport::Concern class_methods do - # A patch over ActiveRecord::Base.transaction that provides + # A patch over ApplicationRecord.transaction that provides # observability into transactional methods. def transaction(**options, &block) transaction_type = get_transaction_type(connection.transaction_open?, options[:requires_new]) diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index ebc3ee240bd..72aa1cfe00b 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -26,6 +26,7 @@ module Gitlab scope :successful_in_execution_order, -> { where.not(finished_at: nil).with_status(:succeeded).order(:finished_at) } scope :with_preloads, -> { preload(:batched_migration) } scope :created_since, ->(date_time) { where('created_at >= ?', date_time) } + scope :blocked_by_max_attempts, -> { where('attempts >= ?', MAX_ATTEMPTS) } state_machine :status, initial: :pending do state :pending, value: 0 @@ -128,7 +129,8 @@ module Gitlab batched_migration.column_name, batch_min_value: min_value, batch_size: new_batch_size, - job_arguments: batched_migration.job_arguments + job_arguments: batched_migration.job_arguments, + job_class: batched_migration.job_class ) midpoint = next_batch_bounds.last diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index d052d5adc4c..9c8db2243f9 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -104,6 +104,12 @@ module Gitlab .sum(:batch_size) end + def reset_attempts_of_blocked_jobs! + batched_jobs.blocked_by_max_attempts.each_batch(of: 100) do |batch| + batch.update_all(attempts: 0) + end + end + def interval_elapsed?(variance: 0) return true unless last_job @@ -199,10 +205,32 @@ module Gitlab BatchOptimizer.new(self).optimize! end + def health_context + HealthStatus::Context.new([table_name]) + end + def hold!(until_time: 10.minutes.from_now) + duration_s = (until_time - Time.current).round + Gitlab::AppLogger.info( + message: "#{self} put on hold until #{until_time}", + migration_id: id, + job_class_name: job_class_name, + duration_s: duration_s + ) + update!(on_hold_until: until_time) end + def on_hold? + return false unless on_hold_until + + on_hold_until > Time.zone.now + end + + def to_s + "BatchedMigration[id: #{id}]" + end + private def validate_batched_jobs_status diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 388eb596ce2..1bc2e931391 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -29,7 +29,8 @@ module Gitlab if next_batched_job = find_or_create_next_batched_job(active_migration) migration_wrapper.perform(next_batched_job) - active_migration.optimize! + adjust_migration(active_migration) + active_migration.failure! if next_batched_job.failed? && active_migration.should_stop? else finish_active_migration(active_migration) @@ -71,12 +72,17 @@ module Gitlab elsif migration.finished? Gitlab::AppLogger.warn "Batched background migration for the given configuration is already finished: #{configuration}" else + migration.reset_attempts_of_blocked_jobs! + migration.finalize! migration.batched_jobs.with_status(:pending).each { |job| migration_wrapper.perform(job) } run_migration_while(migration, :finalizing) - raise FailedToFinalize unless migration.finished? + error_message = "Batched migration #{migration.job_class_name} could not be completed and a manual action is required."\ + "Check the admin panel at (`/admin/background_migrations`) for more details." + + raise FailedToFinalize, error_message unless migration.finished? end end @@ -101,7 +107,8 @@ module Gitlab active_migration.column_name, batch_min_value: batch_min_value, batch_size: active_migration.batch_size, - job_arguments: active_migration.job_arguments) + job_arguments: active_migration.job_arguments, + job_class: active_migration.job_class) return if next_batch_bounds.nil? @@ -135,6 +142,16 @@ module Gitlab migration.reload_last_job end end + + def adjust_migration(active_migration) + signal = HealthStatus.evaluate(active_migration) + + if signal.is_a?(HealthStatus::Signals::Stop) + active_migration.hold! + else + active_migration.optimize! + end + end end end end diff --git a/lib/gitlab/database/background_migration/health_status.rb b/lib/gitlab/database/background_migration/health_status.rb new file mode 100644 index 00000000000..01f9c5eb5fd --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + # Rather than passing along the migration, we use a more explicitly defined context + Context = Struct.new(:tables) + + def self.evaluate(migration, indicator = Indicators::AutovacuumActiveOnTable) + signal = begin + indicator.new(migration.health_context).evaluate + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e, migration_id: migration.id, + job_class_name: migration.job_class_name) + Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})") + end + + log_signal(signal, migration) if signal.log_info? + + signal + end + + def self.log_signal(signal, migration) + Gitlab::AppLogger.info( + message: "#{migration} signaled: #{signal}", + migration_id: migration.id + ) + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/health_status/indicators.rb b/lib/gitlab/database/background_migration/health_status/indicators.rb new file mode 100644 index 00000000000..69503e5b61f --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status/indicators.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + module Indicators + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb b/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb new file mode 100644 index 00000000000..48e12609a13 --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + module Indicators + class AutovacuumActiveOnTable + def initialize(context) + @context = context + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + autovacuum_active_on = active_autovacuums_for(context.tables) + + if autovacuum_active_on.empty? + Signals::Normal.new(self.class, reason: 'no autovacuum running on any relevant tables') + else + Signals::Stop.new(self.class, reason: "autovacuum running on: #{autovacuum_active_on.join(', ')}") + end + end + + private + + attr_reader :context + + def enabled? + Feature.enabled?(:batched_migrations_health_status_autovacuum, type: :ops) + end + + def active_autovacuums_for(tables) + Gitlab::Database::PostgresAutovacuumActivity.for_tables(tables) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/health_status/signals.rb b/lib/gitlab/database/background_migration/health_status/signals.rb new file mode 100644 index 00000000000..6cd0ebd1bd0 --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status/signals.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + module Signals + # Base class for a signal + class Base + attr_reader :indicator_class, :reason + + def initialize(indicator_class, reason:) + @indicator_class = indicator_class + @reason = reason + end + + def to_s + "#{short_name} (indicator: #{indicator_class}; reason: #{reason})" + end + + # :nocov: + def log_info? + false + end + # :nocov: + + private + + def short_name + self.class.name.demodulize + end + end + + # A Signals::Stop is an indication to put a migration on hold or stop it entirely: + # In general, we want to slow down or pause the migration. + class Stop < Base + # :nocov: + def log_info? + true + end + # :nocov: + end + + # A Signals::Normal indicates normal system state: We carry on with the migration + # and may even attempt to optimize its throughput etc. + class Normal < Base; end + + # When given an Signals::Unknown, something unexpected happened while + # we evaluated system indicators. + class Unknown < Base + # :nocov: + def log_info? + true + end + # :nocov: + end + + # No signal could be determined, e.g. because the indicator + # was disabled. + class NotAvailable < Base; end + end + end + end + end +end diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index 0d876f5124f..02f008abf85 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -36,8 +36,7 @@ module Gitlab private def select_base_models(names) - base_models = Gitlab::Database.database_base_models - + base_models = Gitlab::Database.database_base_models_with_gitlab_shared return base_models if names.empty? names.each_with_object(HashWithIndifferentAccess.new) do |name, hash| @@ -48,7 +47,7 @@ module Gitlab end def with_shared_model_connections(shared_model, selected_databases, &blk) - Gitlab::Database.database_base_models.each_pair do |connection_name, connection_model| + Gitlab::Database.database_base_models_with_gitlab_shared.each_pair do |connection_name, connection_model| if shared_model.limit_connection_names next unless shared_model.limit_connection_names.include?(connection_name.to_sym) end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index baf4cc48424..365a4283d4c 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -13,6 +13,8 @@ module Gitlab module Database module GitlabSchema + GITLAB_SCHEMAS_FILE = 'lib/gitlab/database/gitlab_schemas.yml' + # These tables are deleted/renamed, but still referenced by migrations. # This is needed for now, but should be removed in the future DELETED_TABLES = { @@ -93,7 +95,7 @@ module Gitlab end def self.tables_to_schema - @tables_to_schema ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_schemas.yml')) + @tables_to_schema ||= YAML.load_file(Rails.root.join(GITLAB_SCHEMAS_FILE)) end def self.schema_names @@ -102,3 +104,5 @@ module Gitlab end end end + +Gitlab::Database::GitlabSchema.prepend_mod diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 71c323cb393..4a467d18f0a 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -108,10 +108,12 @@ ci_resource_groups: :gitlab_ci ci_resources: :gitlab_ci ci_runner_namespaces: :gitlab_ci ci_runner_projects: :gitlab_ci +ci_runner_versions: :gitlab_ci ci_runners: :gitlab_ci ci_running_builds: :gitlab_ci ci_sources_pipelines: :gitlab_ci ci_secure_files: :gitlab_ci +ci_secure_file_states: :gitlab_ci ci_sources_projects: :gitlab_ci ci_stages: :gitlab_ci ci_subscriptions_projects: :gitlab_ci @@ -131,7 +133,6 @@ cluster_providers_gcp: :gitlab_main clusters_applications_cert_managers: :gitlab_main clusters_applications_cilium: :gitlab_main clusters_applications_crossplane: :gitlab_main -clusters_applications_elastic_stacks: :gitlab_main clusters_applications_helm: :gitlab_main clusters_applications_ingress: :gitlab_main clusters_applications_jupyter: :gitlab_main @@ -139,7 +140,6 @@ clusters_applications_knative: :gitlab_main clusters_applications_prometheus: :gitlab_main clusters_applications_runners: :gitlab_main clusters: :gitlab_main -clusters_integration_elasticstack: :gitlab_main clusters_integration_prometheus: :gitlab_main clusters_kubernetes_namespaces: :gitlab_main commit_user_mentions: :gitlab_main @@ -325,6 +325,7 @@ milestone_releases: :gitlab_main milestones: :gitlab_main namespace_admin_notes: :gitlab_main namespace_aggregation_schedules: :gitlab_main +namespace_bans: :gitlab_main namespace_limits: :gitlab_main namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main @@ -423,6 +424,8 @@ project_incident_management_settings: :gitlab_main project_metrics_settings: :gitlab_main project_mirror_data: :gitlab_main project_pages_metadata: :gitlab_main +project_relation_export_uploads: :gitlab_main +project_relation_exports: :gitlab_main project_repositories: :gitlab_main project_repository_states: :gitlab_main project_repository_storage_moves: :gitlab_main @@ -432,7 +435,6 @@ projects: :gitlab_main projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main -project_tracing_settings: :gitlab_main prometheus_alert_events: :gitlab_main prometheus_alerts: :gitlab_main prometheus_metrics: :gitlab_main @@ -467,6 +469,10 @@ routes: :gitlab_main saml_group_links: :gitlab_main saml_providers: :gitlab_main saved_replies: :gitlab_main +sbom_components: :gitlab_main +sbom_occurrences: :gitlab_main +sbom_component_versions: :gitlab_main +sbom_sources: :gitlab_main schema_migrations: :gitlab_internal scim_identities: :gitlab_main scim_oauth_access_tokens: :gitlab_main @@ -547,6 +553,7 @@ vulnerability_flags: :gitlab_main vulnerability_historical_statistics: :gitlab_main vulnerability_identifiers: :gitlab_main vulnerability_issue_links: :gitlab_main +vulnerability_merge_request_links: :gitlab_main vulnerability_occurrence_identifiers: :gitlab_main vulnerability_occurrence_pipelines: :gitlab_main vulnerability_occurrences: :gitlab_main @@ -571,3 +578,4 @@ zentao_tracker_data: :gitlab_main dingtalk_tracker_data: :gitlab_main zoom_meetings: :gitlab_main batched_background_migration_job_transition_logs: :gitlab_shared +user_namespace_callouts: :gitlab_main diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index 6517923d23e..5f9416fb4db 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -19,7 +19,7 @@ module Gitlab ].freeze def self.base_models - @base_models ||= ::Gitlab::Database.database_base_models.values.freeze + @base_models ||= ::Gitlab::Database.database_base_models_using_load_balancing.values.freeze end def self.each_load_balancer diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 4aaeaa7b365..936b986ea07 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -72,12 +72,6 @@ module Gitlab ) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - if transaction_open? - raise 'The `queue_batched_background_migration` cannot be run inside a transaction. ' \ - 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \ - 'your migration class.' - end - gitlab_schema ||= gitlab_schema_from_context Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information @@ -182,12 +176,6 @@ module Gitlab def delete_batched_background_migration(job_class_name, table_name, column_name, job_arguments) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - if transaction_open? - raise 'The `#delete_batched_background_migration` cannot be run inside a transaction. ' \ - 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \ - 'your migration class.' - end - Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information Gitlab::Database::BackgroundMigration::BatchedMigration diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index 0c6a8d3d856..f38d847b0e8 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -14,7 +14,7 @@ module Gitlab def jobs_by_migration_name Gitlab::Database::BackgroundMigration::BatchedMigration .executable - .created_after(2.days.ago) # Simple way to exclude migrations already running before migration testing + .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing .to_h do |migration| batching_strategy = migration.batch_class.new(connection: connection) diff --git a/lib/gitlab/database/postgres_autovacuum_activity.rb b/lib/gitlab/database/postgres_autovacuum_activity.rb new file mode 100644 index 00000000000..a4dc199c259 --- /dev/null +++ b/lib/gitlab/database/postgres_autovacuum_activity.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresAutovacuumActivity < SharedModel + self.table_name = 'postgres_autovacuum_activity' + self.primary_key = 'table_identifier' + + def self.for_tables(tables) + Gitlab::Database::LoadBalancing::Session.current.use_primary do + where('schema = current_schema()').where(table: tables) + end + end + + def to_s + "table #{table_identifier} (started: #{vacuum_start})" + end + end + end +end diff --git a/lib/gitlab/database_importers/instance_administrators/create_group.rb b/lib/gitlab/database_importers/instance_administrators/create_group.rb index 79244120776..bb489ced3d2 100644 --- a/lib/gitlab/database_importers/instance_administrators/create_group.rb +++ b/lib/gitlab/database_importers/instance_administrators/create_group.rb @@ -77,7 +77,7 @@ module Gitlab def add_group_members(result) group = result[:group] - members = group.add_users(members_to_add(group), Gitlab::Access::MAINTAINER) + members = group.add_members(members_to_add(group), Gitlab::Access::MAINTAINER) errors = members.flat_map { |member| member.errors.full_messages } if errors.any? @@ -112,7 +112,7 @@ module Gitlab def members_to_add(group) # Exclude admins who are already members of group because - # `group.add_users(users)` returns an error if the users parameter contains + # `group.add_members(users)` returns an error if the users parameter contains # users who are already members of the group. instance_admins - group.members.collect(&:user) end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index 36a840372c5..76855f2950d 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -6,9 +6,6 @@ module Gitlab URL_REGEX = %r{https?://[^'" ]+}.freeze GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/.freeze REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze - VALID_LINK_ATTRIBUTES = %w[href rel target].freeze - - include ActionView::Helpers::SanitizeHelper class_attribute :file_type @@ -65,10 +62,9 @@ module Gitlab end def link_tag(name, url) - sanitize( - %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}, - attributes: VALID_LINK_ATTRIBUTES - ) + href_attribute = %{href="#{ERB::Util.html_escape_once(url)}" } if Gitlab::UrlSanitizer.valid_web?(url) + + %{<a #{href_attribute}rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}.html_safe end # Links package names based on regex. diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 8e039d32ef5..8c55652da43 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -373,7 +373,7 @@ module Gitlab end def rendered - return unless use_semantic_ipynb_diff? && ipynb? && modified_file? && !too_large? + return unless use_semantic_ipynb_diff? && ipynb? && modified_file? && !collapsed? && !too_large? strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) } end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 7fa1bd6b5ec..924de132840 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -68,20 +68,12 @@ module Gitlab end end - def diff_file_with_old_path(old_path, a_mode = nil) - if Feature.enabled?(:file_identifier_hash) && a_mode.present? - diff_files.find { |diff_file| diff_file.old_path == old_path && diff_file.a_mode == a_mode } - else - diff_files.find { |diff_file| diff_file.old_path == old_path } - end + def diff_file_with_old_path(old_path) + diff_files.find { |diff_file| diff_file.old_path == old_path } end - def diff_file_with_new_path(new_path, b_mode = nil) - if Feature.enabled?(:file_identifier_hash) && b_mode.present? - diff_files.find { |diff_file| diff_file.new_path == new_path && diff_file.b_mode == b_mode } - else - diff_files.find { |diff_file| diff_file.new_path == new_path } - end + def diff_file_with_new_path(new_path) + diff_files.find { |diff_file| diff_file.new_path == new_path } end def clear_cache diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb index e24150a2330..19fc028594c 100644 --- a/lib/gitlab/diff/formatters/base_formatter.rb +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -6,7 +6,6 @@ module Gitlab class BaseFormatter attr_reader :old_path attr_reader :new_path - attr_reader :file_identifier_hash attr_reader :base_sha attr_reader :start_sha attr_reader :head_sha @@ -16,7 +15,6 @@ module Gitlab attrs[:diff_refs] = diff_file.diff_refs attrs[:old_path] = diff_file.old_path attrs[:new_path] = diff_file.new_path - attrs[:file_identifier_hash] = diff_file.file_identifier_hash end if diff_refs = attrs[:diff_refs] @@ -27,7 +25,6 @@ module Gitlab @old_path = attrs[:old_path] @new_path = attrs[:new_path] - @file_identifier_hash = attrs[:file_identifier_hash] @base_sha = attrs[:base_sha] @start_sha = attrs[:start_sha] @head_sha = attrs[:head_sha] @@ -38,7 +35,7 @@ module Gitlab end def to_h - out = { + { base_sha: base_sha, start_sha: start_sha, head_sha: head_sha, @@ -46,12 +43,6 @@ module Gitlab new_path: new_path, position_type: position_type } - - if Feature.enabled?(:file_identifier_hash) - out[:file_identifier_hash] = file_identifier_hash - end - - out end def position_type diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index f950d01fdf0..8e9dc3a305f 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -193,6 +193,8 @@ module Gitlab results = redis.hmget(key, file_paths) end + record_hit_ratio(results) + results.map! do |result| Gitlab::Json.parse(gzip_decompress(result), symbolize_names: true) unless result.nil? end @@ -215,6 +217,11 @@ module Gitlab def current_transaction ::Gitlab::Metrics::WebTransaction.current end + + def record_hit_ratio(results) + current_transaction&.increment(:gitlab_redis_diff_caching_requests_total) + current_transaction&.increment(:gitlab_redis_diff_caching_hits_total) if results.any?(&:present?) + end end end end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 74c33c46598..40b6ae2f14e 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -9,7 +9,6 @@ module Gitlab delegate :old_path, :new_path, - :file_identifier_hash, :base_sha, :start_sha, :head_sha, @@ -161,11 +160,7 @@ module Gitlab def find_diff_file_from(diffable) diff_files = diffable.diffs(diff_options).diff_files - if Feature.enabled?(:file_identifier_hash) && file_identifier_hash.present? - diff_files.find { |df| df.file_identifier_hash == file_identifier_hash } - else - diff_files.first - end + diff_files.first end def multiline? diff --git a/lib/gitlab/diff/position_tracer/image_strategy.rb b/lib/gitlab/diff/position_tracer/image_strategy.rb index 046a6782dda..aac52b536f7 100644 --- a/lib/gitlab/diff/position_tracer/image_strategy.rb +++ b/lib/gitlab/diff/position_tracer/image_strategy.rb @@ -7,24 +7,21 @@ module Gitlab def trace(position) a_path = position.old_path b_path = position.new_path - diff_file = diff_file(position) - a_mode = diff_file&.a_mode - b_mode = diff_file&.b_mode # If file exists in B->D (e.g. updated, renamed, removed), let the # note become outdated. - bd_diff = bd_diffs.diff_file_with_old_path(b_path, b_mode) + bd_diff = bd_diffs.diff_file_with_old_path(b_path) return { position: new_position(position, bd_diff), outdated: true } if bd_diff # If file still exists in the new diff, update the position. - cd_diff = cd_diffs.diff_file_with_new_path(b_path, b_mode) + cd_diff = cd_diffs.diff_file_with_new_path(b_path) return { position: new_position(position, cd_diff), outdated: false } if cd_diff # If file exists in A->C (e.g. rebased and same changes were present # in target branch), let the note become outdated. - ac_diff = ac_diffs.diff_file_with_old_path(a_path, a_mode) + ac_diff = ac_diffs.diff_file_with_old_path(a_path) return { position: new_position(position, ac_diff), outdated: true } if ac_diff diff --git a/lib/gitlab/diff/position_tracer/line_strategy.rb b/lib/gitlab/diff/position_tracer/line_strategy.rb index 0f0b8f0c4f3..d7a7e3f5425 100644 --- a/lib/gitlab/diff/position_tracer/line_strategy.rb +++ b/lib/gitlab/diff/position_tracer/line_strategy.rb @@ -76,20 +76,16 @@ module Gitlab def trace_added_line(position) b_path = position.new_path b_line = position.new_line - diff_file = diff_file(position) - b_mode = diff_file&.b_mode - bd_diff = bd_diffs.diff_file_with_old_path(b_path, b_mode) + bd_diff = bd_diffs.diff_file_with_old_path(b_path) d_path = bd_diff&.new_path || b_path - d_mode = bd_diff&.b_mode || b_mode d_line = LineMapper.new(bd_diff).old_to_new(b_line) if d_line - cd_diff = cd_diffs.diff_file_with_new_path(d_path, d_mode) + cd_diff = cd_diffs.diff_file_with_new_path(d_path) c_path = cd_diff&.old_path || d_path - c_mode = cd_diff&.a_mode || d_mode c_line = LineMapper.new(cd_diff).new_to_old(d_line) if c_line @@ -102,7 +98,7 @@ module Gitlab else # If the line is no longer in the MR, we unfortunately cannot show # the current state on the CD diff, so we treat it as outdated. - ac_diff = ac_diffs.diff_file_with_new_path(c_path, c_mode) + ac_diff = ac_diffs.diff_file_with_new_path(c_path) { position: new_position(ac_diff, nil, c_line, position.line_range), outdated: true } end @@ -119,26 +115,22 @@ module Gitlab def trace_removed_line(position) a_path = position.old_path a_line = position.old_line - diff_file = diff_file(position) - a_mode = diff_file&.a_mode - ac_diff = ac_diffs.diff_file_with_old_path(a_path, a_mode) + ac_diff = ac_diffs.diff_file_with_old_path(a_path) c_path = ac_diff&.new_path || a_path - c_mode = ac_diff&.b_mode || a_mode c_line = LineMapper.new(ac_diff).old_to_new(a_line) if c_line - cd_diff = cd_diffs.diff_file_with_old_path(c_path, c_mode) + cd_diff = cd_diffs.diff_file_with_old_path(c_path) d_path = cd_diff&.new_path || c_path - d_mode = cd_diff&.b_mode || c_mode d_line = LineMapper.new(cd_diff).old_to_new(c_line) if d_line # If the line is still in C but also in D, it has turned from a # removed line into an unchanged one. - bd_diff = bd_diffs.diff_file_with_new_path(d_path, d_mode) + bd_diff = bd_diffs.diff_file_with_new_path(d_path) { position: new_position(bd_diff, nil, d_line, position.line_range), outdated: true } else @@ -156,21 +148,17 @@ module Gitlab a_line = position.old_line b_path = position.new_path b_line = position.new_line - diff_file = diff_file(position) - a_mode = diff_file&.a_mode - b_mode = diff_file&.b_mode - ac_diff = ac_diffs.diff_file_with_old_path(a_path, a_mode) + ac_diff = ac_diffs.diff_file_with_old_path(a_path) c_path = ac_diff&.new_path || a_path - c_mode = ac_diff&.b_mode || a_mode c_line = LineMapper.new(ac_diff).old_to_new(a_line) - bd_diff = bd_diffs.diff_file_with_old_path(b_path, b_mode) + bd_diff = bd_diffs.diff_file_with_old_path(b_path) d_line = LineMapper.new(bd_diff).old_to_new(b_line) - cd_diff = cd_diffs.diff_file_with_old_path(c_path, c_mode) + cd_diff = cd_diffs.diff_file_with_old_path(c_path) if c_line && d_line # If the line is still in C and D, it is still unchanged. diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb index 0a5b2ec3890..3e1652bd318 100644 --- a/lib/gitlab/diff/rendered/notebook/diff_file.rb +++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb @@ -79,7 +79,7 @@ module Gitlab rescue Timeout::Error => e rendered_timeout.increment(source: Gitlab::Runtime.sidekiq? ? BACKGROUND_EXECUTION : FOREGROUND_EXECUTION) log_event(LOG_IPYNBDIFF_TIMEOUT, e) - rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e + rescue IpynbDiff::InvalidNotebookError => e log_event(LOG_IPYNBDIFF_INVALID, e) end end diff --git a/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb b/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb index 2e1b5ea301d..f381792953e 100644 --- a/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb +++ b/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb @@ -91,7 +91,7 @@ module Gitlab return 0 unless line_in_source.present? - line_in_source + 1 + line_in_source end def image_as_rich_text(line_text) diff --git a/lib/gitlab/elasticsearch/logs/lines.rb b/lib/gitlab/elasticsearch/logs/lines.rb deleted file mode 100644 index ff9185dd331..00000000000 --- a/lib/gitlab/elasticsearch/logs/lines.rb +++ /dev/null @@ -1,157 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Elasticsearch - module Logs - class Lines - InvalidCursor = Class.new(RuntimeError) - - # How many log lines to fetch in a query - LOGS_LIMIT = 500 - - def initialize(client) - @client = client - end - - def pod_logs(namespace, pod_name: nil, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil, chart_above_v2: true) - query = { bool: { must: [] } }.tap do |q| - filter_pod_name(q, pod_name) - filter_namespace(q, namespace) - filter_container_name(q, container_name) - filter_search(q, search) - filter_times(q, start_time, end_time) - end - - body = build_body(query, cursor, chart_above_v2) - response = @client.search body: body - - format_response(response) - end - - private - - def build_body(query, cursor = nil, chart_above_v2 = true) - offset_field = chart_above_v2 ? "log.offset" : "offset" - body = { - query: query, - # reverse order so we can query N-most recent records - sort: [ - { "@timestamp": { order: :desc } }, - { "#{offset_field}": { order: :desc } } - ], - # only return these fields in the response - _source: ["@timestamp", "message", "kubernetes.pod.name"], - # fixed limit for now, we should support paginated queries - size: ::Gitlab::Elasticsearch::Logs::Lines::LOGS_LIMIT - } - - unless cursor.nil? - body[:search_after] = decode_cursor(cursor) - end - - body - end - - def filter_pod_name(query, pod_name) - # We can filter by "all pods" with a null pod_name - return if pod_name.nil? - - query[:bool][:must] << { - match_phrase: { - "kubernetes.pod.name" => { - query: pod_name - } - } - } - end - - def filter_namespace(query, namespace) - query[:bool][:must] << { - match_phrase: { - "kubernetes.namespace" => { - query: namespace - } - } - } - end - - def filter_container_name(query, container_name) - # A pod can contain multiple containers. - # By default we return logs from every container - return if container_name.nil? - - query[:bool][:must] << { - match_phrase: { - "kubernetes.container.name" => { - query: container_name - } - } - } - end - - def filter_search(query, search) - return if search.nil? - - query[:bool][:must] << { - simple_query_string: { - query: search, - fields: [:message], - default_operator: :and - } - } - end - - def filter_times(query, start_time, end_time) - return unless start_time || end_time - - time_range = { range: { :@timestamp => {} } }.tap do |tr| - tr[:range][:@timestamp][:gte] = start_time if start_time - tr[:range][:@timestamp][:lt] = end_time if end_time - end - - query[:bool][:filter] = [time_range] - end - - def format_response(response) - results = response.fetch("hits", {}).fetch("hits", []) - last_result = results.last - results = results.map do |hit| - { - timestamp: hit["_source"]["@timestamp"], - message: hit["_source"]["message"], - pod: hit["_source"]["kubernetes"]["pod"]["name"] - } - end - - # we queried for the N-most recent records but we want them ordered oldest to newest - { - logs: results.reverse, - cursor: last_result.nil? ? nil : encode_cursor(last_result["sort"]) - } - end - - # we want to hide the implementation details of the search_after parameter from the frontend - # behind a single easily transmitted value - def encode_cursor(obj) - obj.join(',') - end - - def decode_cursor(obj) - cursor = obj.split(',').map(&:to_i) - - unless valid_cursor(cursor) - raise InvalidCursor, "invalid cursor format" - end - - cursor - end - - def valid_cursor(cursor) - cursor.instance_of?(Array) && - cursor.length == 2 && - cursor.map {|i| i.instance_of?(Integer)}.reduce(:&) - end - end - end - end -end diff --git a/lib/gitlab/elasticsearch/logs/pods.rb b/lib/gitlab/elasticsearch/logs/pods.rb deleted file mode 100644 index 66499ae956a..00000000000 --- a/lib/gitlab/elasticsearch/logs/pods.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Elasticsearch - module Logs - class Pods - # How many items to fetch in a query - PODS_LIMIT = 500 - CONTAINERS_LIMIT = 500 - - def initialize(client) - @client = client - end - - def pods(namespace) - body = build_body(namespace) - response = @client.search body: body - - format_response(response) - end - - private - - def build_body(namespace) - { - aggs: { - pods: { - aggs: { - containers: { - terms: { - field: 'kubernetes.container.name', - size: ::Gitlab::Elasticsearch::Logs::Pods::CONTAINERS_LIMIT - } - } - }, - terms: { - field: 'kubernetes.pod.name', - size: ::Gitlab::Elasticsearch::Logs::Pods::PODS_LIMIT - } - } - }, - query: { - bool: { - must: { - match_phrase: { - "kubernetes.namespace": namespace - } - } - } - }, - # don't populate hits, only the aggregation is needed - size: 0 - } - end - - def format_response(response) - results = response.dig("aggregations", "pods", "buckets") || [] - results.map do |bucket| - { - name: bucket["key"], - container_names: (bucket.dig("containers", "buckets") || []).map do |cbucket| - cbucket["key"] - end - } - end - end - end - end - end -end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 71b1d4ed8f9..8e2c7559bc1 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -98,7 +98,10 @@ module Gitlab title: mail.subject, description: message_including_template, confidential: true, - external_author: from_address + external_author: from_address, + extra_params: { + cc: mail.cc + } }, spam_params: nil ).execute diff --git a/lib/gitlab/email/message/in_product_marketing/experience.rb b/lib/gitlab/email/message/in_product_marketing/experience.rb deleted file mode 100644 index 7520de6d2a3..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/experience.rb +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class Experience < Base - include Gitlab::Utils::StrongMemoize - - EASE_SCORE_SURVEY_ID = 1 - - def subject_line - s_('InProductMarketing|Do you have a minute?') - end - - def tagline - end - - def title - s_('InProductMarketing|We want your GitLab experience to be great') - end - - def subtitle - s_('InProductMarketing|Take this 1-question survey!') - end - - def body_line1 - s_('InProductMarketing|%{strong_start}Overall, how difficult or easy was it to get started with GitLab?%{strong_end}').html_safe % strong_options - end - - def body_line2 - s_('InProductMarketing|Click on the number below that corresponds with your answer — 1 being very difficult, 5 being very easy.') - end - - def cta_text - end - - def feedback_link(rating) - params = { - onboarding_progress: onboarding_progress, - response: rating, - show_invite_link: show_invite_link, - survey_id: EASE_SCORE_SURVEY_ID - } - - params[:show_incentive] = true if show_incentive? - - "#{gitlab_com_root_url}/-/survey_responses?#{params.to_query}" - end - - def feedback_ratings(rating) - [ - s_('InProductMarketing|Very difficult'), - s_('InProductMarketing|Difficult'), - s_('InProductMarketing|Neutral'), - s_('InProductMarketing|Easy'), - s_('InProductMarketing|Very easy') - ][rating - 1] - end - - def feedback_thanks - s_('InProductMarketing|Feedback from users like you really improves our product. Thanks for your help!') - end - - private - - def onboarding_progress - strong_memoize(:onboarding_progress) do - group.onboarding_progress.number_of_completed_actions - end - end - - def show_invite_link - strong_memoize(:show_invite_link) do - group.max_member_access_for_user(user) >= GroupMember::DEVELOPER && user.preferred_language == 'en' - end - end - - def show_incentive? - show_invite_link && group.member_count > 1 - end - - def gitlab_com_root_url - return root_url.chomp('/') if Rails.env.development? - - Gitlab::Saas.com_url - end - end - end - end - end -end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 35c1a1e73cf..83920182da4 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -20,7 +20,10 @@ module Gitlab ::Gitlab::ErrorTracking::Processor::SidekiqProcessor, ::Gitlab::ErrorTracking::Processor::GrpcErrorProcessor, ::Gitlab::ErrorTracking::Processor::ContextPayloadProcessor, - ::Gitlab::ErrorTracking::Processor::SanitizeErrorMessageProcessor + ::Gitlab::ErrorTracking::Processor::SanitizeErrorMessageProcessor, + # IMPORTANT: this processor must stay at the bottom, right before + # sending the event to Sentry. + ::Gitlab::ErrorTracking::Processor::SanitizerProcessor ].freeze class << self diff --git a/lib/gitlab/error_tracking/error_repository.rb b/lib/gitlab/error_tracking/error_repository.rb index 4ec636703d9..fd2467add20 100644 --- a/lib/gitlab/error_tracking/error_repository.rb +++ b/lib/gitlab/error_tracking/error_repository.rb @@ -15,7 +15,12 @@ module Gitlab # # @return [self] def self.build(project) - strategy = ActiveRecordStrategy.new(project) + strategy = + if Feature.enabled?(:use_click_house_database_for_error_tracking, project) + OpenApiStrategy.new(project) + else + ActiveRecordStrategy.new(project) + end new(strategy) end @@ -72,14 +77,15 @@ module Gitlab # @param sort [String] order list by 'first_seen', 'last_seen', or 'frequency' # @param filters [Hash<Symbol, String>] filter list by # @option filters [String] :status error status + # @params query [String, nil] free text search # @param limit [Integer, String] limit result # @param cursor [Hash] pagination information # # @return [Array<Array<Gitlab::ErrorTracking::Error>, Pagination>] - def list_errors(sort: 'last_seen', filters: {}, limit: 20, cursor: {}) + def list_errors(sort: 'last_seen', filters: {}, query: nil, limit: 20, cursor: {}) limit = [limit.to_i, 100].min - strategy.list_errors(filters: filters, sort: sort, limit: limit, cursor: cursor) + strategy.list_errors(filters: filters, query: query, sort: sort, limit: limit, cursor: cursor) end # Fetches last event for error +id+. @@ -105,6 +111,10 @@ module Gitlab strategy.update_error(id, status: status) end + def dsn_url(public_key) + strategy.dsn_url(public_key) + end + private attr_reader :strategy diff --git a/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb b/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb index e5b532ee0f0..01e7fbda384 100644 --- a/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb +++ b/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb @@ -39,11 +39,12 @@ module Gitlab handle_exceptions(e) end - def list_errors(filters:, sort:, limit:, cursor:) + def list_errors(filters:, query:, sort:, limit:, cursor:) errors = project_errors errors = filter_by_status(errors, filters[:status]) errors = sort(errors, sort) errors = errors.keyset_paginate(cursor: cursor, per_page: limit) + # query is not supported pagination = ErrorRepository::Pagination.new(errors.cursor_for_next_page, errors.cursor_for_previous_page) @@ -60,6 +61,24 @@ module Gitlab project_error(id).update(attributes) end + def dsn_url(public_key) + gitlab = Settings.gitlab + + custom_port = Settings.gitlab_on_standard_port? ? nil : ":#{gitlab.port}" + + base_url = [ + gitlab.protocol, + "://", + public_key, + '@', + gitlab.host, + custom_port, + gitlab.relative_url_root + ].join('') + + "#{base_url}/api/v4/error_tracking/collector/#{project.id}" + end + private attr_reader :project diff --git a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb new file mode 100644 index 00000000000..e3eae20c520 --- /dev/null +++ b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb @@ -0,0 +1,248 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class ErrorRepository + class OpenApiStrategy + def initialize(project) + @project = project + + api_url = configured_api_url + + open_api.configure do |config| + config.scheme = api_url.scheme + config.host = [api_url.host, api_url.port].compact.join(':') + config.server_index = nil + config.logger = Gitlab::AppLogger + end + end + + def report_error( + name:, description:, actor:, platform:, + environment:, level:, occurred_at:, payload: + ) + raise NotImplementedError, 'Use ingestion endpoint' + end + + def find_error(id) + api = open_api::ErrorsApi.new + error = api.get_error(project_id, id) + + to_sentry_detailed_error(error) + rescue ErrorTrackingOpenAPI::ApiError => e + log_exception(e) + nil + end + + def list_errors(filters:, query:, sort:, limit:, cursor:) + opts = { + sort: "#{sort}_desc", + status: filters[:status], + query: query, + cursor: cursor, + limit: limit + }.compact + + api = open_api::ErrorsApi.new + errors, _status, headers = api.list_errors_with_http_info(project_id, opts) + pagination = pagination_from_headers(headers) + + if errors.size < limit + # Don't show next link if amount of errors is less then requested. + # This a workaround until the Golang backend returns link cursor + # only if there is a next page. + pagination.next = nil + end + + [errors.map { to_sentry_error(_1) }, pagination] + rescue ErrorTrackingOpenAPI::ApiError => e + log_exception(e) + [[], ErrorRepository::Pagination.new] + end + + def last_event_for(id) + event = newest_event_for(id) + return unless event + + api = open_api::ErrorsApi.new + error = api.get_error(project_id, id) + return unless error + + to_sentry_error_event(event, error) + rescue ErrorTrackingOpenAPI::ApiError => e + log_exception(e) + nil + end + + def update_error(id, **attributes) + opts = attributes.slice(:status) + + body = open_api::ErrorUpdatePayload.new(opts) + + api = open_api::ErrorsApi.new + api.update_error(project_id, id, body) + + true + rescue ErrorTrackingOpenAPI::ApiError => e + log_exception(e) + false + end + + def dsn_url(public_key) + config = open_api::Configuration.default + + base_url = [ + config.scheme, + "://", + public_key, + '@', + config.host, + config.base_path + ].join('') + + "#{base_url}/projects/api/#{project_id}" + end + + private + + def event_for(id, sort:) + opts = { sort: sort, limit: 1 } + + api = open_api::ErrorsApi.new + api.list_events(project_id, id, opts).first + rescue ErrorTrackingOpenAPI::ApiError => e + log_exception(e) + nil + end + + def newest_event_for(id) + event_for(id, sort: 'occurred_at_desc') + end + + def oldest_event_for(id) + event_for(id, sort: 'occurred_at_asc') + end + + def to_sentry_error(error) + Gitlab::ErrorTracking::Error.new( + id: error.fingerprint.to_s, + title: error.name, + message: error.description, + culprit: error.actor, + first_seen: error.first_seen_at, + last_seen: error.last_seen_at, + status: error.status, + count: error.event_count, + user_count: error.approximated_user_count + ) + end + + def to_sentry_detailed_error(error) + Gitlab::ErrorTracking::DetailedError.new( + id: error.fingerprint.to_s, + title: error.name, + message: error.description, + culprit: error.actor, + first_seen: error.first_seen_at.to_s, + last_seen: error.last_seen_at.to_s, + count: error.event_count, + user_count: error.approximated_user_count, + project_id: error.project_id, + status: error.status, + tags: { level: nil, logger: nil }, + external_url: external_url(error.fingerprint), + external_base_url: external_base_url, + integrated: true, + first_release_version: release_from(oldest_event_for(error.fingerprint)), + last_release_version: release_from(newest_event_for(error.fingerprint)) + ) + end + + def to_sentry_error_event(event, error) + Gitlab::ErrorTracking::ErrorEvent.new( + issue_id: event.fingerprint.to_s, + date_received: error.last_seen_at, + stack_trace_entries: build_stacktrace(event) + ) + end + + def pagination_from_headers(headers) + links = headers['link'].to_s.split(', ') + + pagination_hash = links.map { parse_pagination_link(_1) }.compact.to_h + + ErrorRepository::Pagination.new(pagination_hash['next'], pagination_hash['prev']) + end + + LINK_PATTERN = %r{cursor=(?<cursor>[^&]+).*; rel="(?<direction>\w+)"}.freeze + + def parse_pagination_link(content) + match = LINK_PATTERN.match(content) + return unless match + + [match['direction'], CGI.unescape(match['cursor'])] + end + + def build_stacktrace(event) + payload = parse_json(event.payload) + return [] unless payload + + ::ErrorTracking::StacktraceBuilder.new(payload).stacktrace + end + + def parse_json(payload) + Gitlab::Json.parse(payload) + rescue JSON::ParserError + end + + def release_from(event) + return unless event + + payload = parse_json(event.payload) + return unless payload + + payload['release'] + end + + def project_id + @project.id + end + + def open_api + ErrorTrackingOpenAPI + end + + # For compatibility with sentry integration + def external_url(id) + Gitlab::Routing.url_helpers.details_namespace_project_error_tracking_index_url( + namespace_id: @project.namespace, + project_id: @project, + issue_id: id) + end + + # For compatibility with sentry integration + def external_base_url + Gitlab::Routing.url_helpers.project_url(@project) + end + + def configured_api_url + url = Gitlab::CurrentSettings.current_application_settings.error_tracking_api_url || + 'http://localhost:8080' + + Gitlab::UrlBlocker.validate!(url, schemes: %w[http https], allow_localhost: true) + + URI(url) + end + + def log_exception(exception) + params = { + http_code: exception.code, + response_body: exception.response_body&.truncate(100) + } + + Gitlab::AppLogger.error(Gitlab::Utils::InlineHash.merge_keys(params, prefix: 'open_api')) + end + end + end + end +end diff --git a/lib/gitlab/error_tracking/processor/sanitizer_processor.rb b/lib/gitlab/error_tracking/processor/sanitizer_processor.rb new file mode 100644 index 00000000000..e6114f8e206 --- /dev/null +++ b/lib/gitlab/error_tracking/processor/sanitizer_processor.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + module Processor + module SanitizerProcessor + SANITIZED_HTTP_HEADERS = %w[Authorization Private-Token Job-Token].freeze + SANITIZED_ATTRIBUTES = %i[user contexts extra tags].freeze + + # This processor removes sensitive fields or headers from the event + # before sending. Sentry versions above 4.0 don't support + # sanitized_fields and sanitized_http_headers anymore. The official + # document recommends using before_send instead. + # + # For more information, please visit: + # https://docs.sentry.io/platforms/ruby/guides/rails/configuration/filtering/#using-beforesend + def self.call(event) + # Raven::Event instances don't need this processing. + return event unless event.is_a?(Sentry::Event) + + if event.request.present? + event.request.cookies = {} + event.request.data = {} + end + + if event.request.present? && event.request.headers.is_a?(Hash) + header_filter = ActiveSupport::ParameterFilter.new(SANITIZED_HTTP_HEADERS) + event.request.headers = header_filter.filter(event.request.headers) + end + + attribute_filter = ActiveSupport::ParameterFilter.new(Rails.application.config.filter_parameters) + SANITIZED_ATTRIBUTES.each do |attribute| + event.send("#{attribute}=", attribute_filter.filter(event.send(attribute))) # rubocop:disable GitlabSecurity/PublicSend + end + + if event.request.present? && event.request.query_string.present? + query = Rack::Utils.parse_nested_query(event.request.query_string) + query = attribute_filter.filter(query) + query = Rack::Utils.build_nested_query(query) + event.request.query_string = query + end + + event + end + end + end + end +end diff --git a/lib/gitlab/event_store.rb b/lib/gitlab/event_store.rb index e20ea1c7365..4955e873688 100644 --- a/lib/gitlab/event_store.rb +++ b/lib/gitlab/event_store.rb @@ -35,6 +35,11 @@ module Gitlab store.subscribe ::MergeRequests::UpdateHeadPipelineWorker, to: ::Ci::PipelineCreatedEvent store.subscribe ::Namespaces::UpdateRootStatisticsWorker, to: ::Projects::ProjectDeletedEvent + + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Pages::PageDeployedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Pages::PageDeletedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectDeletedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectCreatedEvent end private_class_method :configure! end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 505d0b8d728..882bd57eb1d 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -18,6 +18,8 @@ module Gitlab UnknownRef = Class.new(BaseError) CommandTimedOut = Class.new(CommandError) InvalidPageToken = Class.new(BaseError) + InvalidRefFormatError = Class.new(BaseError) + ReferencesLockedError = Class.new(BaseError) class << self include Gitlab::EncodingHelper diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index f72217dedde..bb5bbeeb27e 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -128,6 +128,9 @@ module Gitlab @loaded_size = @data.bytesize if @data @loaded_all_data = @loaded_size == size + # Recalculate binary status if we loaded all data + @binary = nil if @loaded_all_data + record_metric_blob_size record_metric_truncated(truncated?) end diff --git a/lib/gitlab/git/conflict/parser.rb b/lib/gitlab/git/conflict/parser.rb index 20de8ebde4e..670305de95b 100644 --- a/lib/gitlab/git/conflict/parser.rb +++ b/lib/gitlab/git/conflict/parser.rb @@ -27,7 +27,7 @@ module Gitlab conflict_end = ">>>>>>> #{their_path}" text.each_line.map do |line| - full_line = line.delete("\n") + full_line = line.chomp if full_line == conflict_start validate_delimiter!(type.nil?) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index df744bd60b4..d7f892ae9d9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -800,9 +800,9 @@ module Gitlab end end - def list_refs + def list_refs(patterns = [Gitlab::Git::BRANCH_REF_PREFIX]) wrapped_gitaly_errors do - gitaly_ref_client.list_refs + gitaly_ref_client.list_refs(patterns) end end diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index 40c003821b9..bc0af12d7e3 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -63,10 +63,7 @@ module Gitlab def tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) tree_entries_from_rugged(repository, sha, path, recursive).tap do |entries| # This was an optimization to reduce N+1 queries for Gitaly - # (https://gitlab.com/gitlab-org/gitaly/issues/530). It - # used to be done lazily in the view via - # TreeHelper#flatten_tree, so it's possible there's a - # performance impact by loading this eagerly. + # (https://gitlab.com/gitlab-org/gitaly/issues/530). rugged_populate_flat_path(repository, sha, path, entries) end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index f376dbce177..996534f4194 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -285,6 +285,8 @@ module Gitlab end def self.enforce_gitaly_request_limits? + return false if ENV["GITALY_DISABLE_REQUEST_LIMITS"] + # We typically don't want to enforce request limits in production # However, we have some production-like test environments, i.e., ones # where `Rails.env.production?` returns `true`. We do want to be able to @@ -293,7 +295,7 @@ module Gitlab # enforce request limits. return true if Feature::Gitaly.enabled?('enforce_requests_limits') - !(Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"]) + !Rails.env.production? end private_class_method :enforce_gitaly_request_limits? @@ -483,6 +485,22 @@ module Gitlab stack_counter.select { |_, v| v == max }.keys end + + def self.decode_detailed_error(err) + # details could have more than one in theory, but we only have one to worry about for now. + detailed_error = err.to_rpc_status&.details&.first + + return unless detailed_error.present? + + prefix = %r{type\.googleapis\.com\/gitaly\.(?<error_type>.+)} + error_type = prefix.match(detailed_error.type_url)[:error_type] + + Gitaly.const_get(error_type, false).decode(detailed_error.value) + rescue NameError, NoMethodError + # Error Class might not be known to ruby yet + nil + end + private_class_method :max_stacks end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index d575c0f470d..35d3ddf5d7f 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -102,7 +102,7 @@ module Gitlab raise Gitlab::Git::PreReceiveError, pre_receive_error end rescue GRPC::BadStatus => e - detailed_error = decode_detailed_error(e) + detailed_error = GitalyClient.decode_detailed_error(e) case detailed_error&.error when :custom_hook @@ -166,7 +166,7 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) rescue GRPC::BadStatus => e - detailed_error = decode_detailed_error(e) + detailed_error = GitalyClient.decode_detailed_error(e) case detailed_error&.error when :access_check @@ -277,7 +277,7 @@ module Gitlab rebase_sha rescue GRPC::BadStatus => e - detailed_error = decode_detailed_error(e) + detailed_error = GitalyClient.decode_detailed_error(e) case detailed_error&.error when :access_check @@ -314,7 +314,7 @@ module Gitlab response.squash_sha rescue GRPC::BadStatus => e - detailed_error = decode_detailed_error(e) + detailed_error = GitalyClient.decode_detailed_error(e) case detailed_error&.error when :resolve_revision, :rebase_conflict @@ -474,7 +474,7 @@ module Gitlab handle_cherry_pick_or_revert_response(response) rescue GRPC::BadStatus => e - detailed_error = decode_detailed_error(e) + detailed_error = GitalyClient.decode_detailed_error(e) case detailed_error&.error when :access_check @@ -483,6 +483,8 @@ module Gitlab raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) when :cherry_pick_conflict raise Gitlab::Git::Repository::CreateTreeError, 'CONFLICT' + when :changes_already_applied + raise Gitlab::Git::Repository::CreateTreeError, 'EMPTY' when :target_branch_diverged raise Gitlab::Git::CommitError, 'branch diverged' else @@ -536,21 +538,6 @@ module Gitlab raise ArgumentError, "Unknown action '#{action[:action]}'" end - def decode_detailed_error(err) - # details could have more than one in theory, but we only have one to worry about for now. - detailed_error = err.to_rpc_status&.details&.first - - return unless detailed_error.present? - - prefix = %r{type\.googleapis\.com\/gitaly\.(?<error_type>.+)} - error_type = prefix.match(detailed_error.type_url)[:error_type] - - Gitaly.const_get(error_type, false).decode(detailed_error.value) - rescue NameError, NoMethodError - # Error Class might not be known to ruby yet - nil - end - def custom_hook_error_message(custom_hook_error) # Custom hooks may return messages via either stdout or stderr which have a specific prefix. If # that prefix is present we'll want to print the hook's output, otherwise we'll want to print the diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index c064811b1e7..31e1406356f 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -132,6 +132,17 @@ module Gitlab response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.medium_timeout) raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error&.error + when :invalid_format + raise Gitlab::Git::InvalidRefFormatError, "references have an invalid format: #{detailed_error.invalid_format.refs.join(",")}" + when :references_locked + raise Gitlab::Git::ReferencesLockedError + else + raise e + end end # Limit: 0 implies no limit, thus all tag names will be returned diff --git a/lib/gitlab/github_import/importer/events/changed_label.rb b/lib/gitlab/github_import/importer/events/changed_label.rb new file mode 100644 index 00000000000..6c408158b02 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/changed_label.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class ChangedLabel + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + create_event(issue_event) + end + + private + + attr_reader :project, :user_id + + def create_event(issue_event) + ResourceLabelEvent.create!( + issue_id: issue_event.issue_db_id, + user_id: user_id, + label_id: label_finder.id_for(issue_event.label_title), + action: action(issue_event.event), + created_at: issue_event.created_at + ) + end + + def label_finder + Gitlab::GithubImport::LabelFinder.new(project) + end + + def action(event_type) + event_type == 'unlabeled' ? 'remove' : 'add' + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/closed.rb b/lib/gitlab/github_import/importer/events/closed.rb new file mode 100644 index 00000000000..8b2136c9b24 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/closed.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Closed + attr_reader :project, :user_id + + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + create_event(issue_event) + create_state_event(issue_event) + end + + private + + def create_event(issue_event) + Event.create!( + project_id: project.id, + author_id: user_id, + action: 'closed', + target_type: Issue.name, + target_id: issue_event.issue_db_id, + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def create_state_event(issue_event) + ResourceStateEvent.create!( + user_id: user_id, + issue_id: issue_event.issue_db_id, + source_commit: issue_event.commit_id, + state: 'closed', + close_after_error_tracking_resolve: false, + close_auto_resolve_prometheus_alert: false, + created_at: issue_event.created_at + ) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb new file mode 100644 index 00000000000..20b902cfe50 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class CrossReferenced + attr_reader :project, :user_id + + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + mentioned_in_record_class = mentioned_in_type(issue_event) + mentioned_in_number = issue_event.source.dig(:issue, :number) + mentioned_in_record = init_mentioned_in( + mentioned_in_record_class, mentioned_in_number + ) + return if mentioned_in_record.nil? + + note_body = cross_reference_note_content(mentioned_in_record.gfm_reference(project)) + track_activity(mentioned_in_record_class) + create_note(issue_event, note_body) + end + + private + + def track_activity(mentioned_in_class) + return if mentioned_in_class != Issue + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event( + Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_CROSS_REFERENCED, + values: user_id + ) + end + + def create_note(issue_event, note_body) + Note.create!( + system: true, + noteable_type: Issue.name, + noteable_id: issue_event.issue_db_id, + project: project, + author_id: user_id, + note: note_body, + system_note_metadata: SystemNoteMetadata.new(action: 'cross_reference'), + created_at: issue_event.created_at + ) + end + + def mentioned_in_type(issue_event) + is_pull_request = issue_event.source.dig(:issue, :pull_request).present? + is_pull_request ? MergeRequest : Issue + end + + # record_class - Issue/MergeRequest + def init_mentioned_in(record_class, iid) + db_id = fetch_mentioned_in_db_id(record_class, iid) + return if db_id.nil? + + record = record_class.new(id: db_id, iid: iid) + record.project = project + record.readonly! + record + end + + # record_class - Issue/MergeRequest + def fetch_mentioned_in_db_id(record_class, number) + sawyer_mentioned_in_adapter = Struct.new(:iid, :issuable_type, keyword_init: true) + mentioned_in_adapter = sawyer_mentioned_in_adapter.new( + iid: number, issuable_type: record_class.name + ) + + Gitlab::GithubImport::IssuableFinder.new(project, mentioned_in_adapter).database_id + end + + def cross_reference_note_content(gfm_reference) + "#{::SystemNotes::IssuablesService.cross_reference_note_prefix}#{gfm_reference}" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/renamed.rb b/lib/gitlab/github_import/importer/events/renamed.rb new file mode 100644 index 00000000000..6a11c492210 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/renamed.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Renamed + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent` + def execute(issue_event) + Note.create!(note_params(issue_event)) + end + + private + + attr_reader :project, :user_id + + def note_params(issue_event) + { + noteable_id: issue_event.issue_db_id, + noteable_type: Issue.name, + project_id: project.id, + author_id: user_id, + note: parse_body(issue_event), + system: true, + created_at: issue_event.created_at, + updated_at: issue_event.created_at, + system_note_metadata: SystemNoteMetadata.new( + { + action: "title", + created_at: issue_event.created_at, + updated_at: issue_event.created_at + } + ) + } + end + + def parse_body(issue_event) + old_diffs, new_diffs = Gitlab::Diff::InlineDiff.new( + issue_event.old_title, issue_event.new_title + ).inline_diffs + + marked_old_title = Gitlab::Diff::InlineDiffMarkdownMarker.new(issue_event.old_title).mark(old_diffs) + marked_new_title = Gitlab::Diff::InlineDiffMarkdownMarker.new(issue_event.new_title).mark(new_diffs) + + "changed title from **#{marked_old_title}** to **#{marked_new_title}**" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/reopened.rb b/lib/gitlab/github_import/importer/events/reopened.rb new file mode 100644 index 00000000000..c0f3802bc46 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/reopened.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Reopened + attr_reader :project, :user_id + + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + create_event(issue_event) + create_state_event(issue_event) + end + + private + + def create_event(issue_event) + Event.create!( + project_id: project.id, + author_id: user_id, + action: 'reopened', + target_type: Issue.name, + target_id: issue_event.issue_db_id, + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def create_state_event(issue_event) + ResourceStateEvent.create!( + user_id: user_id, + issue_id: issue_event.issue_db_id, + state: 'reopened', + created_at: issue_event.created_at + ) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb new file mode 100644 index 00000000000..e451af61ec3 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_event_importer.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueEventImporter + attr_reader :issue_event, :project, :client, :user_finder + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(issue_event, project, client) + @issue_event = issue_event + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + end + + def execute + case issue_event.event + when 'closed' + Gitlab::GithubImport::Importer::Events::Closed.new(project, author_id) + .execute(issue_event) + when 'reopened' + Gitlab::GithubImport::Importer::Events::Reopened.new(project, author_id) + .execute(issue_event) + when 'labeled', 'unlabeled' + Gitlab::GithubImport::Importer::Events::ChangedLabel.new(project, author_id) + .execute(issue_event) + when 'renamed' + Gitlab::GithubImport::Importer::Events::Renamed.new(project, author_id) + .execute(issue_event) + when 'cross-referenced' + Gitlab::GithubImport::Importer::Events::CrossReferenced.new(project, author_id) + .execute(issue_event) + else + Gitlab::GithubImport::Logger.debug( + message: 'UNSUPPORTED_EVENT_TYPE', + event_type: issue_event.event, event_github_id: issue_event.id + ) + end + end + + private + + def author_id + id, _status = user_finder.author_id_for(issue_event, author_key: :actor) + id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index 35fd4bd88a0..e7d41856b04 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -51,6 +51,7 @@ module Gitlab title: issue.truncated_title, author_id: author_id, project_id: project.id, + namespace_id: project.project_namespace_id, description: description, milestone_id: milestone_finder.id_for(issue), state_id: ::Issue.available_states[issue.state], diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 673f56b5753..1410006af26 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -21,14 +21,12 @@ module Gitlab author_id, author_found = user_finder.author_id_for(note) - note_body = MarkdownText.format(note.note, note.author, author_found) - attributes = { noteable_type: note.noteable_type, noteable_id: noteable_id, project_id: project.id, author_id: author_id, - note: note_body, + note: note_body(author_found), discussion_id: note.discussion_id, system: false, created_at: note.created_at, @@ -48,6 +46,13 @@ module Gitlab def find_noteable_id GithubImport::IssuableFinder.new(project, note).database_id end + + private + + def note_body(author_found) + text = MarkdownText.convert_ref_links(note.note, project) + MarkdownText.format(text, note.author, author_found) + end end end end diff --git a/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb index a2c3d1bd057..e1d9ae44065 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb @@ -37,15 +37,15 @@ module Gitlab private - def noteables - project.merge_requests.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + def parent_collection + project.merge_requests.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord end def page_counter_id(merge_request) "merge_request/#{merge_request.id}/#{collection_method}" end - def notes_imported_cache_key + def parent_imported_cache_key "github-importer/merge_request/diff_notes/already-imported/#{project.id}" end end diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb new file mode 100644 index 00000000000..45bbc25e637 --- /dev/null +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class SingleEndpointIssueEventsImporter + include ParallelScheduling + include SingleEndpointNotesImporting + + PROCESSED_PAGE_CACHE_KEY = 'issues/%{issue_iid}/%{collection}' + BATCH_SIZE = 100 + + def initialize(project, client, parallel: true) + @project = project + @client = client + @parallel = parallel + @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % + { project: project.id, collection: collection_method } + end + + def each_associated(parent_record, associated) + compose_associated_id!(parent_record, associated) + return if already_imported?(associated) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + associated.issue_db_id = parent_record.id + yield(associated) + + mark_as_imported(associated) + end + + def importer_class + IssueEventImporter + end + + def representation_class + Representation::IssueEvent + end + + def sidekiq_worker_class + ImportIssueEventWorker + end + + def object_type + :issue_event + end + + def collection_method + :issue_timeline + end + + def parent_collection + project.issues.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord + end + + def parent_imported_cache_key + "github-importer/issues/#{collection_method}/already-imported/#{project.id}" + end + + def page_counter_id(issue) + PROCESSED_PAGE_CACHE_KEY % { issue_iid: issue.iid, collection: collection_method } + end + + def id_for_already_imported_cache(event) + event.id + end + + def collection_options + { state: 'all', sort: 'created', direction: 'asc' } + end + + # Cross-referenced events on Github doesn't have id. + def compose_associated_id!(issue, event) + return if event.event != 'cross-referenced' + + event.id = "cross-reference##{issue.id}-in-#{event.source.issue.id}" + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb index 49569ed52d8..fe64df45700 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb @@ -37,15 +37,15 @@ module Gitlab private - def noteables - project.issues.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + def parent_collection + project.issues.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord end def page_counter_id(issue) "issue/#{issue.id}/#{collection_method}" end - def notes_imported_cache_key + def parent_imported_cache_key "github-importer/issue/notes/already-imported/#{project.id}" end end diff --git a/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb index d837639c14d..3b1991d2b88 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb @@ -37,15 +37,15 @@ module Gitlab private - def noteables - project.merge_requests.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + def parent_collection + project.merge_requests.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord end def page_counter_id(merge_request) "merge_request/#{merge_request.id}/#{collection_method}" end - def notes_imported_cache_key + def parent_imported_cache_key "github-importer/merge_request/notes/already-imported/#{project.id}" end end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index 0b1c221bbec..692016bd005 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -5,8 +5,34 @@ module Gitlab class MarkdownText include Gitlab::EncodingHelper - def self.format(*args) - new(*args).to_s + ISSUE_REF_MATCHER = '%{github_url}/%{import_source}/issues' + PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull' + + class << self + def format(*args) + new(*args).to_s + end + + # Links like `https://domain.github.com/<namespace>/<project>/pull/<iid>` needs to be converted + def convert_ref_links(text, project) + matcher_options = { github_url: github_url, import_source: project.import_source } + issue_ref_matcher = ISSUE_REF_MATCHER % matcher_options + pull_ref_matcher = PULL_REF_MATCHER % matcher_options + + url_helpers = Rails.application.routes.url_helpers + text.gsub(issue_ref_matcher, url_helpers.project_issues_url(project)) + .gsub(pull_ref_matcher, url_helpers.project_merge_requests_url(project)) + end + + private + + # Returns github domain without slash in the end + def github_url + oauth_config = Gitlab::Auth::OAuth::Provider.config_for('github') || {} + url = oauth_config['url'].presence || 'https://github.com' + url = url.chop if url.end_with?('/') + url + end end # text - The Markdown text as a String. diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb new file mode 100644 index 00000000000..9016338db3b --- /dev/null +++ b/lib/gitlab/github_import/representation/issue_event.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class IssueEvent + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :actor, :event, :commit_id, :label_title, :old_title, :new_title, + :source, :created_at + expose_attribute :issue_db_id # set in SingleEndpointIssueEventsImporter#each_associated + + # Builds a event from a GitHub API response. + # + # event - An instance of `Sawyer::Resource` containing the event details. + def self.from_api_response(event) + new( + id: event.id, + actor: event.actor && Representation::User.from_api_response(event.actor), + event: event.event, + commit_id: event.commit_id, + label_title: event.label && event.label[:name], + old_title: event.rename && event.rename[:from], + new_title: event.rename && event.rename[:to], + source: event.source, + issue_db_id: event.issue_db_id, + created_at: event.created_at + ) + end + + # Builds a event using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + hash[:actor] &&= Representation::User.from_json_hash(hash[:actor]) + + new(hash) + end + + # attributes - A Hash containing the event details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { id: id } + end + end + end + end +end diff --git a/lib/gitlab/github_import/single_endpoint_notes_importing.rb b/lib/gitlab/github_import/single_endpoint_notes_importing.rb index 43402ecd165..0a3559adde3 100644 --- a/lib/gitlab/github_import/single_endpoint_notes_importing.rb +++ b/lib/gitlab/github_import/single_endpoint_notes_importing.rb @@ -4,9 +4,14 @@ # - SingleEndpointDiffNotesImporter # - SingleEndpointIssueNotesImporter # - SingleEndpointMergeRequestNotesImporter +# if `github_importer_single_endpoint_notes_import` feature flag is on. # -# `github_importer_single_endpoint_notes_import` -# feature flag is on. +# - SingleEndpointIssueEventsImporter +# if `github_importer_issue_events_import` feature flag is on. +# +# Fetches associated objects page by page to each item of parent collection. +# Currently `associated` is note or event. +# Currently `parent` is MergeRequest or Issue record. # # It fetches 1 PR's associated objects at a time using `issue_comments` or # `pull_request_comments` endpoint, which is slower than `NotesImporter` @@ -18,67 +23,75 @@ module Gitlab module SingleEndpointNotesImporting BATCH_SIZE = 100 - def each_object_to_import - each_notes_page do |page| - page.objects.each do |note| - next if already_imported?(note) + def each_object_to_import(&block) + each_associated_page do |parent_record, associated_page| + associated_page.objects.each do |associated| + each_associated(parent_record, associated, &block) + end + end + end - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + def id_for_already_imported_cache(associated) + associated.id + end - yield(note) + def parent_collection + raise NotImplementedError + end - mark_as_imported(note) - end - end + def parent_imported_cache_key + raise NotImplementedError end - def id_for_already_imported_cache(note) - note.id + def page_counter_id(parent) + raise NotImplementedError end private - def each_notes_page - noteables.each_batch(of: BATCH_SIZE, column: :iid) do |batch| - batch.each do |noteable| - # The page counter needs to be scoped by noteable to avoid skipping - # pages of notes from already imported noteables. - page_counter = PageCounter.new(project, page_counter_id(noteable)) + # Sometimes we need to add some extra info from parent + # to associated record that is not available by default + # in Github API response object. For example: + # lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb:26 + def each_associated(_parent_record, associated) + return if already_imported?(associated) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + yield(associated) + + mark_as_imported(associated) + end + + def each_associated_page + parent_collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| + batch.each do |parent_record| + # The page counter needs to be scoped by parent_record to avoid skipping + # pages of notes from already imported parent_record. + page_counter = PageCounter.new(project, page_counter_id(parent_record)) repo = project.import_source options = collection_options.merge(page: page_counter.current) - client.each_page(collection_method, repo, noteable.iid, options) do |page| + client.each_page(collection_method, repo, parent_record.iid, options) do |page| next unless page_counter.set(page.number) - yield page + yield parent_record, page end - mark_notes_imported(noteable) + mark_parent_imported(parent_record) end end end - def mark_notes_imported(noteable) + def mark_parent_imported(parent) Gitlab::Cache::Import::Caching.set_add( - notes_imported_cache_key, - noteable.iid + parent_imported_cache_key, + parent.iid ) end - def already_imported_noteables - Gitlab::Cache::Import::Caching.values_from_set(notes_imported_cache_key) - end - - def noteables - NotImplementedError - end - - def notes_imported_cache_key - NotImplementedError - end - - def page_counter_id(noteable) - NotImplementedError + def already_imported_parents + Gitlab::Cache::Import::Caching.values_from_set(parent_imported_cache_key) end end end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 93483ee697a..efaa2ce3002 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -39,13 +39,9 @@ module Gitlab # # If the object has no author ID we'll use the ID of the GitLab ghost # user. - def author_id_for(object) - id = - if object&.author - user_id_for(object.author) - else - GithubImport.ghost_user_id - end + def author_id_for(object, author_key: :author) + user_info = author_key == :actor ? object&.actor : object&.author + id = user_info ? user_id_for(user_info) : GithubImport.ghost_user_id if id [id, true] diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index ab7de14b07a..a03aeb9c293 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -3,17 +3,6 @@ module Gitlab module Gpg class Commit < Gitlab::SignedCommit - def signature - super - - return @signature if @signature - - cached_signature = lazy_signature&.itself - return @signature = cached_signature if cached_signature.present? - - @signature = create_cached_signature! - end - def update_signature!(cached_signature) using_keychain do |gpg_key| cached_signature.update!(attributes(gpg_key)) @@ -23,12 +12,8 @@ module Gitlab private - def lazy_signature - BatchLoader.for(@commit.sha).batch do |shas, loader| - CommitSignatures::GpgSignature.by_commit_sha(shas).each do |signature| - loader.call(signature.commit_sha, signature) - end - end + def signature_class + CommitSignatures::GpgSignature end def using_keychain diff --git a/lib/gitlab/grape_logging/loggers/response_logger.rb b/lib/gitlab/grape_logging/loggers/response_logger.rb new file mode 100644 index 00000000000..0465f01f7f5 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/response_logger.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module GrapeLogging + module Loggers + class ResponseLogger < ::GrapeLogging::Loggers::Base + def parameters(_, response) + return {} unless Feature.enabled?(:log_response_length) + + response_bytes = 0 + response.each { |resp| response_bytes += resp.to_s.bytesize } + { + response_bytes: response_bytes + } + end + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index c284160e539..3e119a39e6d 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -199,7 +199,7 @@ module Gitlab field_name = field.try(:attribute_name) || field field_value = node[field_name] ordering[field_name] = if field_value.is_a?(Time) - field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') + field_value.to_s(:inspect) else field_value.to_s end diff --git a/lib/gitlab/harbor/client.rb b/lib/gitlab/harbor/client.rb index 06142ae2b40..ee40725ba95 100644 --- a/lib/gitlab/harbor/client.rb +++ b/lib/gitlab/harbor/client.rb @@ -21,14 +21,44 @@ module Gitlab { success: response.success? } end + def get_repositories(params) + get(url("projects/#{integration.project_name}/repositories"), params) + end + + def get_artifacts(params) + repository_name = params.delete(:repository_name) + get(url("projects/#{integration.project_name}/repositories/#{repository_name}/artifacts"), params) + end + + def get_tags(params) + repository_name = params.delete(:repository_name) + artifact_name = params.delete(:artifact_name) + get( + url("projects/#{integration.project_name}/repositories/#{repository_name}/artifacts/#{artifact_name}/tags"), + params + ) + end + private - def url(path) - Gitlab::Utils.append_path(base_url, path) + def get(path, params = {}) + options = { headers: headers, query: params } + response = Gitlab::HTTP.get(path, options) + + raise Gitlab::Harbor::Client::Error, 'request error' unless response.success? + + { + body: Gitlab::Json.parse(response.body), + total_count: response.headers['x-total-count'].to_i + } + rescue JSON::ParserError + raise Gitlab::Harbor::Client::Error, 'invalid response format' end - def base_url - Gitlab::Utils.append_path(integration.url, '/api/v2.0/') + # url must be used within get method otherwise this would avoid validation by GitLab::HTTP + def url(path) + base_url = Gitlab::Utils.append_path(integration.url, '/api/v2.0/') + Gitlab::Utils.append_path(base_url, path) end def headers diff --git a/lib/gitlab/harbor/query.rb b/lib/gitlab/harbor/query.rb new file mode 100644 index 00000000000..c120810ecf1 --- /dev/null +++ b/lib/gitlab/harbor/query.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +module Gitlab + module Harbor + class Query + include ActiveModel::Validations + + attr_reader :client, :repository_id, :artifact_id, :search, :limit, :sort, :page + + DEFAULT_LIMIT = 10 + SORT_REGEX = %r{\A(creation_time|update_time|name) (asc|desc)\z}.freeze + + validates :page, numericality: { greater_than: 0, integer: true }, allow_blank: true + validates :limit, numericality: { greater_than: 0, less_than_or_equal_to: 25, integer: true }, allow_blank: true + validates :repository_id, format: { + with: /\A[a-zA-Z0-9\_\.\-$]+\z/, + message: 'Id invalid' + }, allow_blank: true + validates :artifact_id, format: { + with: /\A[a-zA-Z0-9\_\.\-$]+\z/, + message: 'Id invalid' + }, allow_blank: true + validates :sort, format: { + with: SORT_REGEX, + message: 'params invalid' + }, allow_blank: true + validates :search, format: { + with: /\A([a-z\_]*=[a-zA-Z0-9\- :]*,*)*\z/, + message: 'params invalid' + }, allow_blank: true + + def initialize(integration, params) + @client = Client.new(integration) + @repository_id = params[:repository_id] + @artifact_id = params[:artifact_id] + @search = params[:search] + @limit = params[:limit] + @sort = params[:sort] + @page = params[:page] + validate + end + + def repositories + result = @client.get_repositories(query_options) + return [] if result[:total_count] == 0 + + Kaminari.paginate_array( + result[:body], + limit: query_page_size, + total_count: result[:total_count] + ) + end + + def artifacts + result = @client.get_artifacts(query_artifacts_options) + return [] if result[:total_count] == 0 + + Kaminari.paginate_array( + result[:body], + limit: query_page_size, + total_count: result[:total_count] + ) + end + + def tags + result = @client.get_tags(query_tags_options) + return [] if result[:total_count] == 0 + + Kaminari.paginate_array( + result[:body], + limit: query_page_size, + total_count: result[:total_count] + ) + end + + private + + def query_artifacts_options + options = query_options + options[:repository_name] = repository_id + options[:with_tag] = true + + options + end + + def query_options + options = { + page: query_page, + page_size: query_page_size + } + + options[:q] = query_search if search.present? + options[:sort] = query_sort if sort.present? + + options + end + + def query_tags_options + options = query_options + options[:repository_name] = repository_id + options[:artifact_name] = artifact_id + + options + end + + def query_page + page.presence || 1 + end + + def query_page_size + (limit.presence || DEFAULT_LIMIT).to_i + end + + def query_search + search.gsub('=', '=~') + end + + def query_sort + match = sort.match(SORT_REGEX) + order = (match[2] == 'asc' ? '' : '-') + + "#{order}#{match[1]}" + end + end + end +end diff --git a/lib/gitlab/hash_digest/facade.rb b/lib/gitlab/hash_digest/facade.rb deleted file mode 100644 index d8efef02893..00000000000 --- a/lib/gitlab/hash_digest/facade.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HashDigest - # Used for rolling out to use OpenSSL::Digest::SHA256 - # for ActiveSupport::Digest - class Facade - class << self - def hexdigest(...) - hash_digest_class.hexdigest(...) - end - - def hash_digest_class - if use_sha256? - ::OpenSSL::Digest::SHA256 - else - ::Digest::MD5 # rubocop:disable Fips/MD5 - end - end - - def use_sha256? - return false unless Feature.feature_flags_available? - - Feature.enabled?(:active_support_hash_digest_sha256) - end - end - end - end -end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 2b4bdbd48bd..b4f90715293 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -62,7 +62,8 @@ module Gitlab assignee_id: merge_request.assignee_ids.first, # This key is deprecated labels: merge_request.labels_hook_attrs, state: merge_request.state, # This key is deprecated - blocking_discussions_resolved: merge_request.mergeable_discussions_state? + blocking_discussions_resolved: merge_request.mergeable_discussions_state?, + first_contribution: merge_request.first_contribution? } merge_request.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 7bb16e071b0..567c4dc899f 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -44,29 +44,19 @@ module Gitlab options end - options[:skip_read_total_timeout] = true if options[:skip_read_total_timeout].nil? && options[:stream_body] - - if options[:skip_read_total_timeout] + if options[:stream_body] return httparty_perform_request(http_method, path, options_with_timeouts, &block) end start_time = nil read_total_timeout = options.fetch(:timeout, DEFAULT_READ_TOTAL_TIMEOUT) - tracked_timeout_error = false httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| start_time ||= Gitlab::Metrics::System.monotonic_time elapsed = Gitlab::Metrics::System.monotonic_time - start_time if elapsed > read_total_timeout - error = ReadTotalTimeout.new("Request timed out after #{elapsed} seconds") - - raise error if options[:use_read_total_timeout] - - unless tracked_timeout_error - Gitlab::ErrorTracking.track_exception(error) - tracked_timeout_error = true - end + raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds" end block.call fragment if block diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 8d9f86d3232..cad0e773b05 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -25,6 +25,7 @@ module Gitlab 'pt_BR' => 'Portuguese (Brazil) - português (Brasil)', 'ro_RO' => 'Romanian - română', 'ru' => 'Russian - русский', + 'si_LK' => 'Sinhalese - සිංහල', 'tr_TR' => 'Turkish - Türkçe', 'uk' => 'Ukrainian - українська', 'zh_CN' => 'Chinese, Simplified - 简体中文', @@ -43,29 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 41, - 'de' => 14, + 'da_DK' => 40, + 'de' => 15, 'en' => 100, 'eo' => 0, - 'es' => 36, + 'es' => 37, 'fil_PH' => 0, - 'fr' => 10, + 'fr' => 11, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, 'ja' => 33, - 'ko' => 12, - 'nb_NO' => 27, + 'ko' => 11, + 'nb_NO' => 26, 'nl_NL' => 0, 'pl_PL' => 4, - 'pt_BR' => 54, - 'ro_RO' => 79, - 'ru' => 29, + 'pt_BR' => 55, + 'ro_RO' => 100, + 'ru' => 28, + 'si_LK' => 11, 'tr_TR' => 12, - 'uk' => 44, - 'zh_CN' => 94, + 'uk' => 49, + 'zh_CN' => 99, 'zh_HK' => 2, - 'zh_TW' => 2 + 'zh_TW' => 4 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index ebabf537ce5..59396c6bad2 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -70,7 +70,11 @@ module Gitlab batch = batch.preload(key_preloads) if key_preloads batch.each do |record| + before_read_callback(record) + items << Raw.new(record.to_json(options)) + + after_read_callback(record) end end end @@ -168,6 +172,20 @@ module Gitlab def read_from_replica_if_available(&block) ::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&block) end + + def before_read_callback(record) + remove_cached_external_diff(record) + end + + def after_read_callback(record) + remove_cached_external_diff(record) + end + + def remove_cached_external_diff(record) + return unless record.is_a?(MergeRequest) + + record.merge_request_diff&.remove_cached_external_diff + end end end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 5a1787218f5..50ff6146174 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -761,6 +761,7 @@ excluded_attributes: - :exported_protected_branches - :repository_size_limit - :external_webhook_token + - :incident_management_issuable_escalation_statuses namespaces: - :runners_token - :runners_token_encrypted @@ -819,6 +820,7 @@ excluded_attributes: - :upvotes_count - :work_item_type_id - :email_message_id + - :incident_management_issuable_escalation_status merge_request: &merge_request_excluded_definition - :milestone_id - :sprint_id diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 8110720fb46..c4b0e24e34a 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -89,6 +89,7 @@ module Gitlab when :'Ci::PipelineSchedule' then setup_pipeline_schedule when :'ProtectedBranch::MergeAccessLevel' then setup_protected_branch_access_level when :'ProtectedBranch::PushAccessLevel' then setup_protected_branch_access_level + when :releases then setup_release end update_project_references @@ -133,7 +134,7 @@ module Gitlab end def setup_diff - diff = @relation_hash.delete('utf8_diff') + diff = @relation_hash.delete('diff_export') || @relation_hash.delete('utf8_diff') parsed_relation_hash['diff'] = diff end @@ -150,6 +151,14 @@ module Gitlab @relation_hash['relative_position'] = compute_relative_position end + def setup_release + # When author is not present for source release set the author as ghost user. + + if @relation_hash['author_id'].blank? + @relation_hash['author_id'] = User.select(:id).ghost.id + end + end + def setup_pipeline_schedule @relation_hash['active'] = false end diff --git a/lib/gitlab/issuable/clone/attributes_rewriter.rb b/lib/gitlab/issuable/clone/attributes_rewriter.rb new file mode 100644 index 00000000000..fd9b2f086fc --- /dev/null +++ b/lib/gitlab/issuable/clone/attributes_rewriter.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Issuable + module Clone + class AttributesRewriter + attr_reader :current_user, :original_entity, :target_parent + + def initialize(current_user, original_entity, target_parent) + raise ArgumentError, 'target_parent cannot be nil' if target_parent.nil? + + @current_user = current_user + @original_entity = original_entity + @target_parent = target_parent + end + + def execute(include_milestone: true) + attributes = { label_ids: cloneable_labels.pluck_primary_key } + + if include_milestone + milestone = matching_milestone(original_entity.milestone&.title) + attributes[:milestone_id] = milestone.id if milestone.present? + end + + attributes + end + + private + + def cloneable_labels + params = { + project_id: project&.id, + group_id: group&.id, + title: original_entity.labels.select(:title), + include_ancestor_groups: true + } + + params[:only_group_labels] = true if target_parent.is_a?(Group) + + LabelsFinder.new(current_user, params).execute + end + + def matching_milestone(title) + return if title.blank? + + params = { title: title, project_ids: project&.id, group_ids: group&.id } + + milestones = MilestonesFinder.new(params).execute + milestones.first + end + + def project + target_parent if target_parent.is_a?(Project) + end + + def group + if target_parent.is_a?(Group) + target_parent + elsif target_parent&.group && current_user.can?(:read_group, target_parent.group) + target_parent.group + end + end + end + end + end +end diff --git a/lib/gitlab/issuable/clone/copy_resource_events_service.rb b/lib/gitlab/issuable/clone/copy_resource_events_service.rb new file mode 100644 index 00000000000..563805fcb01 --- /dev/null +++ b/lib/gitlab/issuable/clone/copy_resource_events_service.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +module Gitlab + module Issuable + module Clone + class CopyResourceEventsService + attr_reader :current_user, :original_entity, :new_entity + + def initialize(current_user, original_entity, new_entity) + @current_user = current_user + @original_entity = original_entity + @new_entity = new_entity + end + + def execute + copy_resource_label_events + copy_resource_milestone_events + copy_resource_state_events + end + + private + + def copy_resource_label_events + copy_events(ResourceLabelEvent.table_name, original_entity.resource_label_events) do |event| + event.attributes + .except('id', 'reference', 'reference_html') + .merge(entity_key => new_entity.id, 'action' => ResourceLabelEvent.actions[event.action]) + end + end + + def copy_resource_milestone_events + return unless milestone_events_supported? + + copy_events(ResourceMilestoneEvent.table_name, original_entity.resource_milestone_events) do |event| + if event.remove? + event_attributes_with_milestone(event, nil) + else + destination_milestone = matching_milestone(event.milestone_title) + + event_attributes_with_milestone(event, destination_milestone) if destination_milestone.present? + end + end + end + + def copy_resource_state_events + return unless state_events_supported? + + copy_events(ResourceStateEvent.table_name, original_entity.resource_state_events) do |event| + event.attributes + .except(*blocked_state_event_attributes) + .merge(entity_key => new_entity.id, + 'state' => ResourceStateEvent.states[event.state]) + end + end + + # Overriden on EE::Gitlab::Issuable::Clone::CopyResourceEventsService + def blocked_state_event_attributes + ['id'] + end + + def event_attributes_with_milestone(event, milestone) + event.attributes + .except('id') + .merge(entity_key => new_entity.id, + 'milestone_id' => milestone&.id, + 'action' => ResourceMilestoneEvent.actions[event.action], + 'state' => ResourceMilestoneEvent.states[event.state]) + end + + def copy_events(table_name, events_to_copy) + events_to_copy.find_in_batches do |batch| + events = batch.map do |event| + yield(event) + end.compact + + ApplicationRecord.legacy_bulk_insert(table_name, events) # rubocop:disable Gitlab/BulkInsert + end + end + + def entity_key + new_entity.class.name.underscore.foreign_key + end + + def milestone_events_supported? + both_respond_to?(:resource_milestone_events) + end + + def state_events_supported? + both_respond_to?(:resource_state_events) + end + + def both_respond_to?(method) + original_entity.respond_to?(method) && + new_entity.respond_to?(method) + end + + def matching_milestone(title) + return if title.blank? || !new_entity.supports_milestone? + + params = { title: title, project_ids: new_entity.project&.id, group_ids: group&.id } + + milestones = MilestonesFinder.new(params).execute + milestones.first + end + + def group + if new_entity.project&.group && current_user.can?(:read_group, new_entity.project.group) + new_entity.project.group + end + end + end + end + end +end + +Gitlab::Issuable::Clone::CopyResourceEventsService.prepend_mod diff --git a/lib/gitlab/jira_import/issue_serializer.rb b/lib/gitlab/jira_import/issue_serializer.rb index ab748d67fbf..70ec6f08fcd 100644 --- a/lib/gitlab/jira_import/issue_serializer.rb +++ b/lib/gitlab/jira_import/issue_serializer.rb @@ -5,10 +5,11 @@ module Gitlab class IssueSerializer attr_reader :jira_issue, :project, :import_owner_id, :params, :formatter - def initialize(project, jira_issue, import_owner_id, params = {}) + def initialize(project, jira_issue, import_owner_id, work_item_type_id, params = {}) @jira_issue = jira_issue @project = project @import_owner_id = import_owner_id + @work_item_type_id = work_item_type_id @params = params @formatter = Gitlab::ImportFormatter.new end @@ -17,6 +18,7 @@ module Gitlab { iid: params[:iid], project_id: project.id, + namespace_id: project.project_namespace_id, description: description, title: title, state_id: map_status(jira_issue.status.statusCategory), @@ -24,7 +26,8 @@ module Gitlab created_at: jira_issue.created, author_id: reporter, assignee_ids: assignees, - label_ids: label_ids + label_ids: label_ids, + work_item_type_id: @work_item_type_id } end @@ -45,9 +48,9 @@ module Gitlab def map_status(jira_status_category) case jira_status_category["key"].downcase when 'done' - Issuable::STATE_ID_MAP[:closed] + ::Issuable::STATE_ID_MAP[:closed] else - Issuable::STATE_ID_MAP[:opened] + ::Issuable::STATE_ID_MAP[:opened] end end diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 8a03162f111..f1ead57c911 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -16,6 +16,7 @@ module Gitlab @start_at = Gitlab::JiraImport.get_issues_next_start_at(project.id) @imported_items_cache_key = JiraImport.already_imported_cache_key(:issues, project.id) @job_waiter = JobWaiter.new + @issue_type_id = WorkItems::Type.default_issue_type.id end def execute @@ -58,8 +59,13 @@ module Gitlab next if already_imported?(jira_issue.id) begin - issue_attrs = IssueSerializer.new(project, jira_issue, running_import.user_id, { iid: next_iid }).execute - + issue_attrs = IssueSerializer.new( + project, + jira_issue, + running_import.user_id, + @issue_type_id, + { iid: next_iid } + ).execute Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key) job_waiter.jobs_remaining += 1 diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 512936bb4f4..ce07752f88c 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -95,7 +95,7 @@ module Gitlab opts = standardize_opts(opts) Oj.load(string, opts) - rescue Oj::ParseError, Encoding::UndefinedConversionError => ex + rescue Oj::ParseError, EncodingError, Encoding::UndefinedConversionError => ex raise parser_error, ex end diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index 84ead5119d5..f8ec58cf217 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -7,7 +7,7 @@ module Gitlab LIMITED_ARRAY_SENTINEL = { key: 'truncated', value: '...' }.freeze IGNORE_PARAMS = Set.new(%w(controller action format)).freeze - KNOWN_PAYLOAD_PARAMS = [:remote_ip, :user_id, :username, :ua, :queue_duration_s, + KNOWN_PAYLOAD_PARAMS = [:remote_ip, :user_id, :username, :ua, :queue_duration_s, :response_bytes, :etag_route, :request_urgency, :target_duration_s] + CLOUDFLARE_CUSTOM_HEADERS.values def self.call(event) @@ -36,6 +36,10 @@ module Gitlab payload[:feature_flag_states] = Feature.logged_states.map { |key, state| "#{key}:#{state ? 1 : 0}" } end + if Feature.disabled?(:log_response_length) + payload.delete(:response_bytes) + end + payload end end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb new file mode 100644 index 00000000000..db75ba8a47d --- /dev/null +++ b/lib/gitlab/memory/watchdog.rb @@ -0,0 +1,192 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + # A background thread that observes Ruby heap fragmentation and calls + # into a handler when the Ruby heap has been fragmented for an extended + # period of time. + # + # See Gitlab::Metrics::Memory for how heap fragmentation is defined. + # + # To decide whether a given fragmentation level is being exceeded, + # the watchdog regularly polls the GC. Whenever a violation occurs + # a strike is issued. If the maximum number of strikes are reached, + # a handler is invoked to deal with the situation. + # + # The duration for which a process may be above a given fragmentation + # threshold is computed as `max_strikes * sleep_time_seconds`. + class Watchdog < Daemon + DEFAULT_SLEEP_TIME_SECONDS = 60 + DEFAULT_HEAP_FRAG_THRESHOLD = 0.5 + DEFAULT_MAX_STRIKES = 5 + + # This handler does nothing. It returns `false` to indicate to the + # caller that the situation has not been dealt with so it will + # receive calls repeatedly if fragmentation remains high. + # + # This is useful for "dress rehearsals" in production since it allows + # us to observe how frequently the handler is invoked before taking action. + class NullHandler + include Singleton + + def on_high_heap_fragmentation(value) + # NOP + false + end + end + + # This handler sends SIGTERM and considers the situation handled. + class TermProcessHandler + def initialize(pid = $$) + @pid = pid + end + + def on_high_heap_fragmentation(value) + Process.kill(:TERM, @pid) + true + end + end + + # This handler invokes Puma's graceful termination handler, which takes + # into account a configurable grace period during which a process may + # remain unresponsive to a SIGTERM. + class PumaHandler + def initialize(puma_options = ::Puma.cli_config.options) + @worker = ::Puma::Cluster::WorkerHandle.new(0, $$, 0, puma_options) + end + + def on_high_heap_fragmentation(value) + @worker.term + true + end + end + + # max_heap_fragmentation: + # The degree to which the Ruby heap is allowed to be fragmented. Range [0,1]. + # max_strikes: + # How many times the process is allowed to be above max_heap_fragmentation before + # a handler is invoked. + # sleep_time_seconds: + # Used to control the frequency with which the watchdog will wake up and poll the GC. + def initialize( + handler: NullHandler.instance, + logger: Logger.new($stdout), + max_heap_fragmentation: ENV['GITLAB_MEMWD_MAX_HEAP_FRAG']&.to_f || DEFAULT_HEAP_FRAG_THRESHOLD, + max_strikes: ENV['GITLAB_MEMWD_MAX_STRIKES']&.to_i || DEFAULT_MAX_STRIKES, + sleep_time_seconds: ENV['GITLAB_MEMWD_SLEEP_TIME_SEC']&.to_i || DEFAULT_SLEEP_TIME_SECONDS, + **options) + super(**options) + + @handler = handler + @logger = logger + @max_heap_fragmentation = max_heap_fragmentation + @sleep_time_seconds = sleep_time_seconds + @max_strikes = max_strikes + + @alive = true + @strikes = 0 + + init_prometheus_metrics(max_heap_fragmentation) + end + + attr_reader :strikes, :max_heap_fragmentation, :max_strikes, :sleep_time_seconds + + def run_thread + @logger.info(log_labels.merge(message: 'started')) + + while @alive + sleep(@sleep_time_seconds) + + monitor_heap_fragmentation if Feature.enabled?(:gitlab_memory_watchdog, type: :ops) + end + + @logger.info(log_labels.merge(message: 'stopped')) + end + + private + + def monitor_heap_fragmentation + heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation + + if heap_fragmentation > @max_heap_fragmentation + @strikes += 1 + @heap_frag_violations.increment + else + @strikes = 0 + end + + if @strikes > @max_strikes + # If the handler returns true, it means the event is handled and we can shut down. + @alive = !handle_heap_fragmentation_limit_exceeded(heap_fragmentation) + @strikes = 0 + end + end + + def handle_heap_fragmentation_limit_exceeded(value) + @logger.warn( + log_labels.merge( + message: 'heap fragmentation limit exceeded', + memwd_cur_heap_frag: value + )) + @heap_frag_violations_handled.increment + + handler.on_high_heap_fragmentation(value) + end + + def handler + # This allows us to keep the watchdog running but turn it into "friendly mode" where + # all that happens is we collect logs and Prometheus events for fragmentation violations. + return NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) + + @handler + end + + def stop_working + @alive = false + end + + def log_labels + { + pid: $$, + worker_id: worker_id, + memwd_handler_class: handler.class.name, + memwd_sleep_time_s: @sleep_time_seconds, + memwd_max_heap_frag: @max_heap_fragmentation, + memwd_max_strikes: @max_strikes, + memwd_cur_strikes: @strikes, + memwd_rss_bytes: process_rss_bytes + } + end + + def worker_id + ::Prometheus::PidProvider.worker_id + end + + def process_rss_bytes + Gitlab::Metrics::System.memory_usage_rss + end + + def init_prometheus_metrics(max_heap_fragmentation) + default_labels = { pid: worker_id } + + @heap_frag_limit = Gitlab::Metrics.gauge( + :gitlab_memwd_heap_frag_limit, + 'The configured limit for how fragmented the Ruby heap is allowed to be', + default_labels + ) + @heap_frag_limit.set({}, max_heap_fragmentation) + + @heap_frag_violations = Gitlab::Metrics.counter( + :gitlab_memwd_heap_frag_violations_total, + 'Total number of times heap fragmentation in a Ruby process exceeded its allowed maximum', + default_labels + ) + @heap_frag_violations_handled = Gitlab::Metrics.counter( + :gitlab_memwd_heap_frag_violations_handled_total, + 'Total number of times heap fragmentation violations in a Ruby process were handled', + default_labels + ) + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index ba2eb729d7b..858a0a120cc 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -7,6 +7,8 @@ module Gitlab module Metrics module Exporter class BaseExporter < Daemon + CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze + attr_reader :server # @param settings [Hash] SettingsLogic hash containing the `*_exporter` config @@ -38,10 +40,16 @@ module Gitlab [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] ] - @server = ::WEBrick::HTTPServer.new( - Port: settings.port, BindAddress: settings.address, - Logger: logger, AccessLog: access_log - ) + server_config = { + Port: settings.port, + BindAddress: settings.address, + Logger: logger, + AccessLog: access_log + } + + server_config.merge!(ssl_config) if settings['tls_enabled'] + + @server = ::WEBrick::HTTPServer.new(server_config) server.mount '/', Rack::Handler::WEBrick, rack_app true @@ -82,6 +90,33 @@ module Gitlab run -> (env) { [404, {}, ['']] } end end + + def ssl_config + # This monkey-patches WEBrick::GenericServer, so never require this unless TLS is enabled. + require 'webrick/ssl' + + certs = load_ca_certs_bundle(File.binread(settings['tls_cert_path'])) + + { + SSLEnable: true, + SSLCertificate: certs.shift, + SSLPrivateKey: OpenSSL::PKey.read(File.binread(settings['tls_key_path'])), + # SSLStartImmediately is true by default according to the docs, but when WEBrick creates the + # SSLServer internally, the switch was always nil for some reason. Setting this explicitly fixes this. + SSLStartImmediately: true, + SSLExtraChainCert: certs + } + end + + # In Ruby OpenSSL v3.0.0, this can be replaced by OpenSSL::X509::Certificate.load + # https://github.com/ruby/openssl/issues/254 + def load_ca_certs_bundle(ca_certs_string) + return [] unless ca_certs_string + + ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string| + OpenSSL::X509::Certificate.new(ca_cert_string) + end + end end end end diff --git a/lib/gitlab/metrics/memory.rb b/lib/gitlab/metrics/memory.rb new file mode 100644 index 00000000000..c165cdec7a3 --- /dev/null +++ b/lib/gitlab/metrics/memory.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Memory + extend self + + HEAP_SLOTS_PER_PAGE = GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] + + def gc_heap_fragmentation(gc_stat = GC.stat) + 1 - (gc_stat[:heap_live_slots] / (HEAP_SLOTS_PER_PAGE * gc_stat[:heap_eden_pages].to_f)) + end + end + end +end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index 848b73792cb..0b513f5521e 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -29,6 +29,7 @@ module Gitlab clear_memoization(:registry) REGISTRY_MUTEX.synchronize do + ::Prometheus::Client.cleanup! ::Prometheus::Client.reset! end end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 4a3ef3711a5..8e002293347 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -39,7 +39,8 @@ module Gitlab process_proportional_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :proportional_memory_bytes), 'Memory used (PSS)', labels), process_start_time_seconds: ::Gitlab::Metrics.gauge(metric_name(:process, :start_time_seconds), 'Process start time seconds'), sampler_duration: ::Gitlab::Metrics.counter(metric_name(:sampler, :duration_seconds_total), 'Sampler time', labels), - gc_duration_seconds: ::Gitlab::Metrics.histogram(metric_name(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS) + gc_duration_seconds: ::Gitlab::Metrics.histogram(metric_name(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS), + heap_fragmentation: ::Gitlab::Metrics.gauge(metric_name(:gc_stat_ext, :heap_fragmentation), 'Ruby heap fragmentation', labels) } GC.stat.keys.each do |key| @@ -76,8 +77,13 @@ module Gitlab end # Collect generic GC stats - GC.stat.each do |key, value| - metrics[key].set(labels, value) + GC.stat.then do |gc_stat| + gc_stat.each do |key, value| + metrics[key].set(labels, value) + end + + # Collect custom GC stats + metrics[:heap_fragmentation].set(labels, Memory.gc_heap_fragmentation(gc_stat)) end end diff --git a/lib/gitlab/metrics/sli.rb b/lib/gitlab/metrics/sli.rb index 2de19514354..15cfe777f4d 100644 --- a/lib/gitlab/metrics/sli.rb +++ b/lib/gitlab/metrics/sli.rb @@ -82,7 +82,7 @@ module Gitlab private def counter_name(suffix) - :"#{COUNTER_PREFIX}:#{name}_apdex:#{suffix}" + [COUNTER_PREFIX, "#{name}_apdex", suffix].join('_').to_sym end def numerator_counter @@ -100,7 +100,7 @@ module Gitlab private def counter_name(suffix) - :"#{COUNTER_PREFIX}:#{name}:#{suffix}" + [COUNTER_PREFIX, name, suffix].join('_').to_sym end def numerator_counter diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 7c22ce64ea2..e3756a8c9f6 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -24,7 +24,7 @@ module Gitlab # This event is published from ActiveRecordBaseTransactionMetrics and # used to record a database transaction duration when calling - # ActiveRecord::Base.transaction {} block. + # ApplicationRecord.transaction {} block. def transaction(event) observe(:gitlab_database_transaction_seconds, event) do buckets TRANSACTION_DURATION_BUCKET @@ -186,7 +186,10 @@ module Gitlab end ::Gitlab::Database.database_base_models.keys.each do |config_name| - counters << compose_metric_key(metric, nil, config_name) # main / ci + counters << compose_metric_key(metric, nil, config_name) # main / ci / geo + end + + ::Gitlab::Database.database_base_models_using_load_balancing.keys.each do |config_name| counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica / ci_replica end end diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb new file mode 100644 index 00000000000..991a1297d03 --- /dev/null +++ b/lib/gitlab/pages/cache_control.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class CacheControl + CACHE_KEY_FORMAT = 'pages_domain_for_%{type}_%{id}' + + attr_reader :cache_key + + class << self + def for_project(project_id) + new(type: :project, id: project_id) + end + + def for_namespace(namespace_id) + new(type: :namespace, id: namespace_id) + end + end + + def initialize(type:, id:) + raise(ArgumentError, "type must be :namespace or :project") unless %i[namespace project].include?(type) + + @cache_key = CACHE_KEY_FORMAT % { type: type, id: id } + end + + def clear_cache + Rails.cache.delete(cache_key) + end + end + end +end diff --git a/lib/gitlab/pages/deployment_update.rb b/lib/gitlab/pages/deployment_update.rb new file mode 100644 index 00000000000..2f5c6938e2a --- /dev/null +++ b/lib/gitlab/pages/deployment_update.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class DeploymentUpdate + include ActiveModel::Validations + + PUBLIC_DIR = 'public' + + validate :validate_state, unless: -> { errors.any? } + validate :validate_outdated_sha, unless: -> { errors.any? } + validate :validate_max_size, unless: -> { errors.any? } + validate :validate_public_folder, unless: -> { errors.any? } + validate :validate_max_entries, unless: -> { errors.any? } + + def initialize(project, build) + @project = project + @build = build + end + + def latest? + # check if sha for the ref is still the most recent one + # this helps in case when multiple deployments happens + sha == latest_sha + end + + def entries_count + # we're using the full archive and pages daemon needs to read it + # so we want the total count from entries, not only "public/" directory + # because it better approximates work we need to do before we can serve the site + @entries_count = build.artifacts_metadata_entry("", recursive: true).entries.count + end + + private + + attr_reader :build, :project + + def validate_state + errors.add(:base, 'missing pages artifacts') unless build.artifacts? + errors.add(:base, 'missing artifacts metadata') unless build.artifacts_metadata? + end + + def validate_max_size + if total_size > max_size + errors.add(:base, "artifacts for pages are too large: #{total_size}") + end + end + + # Calculate page size after extract + def total_size + @total_size ||= build.artifacts_metadata_entry(PUBLIC_DIR + '/', recursive: true).total_size + end + + def max_size_from_settings + Gitlab::CurrentSettings.max_pages_size.megabytes + end + + def max_size + max_pages_size = max_size_from_settings + + return ::Gitlab::Pages::MAX_SIZE if max_pages_size == 0 + + max_pages_size + end + + def validate_max_entries + if pages_file_entries_limit > 0 && entries_count > pages_file_entries_limit + errors.add( + :base, + "pages site contains #{entries_count} file entries, while limit is set to #{pages_file_entries_limit}" + ) + end + end + + def validate_public_folder + if total_size <= 0 + errors.add(:base, 'Error: The `public/` folder is missing, or not declared in `.gitlab-ci.yml`.') + end + end + + def pages_file_entries_limit + project.actual_limits.pages_file_entries + end + + def validate_outdated_sha + return if latest? + + # use pipeline_id in case the build is retried + last_deployed_pipeline_id = project.pages_metadatum&.pages_deployment&.ci_build&.pipeline_id + + return unless last_deployed_pipeline_id + return if last_deployed_pipeline_id <= build.pipeline_id + + errors.add(:base, 'build SHA is outdated for this ref') + end + + def latest_sha + project.commit(build.ref).try(:sha).to_s + ensure + # Close any file descriptors that were opened and free libgit2 buffers + project.cleanup + end + + def sha + build.sha + end + end + end +end + +Gitlab::Pages::DeploymentUpdate.prepend_mod_with('Gitlab::Pages::DeploymentUpdate') diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index f19cdf06d9a..2d9fb0a50fc 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -4,9 +4,18 @@ module Gitlab module Pagination module CursorBasedKeyset SUPPORTED_ORDERING = { - Group => { name: :asc } + Group => { name: :asc }, + AuditEvent => { id: :desc } }.freeze + # Relation types that are enforced in this list + # enforce the use of keyset pagination, thus erroring out requests + # made with offset pagination above a certain limit. + # + # In many cases this could introduce a breaking change + # so enforcement is optional. + ENFORCED_TYPES = [Group].freeze + def self.available_for_type?(relation) SUPPORTED_ORDERING.key?(relation.klass) end @@ -16,6 +25,10 @@ module Gitlab order_satisfied?(relation, cursor_based_request_context) end + def self.enforced_for_type?(relation) + ENFORCED_TYPES.include?(relation.klass) + end + def self.order_satisfied?(relation, cursor_based_request_context) order_by_from_request = cursor_based_request_context.order_by diff --git a/lib/gitlab/pagination/keyset/cursor_based_request_context.rb b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb index e06d7e48ca3..41b90846345 100644 --- a/lib/gitlab/pagination/keyset/cursor_based_request_context.rb +++ b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb @@ -5,6 +5,8 @@ module Gitlab module Keyset class CursorBasedRequestContext DEFAULT_SORT_DIRECTION = :desc + DEFAULT_SORT_COLUMN = :id + attr_reader :request_context delegate :params, to: :request_context @@ -28,7 +30,7 @@ module Gitlab end def order_by - { params[:order_by].to_sym => params[:sort]&.to_sym || DEFAULT_SORT_DIRECTION } + { (params[:order_by]&.to_sym || DEFAULT_SORT_COLUMN) => (params[:sort]&.to_sym || DEFAULT_SORT_DIRECTION) } end end end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index 290e94401b8..0d8e4ea6fee 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -96,7 +96,9 @@ module Gitlab column_definitions.each_with_object({}.with_indifferent_access) do |column_definition, hash| field_value = node[column_definition.attribute_name] hash[column_definition.attribute_name] = if field_value.is_a?(Time) - field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') + # use :inspect formatter to provide specific timezone info + # eg 2022-07-05 21:57:56.041499000 +0800 + field_value.to_s(:inspect) elsif field_value.nil? nil elsif lower_named_function?(column_definition) @@ -107,6 +109,10 @@ module Gitlab end end + def attribute_names + column_definitions.map(&:attribute_name) + end + # This methods builds the conditions for the keyset pagination # # Example: diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 259d9e38d65..3b85d6952a1 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -23,7 +23,7 @@ module Gitlab _('Closed this %{quick_action_target}.') % { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end - types Issuable + types ::Issuable condition do quick_action_target.persisted? && quick_action_target.open? && @@ -45,7 +45,7 @@ module Gitlab _('Reopened this %{quick_action_target}.') % { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end - types Issuable + types ::Issuable condition do quick_action_target.persisted? && quick_action_target.closed? && @@ -63,7 +63,7 @@ module Gitlab _('Changed the title to "%{title_param}".') % { title_param: title_param } end params '<New title>' - types Issuable + types ::Issuable condition do quick_action_target.persisted? && current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) @@ -82,7 +82,7 @@ module Gitlab end end params '~label1 ~"label 2"' - types Issuable + types ::Issuable condition do current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target) && find_labels.any? @@ -102,7 +102,7 @@ module Gitlab end end params '~label1 ~"label 2"' - types Issuable + types ::Issuable condition do quick_action_target.persisted? && quick_action_target.labels.any? && @@ -134,7 +134,7 @@ module Gitlab "Replaces all labels with #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? end params '~label1 ~"label 2"' - types Issuable + types ::Issuable condition do quick_action_target.persisted? && quick_action_target.labels.any? && @@ -147,7 +147,7 @@ module Gitlab desc { _('Add a to do') } explanation { _('Adds a to do.') } execution_message { _('Added a to do.') } - types Issuable + types ::Issuable condition do quick_action_target.persisted? && !TodoService.new.todo_exist?(quick_action_target, current_user) @@ -159,7 +159,7 @@ module Gitlab desc { _('Mark to do as done') } explanation { _('Marks to do as done.') } execution_message { _('Marked to do as done.') } - types Issuable + types ::Issuable condition do quick_action_target.persisted? && TodoService.new.todo_exist?(quick_action_target, current_user) @@ -177,7 +177,7 @@ module Gitlab _('Subscribed to this %{quick_action_target}.') % { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end - types Issuable + types ::Issuable condition do quick_action_target.persisted? && !quick_action_target.subscribed?(current_user, project) @@ -195,7 +195,7 @@ module Gitlab _('Unsubscribed from this %{quick_action_target}.') % { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end - types Issuable + types ::Issuable condition do quick_action_target.persisted? && quick_action_target.subscribed?(current_user, project) @@ -212,7 +212,7 @@ module Gitlab _("Toggled :%{name}: emoji award.") % { name: name } if name end params ':emoji:' - types Issuable + types ::Issuable condition do quick_action_target.persisted? end @@ -228,14 +228,14 @@ module Gitlab desc { _("Append the comment with %{shrug}") % { shrug: SHRUG } } params '<Comment>' - types Issuable + types ::Issuable substitution :shrug do |comment| "#{comment} #{SHRUG}" end desc { _("Append the comment with %{tableflip}") % { tableflip: TABLEFLIP } } params '<Comment>' - types Issuable + types ::Issuable substitution :tableflip do |comment| "#{comment} #{TABLEFLIP}" end diff --git a/lib/gitlab/quick_actions/users_extractor.rb b/lib/gitlab/quick_actions/users_extractor.rb new file mode 100644 index 00000000000..06e04c74312 --- /dev/null +++ b/lib/gitlab/quick_actions/users_extractor.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + class UsersExtractor + MAX_QUICK_ACTION_USERS = 100 + + Error = Class.new(ArgumentError) + TooManyError = Class.new(Error) do + def limit + MAX_QUICK_ACTION_USERS + end + end + + MissingError = Class.new(Error) + TooManyFoundError = Class.new(TooManyError) + TooManyRefsError = Class.new(TooManyError) + + attr_reader :text, :current_user, :project, :group, :target + + def initialize(current_user, project:, group:, target:, text:) + @current_user = current_user + @project = project + @group = group + @target = target + @text = text + end + + def execute + return [] unless text.present? + + users = collect_users + + check_users!(users) + + users + end + + private + + def collect_users + users = [] + users << current_user if me? + users += find_referenced_users if references.any? + + users + end + + def check_users!(users) + raise TooManyFoundError if users.size > MAX_QUICK_ACTION_USERS + + found = found_names(users) + missing = references.filter_map do + "'#{_1}'" unless found.include?(_1.downcase.delete_prefix('@')) + end + + raise MissingError, missing.to_sentence if missing.present? + end + + def found_names(users) + users.map(&:username).map(&:downcase).to_set + end + + def find_referenced_users + raise TooManyRefsError if references.size > MAX_QUICK_ACTION_USERS + + User.by_username(usernames).limit(MAX_QUICK_ACTION_USERS) + end + + def usernames + references.map { _1.delete_prefix('@') } + end + + def references + @references ||= begin + refs = args - ['me'] + # nb: underscores may be passed in escaped to protect them from markdown rendering + refs.map! { _1.gsub(/\\_/, '_') } + refs + end + end + + def args + @args ||= text.split(/\s|,/).map(&:strip).select(&:present?).uniq - ['and'] + end + + def me? + args&.include?('me') + end + end + end +end + +Gitlab::QuickActions::UsersExtractor.prepend_mod_with('Gitlab::QuickActions::UsersExtractor') diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 24c540eea47..94f06e957cf 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -11,8 +11,15 @@ module Gitlab end end class PipelinedDiffError < StandardError + def initialize(result_primary, result_secondary) + @result_primary = result_primary + @result_secondary = result_secondary + end + def message - 'Pipelined command executed on both stores successfully but results differ between them.' + "Pipelined command executed on both stores successfully but results differ between them. " \ + "Result from the primary: #{@result_primary.inspect}. " \ + "Result from the secondary: #{@result_secondary.inspect}." end end class MethodMissingError < StandardError @@ -246,10 +253,12 @@ module Gitlab result_secondary = send_command(secondary_store, command_name, *args, **kwargs, &block) - # Pipelined commands return an array with all results. If they differ, - # log an error - if result_primary != result_secondary - log_error(PipelinedDiffError.new, command_name) + # Pipelined commands return an array with all results. If they differ, log an error + if result_primary && result_primary != result_secondary + error = PipelinedDiffError.new(result_primary, result_secondary) + error.set_backtrace(Thread.current.backtrace[1..]) # Manually set backtrace, since the error is not `raise`d + + log_error(error, command_name) increment_pipelined_command_error_count(command_name) end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index b0f4194b7a0..0534f890152 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -486,6 +486,10 @@ module Gitlab def sep_by_1(separator, part) %r(#{part} (#{separator} #{part})*)x end + + def x509_subject_key_identifier_regex + @x509_subject_key_identifier_regex ||= /\A(?:\h{2}:)*\h{2}\z/.freeze + end end end diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb index 0a4f2ba64a8..4683f611444 100644 --- a/lib/gitlab/saas.rb +++ b/lib/gitlab/saas.rb @@ -48,6 +48,10 @@ module Gitlab def self.about_pricing_faq_url "https://about.gitlab.com/pricing#faq" end + + def self.doc_url + 'https://docs.gitlab.com' + end end end diff --git a/lib/gitlab/security/scan_configuration.rb b/lib/gitlab/security/scan_configuration.rb index 14883a34950..9b09ccdeb8e 100644 --- a/lib/gitlab/security/scan_configuration.rb +++ b/lib/gitlab/security/scan_configuration.rb @@ -18,7 +18,7 @@ module Gitlab # SAST and Secret Detection are always available, but this isn't # reflected by our license model yet. # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/333113 - %i[sast sast_iac secret_detection].include?(type) + %i[sast sast_iac secret_detection container_scanning].include?(type) end def can_enable_by_merge_request? diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index 113076a6a75..cb7d9c6f8a7 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -145,6 +145,8 @@ module Gitlab sleep(CHECK_INTERVAL_SECONDS) refresh_state(:above_soft_limit) + + log_rss_out_of_range(false) end # There are two chances to break from loop: @@ -153,28 +155,49 @@ module Gitlab # When `above hard limit`, it immediately go to `stop_fetching_new_jobs` # So ignore `above hard limit` and always set `above_soft_limit` here refresh_state(:above_soft_limit) - log_rss_out_of_range(@current_rss, @hard_limit_rss, @soft_limit_rss) + log_rss_out_of_range false end - def log_rss_out_of_range(current_rss, hard_limit_rss, soft_limit_rss) + def log_rss_out_of_range(deadline_exceeded = true) + reason = out_of_range_description(@current_rss, + @hard_limit_rss, + @soft_limit_rss, + deadline_exceeded) + Sidekiq.logger.warn( class: self.class.to_s, pid: pid, message: 'Sidekiq worker RSS out of range', - current_rss: current_rss, - hard_limit_rss: hard_limit_rss, - soft_limit_rss: soft_limit_rss, - reason: out_of_range_description(current_rss, hard_limit_rss, soft_limit_rss) - ) + current_rss: @current_rss, + soft_limit_rss: @soft_limit_rss, + hard_limit_rss: @hard_limit_rss, + reason: reason, + running_jobs: running_jobs) end - def out_of_range_description(rss, hard_limit, soft_limit) + def running_jobs + jobs = [] + Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do + jobs = Gitlab::SidekiqDaemon::Monitor.instance.jobs.map do |jid, job| + { + jid: jid, + worker_class: job[:worker_class].name + } + end + end + + jobs + end + + def out_of_range_description(rss, hard_limit, soft_limit, deadline_exceeded) if rss > hard_limit "current_rss(#{rss}) > hard_limit_rss(#{hard_limit})" - else + elsif deadline_exceeded "current_rss(#{rss}) > soft_limit_rss(#{soft_limit}) longer than GRACE_BALLOON_SECONDS(#{GRACE_BALLOON_SECONDS})" + else + "current_rss(#{rss}) > soft_limit_rss(#{soft_limit})" end end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 6eb39981ef4..7ce3f6b5ccb 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -81,12 +81,6 @@ module Gitlab payload['job_status'] = 'fail' Gitlab::ExceptionLogFormatter.format!(job_exception, payload) - - # Deprecated fields for compatibility - # See https://gitlab.com/gitlab-org/gitlab/-/issues/364241 - payload['error_class'] = payload['exception.class'] - payload['error_message'] = payload['exception.message'] - payload['error_backtrace'] = payload['exception.backtrace'] else payload['message'] = "#{message}: done: #{payload['duration_s']} sec" payload['job_status'] = 'done' diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index dc5481289da..ea2b405c934 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -31,6 +31,7 @@ module Gitlab sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_jobs_interrupted_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_interrupted_total, 'Sidekiq jobs interrupted'), sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'), sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), @@ -89,6 +90,10 @@ module Gitlab @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) end + if job['interrupted_count'].present? + @metrics[:sidekiq_jobs_interrupted_total].increment(labels, 1) + end + job_succeeded = false monotonic_time_start = Gitlab::Metrics::System.monotonic_time job_thread_cputime_start = get_thread_cputime diff --git a/lib/gitlab/signed_commit.rb b/lib/gitlab/signed_commit.rb index 7a154978938..410e71f51a1 100644 --- a/lib/gitlab/signed_commit.rb +++ b/lib/gitlab/signed_commit.rb @@ -18,7 +18,18 @@ module Gitlab end def signature - return unless @commit.has_signature? + return @signature if @signature + + cached_signature = lazy_signature&.itself + + return @signature = cached_signature if cached_signature.present? + + @signature = create_cached_signature! + end + + def update_signature!(cached_signature) + cached_signature.update!(attributes) + @signature = cached_signature end def signature_text @@ -32,5 +43,27 @@ module Gitlab @signature_data.itself ? @signature_data[1] : nil end end + + private + + def signature_class + raise NotImplementedError, '`signature_class` must be implmented by subclass`' + end + + def lazy_signature + BatchLoader.for(@commit.sha).batch do |shas, loader| + signature_class.by_commit_sha(shas).each do |signature| + loader.call(signature.commit_sha, signature) + end + end + end + + def create_cached_signature! + return if attributes.nil? + + return signature_class.new(attributes) if Gitlab::Database.read_only? + + signature_class.safe_create!(attributes) + end end end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 0e7812d08b8..04745bafe7c 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -4,13 +4,13 @@ module Gitlab module Tracking class << self def enabled? - snowplow.enabled? + tracker.enabled? end def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace, **extra).to_context, *context] - snowplow.event(category, action, label: label, property: property, value: value, context: contexts) + tracker.event(category, action, label: label, property: property, value: value, context: contexts) rescue StandardError => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) end @@ -31,25 +31,27 @@ module Gitlab end def options(group) - snowplow.options(group) + tracker.options(group) end def collector_hostname - snowplow.hostname + tracker.hostname end def snowplow_micro_enabled? - Rails.env.development? && Gitlab::Utils.to_boolean(ENV['SNOWPLOW_MICRO_ENABLE']) + Rails.env.development? && Gitlab.config.snowplow_micro.enabled + rescue Settingslogic::MissingSetting + Gitlab::Utils.to_boolean(ENV['SNOWPLOW_MICRO_ENABLE']) end private - def snowplow - @snowplow ||= if snowplow_micro_enabled? - Gitlab::Tracking::Destinations::SnowplowMicro.new - else - Gitlab::Tracking::Destinations::Snowplow.new - end + def tracker + @tracker ||= if snowplow_micro_enabled? + Gitlab::Tracking::Destinations::SnowplowMicro.new + else + Gitlab::Tracking::Destinations::Snowplow.new + end end end end diff --git a/lib/gitlab/tracking/destinations/snowplow_micro.rb b/lib/gitlab/tracking/destinations/snowplow_micro.rb index 3553efba1e1..c7a95e88d0b 100644 --- a/lib/gitlab/tracking/destinations/snowplow_micro.rb +++ b/lib/gitlab/tracking/destinations/snowplow_micro.rb @@ -30,8 +30,9 @@ module Gitlab def uri strong_memoize(:snowplow_uri) do - uri = URI(ENV['SNOWPLOW_MICRO_URI'] || DEFAULT_URI) - uri = URI("http://#{ENV['SNOWPLOW_MICRO_URI']}") unless %w[http https].include?(uri.scheme) + base = base_uri + uri = URI(base) + uri = URI("http://#{base}") unless %w[http https].include?(uri.scheme) uri end end @@ -47,6 +48,14 @@ module Gitlab def protocol uri.scheme end + + def base_uri + url = Gitlab.config.snowplow_micro.address + scheme = Gitlab.config.gitlab.https ? 'https' : 'http' + "#{scheme}://#{url}" + rescue Settingslogic::MissingSetting + ENV['SNOWPLOW_MICRO_URI'] || DEFAULT_URI + end end end end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 05ddc7e26cc..50467de44b8 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -7,11 +7,9 @@ module Gitlab GITLAB_RAILS_SOURCE = 'gitlab-rails' def initialize(namespace: nil, project: nil, user: nil, **extra) - if Feature.enabled?(:standard_context_type_check) - check_argument_type(:namespace, namespace, [Namespace]) - check_argument_type(:project, project, [Project, Integer]) - check_argument_type(:user, user, [User, DeployToken]) - end + check_argument_type(:namespace, namespace, [Namespace]) + check_argument_type(:project, project, [Project, Integer]) + check_argument_type(:user, user, [User, DeployToken]) @namespace = namespace @plan = namespace&.actual_plan_name diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index 85f0ba1fd25..72df8b423df 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -8,10 +8,7 @@ module Gitlab CACHE_EXPIRE_IN = 1.hour MAX_OFFSET = 2**31 - attr_reader :commit, :project, :path, :offset, :limit, :user - - attr_reader :resolved_commits - private :resolved_commits + attr_reader :commit, :project, :path, :offset, :limit, :user, :resolved_commits def initialize(commit, project, user, params = {}) @commit = commit @@ -34,44 +31,37 @@ module Gitlab # # - An Array of Hashes containing the following keys: # - file_name: The full path of the tree entry - # - type: One of :blob, :tree, or :submodule # - commit: The last ::Commit to touch this entry in the tree # - commit_path: URI of the commit in the web interface - # - An Array of the unique ::Commit objects in the first value + # - commit_title_html: Rendered commit title def summarize - summary = contents - .tap { |summary| fill_last_commits!(summary) } - - [summary, commits] - end - - def fetch_logs - logs, _ = summarize + commits_hsh = fetch_last_cached_commits_list + prerender_commit_full_titles!(commits_hsh.values) - new_offset = next_offset if more? + commits_hsh.map do |path_key, commit| + commit = cache_commit(commit) - [logs.as_json, new_offset] + { + file_name: File.basename(path_key).force_encoding(Encoding::UTF_8), + commit: commit, + commit_path: commit_path(commit), + commit_title_html: markdown_field(commit, :full_title) + } + end end - # Does the tree contain more entries after the given offset + limit? - def more? - all_contents[next_offset].present? - end + def fetch_logs + logs = summarize - # The offset of the next batch of tree entries. If more? returns false, this - # batch will be empty - def next_offset - [all_contents.size + 1, offset + limit].min + [logs.first(limit).as_json, next_offset(logs.size)] end private - def contents - all_contents[offset, limit] || [] - end + def next_offset(entries_count) + return if entries_count <= limit - def commits - resolved_commits.values + offset + limit end def repository @@ -83,32 +73,12 @@ module Gitlab File.join(*[path, ""]) if path end - def entry_path(entry) - File.join(*[path, entry[:file_name]].compact).force_encoding(Encoding::ASCII_8BIT) - end - - def fill_last_commits!(entries) - commits_hsh = fetch_last_cached_commits_list - prerender_commit_full_titles!(commits_hsh.values) - - entries.each do |entry| - path_key = entry_path(entry) - commit = cache_commit(commits_hsh[path_key]) - - if commit - entry[:commit] = commit - entry[:commit_path] = commit_path(commit) - entry[:commit_title_html] = markdown_field(commit, :full_title) - end - end - end - def fetch_last_cached_commits_list - cache_key = ['projects', project.id, 'last_commits', commit.id, ensured_path, offset, limit] + cache_key = ['projects', project.id, 'last_commits', commit.id, ensured_path, offset, limit + 1] commits = Rails.cache.fetch(cache_key, expires_in: CACHE_EXPIRE_IN) do repository - .list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit, literal_pathspec: true) + .list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit + 1, literal_pathspec: true) .transform_values! { |commit| commit_to_hash(commit) } end @@ -131,26 +101,6 @@ module Gitlab Gitlab::Routing.url_helpers.project_commit_path(project, commit) end - def all_contents - strong_memoize(:all_contents) { cached_contents } - end - - def cached_contents - cache_key = ['projects', project.id, 'content', commit.id, path] - - Rails.cache.fetch(cache_key, expires_in: CACHE_EXPIRE_IN) do - [ - *tree.trees, - *tree.blobs, - *tree.submodules - ].map { |entry| { file_name: entry.name, type: entry.type } } - end - end - - def tree - strong_memoize(:tree) { repository.tree(commit.id, path) } - end - def prerender_commit_full_titles!(commits) # Preload commit authors as they are used in rendering commits.each(&:lazy_author) diff --git a/lib/gitlab/usage/metrics/instrumentations/unique_active_users_metric.rb b/lib/gitlab/usage/metrics/instrumentations/unique_active_users_metric.rb deleted file mode 100644 index 9da30db05dd..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/unique_active_users_metric.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class UniqueActiveUsersMetric < DatabaseMetric - operation :count - relation { ::User.active } - - metric_options do - { - batch_size: 10_000 - } - end - - def time_constraints - case time_frame - when '28d' - monthly_time_range_db_params(column: :last_activity_on) - else - super - end - end - end - end - end - end -end diff --git a/lib/gitlab/usage/service_ping/instrumented_payload.rb b/lib/gitlab/usage/service_ping/instrumented_payload.rb index 6cc67321ba1..3aa6789a010 100644 --- a/lib/gitlab/usage/service_ping/instrumented_payload.rb +++ b/lib/gitlab/usage/service_ping/instrumented_payload.rb @@ -34,6 +34,13 @@ module Gitlab return {} unless definition.present? Gitlab::Usage::Metric.new(definition).method(output_method).call + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + metric_fallback(key_path) + end + + def metric_fallback(key_path) + ::Gitlab::Usage::Metrics::KeyPathProcessor.process(key_path, ::Gitlab::Utils::UsageData::FALLBACK) end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 604fa364aa2..6f36a09fe48 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -105,7 +105,6 @@ module Gitlab clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), clusters_management_project: count(::Clusters::Cluster.with_management_project), - clusters_integrations_elastic_stack: count(::Clusters::Integrations::ElasticStack.enabled), clusters_integrations_prometheus: count(::Clusters::Integrations::Prometheus.enabled), kubernetes_agents: count(::Clusters::Agent), kubernetes_agents_with_token: distinct_count(::Clusters::AgentToken, :agent_id), @@ -135,7 +134,6 @@ module Gitlab projects_creating_incidents: distinct_count(Issue.incident, :project_id), projects_imported_from_github: count(Project.where(import_type: 'github')), projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), - projects_with_tracing_enabled: count(ProjectTracingSetting), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), @@ -558,7 +556,6 @@ module Gitlab operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period), start: minimum_id(User), finish: maximum_id(User)), - projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id), projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id), projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id), projects_with_alert_incidents: distinct_count(::Issue.incident.with_alert_management_alerts.where(time_period), :project_id), @@ -654,8 +651,6 @@ module Gitlab end def with_duration - return yield unless Feature.enabled?(:measure_service_ping_metric_collection) - result = nil duration = Benchmark.realtime do result = yield diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index 8feb24e49ac..5ede840661a 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -10,24 +10,24 @@ module Gitlab EDIT_BY_LIVE_PREVIEW = 'g_edit_by_live_preview' class << self - def track_web_ide_edit_action(author:, time: Time.zone.now) - track_unique_action(EDIT_BY_WEB_IDE, author, time) + def track_web_ide_edit_action(author:, time: Time.zone.now, project:) + track_unique_action(EDIT_BY_WEB_IDE, author, time, project) end def count_web_ide_edit_actions(date_from:, date_to:) count_unique(EDIT_BY_WEB_IDE, date_from, date_to) end - def track_sfe_edit_action(author:, time: Time.zone.now) - track_unique_action(EDIT_BY_SFE, author, time) + def track_sfe_edit_action(author:, time: Time.zone.now, project:) + track_unique_action(EDIT_BY_SFE, author, time, project) end def count_sfe_edit_actions(date_from:, date_to:) count_unique(EDIT_BY_SFE, date_from, date_to) end - def track_snippet_editor_edit_action(author:, time: Time.zone.now) - track_unique_action(EDIT_BY_SNIPPET_EDITOR, author, time) + def track_snippet_editor_edit_action(author:, time: Time.zone.now, project:) + track_unique_action(EDIT_BY_SNIPPET_EDITOR, author, time, project) end def count_snippet_editor_edit_actions(date_from:, date_to:) @@ -39,15 +39,25 @@ module Gitlab count_unique(events, date_from, date_to) end - def track_live_preview_edit_action(author:, time: Time.zone.now) - track_unique_action(EDIT_BY_LIVE_PREVIEW, author, time) + def track_live_preview_edit_action(author:, time: Time.zone.now, project:) + track_unique_action(EDIT_BY_LIVE_PREVIEW, author, time, project) end private - def track_unique_action(action, author, time) + def track_unique_action(action, author, time, project = nil) return unless author + if Feature.enabled?(:route_hll_to_snowplow_phase2) + Gitlab::Tracking.event( + 'ide_edit', + action.to_s, + project: project, + namespace: project&.namespace, + user: author + ) + end + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 0ace6e99c59..40581bda81b 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -33,10 +33,24 @@ module Gitlab pipeline_authoring quickactions search - testing user_packages ].freeze + CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS = %w[ + ci_users + error_tracking + ide_edit + importer + incident_management_alerts + pipeline_authoring + secure + snippets + source_code + terraform + testing + work_items + ].freeze + # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # @@ -114,7 +128,7 @@ module Gitlab # - Most of the metrics have weekly aggregation. We recommend this as it generates fewer keys in Redis to store. # - The aggregation used doesn't affect data granulation. def unique_events_data - categories.each_with_object({}) do |category, category_results| + categories_pending_migration.each_with_object({}) do |category, category_results| events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| @@ -148,6 +162,14 @@ module Gitlab private + def categories_pending_migration + if ::Feature.enabled?(:use_redis_hll_instrumentation_classes) + (categories - CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS) + else + categories + end + end + def track(values, event_name, context: '', time: Time.zone.now) return unless ::ServicePing::ServicePingSettings.enabled? diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 083de402175..9d463e11772 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -149,6 +149,19 @@ module Gitlab Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id) end + + def track_snowplow_action(action, author, project) + return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project&.namespace) + return unless author + + Gitlab::Tracking.event( + ISSUE_CATEGORY, + action.to_s, + project: project, + namespace: project&.namespace, + user: author + ) + end end end end diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml index 5a1e7f03278..76c97a974d7 100644 --- a/lib/gitlab/usage_data_counters/known_events/analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -78,3 +78,31 @@ category: analytics redis_slot: analytics aggregation: weekly +- name: p_analytics_ci_cd_time_to_restore_service + category: analytics + redis_slot: analytics + aggregation: weekly +- name: p_analytics_ci_cd_change_failure_rate + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_analytics_ci_cd_release_statistics + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_analytics_ci_cd_deployment_frequency + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_analytics_ci_cd_lead_time + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_analytics_ci_cd_time_to_restore_service + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_analytics_ci_cd_change_failure_rate + category: analytics + redis_slot: analytics + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_users.yml b/lib/gitlab/usage_data_counters/known_events/ci_users.yml index 5159dcf62ab..b012d61eef5 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_users.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_users.yml @@ -3,3 +3,8 @@ redis_slot: ci_users aggregation: weekly feature_flag: +- name: ci_users_executing_verify_environment_job + category: ci_users + redis_slot: ci_users + aggregation: weekly + feature_flag: diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 0dcbaf59c9c..88c9f44c165 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -157,34 +157,6 @@ category: testing redis_slot: testing aggregation: weekly -- name: i_testing_metrics_report_widget_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_group_code_coverage_visit_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_full_code_quality_report_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_web_performance_widget_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_group_code_coverage_project_click_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_load_performance_widget_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_metrics_report_artifact_uploaders - category: testing - redis_slot: testing - aggregation: weekly - name: i_testing_summary_widget_total category: testing redis_slot: testing @@ -390,3 +362,8 @@ category: growth redis_slot: users aggregation: weekly +# Manage +- name: unique_active_user + category: manage + aggregation: weekly + expiry: 42 diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 9c0f8fe9a80..fbb03a31a6f 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -76,8 +76,19 @@ module Gitlab track_unique_action_by_user(MR_REOPEN_ACTION, user) end - def track_approve_mr_action(user:) + def track_approve_mr_action(user:, merge_request:) track_unique_action_by_user(MR_APPROVE_ACTION, user) + + project = merge_request.target_project + return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) + + Gitlab::Tracking.event( + 'merge_requests', + MR_APPROVE_ACTION, + project: project, + namespace: project.namespace, + user: user + ) end def track_unapprove_mr_action(user:) diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb index 6f5300405c7..51bca8b51fe 100644 --- a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -26,3 +26,7 @@ module Gitlab end end end + +# rubocop:disable Layout/LineLength +Gitlab::UsageDataCounters::WorkItemActivityUniqueCounter.prepend_mod_with('Gitlab::UsageDataCounters::WorkItemActivityUniqueCounter') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/version_info.rb b/lib/gitlab/version_info.rb index aa6d5310161..f967a12b959 100644 --- a/lib/gitlab/version_info.rb +++ b/lib/gitlab/version_info.rb @@ -6,20 +6,27 @@ module Gitlab attr_reader :major, :minor, :patch - def self.parse(str) - if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/) - VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i) + VERSION_REGEX = /(\d+)\.(\d+)\.(\d+)/.freeze + + def self.parse(str, parse_suffix: false) + if str.is_a?(self.class) + str + elsif str && m = str.match(VERSION_REGEX) + VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i, parse_suffix ? m.post_match : nil) else VersionInfo.new end end - def initialize(major = 0, minor = 0, patch = 0) + def initialize(major = 0, minor = 0, patch = 0, suffix = nil) @major = major @minor = minor @patch = patch + @suffix_s = suffix.to_s end + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity def <=>(other) return unless other.is_a? VersionInfo return unless valid? && other.valid? @@ -36,21 +43,49 @@ module Gitlab 1 elsif @patch < other.patch -1 + elsif @suffix_s.empty? && other.suffix.present? + 1 + elsif other.suffix.empty? && @suffix_s.present? + -1 else - 0 + suffix <=> other.suffix end end + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/PerceivedComplexity def to_s if valid? - "%d.%d.%d" % [@major, @minor, @patch] + "%d.%d.%d%s" % [@major, @minor, @patch, @suffix_s] else - "Unknown" + 'Unknown' end end + def suffix + @suffix ||= @suffix_s.strip.gsub('-', '.pre.').scan(/\d+|[a-z]+/i).map do |s| + /^\d+$/ =~ s ? s.to_i : s + end.freeze + end + def valid? @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 end + + def hash + [self.class, to_s].hash + end + + def eql?(other) + (self <=> other) == 0 + end + + def same_minor_version?(other) + @major == other.major && @minor == other.minor + end + + def without_patch + self.class.new(@major, @minor, 0) + end end end diff --git a/lib/gitlab/wiki_pages/front_matter_parser.rb b/lib/gitlab/wiki_pages/front_matter_parser.rb index ee30fa907f4..071b0dde619 100644 --- a/lib/gitlab/wiki_pages/front_matter_parser.rb +++ b/lib/gitlab/wiki_pages/front_matter_parser.rb @@ -3,6 +3,8 @@ module Gitlab module WikiPages class FrontMatterParser + FEATURE_FLAG = :wiki_front_matter + # We limit the maximum length of text we are prepared to parse as YAML, to # avoid exploitations and attempts to consume memory and CPU. We allow for: # - a title line @@ -28,12 +30,18 @@ module Gitlab end # @param [String] wiki_content - def initialize(wiki_content) + # @param [FeatureGate] feature_gate The scope for feature availability + def initialize(wiki_content, feature_gate) @wiki_content = wiki_content + @feature_gate = feature_gate + end + + def self.enabled?(gate = nil) + Feature.enabled?(FEATURE_FLAG, gate) end def parse - return empty_result unless wiki_content.present? + return empty_result unless enabled? && wiki_content.present? return empty_result(block.error) unless block.valid? Result.new(front_matter: block.data, content: strip_front_matter_block) @@ -86,12 +94,16 @@ module Gitlab private - attr_reader :wiki_content + attr_reader :wiki_content, :feature_gate def empty_result(reason = nil, error = nil) Result.new(content: wiki_content, reason: reason, error: error) end + def enabled? + self.class.enabled?(feature_gate) + end + def block @block ||= parse_front_matter_block end diff --git a/lib/gitlab/x509/certificate.rb b/lib/gitlab/x509/certificate.rb index 752f3c6b004..98688f504eb 100644 --- a/lib/gitlab/x509/certificate.rb +++ b/lib/gitlab/x509/certificate.rb @@ -23,6 +23,18 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize end + def self.default_cert_dir + strong_memoize(:default_cert_dir) do + ENV.fetch('SSL_CERT_DIR', OpenSSL::X509::DEFAULT_CERT_DIR) + end + end + + def self.default_cert_file + strong_memoize(:default_cert_file) do + ENV.fetch('SSL_CERT_FILE', OpenSSL::X509::DEFAULT_CERT_FILE) + end + end + def self.from_strings(key_string, cert_string, ca_certs_string = nil) key = OpenSSL::PKey::RSA.new(key_string) cert = OpenSSL::X509::Certificate.new(cert_string) @@ -39,10 +51,10 @@ module Gitlab # Returns all top-level, readable files in the default CA cert directory def self.ca_certs_paths - cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"].select do |path| + cert_paths = Dir["#{default_cert_dir}/*"].select do |path| !File.directory?(path) && File.readable?(path) end - cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE + cert_paths << default_cert_file if File.exist? default_cert_file cert_paths end @@ -61,6 +73,11 @@ module Gitlab clear_memoization(:ca_certs_bundle) end + def self.reset_default_cert_paths + clear_memoization(:default_cert_dir) + clear_memoization(:default_cert_file) + end + # Returns an array of OpenSSL::X509::Certificate objects, empty array if none found # # Ruby OpenSSL::X509::Certificate.new will only load the first diff --git a/lib/gitlab/x509/commit.rb b/lib/gitlab/x509/commit.rb index c7f4b7cbdf5..3636e776a44 100644 --- a/lib/gitlab/x509/commit.rb +++ b/lib/gitlab/x509/commit.rb @@ -5,30 +5,10 @@ require 'digest' module Gitlab module X509 class Commit < Gitlab::SignedCommit - def signature - super - - return @signature if @signature - - cached_signature = lazy_signature&.itself - return @signature = cached_signature if cached_signature.present? - - @signature = create_cached_signature! - end - - def update_signature!(cached_signature) - cached_signature.update!(attributes) - @signature = cached_signature - end - private - def lazy_signature - BatchLoader.for(@commit.sha).batch do |shas, loader| - CommitSignatures::X509CommitSignature.by_commit_sha(shas).each do |signature| - loader.call(signature.commit_sha, signature) - end - end + def signature_class + CommitSignatures::X509CommitSignature end def attributes @@ -45,14 +25,6 @@ module Gitlab verification_status: signature.verification_status } end - - def create_cached_signature! - return if attributes.nil? - - return CommitSignatures::X509CommitSignature.new(attributes) if Gitlab::Database.read_only? - - CommitSignatures::X509CommitSignature.safe_create!(attributes) - end end end end diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index a6761e211fa..8acbfc144e9 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -59,7 +59,7 @@ module Gitlab if Feature.enabled?(:x509_forced_cert_loading, type: :ops) # Forcibly load the default cert file because the OpenSSL library seemingly ignores it - store.add_file(OpenSSL::X509::DEFAULT_CERT_FILE) if File.exist?(OpenSSL::X509::DEFAULT_CERT_FILE) + store.add_file(Gitlab::X509::Certificate.default_cert_file) if File.exist?(Gitlab::X509::Certificate.default_cert_file) # rubocop:disable Layout/LineLength end # valid_signing_time? checks the time attributes already diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 360c9a6c52f..c46ca2783bf 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -8,6 +8,7 @@ require 'google/apis/cloudbilling_v1' require 'google/apis/cloudresourcemanager_v1' require 'google/apis/iam_v1' require 'google/apis/serviceusage_v1' +require 'google/apis/sqladmin_v1beta4' module GoogleApi module CloudPlatform @@ -152,6 +153,22 @@ module GoogleApi Gitlab::HTTP.post(uri, body: { 'token' => access_token }) end + def create_cloudsql_database(gcp_project_id, instance_name, database_name) + database = Google::Apis::SqladminV1beta4::Database.new(name: database_name) + sql_admin_service.insert_database(gcp_project_id, instance_name, database) + end + + def create_cloudsql_user(gcp_project_id, instance_name, username, password) + user = Google::Apis::SqladminV1beta4::User.new + user.name = username + user.password = password + sql_admin_service.insert_user(gcp_project_id, instance_name, user) + end + + def get_cloudsql_instance(gcp_project_id, instance_name) + sql_admin_service.get_instance(gcp_project_id, instance_name) + end + private def enable_service(gcp_project_id, service_name) @@ -219,6 +236,10 @@ module GoogleApi def cloud_resource_manager_service @gpc_service ||= Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new.tap { |s| s.authorization = access_token } end + + def sql_admin_service + @sql_admin_service ||= Google::Apis::SqladminV1beta4::SQLAdminService.new.tap { |s| s.authorization = access_token } + end end end end diff --git a/lib/initializer_connections.rb b/lib/initializer_connections.rb new file mode 100644 index 00000000000..c8a6bb6c511 --- /dev/null +++ b/lib/initializer_connections.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module InitializerConnections + # Prevents any database connections within the block + # by using an empty connection handler + # rubocop:disable Database/MultipleDatabases + def self.with_disabled_database_connections + return yield if Gitlab::Utils.to_boolean(ENV['SKIP_RAISE_ON_INITIALIZE_CONNECTIONS']) + + original_handler = ActiveRecord::Base.connection_handler + + dummy_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new + ActiveRecord::Base.connection_handler = dummy_handler + + yield + + if dummy_handler&.connection_pool_names&.present? + raise "Unxpected connection_pools (#{dummy_handler.connection_pool_names}) ! Call `connects_to` before this block" + end + rescue ActiveRecord::ConnectionNotEstablished + message = "Database connection should not be called during initializers. Read more at https://docs.gitlab.com/ee/development/rails_initializers.html#database-connections-in-initializers" + + raise message + ensure + ActiveRecord::Base.connection_handler = original_handler + dummy_handler&.clear_all_connections! + end + # rubocop:enable Database/MultipleDatabases +end diff --git a/lib/learn_gitlab/onboarding.rb b/lib/learn_gitlab/onboarding.rb index 42415aacbee..54af01a21fe 100644 --- a/lib/learn_gitlab/onboarding.rb +++ b/lib/learn_gitlab/onboarding.rb @@ -3,6 +3,7 @@ module LearnGitlab class Onboarding include Gitlab::Utils::StrongMemoize + include Gitlab::Experiment::Dsl ACTION_ISSUE_IDS = { pipeline_created: 7, @@ -15,12 +16,12 @@ module LearnGitlab :issue_created, :git_write, :merge_request_created, - :user_added, - :security_scan_enabled + :user_added ].freeze - def initialize(namespace) + def initialize(namespace, current_user = nil) @namespace = namespace + @current_user = current_user end def completed_percentage @@ -49,9 +50,20 @@ module LearnGitlab end def tracked_actions - ACTION_ISSUE_IDS.keys + ACTION_PATHS + ACTION_ISSUE_IDS.keys + ACTION_PATHS + deploy_section_tracked_actions end - attr_reader :namespace + def deploy_section_tracked_actions + experiment(:security_actions_continuous_onboarding, + namespace: namespace, + user: current_user, + sticky_to: current_user + ) do |e| + e.control { [:security_scan_enabled] } + e.candidate { [:license_scanning_run, :secure_dependency_scanning_run, :secure_dast_run] } + end.run + end + + attr_reader :namespace, :current_user end end diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb index 4c21845ef18..fda90406e0a 100644 --- a/lib/sidebars/groups/menus/packages_registries_menu.rb +++ b/lib/sidebars/groups/menus/packages_registries_menu.rb @@ -54,7 +54,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Harbor Registry'), - link: group_harbor_registries_path(context.group), + link: group_harbor_repositories_path(context.group), active_routes: { controller: 'groups/harbor/repositories' }, item_id: :harbor_registry ) diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index a98cc20d51a..1c04a7b117d 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -88,8 +88,14 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Google Cloud'), - link: project_google_cloud_index_path(context.project), - active_routes: { controller: [:google_cloud, :service_accounts, :deployments, :gcp_regions] }, + link: project_google_cloud_configuration_path(context.project), + active_routes: { controller: [ + :configuration, + :service_accounts, + :databases, + :deployments, + :gcp_regions + ] }, item_id: :google_cloud ) end diff --git a/lib/sidebars/projects/menus/learn_gitlab_menu.rb b/lib/sidebars/projects/menus/learn_gitlab_menu.rb index 5de70ea7d7f..d2bc2fa0681 100644 --- a/lib/sidebars/projects/menus/learn_gitlab_menu.rb +++ b/lib/sidebars/projects/menus/learn_gitlab_menu.rb @@ -29,7 +29,10 @@ module Sidebars override :pill_count def pill_count strong_memoize(:pill_count) do - percentage = LearnGitlab::Onboarding.new(context.project.namespace).completed_percentage + percentage = LearnGitlab::Onboarding.new( + context.project.namespace, + context.current_user + ).completed_percentage "#{percentage}%" end diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb index c35bc1f5481..23e1a95c401 100644 --- a/lib/sidebars/projects/menus/monitor_menu.rb +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -9,8 +9,6 @@ module Sidebars return false unless context.project.feature_available?(:operations, context.current_user) add_item(metrics_dashboard_menu_item) - add_item(logs_menu_item) - add_item(tracing_menu_item) add_item(error_tracking_menu_item) add_item(alert_management_menu_item) add_item(incidents_menu_item) @@ -57,36 +55,6 @@ module Sidebars ) end - def logs_menu_item - if !Feature.enabled?(:monitor_logging, context.project) || - !can?(context.current_user, :read_environment, context.project) || - !can?(context.current_user, :read_pod_logs, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :logs) - end - - ::Sidebars::MenuItem.new( - title: _('Logs'), - link: project_logs_path(context.project), - active_routes: { path: 'logs#index' }, - item_id: :logs - ) - end - - def tracing_menu_item - if !Feature.enabled?(:monitor_tracing, context.project) || - !can?(context.current_user, :read_environment, context.project) || - !can?(context.current_user, :admin_project, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :tracing) - end - - ::Sidebars::MenuItem.new( - title: _('Tracing'), - link: project_tracing_path(context.project), - active_routes: { path: 'tracings#show' }, - item_id: :tracing - ) - end - def error_tracking_menu_item unless can?(context.current_user, :read_sentry_issue, context.project) return ::Sidebars::NilMenuItem.new(item_id: :error_tracking) diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb index d82a02a342f..914368e6fec 100644 --- a/lib/sidebars/projects/menus/packages_registries_menu.rb +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -70,8 +70,8 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Harbor Registry'), - link: project_harbor_registry_index_path(context.project), - active_routes: { controller: 'projects/harbor/repositories' }, + link: project_harbor_repositories_path(context.project), + active_routes: { controller: :harbor_registry }, item_id: :harbor_registry ) end diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb index d907c041ad8..a6ff2405390 100644 --- a/lib/system_check/app/redis_version_check.rb +++ b/lib/system_check/app/redis_version_check.rb @@ -8,7 +8,7 @@ module SystemCheck # Redis 5.x will be deprecated # https://gitlab.com/gitlab-org/gitlab/-/issues/331468 MIN_REDIS_VERSION = '5.0.0' - RECOMMENDED_REDIS_VERSION = '5.0.0' + RECOMMENDED_REDIS_VERSION = "6.0.0" set_name "Redis version >= #{RECOMMENDED_REDIS_VERSION}?" @custom_error_message = '' diff --git a/lib/tasks/contracts.rake b/lib/tasks/contracts/merge_requests.rake index 6bb7f30ad57..05ed9c30495 100644 --- a/lib/tasks/contracts.rake +++ b/lib/tasks/contracts/merge_requests.rake @@ -4,12 +4,12 @@ return if Rails.env.production? require 'pact/tasks/verification_task' -contracts = File.expand_path('../../spec/contracts', __dir__) +contracts = File.expand_path('../../../spec/contracts', __dir__) provider = File.expand_path('provider', contracts) # rubocop:disable Rails/RakeEnvironment namespace :contracts do - namespace :mr do + namespace :merge_requests do Pact::VerificationTask.new(:diffs_batch) do |pact| pact.uri( "#{contracts}/contracts/project/merge_request/show/mergerequest#show-merge_request_diffs_batch_endpoint.json", @@ -33,14 +33,11 @@ namespace :contracts do end desc 'Run all merge request contract tests' - task 'test:merge_request', :contract_mr do |_t, arg| - raise(ArgumentError, 'Merge request contract tests require contract_mr to be set') unless arg[:contract_mr] - - ENV['CONTRACT_MR'] = arg[:contract_mr] - errors = %w[metadata discussions diffs].each_with_object([]) do |task, err| + task 'test:merge_requests', :contract_mr do |_t, arg| + errors = %w[diffs_batch diffs_metadata discussions].each_with_object([]) do |task, err| Rake::Task["contracts:mr:pact:verify:#{task}"].execute rescue StandardError, SystemExit - err << "contracts:mr:pact:verify:#{task}" + err << "contracts:merge_requests:pact:verify:#{task}" end raise StandardError, "Errors in tasks #{errors.join(', ')}" unless errors.empty? diff --git a/lib/tasks/contracts/pipelines.rake b/lib/tasks/contracts/pipelines.rake new file mode 100644 index 00000000000..c018645722e --- /dev/null +++ b/lib/tasks/contracts/pipelines.rake @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +require 'pact/tasks/verification_task' + +contracts = File.expand_path('../../../spec/contracts', __dir__) +provider = File.expand_path('provider', contracts) + +# rubocop:disable Rails/RakeEnvironment +namespace :contracts do + namespace :pipelines do + Pact::VerificationTask.new(:get_list_project_pipelines) do |pact| + pact.uri( + "#{contracts}/contracts/project/pipeline/index/pipelines#index-get_list_project_pipelines.json", + pact_helper: "#{provider}/pact_helpers/project/pipeline/get_list_project_pipelines_helper.rb" + ) + end + + Pact::VerificationTask.new(:get_pipeline_header_data) do |pact| + pact.uri( + "#{contracts}/contracts/project/pipeline/show/pipelines#show-get_pipeline_header_data.json", + pact_helper: "#{provider}/pact_helpers/project/pipeline/get_pipeline_header_data_helper.rb" + ) + end + + desc 'Run all pipeline contract tests' + task 'test:pipelines', :contract_mr do |_t, arg| + errors = %w[get_list_project_pipelines get_pipeline_header_data].each_with_object([]) do |task, err| + Rake::Task["contracts:pipelines:pact:verify:#{task}"].execute + rescue StandardError, SystemExit + err << "contracts:pipelines:pact:verify:#{task}" + end + + raise StandardError, "Errors in tasks #{errors.join(', ')}" unless errors.empty? + end + end +end +# rubocop:enable Rails/RakeEnvironment diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 08a11100431..48bf49ff284 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -50,6 +50,9 @@ namespace :dev do connection.execute(cmd) rescue ActiveRecord::NoDatabaseError end + + # Clear connections opened by this rake task too + ActiveRecord::Base.clear_all_connections! # rubocop:disable Database/MultipleDatabases end end diff --git a/lib/tasks/gems.rake b/lib/tasks/gems.rake new file mode 100644 index 00000000000..c6be6d9eead --- /dev/null +++ b/lib/tasks/gems.rake @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +namespace :gems do + # :nocov: + namespace :error_tracking_open_api do + desc 'Generate OpenAPI client for Error Tracking' + # rubocop:disable Rails/RakeEnvironment + task :generate do |task| + # Configuration + api_url = 'https://gitlab.com/gitlab-org/opstrace/opstrace/-/raw/main/go/pkg/errortracking/swagger.yaml' + gem_name = 'error_tracking_open_api' + module_name = 'ErrorTrackingOpenAPI' # Namespacing is not supported like `ErrorTracking::OpenAPI` + docker_image = 'openapitools/openapi-generator-cli:v6.0.0' + + vendor_gem_dir = Pathname.new(root_directory) + gem_dir = vendor_gem_dir / gem_name + + # Always start with a clean state. + rm_rf(gem_dir) + + generate_gem( + vendor_gem_dir: vendor_gem_dir, + api_url: api_url, + gem_name: gem_name, + module_name: module_name, + docker_image: docker_image + ) + + post_process(gem_dir: gem_dir, gem_name: gem_name, task: task) + end + # rubocop:enable Rails/RakeEnvironment + + def root_directory + File.expand_path('../../vendor/gems', __dir__) + end + + def generate_gem(vendor_gem_dir:, api_url:, gem_name:, module_name:, docker_image:) + user_id = File.stat(vendor_gem_dir).uid + + Kernel.system('docker', 'run', + "--user=#{user_id}", '--rm', "--volume=#{vendor_gem_dir}:/code", docker_image, + 'generate', + '--input-spec', api_url, + '--generator-name', 'ruby', + '--output', "/code/#{gem_name}", + "--additional-properties=moduleName=#{module_name}" + ) + end + + def post_process(gem_dir:, gem_name:, task:) + write_file(gem_dir / 'README.md') do |content| + readme_banner(task) + content + end + + write_file(gem_dir / 'LICENSE', license) + write_file(gem_dir / "#{gem_name}.gemspec") do |content| + replace_string(content, 'Unlicense', 'MIT') + replace_string(content, /(\.files\s*=).*/, '\1 Dir.glob("lib/**/*")') + replace_string(content, /(\.test_files\s*=).*/, '\1 []') + end + + remove_entry_secure(gem_dir / 'Gemfile') + remove_entry_secure(gem_dir / '.rubocop.yml') + remove_entry_secure(gem_dir / '.travis.yml') + remove_entry_secure(gem_dir / 'git_push.sh') + remove_entry_secure(gem_dir / 'spec') + remove_entry_secure(gem_dir / '.rspec') + end + + def write_file(full_path, content = nil, &block) + content ||= yield(File.read(full_path)) + + File.write(full_path, content) + end + + def replace_string(content, from, to) + raise "#{from.inspect} not found" unless content.gsub!(from, to) + + content + end + + def readme_banner(task) + # rubocop:disable Rails/TimeZone + <<~BANNER + # Generated by `rake #{task.name}` on #{Time.now.strftime('%Y-%m-%d')} + + See https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/rake_tasks.md#update-openapi-client-for-error-tracking-feature + + BANNER + # rubocop:enable Rails/TimeZone + end + + def license + year = [2022, Date.today.year].uniq.join('-') + + <<~LICENSE + Copyright #{year} GitLab B.V. + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + LICENSE + end + end + # :nocov: +end diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake index df0c6a260a2..a903c743ea2 100644 --- a/lib/tasks/gitlab/bulk_add_permission.rake +++ b/lib/tasks/gitlab/bulk_add_permission.rake @@ -9,10 +9,10 @@ namespace :gitlab do project_ids = Project.pluck(:id) puts "Importing #{user_ids.size} users into #{project_ids.size} projects" - ProjectMember.add_users_to_projects(project_ids, user_ids, ProjectMember::DEVELOPER) + ProjectMember.add_members_to_projects(project_ids, user_ids, ProjectMember::DEVELOPER) puts "Importing #{admin_ids.size} admins into #{project_ids.size} projects" - ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MAINTAINER) + ProjectMember.add_members_to_projects(project_ids, admin_ids, ProjectMember::MAINTAINER) end desc "GitLab | Import | Add a specific user to all projects (as a developer)" @@ -20,7 +20,7 @@ namespace :gitlab do user = User.find_by(email: args.email) project_ids = Project.pluck(:id) puts "Importing #{user.email} users into #{project_ids.size} projects" - ProjectMember.add_users_to_projects(project_ids, Array.wrap(user.id), ProjectMember::DEVELOPER) + ProjectMember.add_members_to_projects(project_ids, Array.wrap(user.id), ProjectMember::DEVELOPER) end desc "GitLab | Import | Add all users to all groups (admin users are added as owners)" @@ -32,8 +32,8 @@ namespace :gitlab do puts "Importing #{user_ids.size} users into #{groups.size} groups" puts "Importing #{admin_ids.size} admins into #{groups.size} groups" groups.each do |group| - group.add_users(user_ids, GroupMember::DEVELOPER) - group.add_users(admin_ids, GroupMember::OWNER) + group.add_members(user_ids, GroupMember::DEVELOPER) + group.add_members(admin_ids, GroupMember::OWNER) end end @@ -43,7 +43,7 @@ namespace :gitlab do groups = Group.all puts "Importing #{user.email} users into #{groups.size} groups" groups.each do |group| - group.add_users(Array.wrap(user.id), GroupMember::DEVELOPER) + group.add_members(Array.wrap(user.id), GroupMember::DEVELOPER) end end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index a446a17dfc3..5ed54bb6921 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -94,7 +94,7 @@ namespace :gitlab do connection = Gitlab::Database.database_base_models['main'].connection databases_loaded << configure_database(connection) else - Gitlab::Database.database_base_models.each do |name, model| + Gitlab::Database.database_base_models_with_gitlab_shared.each do |name, model| next unless databases_with_tasks.any? { |db_with_tasks| db_with_tasks.name == name } databases_loaded << configure_database(model.connection, database_name: name) @@ -367,5 +367,69 @@ namespace :gitlab do Rake::Task['gitlab:db:execute_batched_migrations'].invoke end end + + namespace :dictionary do + DB_DOCS_PATH = File.join(Rails.root, 'db', 'docs') + + desc 'Generate database docs yaml' + task generate: :environment do + FileUtils.mkdir_p(DB_DOCS_PATH) unless Dir.exist?(DB_DOCS_PATH) + + Rails.application.eager_load! + + tables = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.tables } + classes = tables.to_h { |t| [t, []] } + + Gitlab::Database.database_base_models.each do |_, model_class| + model_class + .descendants + .reject(&:abstract_class) + .reject { |c| c.name =~ /^(?:EE::)?Gitlab::(?:BackgroundMigration|DatabaseImporters)::/ } + .reject { |c| c.name =~ /^HABTM_/ } + .each { |c| classes[c.table_name] << c.name if classes.has_key?(c.table_name) } + end + + version = Gem::Version.new(File.read('VERSION')) + milestone = version.release.segments[0..1].join('.') + + tables.each do |table_name| + file = File.join(DB_DOCS_PATH, "#{table_name}.yml") + + table_metadata = { + 'table_name' => table_name, + 'classes' => classes[table_name]&.sort&.uniq, + 'feature_categories' => [], + 'description' => nil, + 'introduced_by_url' => nil, + 'milestone' => milestone + } + + if File.exist?(file) + outdated = false + + existing_metadata = YAML.safe_load(File.read(file)) + + if existing_metadata['table_name'] != table_metadata['table_name'] + existing_metadata['table_name'] = table_metadata['table_name'] + outdated = true + end + + if existing_metadata['classes'].difference(table_metadata['classes']).any? + existing_metadata['classes'] = table_metadata['classes'] + outdated = true + end + + File.write(file, existing_metadata.to_yaml) if outdated + else + File.write(file, table_metadata.to_yaml) + end + end + end + + # Temporary disable this, see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85760#note_998452069 + # Rake::Task['db:migrate'].enhance do + # Rake::Task['gitlab:db:dictionary:generate'].invoke if Rails.env.development? + # end + end end end diff --git a/lib/tasks/gitlab/db/lock_writes.rake b/lib/tasks/gitlab/db/lock_writes.rake index b57c2860fe3..3a083036781 100644 --- a/lib/tasks/gitlab/db/lock_writes.rake +++ b/lib/tasks/gitlab/db/lock_writes.rake @@ -11,6 +11,9 @@ namespace :gitlab do schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 + next if schema_name == :gitlab_geo + if schemas_for_connection.include?(schema_name.to_sym) drop_write_trigger(database_name, connection, table_name) else @@ -24,6 +27,9 @@ namespace :gitlab do task unlock_writes: :environment do Gitlab::Database::EachDatabase.each_database_connection do |connection, database_name| Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 + next if schema_name == :gitlab_geo + drop_write_trigger(database_name, connection, table_name) end drop_write_trigger_function(connection) diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index b9137aa0d4c..a05b749a60e 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -15,10 +15,8 @@ namespace :gitlab do # Also avoids pipeline failures in case developer # dumps schema with flags disabled locally before pushing task enable_feature_flags: :environment do - class Feature - def self.enabled?(*args) - true - end + def Feature.enabled?(*args) + true end end diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake index 091743485c9..fc17c7d0177 100644 --- a/lib/tasks/gitlab/web_hook.rake +++ b/lib/tasks/gitlab/web_hook.rake @@ -39,7 +39,8 @@ namespace :gitlab do web_hooks.find_each do |hook| next unless hook.url == web_hook_url - result = WebHooks::DestroyService.new(nil).sync_destroy(hook) + user = hook.parent.owners.first + result = WebHooks::DestroyService.new(user).execute(hook) raise "Unable to destroy Web hook" unless result[:status] == :success diff --git a/lib/unnested_in_filters/dsl.rb b/lib/unnested_in_filters/dsl.rb new file mode 100644 index 00000000000..f5f358c729e --- /dev/null +++ b/lib/unnested_in_filters/dsl.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +# Including the `UnnestedInFilters::Dsl` module to an ActiveRecord +# model extends the interface of the following class instances to be +# able to use the `use_unnested_filters` method; +# +# - Model relation; +# `Model.where(...).use_unnested_filters` +# - All the association proxies +# `project.model_association.use_unnested_filters` +# - All the relation instances of the association +# `project.model_association.where(...).use_unnested_filters +# +# Note: The interface of the model itself won't be extended as we don't +# have a use-case for now(`Model.use_unnested_filters` won't work). +# +# Example usage of the API; +# +# relation = Vulnerabilities::Read.where(state: [1, 4]) +# .use_unnested_filters +# .order(severity: :desc, vulnerability_id: :desc) +# +# relation.to_a # => Will load records by using the optimized query +# +# See `UnnestedInFilters::Rewriter` for the details about the optimizations applied. +# +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module UnnestedInFilters + module Dsl + extend ActiveSupport::Concern + + MODULES_TO_EXTEND = [ + ActiveRecord::Relation, + ActiveRecord::Associations::CollectionProxy, + ActiveRecord::AssociationRelation + ].freeze + + included do + MODULES_TO_EXTEND.each do |mod| + delegate_mod = relation_delegate_class(mod) + delegate_mod.prepend(UnnestedInFilters::Dsl::Relation) + end + end + + module Relation + def use_unnested_filters + spawn.use_unnested_filters! + end + + def use_unnested_filters! + assert_mutability! + @values[:unnested_filters] = true + + self + end + + def use_unnested_filters? + @values.fetch(:unnested_filters, false) + end + + def load(*) + return super if loaded? || !rewrite_query? + + @records = unnested_filter_rewriter.rewrite.to_a + @loaded = true + + self + end + + def exists?(*) + return super unless rewrite_query? + + unnested_filter_rewriter.rewrite.exists? + end + + private + + def rewrite_query? + use_unnested_filters? && unnested_filter_rewriter.rewrite? + end + + def unnested_filter_rewriter + @unnested_filter_rewriter ||= UnnestedInFilters::Rewriter.new(self) + end + end + end +end diff --git a/lib/unnested_in_filters/rewriter.rb b/lib/unnested_in_filters/rewriter.rb new file mode 100644 index 00000000000..cba002a5632 --- /dev/null +++ b/lib/unnested_in_filters/rewriter.rb @@ -0,0 +1,191 @@ +# frozen_string_literal: true + +# rubocop:disable CodeReuse/ActiveRecord (This module is generating ActiveRecord relations therefore using AR methods is necessary) +module UnnestedInFilters + class Rewriter + include Gitlab::Utils::StrongMemoize + + class ValueTable + def initialize(model, attribute, values) + @model = model + @attribute = attribute.to_s + @values = values + end + + def to_sql + "unnest(#{serialized_values}::#{sql_type}[]) AS #{table_name}(#{column_name})" + end + + def as_predicate + "#{model.table_name}.#{column_name} = #{table_name}.#{column_name}" + end + + private + + attr_reader :model, :attribute, :values + + delegate :connection, :columns, :attribute_types, to: :model, private: true + delegate :quote, :quote_table_name, :quote_column_name, to: :connection + + def table_name + quote_table_name(attribute.pluralize) + end + + def column_name + quote_column_name(attribute) + end + + def serialized_values + array_type.serialize(values) + .then { |array| quote(array) } + end + + def array_type + ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Array.new(attribute_types[attribute]) + end + + def sql_type + column.sql_type_metadata.sql_type + end + + def column + columns.find { _1.name == attribute } + end + end + + def initialize(relation) + @relation = relation + end + + # Rewrites the given ActiveRecord::Relation object to + # utilize the DB indices efficiently. + # + # Example usage; + # + # relation = Vulnerabilities::Read.where(state: [1, 4]) + # relation = relation.order(severity: :desc, vulnerability_id: :desc) + # + # rewriter = UnnestedInFilters::Rewriter.new(relation) + # optimized_relation = rewriter.rewrite + # + # In the above example. the `relation` object would produce the following SQL query; + # + # SELECT + # "vulnerability_reads".* + # FROM + # "vulnerability_reads" + # WHERE + # "vulnerability_reads"."state" IN (1, 4) + # ORDER BY + # "vulnerability_reads"."severity" DESC, + # "vulnerability_reads"."vulnerability_id" DESC + # LIMIT 20; + # + # And the `optimized_relation` object would would produce the following query to + # utilize the index on (state, severity, vulnerability_id); + # + # SELECT + # "vulnerability_reads".* + # FROM + # unnest('{1, 4}'::smallint[]) AS "states" ("state"), + # LATERAL ( + # SELECT + # "vulnerability_reads".* + # FROM + # "vulnerability_reads" + # WHERE + # (vulnerability_reads."state" = "states"."state") + # ORDER BY + # "vulnerability_reads"."severity" DESC, + # "vulnerability_reads"."vulnerability_id" DESC + # LIMIT 20) AS vulnerability_reads + # ORDER BY + # "vulnerability_reads"."severity" DESC, + # "vulnerability_reads"."vulnerability_id" DESC + # LIMIT 20 + # + def rewrite + log_rewrite + + model.from(from) + .limit(limit_value) + .order(order_values) + .includes(relation.includes_values) + .preload(relation.preload_values) + .eager_load(relation.eager_load_values) + end + + def rewrite? + strong_memoize(:rewrite) do + in_filters.present? && has_index_coverage? + end + end + + private + + attr_reader :relation + + delegate :model, :order_values, :limit_value, :where_values_hash, to: :relation, private: true + + def log_rewrite + ::Gitlab::AppLogger.info(message: 'Query is being rewritten by `UnnestedInFilters`', model: model.name) + end + + def from + [value_tables.map(&:to_sql) + [lateral]].join(', ') + end + + def lateral + "LATERAL (#{join_relation.to_sql}) AS #{model.table_name}" + end + + def join_relation + value_tables.reduce(unscoped_relation) do |memo, tmp_table| + memo.where(tmp_table.as_predicate) + end + end + + def unscoped_relation + relation.unscope(where: in_filters.keys) + end + + def in_filters + @in_filters ||= where_values_hash.select { _2.is_a?(Array) } + end + + def has_index_coverage? + indices.any? do |index| + (filter_attributes - Array(index.columns)).empty? && # all the filter attributes are indexed + index.columns.last(order_attributes.length) == order_attributes && # index can be used in sorting + (index.columns - (filter_attributes + order_attributes)).empty? # there is no other columns in the index + end + end + + def filter_attributes + @filter_attributes ||= where_values_hash.keys + end + + def order_attributes + @order_attributes ||= order_values.flat_map(&method(:extract_column_name)) + end + + def extract_column_name(order_value) + case order_value + when Arel::Nodes::Ordering + order_value.expr.name + when ::Gitlab::Pagination::Keyset::Order + order_value.attribute_names + end + end + + def indices + model.connection.schema_cache.indexes(model.table_name) + end + + def value_tables + @value_tables ||= in_filters.map do |attribute, values| + ValueTable.new(model, attribute, values) + end + end + end +end |