diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-20 13:37:47 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-20 13:37:47 +0000 |
| commit | aee0a117a889461ce8ced6fcf73207fe017f1d99 (patch) | |
| tree | 891d9ef189227a8445d83f35c1b0fc99573f4380 /lib | |
| parent | 8d46af3258650d305f53b819eabf7ab18d22f59e (diff) | |
| download | gitlab-ce-aee0a117a889461ce8ced6fcf73207fe017f1d99.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-6-stable-eev14.6.0-rc42
Diffstat (limited to 'lib')
407 files changed, 4148 insertions, 4020 deletions
diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb deleted file mode 100644 index cbeaea97951..00000000000 --- a/lib/after_commit_queue.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module AfterCommitQueue - extend ActiveSupport::Concern - - included do - after_commit :_run_after_commit_queue - after_rollback :_clear_after_commit_queue - end - - def run_after_commit(&block) - _after_commit_queue << block if block - - true - end - - def run_after_commit_or_now(&block) - if ApplicationRecord.inside_transaction? - if ActiveRecord::Base.connection.current_transaction.records&.include?(self) - run_after_commit(&block) - else - # If the current transaction does not include this record, we can run - # the block now, even if it queues a Sidekiq job. - Sidekiq::Worker.skipping_transaction_check do - instance_eval(&block) - end - end - else - instance_eval(&block) - end - - true - end - - protected - - def _run_after_commit_queue - while action = _after_commit_queue.pop - self.instance_eval(&action) - end - end - - def _after_commit_queue - @after_commit_queue ||= [] - end - - def _clear_after_commit_queue - _after_commit_queue.clear - end -end diff --git a/lib/api/admin/plan_limits.rb b/lib/api/admin/plan_limits.rb index ab6a4e4a04a..d595b5b2e09 100644 --- a/lib/api/admin/plan_limits.rb +++ b/lib/api/admin/plan_limits.rb @@ -37,6 +37,7 @@ module API optional :conan_max_file_size, type: Integer, desc: 'Maximum Conan package file size in bytes' optional :generic_packages_max_file_size, type: Integer, desc: 'Maximum generic package file size in bytes' + optional :helm_max_file_size, type: Integer, desc: 'Maximum Helm chart file size in bytes' optional :maven_max_file_size, type: Integer, desc: 'Maximum Maven package file size in bytes' optional :npm_max_file_size, type: Integer, desc: 'Maximum NPM package file size in bytes' optional :nuget_max_file_size, type: Integer, desc: 'Maximum NuGet package file size in bytes' diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb index dabb6c7ab3a..72c388160b4 100644 --- a/lib/api/ci/helpers/runner.rb +++ b/lib/api/ci/helpers/runner.rb @@ -29,7 +29,7 @@ module API def get_runner_details_from_request return get_runner_ip unless params['info'].present? - attributes_for_keys(%w(name version revision platform architecture), params['info']) + attributes_for_keys(%w(name version revision platform architecture executor), params['info']) .merge(get_runner_config_from_request) .merge(get_runner_ip) end @@ -52,7 +52,7 @@ module API # HTTP status codes to terminate the job on GitLab Runner: # - 403 - def authenticate_job!(require_running: true) + def authenticate_job!(require_running: true, heartbeat_runner: false) job = current_job # 404 is not returned here because we want to terminate the job if it's @@ -70,7 +70,17 @@ module API job_forbidden!(job, 'Job is not running') unless job.running? end - job.runner&.heartbeat(get_runner_ip) + # Only some requests (like updating the job or patching the trace) should trigger + # runner heartbeat. Operations like artifacts uploading are executed in context of + # the running job and in the job environment, which in many cases will cause the IP + # to be updated to not the expected value. And operations like artifacts downloads can + # be done even after the job is finished and from totally different runners - while + # they would then update the connection status of not the runner that they should. + # Runner requests done in context of job authentication should explicitly define when + # the heartbeat should be triggered. + if heartbeat_runner + job.runner&.heartbeat(get_runner_ip) + end job end diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 03b59e7e6ad..4e5d6c264bf 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -166,7 +166,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - get ':id/pipelines/:pipeline_id/variables', feature_category: :pipeline_authoring do + get ':id/pipelines/:pipeline_id/variables', feature_category: :pipeline_authoring, urgency: :low do authorize! :read_pipeline_variable, pipeline present pipeline.variables, with: Entities::Ci::Variable diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index aabcf34952c..4317789f7aa 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -176,7 +176,7 @@ module API optional :exit_code, type: Integer, desc: %q(Job's exit code) end put '/:id', feature_category: :continuous_integration do - job = authenticate_job! + job = authenticate_job!(heartbeat_runner: true) Gitlab::Metrics.add_event(:update_build) @@ -203,7 +203,7 @@ module API optional :token, type: String, desc: %q(Job's authentication token) end patch '/:id/trace', feature_category: :continuous_integration do - job = authenticate_job! + job = authenticate_job!(heartbeat_runner: true) error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 1785362656e..c89abf72e2d 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -78,13 +78,16 @@ module API name = params[:name] || params[:context] || 'default' - pipeline ||= user_project.ci_pipelines.create!( + pipeline ||= user_project.ci_pipelines.build( source: :external, sha: commit.sha, ref: ref, user: current_user, protected: user_project.protected_for?(ref)) + pipeline.ensure_project_iid! + pipeline.save! + authorize! :update_pipeline, pipeline status = GenericCommitStatus.running_or_pending.find_or_initialize_by( diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index 94cad7e6c65..0e6e04d2645 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -70,7 +70,7 @@ module API end desc 'Composer packages endpoint at group level' - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/packages' do presenter.root end @@ -79,7 +79,7 @@ module API params do requires :sha, type: String, desc: 'Shasum of current json' end - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/p/:sha' do presenter.provider end @@ -88,7 +88,7 @@ module API params do requires :package_name, type: String, file_path: true, desc: 'The Composer package name' end - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/p2/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true do not_found! if packages.empty? @@ -99,7 +99,7 @@ module API params do requires :package_name, type: String, file_path: true, desc: 'The Composer package name' end - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true do not_found! if packages.empty? not_found! if params[:sha].blank? @@ -119,7 +119,7 @@ module API desc 'Composer packages endpoint for registering packages' namespace ':id/packages/composer' do - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true params do optional :branch, type: String, desc: 'The name of the branch' diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb index 3194cdebde8..edf20b6aebe 100644 --- a/lib/api/concerns/packages/conan_endpoints.rb +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -27,6 +27,7 @@ module API PACKAGE_COMPONENT_REGEX = Gitlab::Regex.conan_recipe_component_regex CONAN_REVISION_REGEX = Gitlab::Regex.conan_revision_regex + CONAN_REVISION_USER_CHANNEL_REGEX = Gitlab::Regex.conan_recipe_user_channel_regex CONAN_FILES = (Gitlab::Regex::Packages::CONAN_RECIPE_FILES + Gitlab::Regex::Packages::CONAN_PACKAGE_FILES).freeze @@ -105,10 +106,14 @@ module API params do requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' - requires :package_username, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package username' - requires :package_channel, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package channel' + requires :package_username, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package username' + requires :package_channel, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package channel' end namespace 'conans/:package_name/:package_version/:package_username/:package_channel', requirements: PACKAGE_REQUIREMENTS do + after_validation do + check_username_channel + end + # Get the snapshot # # the snapshot is a hash of { filename: md5 hash } @@ -264,8 +269,8 @@ module API params do requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' - requires :package_username, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package username' - requires :package_channel, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package channel' + requires :package_username, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package username' + requires :package_channel, type: String, regexp: CONAN_REVISION_USER_CHANNEL_REGEX, desc: 'Package channel' requires :recipe_revision, type: String, regexp: CONAN_REVISION_REGEX, desc: 'Conan Recipe Revision' end namespace 'files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision', requirements: PACKAGE_REQUIREMENTS do @@ -273,6 +278,10 @@ module API authenticate_non_get! end + after_validation do + check_username_channel + end + params do requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES end diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index 7a657be5bf3..d6e006df976 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -121,9 +121,7 @@ module API not_found!('Packages') if packages.empty? - include_metadata = Feature.enabled?(:packages_npm_abbreviated_metadata, project, default_enabled: :yaml) - - present ::Packages::Npm::PackagePresenter.new(package_name, packages, include_metadata: include_metadata), + present ::Packages::Npm::PackagePresenter.new(package_name, packages), with: ::API::Entities::NpmPackage end end diff --git a/lib/api/dependency_proxy.rb b/lib/api/dependency_proxy.rb index 185b8d5a15d..9d0b1bf4423 100644 --- a/lib/api/dependency_proxy.rb +++ b/lib/api/dependency_proxy.rb @@ -6,15 +6,6 @@ module API feature_category :dependency_proxy - helpers do - def obtain_new_purge_cache_lease - Gitlab::ExclusiveLease - .new("dependency_proxy:delete_group_blobs:#{user_group.id}", - timeout: 1.hour) - .try_obtain - end - end - after_validation do authorize! :admin_group, user_group end @@ -29,9 +20,6 @@ module API delete ':id/dependency_proxy/cache' do not_found! unless user_group.dependency_proxy_feature_available? - message = 'This request has already been made. It may take some time to purge the cache. You can run this at most once an hour for a given group' - render_api_error!(message, 409) unless obtain_new_purge_cache_lease - # rubocop:disable CodeReuse/Worker PurgeDependencyProxyCacheWorker.perform_async(current_user.id, user_group.id) # rubocop:enable CodeReuse/Worker diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index cf4b2348458..0709a8c2036 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -8,6 +8,13 @@ module API before { authenticate! } + urgency :low, [ + '/projects/:id/merge_requests/:noteable_id/discussions', + '/projects/:id/merge_requests/:noteable_id/discussions/:discussion_id', + '/projects/:id/merge_requests/:noteable_id/discussions/:discussion_id/notes', + '/projects/:id/merge_requests/:noteable_id/discussions/:discussion_id/notes/:note_id' + ] + Helpers::DiscussionsHelpers.feature_category_per_noteable_type.each do |noteable_type, feature_category| parent_type = noteable_type.parent_class.to_s.underscore noteables_str = noteable_type.to_s.underscore.pluralize diff --git a/lib/api/entities/changelog.rb b/lib/api/entities/changelog.rb new file mode 100644 index 00000000000..f8ca5826418 --- /dev/null +++ b/lib/api/entities/changelog.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module API + module Entities + class Changelog < Grape::Entity + expose :to_s, as: :notes + end + end +end diff --git a/lib/api/entities/ci/job_basic.rb b/lib/api/entities/ci/job_basic.rb index c31340f1ff0..0badde4089e 100644 --- a/lib/api/entities/ci/job_basic.rb +++ b/lib/api/entities/ci/job_basic.rb @@ -13,6 +13,7 @@ module API expose :user, with: ::API::Entities::User expose :commit, with: ::API::Entities::Commit expose :pipeline, with: ::API::Entities::Ci::PipelineBasic + expose :failure_reason, if: -> (job) { job.failed? } expose :web_url do |job, _options| Gitlab::Routing.url_helpers.project_job_url(job.project, job) diff --git a/lib/api/entities/ci/pipeline.rb b/lib/api/entities/ci/pipeline.rb index 11336ae070d..a8033a21044 100644 --- a/lib/api/entities/ci/pipeline.rb +++ b/lib/api/entities/ci/pipeline.rb @@ -10,7 +10,9 @@ module API expose :created_at, :updated_at, :started_at, :finished_at, :committed_at expose :duration expose :queued_duration - expose :coverage + expose :coverage do |pipeline| + pipeline.present.coverage + end expose :detailed_status, using: DetailedStatusEntity do |pipeline, options| pipeline.detailed_status(options[:current_user]) end diff --git a/lib/api/entities/ci/pipeline_basic.rb b/lib/api/entities/ci/pipeline_basic.rb index 4d56176bdb3..a2a5a98920a 100644 --- a/lib/api/entities/ci/pipeline_basic.rb +++ b/lib/api/entities/ci/pipeline_basic.rb @@ -4,7 +4,7 @@ module API module Entities module Ci class PipelineBasic < Grape::Entity - expose :id, :project_id, :sha, :ref, :status, :source + expose :id, :iid, :project_id, :sha, :ref, :status, :source expose :created_at, :updated_at expose :web_url do |pipeline, _options| diff --git a/lib/api/entities/ci/runner.rb b/lib/api/entities/ci/runner.rb index 60193fe1df4..c17ff513479 100644 --- a/lib/api/entities/ci/runner.rb +++ b/lib/api/entities/ci/runner.rb @@ -14,7 +14,7 @@ module API expose :online?, as: :online # DEPRECATED # TODO Remove in %15.0 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 - expose :status, as: :deprecated_rest_status + expose :deprecated_rest_status, as: :status end end end diff --git a/lib/api/entities/commit_signature.rb b/lib/api/entities/commit_signature.rb index 505ce462edf..0d8e977a9f5 100644 --- a/lib/api/entities/commit_signature.rb +++ b/lib/api/entities/commit_signature.rb @@ -6,9 +6,9 @@ module API expose :signature_type expose :signature, merge: true do |commit, options| - if commit.signature.is_a?(GpgSignature) || commit.raw_commit_from_rugged? + if commit.signature.is_a?(::CommitSignatures::GpgSignature) || commit.raw_commit_from_rugged? ::API::Entities::GpgCommitSignature.represent commit_signature(commit), options - elsif commit.signature.is_a?(X509CommitSignature) + elsif commit.signature.is_a?(::CommitSignatures::X509CommitSignature) ::API::Entities::X509Signature.represent commit.signature, options end end diff --git a/lib/api/entities/issue_basic.rb b/lib/api/entities/issue_basic.rb index ab248523028..6125dc05a6e 100644 --- a/lib/api/entities/issue_basic.rb +++ b/lib/api/entities/issue_basic.rb @@ -23,7 +23,7 @@ module API expose :issue_type, as: :type, format_with: :upcase, - documentation: { type: "String", desc: "One of #{::WorkItem::Type.base_types.keys.map(&:upcase)}" } + documentation: { type: "String", desc: "One of #{::WorkItem::Type.allowed_types_for_issues.map(&:upcase)}" } expose :assignee, using: ::API::Entities::UserBasic do |issue| issue.assignees.first diff --git a/lib/api/entities/personal_access_token.rb b/lib/api/entities/personal_access_token.rb index 3846929c903..55764daef9d 100644 --- a/lib/api/entities/personal_access_token.rb +++ b/lib/api/entities/personal_access_token.rb @@ -3,7 +3,7 @@ module API module Entities class PersonalAccessToken < Grape::Entity - expose :id, :name, :revoked, :created_at, :scopes, :user_id + expose :id, :name, :revoked, :created_at, :scopes, :user_id, :last_used_at expose :active?, as: :active expose :expires_at do |personal_access_token| personal_access_token.expires_at ? personal_access_token.expires_at.strftime("%Y-%m-%d") : nil diff --git a/lib/api/entities/plan_limit.rb b/lib/api/entities/plan_limit.rb index 04ec44b5167..9f4d1635998 100644 --- a/lib/api/entities/plan_limit.rb +++ b/lib/api/entities/plan_limit.rb @@ -5,6 +5,7 @@ module API class PlanLimit < Grape::Entity expose :conan_max_file_size expose :generic_packages_max_file_size + expose :helm_max_file_size expose :maven_max_file_size expose :npm_max_file_size expose :nuget_max_file_size diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 662ca59852e..1b9299ed17e 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -117,6 +117,7 @@ module API expose :squash_option expose :suggestion_commit_message expose :merge_commit_template + expose :squash_commit_template expose :statistics, using: 'API::Entities::ProjectStatistics', if: -> (project, options) { options[:statistics] && Ability.allowed?(options[:current_user], :read_statistics, project) } diff --git a/lib/api/entities/project_import_failed_relation.rb b/lib/api/entities/project_import_failed_relation.rb index b8f842c1646..26cfae7260c 100644 --- a/lib/api/entities/project_import_failed_relation.rb +++ b/lib/api/entities/project_import_failed_relation.rb @@ -10,6 +10,7 @@ module API end expose :relation_key, as: :relation_name + expose :relation_index, as: :line_number end end end diff --git a/lib/api/entities/project_import_status.rb b/lib/api/entities/project_import_status.rb index e79c1cdf1a2..5daae4a70f2 100644 --- a/lib/api/entities/project_import_status.rb +++ b/lib/api/entities/project_import_status.rb @@ -4,6 +4,7 @@ module API module Entities class ProjectImportStatus < ProjectIdentity expose :import_status + expose :import_type expose :correlation_id do |project, _options| project.import_state&.correlation_id end @@ -15,6 +16,12 @@ module API expose :import_error do |project, _options| project.import_state&.last_error end + + expose :stats do |project, _options| + if project.github_import? + ::Gitlab::GithubImport::ObjectCounter.summary(project) + end + end end end end diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 25cc4e53bd2..f0c0182a02f 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -18,7 +18,7 @@ module API detail 'This feature was introduced in GitLab 12.5.' end get ':id/export/download' do - check_rate_limit! :group_download_export, [current_user, user_group] + check_rate_limit! :group_download_export, scope: [current_user, user_group] if user_group.export_file_exists? if user_group.export_archive_exists? @@ -35,7 +35,7 @@ module API detail 'This feature was introduced in GitLab 12.5.' end post ':id/export' do - check_rate_limit! :group_export, [current_user] + check_rate_limit! :group_export, scope: current_user export_service = ::Groups::ImportExport::ExportService.new(group: user_group, user: current_user) diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 680e3a6e994..d3d1f03585b 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -109,7 +109,7 @@ module API end def present_groups_with_pagination_strategies(params, groups) - return present_groups(params, groups) if current_user.present? || Feature.disabled?(:keyset_pagination_for_groups_api) + return present_groups(params, groups) if current_user.present? options = { with: Entities::Group, @@ -382,6 +382,28 @@ module API end end + desc 'Transfer a group to a new parent group or promote a subgroup to a root group' + params do + optional :group_id, type: Integer, + desc: 'The ID of the target group to which the group needs to be transferred to.'\ + 'If not provided, the source group will be promoted to a root group.' + end + post ':id/transfer' do + group = find_group!(params[:id]) + authorize! :admin_group, group + + new_parent_group = find_group!(params[:group_id]) if params[:group_id].present? + + service = ::Groups::TransferService.new(group, current_user) + + if service.execute(new_parent_group) + group.preload_shared_group_links + present group, with: Entities::GroupDetail, current_user: current_user + else + render_api_error!(service.error, 400) + end + end + desc 'Share a group with a group' do success Entities::GroupDetail end diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb index da0ee8f207e..02613cbf9b9 100644 --- a/lib/api/helpers/label_helpers.rb +++ b/lib/api/helpers/label_helpers.rb @@ -105,7 +105,11 @@ module API end def promote_label(parent) - authorize! :admin_label, parent + unless parent.group + render_api_error!('Failed to promote project label to group label', 400) + end + + authorize! :admin_label, parent.group label = find_label(parent, params[:name], include_ancestor_groups: false) diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 1e89f9f97a2..c2710be6c03 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -8,6 +8,9 @@ module API params :optional_filter_params_ee do end + params :optional_state_filter_ee do + end + def find_source(source_type, id) public_send("find_#{source_type}!", id) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index 4b6dac39348..031c29e7472 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -7,6 +7,21 @@ module API module ApiHelpers include Gitlab::Utils::StrongMemoize + def check_username_channel + username = declared(params)[:package_username] + channel = declared(params)[:package_channel] + + if username == ::Packages::Conan::Metadatum::NONE_VALUE && package_scope == :instance + # at the instance level, username must not be empty (naming convention) + # don't try to process the empty username and eagerly return not found. + not_found! + end + + ::Packages::Conan::Metadatum.validate_username_and_channel(username, channel) do |none_field| + bad_request!("#{none_field} can't be solely blank") + end + end + def present_download_urls(entity) authorize!(:read_package, project) diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 42d1c40dd11..d7de8bd8b8b 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -62,6 +62,7 @@ module API optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' optional :suggestion_commit_message, type: String, desc: 'The commit message used to apply merge request suggestions' optional :merge_commit_template, type: String, desc: 'Template used to create merge commit message' + optional :squash_commit_template, type: String, desc: 'Template used to create squash commit message' optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" optional :ci_default_git_depth, type: Integer, desc: 'Default number of revisions for shallow cloning' optional :auto_devops_enabled, type: Boolean, desc: 'Flag indication if Auto DevOps is enabled' @@ -162,6 +163,7 @@ module API :avatar, :suggestion_commit_message, :merge_commit_template, + :squash_commit_template, :repository_storage, :compliance_framework_setting, :packages_enabled, @@ -181,9 +183,10 @@ module API def filter_attributes_using_license!(attrs) end - def validate_git_import_url!(import_url, import_enabled: true) + def validate_git_import_url!(import_url) return if import_url.blank? - return unless import_enabled + + yield if block_given? result = Import::ValidateRemoteGitEndpointService.new(url: import_url).execute # network call diff --git a/lib/api/helpers/rate_limiter.rb b/lib/api/helpers/rate_limiter.rb index 3a16aef6a74..7d87c74097d 100644 --- a/lib/api/helpers/rate_limiter.rb +++ b/lib/api/helpers/rate_limiter.rb @@ -2,26 +2,27 @@ module API module Helpers + # == RateLimiter + # + # Helper that checks if the rate limit for a given endpoint is throttled by calling the + # Gitlab::ApplicationRateLimiter class. If the action is throttled for the current user, the request + # will be logged and an error message will be rendered with a Too Many Requests response status. + # See app/controllers/concerns/check_rate_limit.rb for Rails controllers version module RateLimiter - def check_rate_limit!(key, scope, users_allowlist = nil) - if rate_limiter.throttled?(key, scope: scope, users_allowlist: users_allowlist) - log_request(key) - render_exceeded_limit_error! - end - end + def check_rate_limit!(key, scope:, **options) + return unless rate_limiter.throttled?(key, scope: scope, **options) - private + rate_limiter.log_request(request, "#{key}_request_limit".to_sym, current_user) - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end + return yield if block_given? - def render_exceeded_limit_error! render_api_error!({ error: _('This endpoint has been requested too many times. Try again later.') }, 429) end - def log_request(key) - rate_limiter.log_request(request, "#{key}_request_limit".to_sym, current_user) + private + + def rate_limiter + ::Gitlab::ApplicationRateLimiter end end end diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index f7f5af07378..d78576b5d5b 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -24,7 +24,6 @@ module API requires :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'invitations-api' - optional :areas_of_focus, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Areas the inviter wants the member to focus upon' optional :tasks_to_be_done, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Tasks the inviter wants the member to do' optional :tasks_project_id, type: Integer, desc: 'The project ID in which to create the task issues' end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 9958526fa7f..4d67cbd1272 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -82,7 +82,7 @@ module API desc: 'Return issues sorted in `asc` or `desc` order.' optional :due_date, type: String, values: %w[0 overdue week month next_month_and_previous_two_weeks] << '', desc: 'Return issues that have no due date (`0`), or whose due date is this week, this month, between two weeks ago and next month, or which are overdue. Accepts: `overdue`, `week`, `month`, `next_month_and_previous_two_weeks`, `0`' - optional :issue_type, type: String, values: WorkItem::Type.base_types.keys, desc: "The type of the issue. Accepts: #{WorkItem::Type.base_types.keys.join(', ')}" + optional :issue_type, type: String, values: WorkItem::Type.allowed_types_for_issues, desc: "The type of the issue. Accepts: #{WorkItem::Type.allowed_types_for_issues.join(', ')}" use :issues_stats_params use :pagination @@ -99,7 +99,7 @@ module API optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" - optional :issue_type, type: String, values: WorkItem::Type.base_types.keys, desc: "The type of the issue. Accepts: #{WorkItem::Type.base_types.keys.join(', ')}" + optional :issue_type, type: String, values: WorkItem::Type.allowed_types_for_issues, desc: "The type of the issue. Accepts: #{WorkItem::Type.allowed_types_for_issues.join(', ')}" use :optional_issue_params_ee end @@ -262,7 +262,7 @@ module API post ':id/issues' do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/21140') - check_rate_limit! :issues_create, [current_user] if Feature.disabled?("rate_limited_service_issues_create", user_project, default_enabled: :yaml) + check_rate_limit!(:issues_create, scope: current_user) if Feature.disabled?("rate_limited_service_issues_create", user_project, default_enabled: :yaml) authorize! :create_issue, user_project @@ -472,7 +472,7 @@ module API end get ':id/issues/:issue_iid/participants' do issue = find_project_issue(params[:issue_iid]) - participants = ::Kaminari.paginate_array(issue.participants) + participants = ::Kaminari.paginate_array(issue.visible_participants(current_user)) present paginate(participants), with: Entities::UserBasic, current_user: current_user, project: user_project end diff --git a/lib/api/lint.rb b/lib/api/lint.rb index 3655cb56564..bfd457a3092 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -43,7 +43,7 @@ module API optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check.' optional :include_jobs, type: Boolean, desc: 'Whether or not to include CI jobs in the response' end - get ':id/ci/lint' do + get ':id/ci/lint', urgency: :low do authorize! :download_code, user_project content = user_project.repository.gitlab_ci_yml_for(user_project.commit.id, user_project.ci_config_path_or_default) @@ -64,7 +64,7 @@ module API optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check.' optional :include_jobs, type: Boolean, desc: 'Whether or not to include CI jobs in the response' end - post ':id/ci/lint' do + post ':id/ci/lint', urgency: :low do authorize! :create_pipeline, user_project result = Gitlab::Ci::Lint diff --git a/lib/api/members.rb b/lib/api/members.rb index f488c8c26fc..4798edc4ddf 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -41,6 +41,7 @@ module API optional :query, type: String, desc: 'A query string to search for members' optional :user_ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Array of user ids to look up for membership' optional :show_seat_info, type: Boolean, desc: 'Show seat information for members' + use :optional_state_filter_ee use :pagination end @@ -94,7 +95,6 @@ module API requires :user_id, types: [Integer, String], desc: 'The user ID of the new member or multiple IDs separated by commas.' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'members-api' - optional :areas_of_focus, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Areas the inviter wants the member to focus upon' optional :tasks_to_be_done, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Tasks the inviter wants the member to do' optional :tasks_project_id, type: Integer, desc: 'The project ID in which to create the task issues' end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 8fa7138af42..87623568a04 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -38,7 +38,7 @@ module API requires :version_id, type: Integer, desc: 'The ID of a merge request diff version' end - get ":id/merge_requests/:merge_request_iid/versions/:version_id" do + get ":id/merge_requests/:merge_request_iid/versions/:version_id", urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) present_cached merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull, cache_context: nil diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 96d1a69c03a..3f39af7f909 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -134,7 +134,7 @@ module API use :merge_requests_params use :optional_scope_param end - get feature_category: :code_review do + get feature_category: :code_review, urgency: :low do authenticate! unless params[:scope] == 'all' validate_anonymous_search_access! if params[:search].present? merge_requests = find_merge_requests @@ -155,7 +155,7 @@ module API optional :non_archived, type: Boolean, desc: 'Return merge requests from non archived projects', default: true end - get ":id/merge_requests", feature_category: :code_review do + get ":id/merge_requests", feature_category: :code_review, urgency: :low do validate_anonymous_search_access! if declared_params[:search].present? merge_requests = find_merge_requests(group_id: user_group.id, include_subgroups: true) @@ -195,7 +195,7 @@ module API use :merge_requests_params optional :iids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The IID array of merge requests' end - get ":id/merge_requests", feature_category: :code_review do + get ":id/merge_requests", feature_category: :code_review, urgency: :low do authorize! :read_merge_request, user_project validate_anonymous_search_access! if declared_params[:search].present? @@ -222,7 +222,7 @@ module API desc: 'The target project of the merge request defaults to the :id of the project' use :optional_params end - post ":id/merge_requests", feature_category: :code_review do + post ":id/merge_requests", feature_category: :code_review, urgency: :low do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20770') authorize! :create_merge_request_from, user_project @@ -244,7 +244,7 @@ module API params do requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' end - delete ":id/merge_requests/:merge_request_iid", feature_category: :code_review do + delete ":id/merge_requests/:merge_request_iid", feature_category: :code_review, urgency: :low do merge_request = find_project_merge_request(params[:merge_request_iid]) authorize!(:destroy_merge_request, merge_request) @@ -263,7 +263,7 @@ module API desc 'Get a single merge request' do success Entities::MergeRequest end - get ':id/merge_requests/:merge_request_iid', feature_category: :code_review do + get ':id/merge_requests/:merge_request_iid', feature_category: :code_review, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) present merge_request, @@ -279,10 +279,10 @@ module API desc 'Get the participants of a merge request' do success Entities::UserBasic end - get ':id/merge_requests/:merge_request_iid/participants', feature_category: :code_review do + get ':id/merge_requests/:merge_request_iid/participants', feature_category: :code_review, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - participants = ::Kaminari.paginate_array(merge_request.participants) + participants = ::Kaminari.paginate_array(merge_request.visible_participants(current_user)) present paginate(participants), with: Entities::UserBasic end @@ -290,7 +290,7 @@ module API desc 'Get the commits of a merge request' do success Entities::Commit end - get ':id/merge_requests/:merge_request_iid/commits', feature_category: :code_review do + get ':id/merge_requests/:merge_request_iid/commits', feature_category: :code_review, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) commits = @@ -303,7 +303,7 @@ module API desc 'Get the context commits of a merge request' do success Entities::Commit end - get ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do + get ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review, urgency: :high do merge_request = find_merge_request_with_access(params[:merge_request_iid]) project = merge_request.project @@ -371,7 +371,7 @@ module API desc 'Show the merge request changes' do success Entities::MergeRequestChanges end - get ':id/merge_requests/:merge_request_iid/changes', feature_category: :code_review do + get ':id/merge_requests/:merge_request_iid/changes', feature_category: :code_review, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) present merge_request, @@ -422,7 +422,7 @@ module API use :optional_params at_least_one_of(*::API::MergeRequests.update_params_at_least_one_of) end - put ':id/merge_requests/:merge_request_iid', feature_category: :code_review do + put ':id/merge_requests/:merge_request_iid', feature_category: :code_review, urgency: :low do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20772') merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request) @@ -454,7 +454,7 @@ module API optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge' end - put ':id/merge_requests/:merge_request_iid/merge', feature_category: :code_review do + put ':id/merge_requests/:merge_request_iid/merge', feature_category: :code_review, urgency: :low do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/4796') merge_request = find_project_merge_request(params[:merge_request_iid]) @@ -524,7 +524,7 @@ module API params do optional :skip_ci, type: Boolean, desc: 'Do not create CI pipeline' end - put ':id/merge_requests/:merge_request_iid/rebase', feature_category: :code_review do + put ':id/merge_requests/:merge_request_iid/rebase', feature_category: :code_review, urgency: :low do merge_request = find_project_merge_request(params[:merge_request_iid]) authorize_push_to_merge_request!(merge_request) @@ -543,7 +543,7 @@ module API params do use :pagination end - get ':id/merge_requests/:merge_request_iid/closes_issues', feature_category: :code_review do + get ':id/merge_requests/:merge_request_iid/closes_issues', feature_category: :code_review, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) issues = ::Kaminari.paginate_array(merge_request.visible_closing_issues_for(current_user)) issues = paginate(issues) diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 656eaa2b2bb..93ef77d5a62 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -7,6 +7,11 @@ module API before { authenticate! } + urgency :low, [ + '/projects/:id/merge_requests/:noteable_id/notes', + '/projects/:id/merge_requests/:noteable_id/notes/:note_id' + ] + Helpers::NotesHelpers.feature_category_per_noteable_type.each do |noteable_type, feature_category| parent_type = noteable_type.parent_class.to_s.underscore noteables_str = noteable_type.to_s.underscore.pluralize @@ -74,7 +79,7 @@ module API post ":id/#{noteables_str}/:noteable_id/notes", feature_category: feature_category do allowlist = Gitlab::CurrentSettings.current_application_settings.notes_create_limit_allowlist - check_rate_limit! :notes_create, [current_user], allowlist + check_rate_limit! :notes_create, scope: current_user, users_allowlist: allowlist noteable = find_noteable(noteable_type, params[:noteable_id]) opts = { diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index e01c195dbc4..843f72c0e1d 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -25,7 +25,7 @@ module API detail 'This feature was introduced in GitLab 10.6.' end get ':id/export/download' do - check_rate_limit! :project_download_export, [current_user, user_project] + check_rate_limit! :project_download_export, scope: [current_user, user_project] if user_project.export_file_exists? if user_project.export_archive_exists? @@ -49,7 +49,7 @@ module API end end post ':id/export' do - check_rate_limit! :project_export, [current_user] + check_rate_limit! :project_export, scope: current_user user_project.remove_exports diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index e7c532e2483..7bdcaa5a26f 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -81,7 +81,7 @@ module API post 'import' do require_gitlab_workhorse! - check_rate_limit! :project_import, [current_user, :project_import] + check_rate_limit! :project_import, scope: [current_user, :project_import] Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/21041') @@ -107,7 +107,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - desc 'Get a project export status' do + desc 'Get a project import status' do detail 'This feature was introduced in GitLab 10.6.' success Entities::ProjectImportStatus end @@ -135,7 +135,7 @@ module API post 'remote-import' do not_found! unless ::Feature.enabled?(:import_project_from_remote_file) - check_rate_limit! :project_import, [current_user, :project_import] + check_rate_limit! :project_import, scope: [current_user, :project_import] response = ::Import::GitlabProjects::CreateProjectFromRemoteFileService.new( current_user, diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 9f0077d23d8..67f0b7af7a9 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -270,7 +270,7 @@ module API attrs = translate_params_for_compatibility(attrs) filter_attributes_using_license!(attrs) - validate_git_import_url!(params[:import_url], import_enabled: check_import_by_url_is_enabled) + validate_git_import_url!(params[:import_url]) { check_import_by_url_is_enabled } project = ::Projects::CreateService.new(current_user, attrs).execute diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 2dd0e40afba..fc976c23726 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -10,6 +10,32 @@ module API helpers ::API::Helpers::HeadersHelpers + helpers do + params :release_params do + requires :version, + type: String, + regexp: Gitlab::Regex.unbounded_semver_regex, + desc: 'The version of the release, using the semantic versioning format' + + optional :from, + type: String, + desc: 'The first commit in the range of commits to use for the changelog' + + optional :to, + type: String, + desc: 'The last commit in the range of commits to use for the changelog' + + optional :date, + type: DateTime, + desc: 'The date and time of the release' + + optional :trailer, + type: String, + desc: 'The Git trailer to use for determining if commits are to be included in the changelog', + default: ::Repositories::ChangelogService::DEFAULT_TRAILER + end + end + before { authorize! :download_code, user_project } feature_category :source_code_management @@ -19,7 +45,7 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do - include ::Gitlab::RateLimitHelpers + include Gitlab::RepositoryArchiveRateLimiter def handle_project_member_errors(errors) if errors[:project_access].any? @@ -124,8 +150,8 @@ module API optional :path, type: String, desc: 'Subfolder of the repository to be downloaded' end get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do - if archive_rate_limit_reached?(current_user, user_project) - render_api_error!({ error: ::Gitlab::RateLimitHelpers::ARCHIVE_RATE_LIMIT_REACHED_MESSAGE }, 429) + check_archive_rate_limit!(current_user, user_project) do + render_api_error!({ error: _('This archive has been requested too many times. Try again later.') }, 429) end not_acceptable! if Gitlab::HotlinkingDetector.intercept_hotlinking?(request) @@ -208,36 +234,33 @@ module API end end - desc 'Generates a changelog section for a release' do - detail 'This feature was introduced in GitLab 13.9' + desc 'Generates a changelog section for a release and returns it' do + detail 'This feature was introduced in GitLab 14.6' end params do - requires :version, - type: String, - regexp: Gitlab::Regex.unbounded_semver_regex, - desc: 'The version of the release, using the semantic versioning format' - - optional :from, - type: String, - desc: 'The first commit in the range of commits to use for the changelog' + use :release_params + end + get ':id/repository/changelog' do + service = ::Repositories::ChangelogService.new( + user_project, + current_user, + **declared_params(include_missing: false) + ) + changelog = service.execute(commit_to_changelog: false) - optional :to, - type: String, - desc: 'The last commit in the range of commits to use for the changelog' + present changelog, with: Entities::Changelog + end - optional :date, - type: DateTime, - desc: 'The date and time of the release' + desc 'Generates a changelog section for a release and commits it in a changelog file' do + detail 'This feature was introduced in GitLab 13.9' + end + params do + use :release_params optional :branch, type: String, desc: 'The branch to commit the changelog changes to' - optional :trailer, - type: String, - desc: 'The Git trailer to use for determining if commits are to be included in the changelog', - default: ::Repositories::ChangelogService::DEFAULT_TRAILER - optional :file, type: String, desc: 'The file to commit the changelog changes to', @@ -261,7 +284,7 @@ module API **declared_params(include_missing: false) ) - service.execute + service.execute(commit_to_changelog: true) status(200) rescue Gitlab::Changelog::Error => ex render_api_error!("Failed to generate the changelog: #{ex.message}", 422) @@ -269,3 +292,5 @@ module API end end end + +API::Repositories.prepend_mod diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb index 33589f6c393..cd56809f45a 100644 --- a/lib/api/resource_label_events.rb +++ b/lib/api/resource_label_events.rb @@ -24,7 +24,7 @@ module API use :pagination end - get ":id/#{eventables_str}/:eventable_id/resource_label_events", feature_category: feature_category do + get ":id/#{eventables_str}/:eventable_id/resource_label_events", feature_category: feature_category, urgency: :low do eventable = find_noteable(eventable_type, params[:eventable_id]) events = eventable.resource_label_events.inc_relations diff --git a/lib/api/resource_milestone_events.rb b/lib/api/resource_milestone_events.rb index c0483ca59c2..04d71faa56a 100644 --- a/lib/api/resource_milestone_events.rb +++ b/lib/api/resource_milestone_events.rb @@ -26,7 +26,7 @@ module API use :pagination end - get ":id/#{eventables_str}/:eventable_id/resource_milestone_events", feature_category: feature_category do + get ":id/#{eventables_str}/:eventable_id/resource_milestone_events", feature_category: feature_category, urgency: :low do eventable = find_noteable(eventable_type, params[:eventable_id]) events = ResourceMilestoneEventFinder.new(current_user, eventable).execute diff --git a/lib/api/resource_state_events.rb b/lib/api/resource_state_events.rb index 9b6f6a954b4..4b92f320d6f 100644 --- a/lib/api/resource_state_events.rb +++ b/lib/api/resource_state_events.rb @@ -25,7 +25,7 @@ module API use :pagination end - get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events", feature_category: feature_category do + get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events", feature_category: feature_category, urgency: :low do eventable = find_noteable(eventable_class, params[:eventable_iid]) events = ResourceStateEventFinder.new(current_user, eventable).execute diff --git a/lib/api/search.rb b/lib/api/search.rb index 3c5801366a8..fbdbe3476db 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -8,6 +8,10 @@ module API feature_category :global_search + rescue_from ActiveRecord::QueryCanceled do |e| + render_api_error!({ error: 'Request timed out' }, 408) + end + helpers do SCOPE_ENTITY = { merge_requests: Entities::MergeRequestBasic, diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 12e1d21a00d..508ccdb4b33 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -154,7 +154,6 @@ module API optional :spam_check_endpoint_enabled, type: Boolean, desc: 'Enable Spam Check via external API endpoint' given spam_check_endpoint_enabled: ->(val) { val } do requires :spam_check_endpoint_url, type: String, desc: 'The URL of the external Spam Check service endpoint' - requires :spam_check_api_key, type: String, desc: 'The API key used by GitLab for accessing the Spam Check service endpoint' end optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb index 7921700e365..0697169b49a 100644 --- a/lib/api/suggestions.rb +++ b/lib/api/suggestions.rb @@ -14,7 +14,7 @@ module API requires :id, type: String, desc: 'The suggestion ID' optional :commit_message, type: String, desc: "A custom commit message to use instead of the default generated message or the project's default message" end - put ':id/apply' do + put ':id/apply', urgency: :low do suggestion = Suggestion.find_by_id(params[:id]) if suggestion @@ -31,7 +31,7 @@ module API requires :ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: "An array of suggestion ID's" optional :commit_message, type: String, desc: "A custom commit message to use instead of the default generated message or the project's default message" end - put 'batch_apply' do + put 'batch_apply', urgency: :low do ids = params[:ids] suggestions = Suggestion.id_in(ids) diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb index f6dfbcafbb6..29e71611092 100644 --- a/lib/api/terraform/state.rb +++ b/lib/api/terraform/state.rb @@ -11,6 +11,13 @@ module API default_format :json + rescue_from( + ::ActiveRecord::RecordNotUnique, + ::PG::UniqueViolation + ) do |e| + render_api_error!(e.message, 422) + end + before do authenticate! authorize! :read_terraform_state, user_project diff --git a/lib/api/topics.rb b/lib/api/topics.rb index bd28ebe58a9..b9c2bcc2da8 100644 --- a/lib/api/topics.rb +++ b/lib/api/topics.rb @@ -69,6 +69,8 @@ module API topic = ::Projects::Topic.find(params[:id]) + topic.remove_avatar! if params.key?(:avatar) && params[:avatar].nil? + if topic.update(declared_params(include_missing: false)) present topic, with: Entities::Projects::Topic else diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index 677d0840208..d6c026963e1 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -101,8 +101,6 @@ module API # of time after a Gitaly timeout, to mitigate frequent Gitaly timeouts # for some Commit diffs. def diff_files(commit) - return commit.diffs.diff_files unless Feature.enabled?(:api_v3_commits_skip_diff_files, commit.project, default_enabled: :yaml) - cache_key = [ GITALY_TIMEOUT_CACHE_KEY, commit.project.id, diff --git a/lib/api/validations/types/workhorse_file.rb b/lib/api/validations/types/workhorse_file.rb index e65e94fc8db..23f402596d5 100644 --- a/lib/api/validations/types/workhorse_file.rb +++ b/lib/api/validations/types/workhorse_file.rb @@ -5,6 +5,7 @@ module API module Types class WorkhorseFile def self.parse(value) + return if value.blank? raise "#{value.class} is not an UploadedFile type" unless parsed?(value) value diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb index c63453f94ca..7ea32c4b1e7 100644 --- a/lib/banzai/filter/base_sanitization_filter.rb +++ b/lib/banzai/filter/base_sanitization_filter.rb @@ -59,7 +59,11 @@ module Banzai def remove_rel lambda do |env| if env[:node_name] == 'a' - env[:node].remove_attribute('rel') + # we allow rel="license" to support the Rel-license microformat + # http://microformats.org/wiki/rel-license + unless env[:node].attribute('rel')&.value == 'license' + env[:node].remove_attribute('rel') + end end end end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 67019454e44..dc65e2abb46 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -112,7 +112,9 @@ module Banzai def add_nofollow!(uri, node) if SCHEMES.include?(uri&.scheme) + license = true if node.attribute('rel')&.value == 'license' node.set_attribute('rel', 'nofollow noreferrer noopener') + node.kwattr_append('rel', 'license') if license node.set_attribute('target', '_blank') end end diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb index 39c42ceaf9b..00a38f02141 100644 --- a/lib/banzai/filter/footnote_filter.rb +++ b/lib/banzai/filter/footnote_filter.rb @@ -21,9 +21,9 @@ module Banzai FOOTNOTE_LI_REFERENCE_PATTERN = /\A#{FOOTNOTE_ID_PREFIX}.+\z/.freeze FOOTNOTE_LINK_REFERENCE_PATTERN = /\A#{FOOTNOTE_LINK_ID_PREFIX}.+\z/.freeze - CSS_SECTION = "ol > li a[href^=\"\##{FOOTNOTE_LINK_ID_PREFIX}\"]" + CSS_SECTION = "section[data-footnotes]" XPATH_SECTION = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_SECTION).freeze - CSS_FOOTNOTE = 'sup > a[id]' + CSS_FOOTNOTE = 'sup > a[data-footnote-ref]' XPATH_FOOTNOTE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_FOOTNOTE).freeze # only needed when feature flag use_cmark_renderer is turned off @@ -37,39 +37,47 @@ module Banzai XPATH_SECTION_OLD = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_SECTION_OLD).freeze def call - xpath_section = Feature.enabled?(:use_cmark_renderer) ? XPATH_SECTION : XPATH_SECTION_OLD - return doc unless first_footnote = doc.at_xpath(xpath_section) + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) + # Sanitization stripped off the section class - add it back in + return doc unless section_node = doc.at_xpath(XPATH_SECTION) - # Sanitization stripped off the section wrapper - add it back in - if Feature.enabled?(:use_cmark_renderer) - first_footnote.parent.parent.parent.wrap('<section class="footnotes" data-footnotes>') + section_node.append_class('footnotes') else + return doc unless first_footnote = doc.at_xpath(XPATH_SECTION_OLD) + return doc unless first_footnote.parent + first_footnote.parent.wrap('<section class="footnotes">') end rand_suffix = "-#{random_number}" modified_footnotes = {} - doc.xpath(XPATH_FOOTNOTE).each do |link_node| - if Feature.enabled?(:use_cmark_renderer) + xpath_footnote = if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) + XPATH_FOOTNOTE + else + Gitlab::Utils::Nokogiri.css_to_xpath('sup > a[id]') + end + + doc.xpath(xpath_footnote).each do |link_node| + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX) ref_num.gsub!(/[[:punct:]]/, '\\\\\&') else ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX_OLD) end - node_xpath = Gitlab::Utils::Nokogiri.css_to_xpath("li[id=#{fn_id(ref_num)}]") + css = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? "section[data-footnotes] li[id=#{fn_id(ref_num)}]" : "li[id=#{fn_id(ref_num)}]" + node_xpath = Gitlab::Utils::Nokogiri.css_to_xpath(css) footnote_node = doc.at_xpath(node_xpath) if footnote_node || modified_footnotes[ref_num] - next if Feature.disabled?(:use_cmark_renderer) && !INTEGER_PATTERN.match?(ref_num) + next if Feature.disabled?(:use_cmark_renderer, default_enabled: :yaml) && !INTEGER_PATTERN.match?(ref_num) link_node[:href] += rand_suffix link_node[:id] += rand_suffix # Sanitization stripped off class - add it back in link_node.parent.append_class('footnote-ref') - link_node['data-footnote-ref'] = nil if Feature.enabled?(:use_cmark_renderer) unless modified_footnotes[ref_num] footnote_node[:id] += rand_suffix @@ -78,7 +86,6 @@ module Banzai if backref_node backref_node[:href] += rand_suffix backref_node.append_class('footnote-backref') - backref_node['data-footnote-backref'] = nil if Feature.enabled?(:use_cmark_renderer) end modified_footnotes[ref_num] = true @@ -96,12 +103,12 @@ module Banzai end def fn_id(num) - prefix = Feature.enabled?(:use_cmark_renderer) ? FOOTNOTE_ID_PREFIX : FOOTNOTE_ID_PREFIX_OLD + prefix = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? FOOTNOTE_ID_PREFIX : FOOTNOTE_ID_PREFIX_OLD "#{prefix}#{num}" end def fnref_id(num) - prefix = Feature.enabled?(:use_cmark_renderer) ? FOOTNOTE_LINK_ID_PREFIX : FOOTNOTE_LINK_ID_PREFIX_OLD + prefix = Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? FOOTNOTE_LINK_ID_PREFIX : FOOTNOTE_LINK_ID_PREFIX_OLD "#{prefix}#{num}" end end diff --git a/lib/banzai/filter/issuable_reference_expansion_filter.rb b/lib/banzai/filter/issuable_reference_expansion_filter.rb new file mode 100644 index 00000000000..6822e36c9be --- /dev/null +++ b/lib/banzai/filter/issuable_reference_expansion_filter.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML filter that appends extra information to issuable links. + # Runs as a post-process filter as issuable might change while + # Markdown is in the cache. + # + # This filter supports cross-project references. + class IssuableReferenceExpansionFilter < HTML::Pipeline::Filter + include Gitlab::Utils::StrongMemoize + + VISIBLE_STATES = %w(closed merged).freeze + + def call + return doc unless context[:issuable_reference_expansion_enabled] + + context = RenderContext.new(project, current_user) + extractor = Banzai::IssuableExtractor.new(context) + issuables = extractor.extract([doc]) + + issuables.each do |node, issuable| + next if !can_read_cross_project? && cross_referenced?(issuable) + next unless should_expand?(node, issuable) + + case node.attr('data-reference-format') + when '+' + expand_reference_with_title_and_state(node, issuable) + else + expand_reference_with_state(node, issuable) + end + end + + doc + end + + private + + # Example: Issue Title (#123 - closed) + def expand_reference_with_title_and_state(node, issuable) + node.content = "#{issuable.title.truncate(50)} (#{node.content}" + node.content += " - #{issuable_state_text(issuable)}" if VISIBLE_STATES.include?(issuable.state) + node.content += ')' + end + + # Example: #123 (closed) + def expand_reference_with_state(node, issuable) + node.content += " (#{issuable_state_text(issuable)})" + end + + def issuable_state_text(issuable) + moved_issue?(issuable) ? s_("IssuableStatus|moved") : issuable.state + end + + def moved_issue?(issuable) + issuable.instance_of?(Issue) && issuable.moved? + end + + def should_expand?(node, issuable) + # We add this extra check to avoid unescaping HTML and generating reference link text for every reference + return unless node.attr('data-reference-format').present? || VISIBLE_STATES.include?(issuable.state) + + CGI.unescapeHTML(node.inner_html) == issuable.reference_link_text(project || group) + end + + def cross_referenced?(issuable) + return true if issuable.project != project + return true if issuable.respond_to?(:group) && issuable.group != group + + false + end + + def can_read_cross_project? + strong_memoize(:can_read_cross_project) do + Ability.allowed?(current_user, :read_cross_project) + end + end + + def current_user + context[:current_user] + end + + def project + context[:project] + end + + def group + context[:group] + end + end + end +end diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb deleted file mode 100644 index a88629ac105..00000000000 --- a/lib/banzai/filter/issuable_state_filter.rb +++ /dev/null @@ -1,66 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that appends state information to issuable links. - # Runs as a post-process filter as issuable state might change while - # Markdown is in the cache. - # - # This filter supports cross-project references. - class IssuableStateFilter < HTML::Pipeline::Filter - VISIBLE_STATES = %w(closed merged).freeze - - def call - return doc unless context[:issuable_state_filter_enabled] - - context = RenderContext.new(project, current_user) - extractor = Banzai::IssuableExtractor.new(context) - issuables = extractor.extract([doc]) - - issuables.each do |node, issuable| - next if !can_read_cross_project? && cross_referenced?(issuable) - - if VISIBLE_STATES.include?(issuable.state) && issuable_reference?(node.inner_html, issuable) - state = moved_issue?(issuable) ? s_("IssuableStatus|moved") : issuable.state - node.content += " (#{state})" - end - end - - doc - end - - private - - def moved_issue?(issuable) - issuable.instance_of?(Issue) && issuable.moved? - end - - def issuable_reference?(text, issuable) - CGI.unescapeHTML(text) == issuable.reference_link_text(project || group) - end - - def cross_referenced?(issuable) - return true if issuable.project != project - return true if issuable.respond_to?(:group) && issuable.group != group - - false - end - - def can_read_cross_project? - Ability.allowed?(current_user, :read_cross_project) - end - - def current_user - context[:current_user] - end - - def project - context[:project] - end - - def group - context[:group] - end - end - end -end diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb index a25ebedf029..dc94e3c925a 100644 --- a/lib/banzai/filter/markdown_engines/common_mark.rb +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -42,11 +42,11 @@ module Banzai def initialize(context) @context = context - @renderer = Banzai::Renderer::CommonMark::HTML.new(options: render_options) if Feature.disabled?(:use_cmark_renderer) + @renderer = Banzai::Renderer::CommonMark::HTML.new(options: render_options) if Feature.disabled?(:use_cmark_renderer, default_enabled: :yaml) end def render(text) - if Feature.enabled?(:use_cmark_renderer) + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) CommonMarker.render_html(text, render_options, extensions) else doc = CommonMarker.render_doc(text, PARSE_OPTIONS, extensions) @@ -58,7 +58,7 @@ module Banzai private def extensions - if Feature.enabled?(:use_cmark_renderer) + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) EXTENSIONS else EXTENSIONS + [ @@ -72,7 +72,7 @@ module Banzai end def render_options_no_sourcepos - Feature.enabled?(:use_cmark_renderer) ? RENDER_OPTIONS_C : RENDER_OPTIONS_RUBY + Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) ? RENDER_OPTIONS_C : RENDER_OPTIONS_RUBY end def render_options_sourcepos diff --git a/lib/banzai/filter/markdown_post_escape_filter.rb b/lib/banzai/filter/markdown_post_escape_filter.rb index ccffe1bfbb1..b979b7573ae 100644 --- a/lib/banzai/filter/markdown_post_escape_filter.rb +++ b/lib/banzai/filter/markdown_post_escape_filter.rb @@ -42,7 +42,7 @@ module Banzai private def lang_tag - if Feature.enabled?(:use_cmark_renderer) + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) Gitlab::Utils::Nokogiri.css_to_xpath('pre') else Gitlab::Utils::Nokogiri.css_to_xpath('code') diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index 53dafe45fb3..6859d67c9d8 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -39,7 +39,7 @@ module Banzai code[:class] = INLINE_CLASSES code[STYLE_ATTRIBUTE] = 'inline' - closing.content = closing.content[1..-1] + closing.content = closing.content[1..] opening.content = opening.content[0..-2] end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index e67cdc7df12..3f160960d23 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -26,7 +26,7 @@ module Banzai def lang_tag @lang_tag ||= - if Feature.enabled?(:use_cmark_renderer) + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) Gitlab::Utils::Nokogiri.css_to_xpath('pre[lang="plantuml"] > code').freeze else Gitlab::Utils::Nokogiri.css_to_xpath('pre > code[lang="plantuml"]').freeze diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb index cae0a8b424a..7a23326bafa 100644 --- a/lib/banzai/filter/references/abstract_reference_filter.rb +++ b/lib/banzai/filter/references/abstract_reference_filter.rb @@ -205,6 +205,8 @@ module Banzai data_attributes = data_attributes_for(link_content || match, parent, object, link_content: !!link_content, link_reference: link_reference) + data_attributes[:reference_format] = matches[:format] if matches.names.include?("format") + data = data_attribute(data_attributes) url = diff --git a/lib/banzai/filter/references/reference_cache.rb b/lib/banzai/filter/references/reference_cache.rb index 259958f1598..c8370d5f9c1 100644 --- a/lib/banzai/filter/references/reference_cache.rb +++ b/lib/banzai/filter/references/reference_cache.rb @@ -29,7 +29,7 @@ module Banzai @references_per_parent[parent_type] ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } - prepare_doc_for_scan(filter.doc).to_enum(:scan, regex).each do + prepare_doc_for_scan.to_enum(:scan, regex).each do parent_path = if parent_type == :project full_project_path($~[:namespace], $~[:project]) else @@ -184,14 +184,12 @@ module Banzai Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} end - def prepare_doc_for_scan(doc) - html = if Feature.enabled?(:reference_cache_memoization, project, default_enabled: :yaml) - result[:rendered_html] ||= doc.to_html - else - doc.to_html - end + def prepare_doc_for_scan + filter.requires_unescaping? ? unescape_html_entities(html_content) : html_content + end - filter.requires_unescaping? ? unescape_html_entities(html) : html + def html_content + result[:rendered_html] ||= filter.doc.to_html end def unescape_html_entities(text) diff --git a/lib/banzai/filter/references/reference_filter.rb b/lib/banzai/filter/references/reference_filter.rb index 6c2c993cc01..97ef71036a2 100644 --- a/lib/banzai/filter/references/reference_filter.rb +++ b/lib/banzai/filter/references/reference_filter.rb @@ -184,7 +184,9 @@ module Banzai end def unescape_link(href) - CGI.unescape(href) + # We cannot use CGI.unescape here because it also converts `+` to spaces. + # We need to keep the `+` for expanded reference formats. + Addressable::URI.unescape(href) end def unescape_html_entities(text) diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb index 04bbcabd93f..408e6dc685d 100644 --- a/lib/banzai/filter/repository_link_filter.rb +++ b/lib/banzai/filter/repository_link_filter.rb @@ -174,7 +174,7 @@ module Banzai def build_relative_path(path, request_path) return request_path if path.empty? return path unless request_path - return path[1..-1] if path.start_with?('/') + return path[1..] if path.start_with?('/') parts = request_path.split('/') diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 7afbc1a1c9c..d5f45ff7689 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -28,6 +28,13 @@ module Banzai allowlist[:attributes]['li'] = %w[id] allowlist[:transformers].push(self.class.remove_non_footnote_ids) + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) + # Allow section elements with data-footnotes attribute + allowlist[:elements].push('section') + allowlist[:attributes]['section'] = %w(data-footnotes) + allowlist[:attributes]['a'].push('data-footnote-ref', 'data-footnote-backref') + end + allowlist end @@ -54,7 +61,7 @@ module Banzai return unless node.name == 'a' || node.name == 'li' return unless node.has_attribute?('id') - if Feature.enabled?(:use_cmark_renderer) + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN else diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 66bd86c5bb4..9fcfcf4acc4 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -58,10 +58,10 @@ module Banzai sourcepos_attr = sourcepos ? "data-sourcepos=\"#{sourcepos}\"" : '' - highlighted = %(<pre #{sourcepos_attr} class="#{css_classes}" + highlighted = %(<div class="gl-relative markdown-code-block js-markdown-code"><pre #{sourcepos_attr} class="#{css_classes}" lang="#{language}" #{lang_params} - v-pre="true"><code>#{code}</code></pre>) + v-pre="true"><code>#{code}</code></pre><copy-code></copy-code></div>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) @@ -70,7 +70,7 @@ module Banzai private def parse_lang_params(node) - node = node.parent if Feature.enabled?(:use_cmark_renderer) + node = node.parent if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) # Commonmarker's FULL_INFO_STRING render option works with the space delimiter. # But the current behavior of GitLab's markdown renderer is different - it grabs everything as the single @@ -92,7 +92,7 @@ module Banzai language, language_params = language.split(LANG_PARAMS_DELIMITER, 2) - if Feature.enabled?(:use_cmark_renderer) + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) language_params = [node.attr('data-meta'), language_params].compact.join(' ') end diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 889574cf6bf..da2262cdf83 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -19,7 +19,7 @@ module Banzai # prevent unnecessary Gitaly calls from being made. Filter::UploadLinkFilter, Filter::RepositoryLinkFilter, - Filter::IssuableStateFilter, + Filter::IssuableReferenceExpansionFilter, Filter::SuggestionFilter ] end diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index 90414a875c6..eb3d551d1d7 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -123,7 +123,7 @@ module BulkImports def with_error_handling response = yield - raise ::BulkImports::NetworkError.new(response: response) unless response.success? + raise ::BulkImports::NetworkError.new("Unsuccessful response #{response.code} from #{response.request.path.path}", response: response) unless response.success? response rescue *Gitlab::HTTP::HTTP_ERRORS => e diff --git a/lib/bulk_imports/common/pipelines/badges_pipeline.rb b/lib/bulk_imports/common/pipelines/badges_pipeline.rb new file mode 100644 index 00000000000..33a24e61a3f --- /dev/null +++ b/lib/bulk_imports/common/pipelines/badges_pipeline.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Pipelines + class BadgesPipeline + include Pipeline + + extractor BulkImports::Common::Extractors::RestExtractor, + query: BulkImports::Common::Rest::GetBadgesQuery + + transformer Common::Transformers::ProhibitedAttributesTransformer + + def transform(context, data) + return if data.blank? + # Project badges API returns badges of both group and project kind. To avoid creation of duplicates for the group we skip group badges when it's a project. + return if context.entity.project? && group_badge?(data) + + { + name: data['name'], + link_url: data['link_url'], + image_url: data['image_url'] + } + end + + def load(context, data) + return if data.blank? + + if context.entity.project? + context.portable.project_badges.create!(data) + else + context.portable.badges.create!(data) + end + end + + private + + def group_badge?(data) + data['kind'] == 'group' + end + end + end + end +end diff --git a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb index 15e126e1646..49c16209661 100644 --- a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb @@ -8,6 +8,9 @@ module BulkImports include Gitlab::ImportExport::CommandLineUtil FILENAME = 'uploads.tar.gz' + AVATAR_PATTERN = %r{.*\/#{BulkImports::UploadsExportService::AVATAR_PATH}\/(?<identifier>.*)}.freeze + + AvatarLoadingError = Class.new(StandardError) def extract(context) download_service(tmp_dir, context).execute @@ -18,14 +21,18 @@ module BulkImports end def load(context, file_path) - dynamic_path = FileUploader.extract_dynamic_path(file_path) + avatar_path = AVATAR_PATTERN.match(file_path) + + return save_avatar(file_path) if avatar_path + + dynamic_path = file_uploader.extract_dynamic_path(file_path) return unless dynamic_path return if File.directory?(file_path) named_captures = dynamic_path.named_captures.symbolize_keys - UploadService.new(context.portable, File.open(file_path, 'r'), FileUploader, **named_captures).execute + UploadService.new(context.portable, File.open(file_path, 'r'), file_uploader, **named_captures).execute end def after_run(_) @@ -46,6 +53,24 @@ module BulkImports def tmp_dir @tmp_dir ||= Dir.mktmpdir('bulk_imports') end + + def file_uploader + @file_uploader ||= if context.entity.group? + NamespaceFileUploader + else + FileUploader + end + end + + def save_avatar(file_path) + File.open(file_path) do |avatar| + service = context.entity.update_service.new(portable, current_user, avatar: avatar) + + unless service.execute + raise AvatarLoadingError, portable.errors.full_messages.to_sentence + end + end + end end end end diff --git a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb index ccab0b979b2..6900835b14d 100644 --- a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb @@ -7,7 +7,9 @@ module BulkImports include Pipeline def extract(*) - BulkImports::Pipeline::ExtractedData.new(data: { url: url_from_parent_path(context.entity.source_full_path) }) + url = url_from_parent_path(context.entity.source_full_path) if source_wiki_exists? + + BulkImports::Pipeline::ExtractedData.new(data: { url: url }) end def transform(_, data) @@ -15,14 +17,15 @@ module BulkImports end def load(context, data) - return unless context.portable.wiki + return unless data&.dig(:url) + wiki = context.portable.wiki url = data[:url].sub("://", "://oauth2:#{context.configuration.access_token}@") Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?) - context.portable.wiki.ensure_repository - context.portable.wiki.repository.fetch_as_mirror(url) + wiki.ensure_repository + wiki.repository.fetch_as_mirror(url) end private @@ -36,6 +39,16 @@ module BulkImports def allow_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end + + def source_wiki_exists? + wikis = client.get(context.entity.wikis_url_path).parsed_response + + wikis.any? + end + + def client + BulkImports::Clients::HTTP.new(url: context.configuration.url, token: context.configuration.access_token) + end end end end diff --git a/lib/bulk_imports/groups/rest/get_badges_query.rb b/lib/bulk_imports/common/rest/get_badges_query.rb index 79ffdd9a1f6..60b2ebcc552 100644 --- a/lib/bulk_imports/groups/rest/get_badges_query.rb +++ b/lib/bulk_imports/common/rest/get_badges_query.rb @@ -1,16 +1,17 @@ # frozen_string_literal: true module BulkImports - module Groups + module Common module Rest module GetBadgesQuery extend self def to_h(context) + resource = context.entity.pluralized_name encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path) { - resource: ['groups', encoded_full_path, 'badges'].join('/'), + resource: [resource, encoded_full_path, 'badges'].join('/'), query: { page: context.tracker.next_page } diff --git a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb b/lib/bulk_imports/groups/pipelines/badges_pipeline.rb deleted file mode 100644 index 8569ff3f77a..00000000000 --- a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Pipelines - class BadgesPipeline - include Pipeline - - extractor BulkImports::Common::Extractors::RestExtractor, - query: BulkImports::Groups::Rest::GetBadgesQuery - - transformer Common::Transformers::ProhibitedAttributesTransformer - - def transform(_, data) - return if data.blank? - - { - name: data['name'], - link_url: data['link_url'], - image_url: data['image_url'] - } - end - - def load(context, data) - return if data.blank? - - context.group.badges.create!(data) - end - end - end - end -end diff --git a/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb deleted file mode 100644 index 6de8bbbc910..00000000000 --- a/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Pipelines - class GroupAvatarPipeline - include Pipeline - - ALLOWED_AVATAR_DOWNLOAD_TYPES = (AvatarUploader::MIME_WHITELIST + %w(application/octet-stream)).freeze - - GroupAvatarLoadingError = Class.new(StandardError) - - def extract(context) - context.extra[:tmpdir] = Dir.mktmpdir - - filepath = BulkImports::FileDownloadService.new( - configuration: context.configuration, - relative_url: "/groups/#{context.entity.encoded_source_full_path}/avatar", - dir: context.extra[:tmpdir], - file_size_limit: Avatarable::MAXIMUM_FILE_SIZE, - allowed_content_types: ALLOWED_AVATAR_DOWNLOAD_TYPES - ).execute - - BulkImports::Pipeline::ExtractedData.new(data: { filepath: filepath }) - end - - def load(context, data) - return if data.blank? - - File.open(data[:filepath]) do |avatar| - service = ::Groups::UpdateService.new( - portable, - current_user, - avatar: avatar - ) - - unless service.execute - raise GroupAvatarLoadingError, portable.errors.full_messages.first - end - end - end - - def after_run(_) - FileUtils.remove_entry(context.extra[:tmpdir]) if context.extra[:tmpdir].present? - end - end - end - end -end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index 241dd428dd5..1a3babe1679 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -11,10 +11,6 @@ module BulkImports pipeline: BulkImports::Groups::Pipelines::GroupPipeline, stage: 0 }, - avatar: { - pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline, - stage: 1 - }, subgroups: { pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, stage: 1 @@ -32,13 +28,17 @@ module BulkImports stage: 1 }, badges: { - pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, + pipeline: BulkImports::Common::Pipelines::BadgesPipeline, stage: 1 }, boards: { pipeline: BulkImports::Common::Pipelines::BoardsPipeline, stage: 2 }, + uploads: { + pipeline: BulkImports::Common::Pipelines::UploadsPipeline, + stage: 2 + }, finisher: { pipeline: BulkImports::Common::Pipelines::EntityFinisher, stage: 3 diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb index 6cc29d63919..d5475a8b324 100644 --- a/lib/bulk_imports/ndjson_pipeline.rb +++ b/lib/bulk_imports/ndjson_pipeline.rb @@ -10,7 +10,12 @@ module BulkImports ndjson_pipeline! def transform(context, data) + return unless data + relation_hash, relation_index = data + + return unless relation_hash + relation_definition = import_export_config.top_relation_tree(relation) relation_object = deep_transform_relation!(relation_hash, relation, relation_definition) do |key, hash| @@ -31,9 +36,7 @@ module BulkImports end def load(_, object) - return unless object - - object.save! unless object.persisted? + object&.save! end def deep_transform_relation!(relation_hash, relation_key, relation_definition, &block) diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb index 2aec496880f..04ac0916bbc 100644 --- a/lib/bulk_imports/projects/graphql/get_project_query.rb +++ b/lib/bulk_imports/projects/graphql/get_project_query.rb @@ -4,6 +4,7 @@ module BulkImports module Projects module Graphql module GetProjectQuery + extend Queryable extend self def to_s @@ -28,22 +29,6 @@ module BulkImports } GRAPHQL end - - def variables(context) - { full_path: context.entity.source_full_path } - end - - def base_path - %w[data project] - end - - def data_path - base_path - end - - def page_info_path - base_path << 'page_info' - end end end end diff --git a/lib/bulk_imports/projects/graphql/get_repository_query.rb b/lib/bulk_imports/projects/graphql/get_repository_query.rb index d3e377c1175..24efce9e276 100644 --- a/lib/bulk_imports/projects/graphql/get_repository_query.rb +++ b/lib/bulk_imports/projects/graphql/get_repository_query.rb @@ -4,6 +4,7 @@ module BulkImports module Projects module Graphql module GetRepositoryQuery + extend Queryable extend self def to_s @@ -15,22 +16,6 @@ module BulkImports } GRAPHQL end - - def variables(context) - { full_path: context.entity.source_full_path } - end - - def base_path - %w[data project] - end - - def data_path - base_path - end - - def page_info_path - base_path << 'page_info' - end end end end diff --git a/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb b/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb new file mode 100644 index 00000000000..1ba57789612 --- /dev/null +++ b/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Graphql + module GetSnippetRepositoryQuery + extend Queryable + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!) { + project(fullPath: $full_path) { + snippets { + page_info: pageInfo { + next_page: endCursor + has_next_page: hasNextPage + } + nodes { + title + createdAt + httpUrlToRepo + } + } + } + } + GRAPHQL + end + + def variables(context) + { + full_path: context.entity.source_full_path, + cursor: context.tracker.next_page, + per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE + } + end + + def base_path + %w[data project snippets] + end + + def data_path + base_path << 'nodes' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/graphql/queryable.rb b/lib/bulk_imports/projects/graphql/queryable.rb new file mode 100644 index 00000000000..a897632dff3 --- /dev/null +++ b/lib/bulk_imports/projects/graphql/queryable.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Graphql + module Queryable + def variables(context) + { full_path: context.entity.source_full_path } + end + + def base_path + %w[data project] + end + + def data_path + base_path + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb b/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb new file mode 100644 index 00000000000..1e54ca7017d --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class AutoDevopsPipeline + include NdjsonPipeline + + relation_name 'auto_devops' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/ci_pipelines_pipeline.rb b/lib/bulk_imports/projects/pipelines/ci_pipelines_pipeline.rb new file mode 100644 index 00000000000..4487835b88e --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/ci_pipelines_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class CiPipelinesPipeline + include NdjsonPipeline + + relation_name 'ci_pipelines' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/container_expiration_policy_pipeline.rb b/lib/bulk_imports/projects/pipelines/container_expiration_policy_pipeline.rb new file mode 100644 index 00000000000..796e2bd5293 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/container_expiration_policy_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ContainerExpirationPolicyPipeline + include NdjsonPipeline + + relation_name 'container_expiration_policy' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/pipeline_schedules_pipeline.rb b/lib/bulk_imports/projects/pipelines/pipeline_schedules_pipeline.rb new file mode 100644 index 00000000000..67053f4e0d4 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/pipeline_schedules_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class PipelineSchedulesPipeline + include NdjsonPipeline + + relation_name 'pipeline_schedules' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb new file mode 100644 index 00000000000..4d742225ff7 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ProjectAttributesPipeline + include Pipeline + + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer + + def extract(context) + download_service(tmp_dir, context).execute + decompression_service(tmp_dir).execute + project_attributes = json_decode(json_attributes) + + BulkImports::Pipeline::ExtractedData.new(data: project_attributes) + end + + def transform(_, data) + subrelations = config.portable_relations_tree.keys.map(&:to_s) + + Gitlab::ImportExport::AttributeCleaner.clean( + relation_hash: data, + relation_class: Project, + excluded_keys: config.relation_excluded_keys(:project) + ).except(*subrelations) + end + + def load(_, data) + portable.assign_attributes(data) + portable.reconcile_shared_runners_setting! + portable.drop_visibility_level! + portable.save! + end + + def after_run(_) + FileUtils.remove_entry(tmp_dir) + end + + def json_attributes + @json_attributes ||= File.read(File.join(tmp_dir, filename)) + end + + private + + def tmp_dir + @tmp_dir ||= Dir.mktmpdir + end + + def config + @config ||= BulkImports::FileTransfer.config_for(portable) + end + + def download_service(tmp_dir, context) + @download_service ||= BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path(BulkImports::FileTransfer::BaseConfig::SELF_RELATION), + dir: tmp_dir, + filename: compressed_filename + ) + end + + def decompression_service(tmp_dir) + @decompression_service ||= BulkImports::FileDecompressionService.new(dir: tmp_dir, filename: compressed_filename) + end + + def compressed_filename + "#{filename}.gz" + end + + def filename + "#{BulkImports::FileTransfer::BaseConfig::SELF_RELATION}.json" + end + + def json_decode(string) + Gitlab::Json.parse(string) + rescue JSON::ParserError => e + Gitlab::ErrorTracking.log_exception(e) + + raise BulkImports::Error, 'Incorrect JSON format' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_feature_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_feature_pipeline.rb new file mode 100644 index 00000000000..ff5437efeef --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/project_feature_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ProjectFeaturePipeline + include NdjsonPipeline + + relation_name 'project_feature' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/repository_pipeline.rb b/lib/bulk_imports/projects/pipelines/repository_pipeline.rb index 6bbd4d0688b..f5ccc1dd922 100644 --- a/lib/bulk_imports/projects/pipelines/repository_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/repository_pipeline.rb @@ -16,6 +16,8 @@ module BulkImports def load(context, data) url = data['httpUrlToRepo'] + return unless url.present? + url = url.sub("://", "://oauth2:#{context.configuration.access_token}@") project = context.portable diff --git a/lib/bulk_imports/projects/pipelines/service_desk_setting_pipeline.rb b/lib/bulk_imports/projects/pipelines/service_desk_setting_pipeline.rb new file mode 100644 index 00000000000..a50b5423366 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/service_desk_setting_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ServiceDeskSettingPipeline + include NdjsonPipeline + + relation_name 'service_desk_setting' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/snippets_pipeline.rb b/lib/bulk_imports/projects/pipelines/snippets_pipeline.rb new file mode 100644 index 00000000000..d543bcec383 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/snippets_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class SnippetsPipeline + include NdjsonPipeline + + relation_name 'snippets' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb b/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb new file mode 100644 index 00000000000..6d423717a51 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class SnippetsRepositoryPipeline + include Pipeline + + extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetSnippetRepositoryQuery + + def transform(_context, data) + data.tap do |d| + d['createdAt'] = DateTime.parse(data['createdAt']) + end + end + + def load(context, data) + return unless data['httpUrlToRepo'].present? + + oauth2_url = oauth2(data['httpUrlToRepo']) + validate_url(oauth2_url) + + matched_snippet = find_matched_snippet(data) + # Skip snippets that we couldn't find a match. Probably because more snippets were + # added after the migration had already started, namely after the SnippetsPipeline + # has already run. + return unless matched_snippet + + matched_snippet.create_repository + matched_snippet.repository.fetch_as_mirror(oauth2_url) + response = Snippets::RepositoryValidationService.new(nil, matched_snippet).execute + + # skips matched_snippet repository creation if repository is invalid + return cleanup_snippet_repository(matched_snippet) if response.error? + + Snippets::UpdateStatisticsService.new(matched_snippet).execute + end + + private + + def find_matched_snippet(data) + Snippet.find_by_project_title_trunc_created_at( + context.portable, data['title'], data['createdAt']) + end + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + + def oauth2(url) + url.sub("://", "://oauth2:#{context.configuration.access_token}@") + end + + def validate_url(url) + Gitlab::UrlBlocker.validate!( + url, + allow_local_network: allow_local_requests?, + allow_localhost: allow_local_requests?) + end + + def cleanup_snippet_repository(snippet) + snippet.repository.remove + snippet.snippet_repository.delete + snippet.repository.expire_exists_cache + end + end + end + end +end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index 9ccc9efff1d..0556395ca66 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -15,6 +15,10 @@ module BulkImports pipeline: BulkImports::Projects::Pipelines::RepositoryPipeline, stage: 1 }, + project_attributes: { + pipeline: BulkImports::Projects::Pipelines::ProjectAttributesPipeline, + stage: 1 + }, labels: { pipeline: BulkImports::Common::Pipelines::LabelsPipeline, stage: 2 @@ -23,10 +27,22 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::MilestonesPipeline, stage: 2 }, + badges: { + pipeline: BulkImports::Common::Pipelines::BadgesPipeline, + stage: 2 + }, issues: { pipeline: BulkImports::Projects::Pipelines::IssuesPipeline, stage: 3 }, + snippets: { + pipeline: BulkImports::Projects::Pipelines::SnippetsPipeline, + stage: 3 + }, + snippets_repository: { + pipeline: BulkImports::Projects::Pipelines::SnippetsRepositoryPipeline, + stage: 4 + }, boards: { pipeline: BulkImports::Common::Pipelines::BoardsPipeline, stage: 4 @@ -43,6 +59,22 @@ module BulkImports pipeline: BulkImports::Projects::Pipelines::ProtectedBranchesPipeline, stage: 4 }, + ci_pipelines: { + pipeline: BulkImports::Projects::Pipelines::CiPipelinesPipeline, + stage: 4 + }, + project_feature: { + pipeline: BulkImports::Projects::Pipelines::ProjectFeaturePipeline, + stage: 4 + }, + container_expiration_policy: { + pipeline: BulkImports::Projects::Pipelines::ContainerExpirationPolicyPipeline, + stage: 4 + }, + service_desk_setting: { + pipeline: BulkImports::Projects::Pipelines::ServiceDeskSettingPipeline, + stage: 4 + }, wiki: { pipeline: BulkImports::Common::Pipelines::WikiPipeline, stage: 5 @@ -51,6 +83,14 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::UploadsPipeline, stage: 5 }, + auto_devops: { + pipeline: BulkImports::Projects::Pipelines::AutoDevopsPipeline, + stage: 5 + }, + pipeline_schedules: { + pipeline: BulkImports::Projects::Pipelines::PipelineSchedulesPipeline, + stage: 5 + }, finisher: { pipeline: BulkImports::Common::Pipelines::EntityFinisher, stage: 6 diff --git a/lib/extracts_ref.rb b/lib/extracts_ref.rb index d130a9d6f82..daba0452318 100644 --- a/lib/extracts_ref.rb +++ b/lib/extracts_ref.rb @@ -113,7 +113,7 @@ module ExtractsRef best_match = valid_refs.max_by(&:length) # Partition the string into the ref and the path, ignoring the empty first value - id.partition(best_match)[1..-1] + id.partition(best_match)[1..] end def use_first_path_segment?(ref) diff --git a/lib/feature.rb b/lib/feature.rb index 8186fbc40fa..f301f206b46 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -83,7 +83,12 @@ class Feature # `persisted?` can potentially generate DB queries and also checks for inclusion # in an array of feature names (177 at last count), possibly reducing performance by half. # So we only perform the `persisted` check if `default_enabled: true` - !default_enabled || Feature.persisted_name?(feature.name) ? feature.enabled?(thing) : true + feature_value = !default_enabled || Feature.persisted_name?(feature.name) ? feature.enabled?(thing) : true + + # If we don't filter out this flag here we will enter an infinite loop + log_feature_flag_state(key, feature_value) if log_feature_flag_states?(key) + + feature_value end def disabled?(key, thing = nil, type: :development, default_enabled: false) @@ -153,6 +158,18 @@ class Feature @logger ||= Feature::Logger.build end + def log_feature_flag_states?(key) + Feature::Definition.log_states?(key) + end + + def log_feature_flag_state(key, feature_value) + logged_states[key] ||= feature_value + end + + def logged_states + RequestStore.fetch(:feature_flag_events) { {} } + end + private def flipper diff --git a/lib/feature/definition.rb b/lib/feature/definition.rb index cd2f5cb07a2..61f7e395769 100644 --- a/lib/feature/definition.rb +++ b/lib/feature/definition.rb @@ -82,6 +82,16 @@ class Feature attributes end + def for_upcoming_milestone? + return false unless milestone + + Gitlab::VersionInfo.parse(milestone + '.999') >= Gitlab.version_info + end + + def force_log_state_changes? + attributes[:log_state_changes] + end + class << self def paths @paths ||= [Rails.root.join('config', 'feature_flags', '**', '*.yml')] @@ -106,6 +116,14 @@ class Feature definitions.has_key?(key.to_sym) end + def log_states?(key) + return false if key == :feature_flag_state_logs + return false if Feature.disabled?(:feature_flag_state_logs, type: :ops) + return false unless (feature = get(key)) + + feature.force_log_state_changes? || feature.for_upcoming_milestone? + end + def valid_usage!(key, type:, default_enabled:) if definition = get(key) definition.valid_usage!(type_in_code: type, default_enabled_in_code: default_enabled) diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index 70e5b523adf..2ce078b2f02 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -58,6 +58,7 @@ class Feature introduced_by_url rollout_issue_url milestone + log_state_changes type group default_enabled diff --git a/lib/flowdock/git/builder.rb b/lib/flowdock/git/builder.rb index 6f4428d1f42..88d9814950a 100644 --- a/lib/flowdock/git/builder.rb +++ b/lib/flowdock/git/builder.rb @@ -51,7 +51,7 @@ module Flowdock end def body - content = @commit[:message][first_line.size..-1] + content = @commit[:message][first_line.size..] content.strip! if content "<pre>#{content}</pre>" unless content.empty? end diff --git a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb index e343e2dcf91..58d3257d07e 100644 --- a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb +++ b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb @@ -12,11 +12,11 @@ module Gitlab class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if metric is for ee' def create_metrics - weekly_params = ["#{key_path}_weekly", '--dir', '7d'] + weekly_params = ["#{key_path}_weekly", '--dir', '7d', '--class_name', 'RedisHLLMetric'] weekly_params << '--ee' if ee? Gitlab::UsageMetricDefinitionGenerator.start(weekly_params) - monthly_params = ["#{key_path}_monthly", '--dir', '28d'] + monthly_params = ["#{key_path}_monthly", '--dir', '28d', '--class_name', 'RedisHLLMetric'] monthly_params << '--ee' if ee? Gitlab::UsageMetricDefinitionGenerator.start(monthly_params) end diff --git a/lib/generators/gitlab/usage_metric_definition_generator.rb b/lib/generators/gitlab/usage_metric_definition_generator.rb index bd34ab0a16f..4ddbe8b9f09 100644 --- a/lib/generators/gitlab/usage_metric_definition_generator.rb +++ b/lib/generators/gitlab/usage_metric_definition_generator.rb @@ -35,6 +35,7 @@ module Gitlab class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if metric is for ee' class_option :dir, type: :string, desc: "Indicates the metric location. It must be one of: #{VALID_INPUT_DIRS.join(', ')}" + class_option :class_name, type: :string, optional: true, desc: 'Instrumentation class name, e.g.: CountIssues' argument :key_paths, type: :array, desc: 'Unique JSON key paths for the metrics' @@ -66,6 +67,10 @@ module Gitlab Gitlab::VERSION.match('(\d+\.\d+)').captures.first end + def class_name + options[:class_name] + end + private def metric_name_suggestion(key_path) diff --git a/lib/generators/gitlab/usage_metric_generator.rb b/lib/generators/gitlab/usage_metric_generator.rb index c0fdcf21f20..0656dfbc312 100644 --- a/lib/generators/gitlab/usage_metric_generator.rb +++ b/lib/generators/gitlab/usage_metric_generator.rb @@ -5,9 +5,9 @@ require 'rails/generators' module Gitlab class UsageMetricGenerator < Rails::Generators::Base CE_DIR = 'lib/gitlab/usage/metrics/instrumentations' - EE_DIR = 'ee/lib/ee/gitlab/usage/metrics/instrumentations' + EE_DIR = 'ee/lib/gitlab/usage/metrics/instrumentations' SPEC_CE_DIR = 'spec/lib/gitlab/usage/metrics/instrumentations' - SPEC_EE_DIR = 'ee/spec/lib/ee/gitlab/usage/metrics/instrumentations' + SPEC_EE_DIR = 'ee/spec/lib/gitlab/usage/metrics/instrumentations' ALLOWED_SUPERCLASSES = { generic: 'Generic', diff --git a/lib/gitlab/abuse.rb b/lib/gitlab/abuse.rb new file mode 100644 index 00000000000..cc95d3c1e0c --- /dev/null +++ b/lib/gitlab/abuse.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Abuse + CONFIDENCE_LEVELS = { + certain: 1.0, + likely: 0.8, + uncertain: 0.5, + unknown: 0.0 + }.freeze + + class << self + def confidence(rating) + CONFIDENCE_LEVELS.fetch(rating.to_sym) + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb index c8b11ecb4a8..2441b7a7497 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb @@ -52,3 +52,5 @@ module Gitlab end end end + +Gitlab::Analytics::CycleAnalytics::Aggregated::DataCollector.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::Aggregated::DataCollector') diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb index 181ee20948b..bf8f9f4dcc7 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb @@ -15,7 +15,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def seconds - @query = @query.select(median_duration_in_seconds.as('median')).reorder(nil) + @query = @query.select(duration_in_seconds(percentile_cont).as('median')).reorder(nil) result = @query.take || {} result['median'] || nil diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb index 7dce757cdc8..1742d396c10 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb @@ -36,7 +36,7 @@ module Gitlab def serialized_records strong_memoize(:serialized_records) do - records = ordered_and_limited_query.select(stage_event_model.arel_table[Arel.star], duration.as('total_time')) + records = ordered_and_limited_query.select(stage_event_model.arel_table[Arel.star], duration_in_seconds.as('total_time')) yield records if block_given? issuables_and_records = load_issuables(records) diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb index f23d1832df9..b00925495f2 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb @@ -27,13 +27,13 @@ module Gitlab end end - def median_duration_in_seconds - Arel::Nodes::Extract.new(percentile_cont, :epoch) - end - def in_progress? params[:end_event_filter] == :in_progress end + + def duration_in_seconds(duration_expression = duration) + Arel::Nodes::Extract.new(duration_expression, :epoch) + end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index 945cecfcf8c..e99ad42b0b2 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -19,7 +19,7 @@ module Gitlab raise NotImplementedError end - def markdown_description + def html_description(options = {}) self.class.name end diff --git a/lib/gitlab/anonymous_session.rb b/lib/gitlab/anonymous_session.rb index 911825eef3a..e58240e16b4 100644 --- a/lib/gitlab/anonymous_session.rb +++ b/lib/gitlab/anonymous_session.rb @@ -2,27 +2,29 @@ module Gitlab class AnonymousSession + include ::Gitlab::Redis::SessionsStoreHelper + def initialize(remote_ip) @remote_ip = remote_ip end def count_session_ip - Gitlab::Redis::SharedState.with do |redis| - redis.pipelined do - redis.incr(session_lookup_name) - redis.expire(session_lookup_name, 24.hours) + redis_store_class.with do |redis| + redis.pipelined do |pipeline| + pipeline.incr(session_lookup_name) + pipeline.expire(session_lookup_name, 24.hours) end end end def session_count - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| redis.get(session_lookup_name).to_i end end def cleanup_session_per_ip_count - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| redis.del(session_lookup_name) end end @@ -32,7 +34,7 @@ module Gitlab attr_reader :remote_ip def session_lookup_name - @session_lookup_name ||= "#{Gitlab::Redis::SharedState::IP_SESSIONS_LOOKUP_NAMESPACE}:#{remote_ip}" + @session_lookup_name ||= "#{Gitlab::Redis::Sessions::IP_SESSIONS_LOOKUP_NAMESPACE}:#{remote_ip}" end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index aa33f56582b..c3415c45b28 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -124,11 +124,8 @@ module Gitlab strong_memoize(:runner_project) do next unless runner&.project_type? - projects = ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/342147') do - runner.projects.take(2) # rubocop: disable CodeReuse/ActiveRecord - end - - projects.first if projects.one? + runner_projects = runner.runner_projects.take(2) # rubocop: disable CodeReuse/ActiveRecord + runner_projects.first.project if runner_projects.one? end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 3db2f1295f9..fb90ad9e275 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -4,12 +4,7 @@ module Gitlab # This class implements a simple rate limiter that can be used to throttle # certain actions. Unlike Rack Attack and Rack::Throttle, which operate at # the middleware level, this can be used at the controller or API level. - # - # @example - # if Gitlab::ApplicationRateLimiter.throttled?(:project_export, scope: [@project, @current_user]) - # flash[:alert] = 'error!' - # redirect_to(edit_project_path(@project), status: :too_many_requests) - # end + # See CheckRateLimit concern for usage. class ApplicationRateLimiter InvalidKeyError = Class.new(StandardError) @@ -47,7 +42,7 @@ module Gitlab project_import: { threshold: -> { application_settings.project_import_limit }, interval: 1.minute }, project_testing_hook: { threshold: 5, interval: 1.minute }, play_pipeline_schedule: { threshold: 1, interval: 1.minute }, - show_raw_controller: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute }, + raw_blob: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute }, group_export: { threshold: -> { application_settings.group_export_limit }, interval: 1.minute }, group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute }, group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, @@ -64,45 +59,47 @@ module Gitlab # be throttled. # # @param key [Symbol] Key attribute registered in `.rate_limits` - # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) - # @option threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` - # @option users_allowlist [Array<String>] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. + # @param scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) + # @param threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` + # @param users_allowlist [Array<String>] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. + # @param peek [Boolean] Optional. When true the key will not be incremented but the current throttled state will be returned. # # @return [Boolean] Whether or not a request should be throttled - def throttled?(key, **options) + def throttled?(key, scope:, threshold: nil, users_allowlist: nil, peek: false) raise InvalidKeyError unless rate_limits[key] - return if scoped_user_in_allowlist?(options) + return false if scoped_user_in_allowlist?(scope, users_allowlist) - threshold_value = options[:threshold] || threshold(key) - threshold_value > 0 && - increment(key, options[:scope]) > threshold_value - end + threshold_value = threshold || threshold(key) - # Increments a cache key that is based on the current time and interval. - # So that when time passes to the next interval, the key changes and the count starts again from 0. - # - # Based on https://github.com/rack/rack-attack/blob/886ba3a18d13c6484cd511a4dc9b76c0d14e5e96/lib/rack/attack/cache.rb#L63-L68 - # - # @param key [Symbol] Key attribute registered in `.rate_limits` - # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) - # - # @return [Integer] incremented value - def increment(key, scope) - interval_value = interval(key) + return false if threshold_value == 0 + interval_value = interval(key) + # `period_key` is based on the current time and interval so when time passes to the next interval + # the key changes and the rate limit count starts again from 0. + # Based on https://github.com/rack/rack-attack/blob/886ba3a18d13c6484cd511a4dc9b76c0d14e5e96/lib/rack/attack/cache.rb#L63-L68 period_key, time_elapsed_in_period = Time.now.to_i.divmod(interval_value) + cache_key = cache_key(key, scope, period_key) - cache_key = "#{action_key(key, scope)}:#{period_key}" - # We add a 1 second buffer to avoid timing issues when we're at the end of a period - expiry = interval_value - time_elapsed_in_period + 1 + value = if peek + read(cache_key) + else + increment(cache_key, interval_value, time_elapsed_in_period) + end - ::Gitlab::Redis::RateLimiting.with do |redis| - redis.pipelined do - redis.incr(cache_key) - redis.expire(cache_key, expiry) - end.first - end + value > threshold_value + end + + # Returns the current rate limited state without incrementing the count. + # + # @param key [Symbol] Key attribute registered in `.rate_limits` + # @param scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) + # @param threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` + # @param users_allowlist [Array<String>] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. + # + # @return [Boolean] Whether or not a request is currently throttled + def peek(key, scope:, threshold: nil, users_allowlist: nil) + throttled?(key, peek: true, scope: scope, threshold: threshold, users_allowlist: users_allowlist) end # Logs request using provided logger @@ -150,7 +147,28 @@ module Gitlab action[setting] if action end - def action_key(key, scope) + # Increments the rate limit count and returns the new count value. + def increment(cache_key, interval_value, time_elapsed_in_period) + # We add a 1 second buffer to avoid timing issues when we're at the end of a period + expiry = interval_value - time_elapsed_in_period + 1 + + ::Gitlab::Redis::RateLimiting.with do |redis| + redis.pipelined do + redis.incr(cache_key) + redis.expire(cache_key, expiry) + end.first + end + end + + # Returns the rate limit count. + # Will be 0 if there is no data in the cache. + def read(cache_key) + ::Gitlab::Redis::RateLimiting.with do |redis| + redis.get(cache_key).to_i + end + end + + def cache_key(key, scope, period_key) composed_key = [key, scope].flatten.compact serialized = composed_key.map do |obj| @@ -161,20 +179,20 @@ module Gitlab end end.join(":") - "application_rate_limiter:#{serialized}" + "application_rate_limiter:#{serialized}:#{period_key}" end def application_settings Gitlab::CurrentSettings.current_application_settings end - def scoped_user_in_allowlist?(options) - return unless options[:users_allowlist].present? + def scoped_user_in_allowlist?(scope, users_allowlist) + return unless users_allowlist.present? - scoped_user = [options[:scope]].flatten.find { |s| s.is_a?(User) } + scoped_user = [scope].flatten.find { |s| s.is_a?(User) } return unless scoped_user - scoped_user.username.downcase.in?(options[:users_allowlist]) + scoped_user.username.downcase.in?(users_allowlist) end end diff --git a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb index 6dbe6f691f6..3ada3f947ee 100644 --- a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb +++ b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb @@ -7,7 +7,7 @@ module Gitlab register_for 'gitlab-html-pipeline' def format(node, lang, opts) - if Feature.enabled?(:use_cmark_renderer) + if Feature.enabled?(:use_cmark_renderer, default_enabled: :yaml) %(<pre #{lang ? %[lang="#{lang}"] : ''}><code>#{node.content}</code></pre>) else %(<pre><code #{lang ? %[ lang="#{lang}"] : ''}>#{node.content}</code></pre>) diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 0970b92723b..3e982168339 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -189,7 +189,7 @@ module Gitlab user = User.id_in(token.resource_owner_id).first return unless user && can_user_login_with_non_expired_password?(user) - Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) + Gitlab::Auth::Result.new(user, nil, :oauth, abilities_for_scopes(token.scopes)) end end end @@ -230,7 +230,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def valid_oauth_token?(token) - token && token.accessible? && valid_scoped_token?(token, [:api]) + token && token.accessible? && valid_scoped_token?(token, Doorkeeper.configuration.scopes) end def valid_scoped_token?(token, scopes) diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb index 904759919ae..ff6dc7313bb 100644 --- a/lib/gitlab/auth/user_access_denied_reason.rb +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -15,7 +15,7 @@ module Gitlab "Your account is pending approval from your administrator and hence blocked." when :terms_not_accepted "You (#{@user.to_reference}) must accept the Terms of Service in order to perform this action. "\ - "Please access GitLab from a web browser to accept these terms." + "To accept these terms, please access GitLab from a web browser at #{Gitlab.config.gitlab.url}." when :deactivated "Your account has been deactivated by your administrator. "\ "Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url}" diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index 22b4b685f81..ed58508182a 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -2,11 +2,13 @@ module Gitlab module BackgroundMigration + DEFAULT_TRACKING_DATABASE = Gitlab::Database::MAIN_DATABASE_NAME + def self.coordinator_for_database(database) - JobCoordinator.for_database(database) + JobCoordinator.for_tracking_database(database) end - def self.queue(database: :main) + def self.queue(database: DEFAULT_TRACKING_DATABASE) coordinator_for_database(database).queue end @@ -22,7 +24,7 @@ module Gitlab # steal_class - The name of the class for which to steal jobs. # retry_dead_jobs - Flag to control whether jobs in Sidekiq::RetrySet or Sidekiq::DeadSet are retried. # database - tracking database this migration executes against - def self.steal(steal_class, retry_dead_jobs: false, database: :main, &block) + def self.steal(steal_class, retry_dead_jobs: false, database: DEFAULT_TRACKING_DATABASE, &block) coordinator_for_database(database).steal(steal_class, retry_dead_jobs: retry_dead_jobs, &block) end @@ -35,15 +37,15 @@ module Gitlab # arguments - The arguments to pass to the background migration's "perform" # method. # database - tracking database this migration executes against - def self.perform(class_name, arguments, database: :main) + def self.perform(class_name, arguments, database: DEFAULT_TRACKING_DATABASE) coordinator_for_database(database).perform(class_name, arguments) end - def self.exists?(migration_class, additional_queues = [], database: :main) + def self.exists?(migration_class, additional_queues = [], database: DEFAULT_TRACKING_DATABASE) coordinator_for_database(database).exists?(migration_class, additional_queues) # rubocop:disable CodeReuse/ActiveRecord end - def self.remaining(database: :main) + def self.remaining(database: DEFAULT_TRACKING_DATABASE) coordinator_for_database(database).remaining end end diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb deleted file mode 100644 index 5b9ee8a0ee2..00000000000 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class AddMergeRequestDiffCommitsCount - class MergeRequestDiff < ActiveRecord::Base - self.table_name = 'merge_request_diffs' - end - - def perform(start_id, stop_id) - Gitlab::AppLogger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") - - update = ' - commits_count = ( - SELECT count(*) - FROM merge_request_diff_commits - WHERE merge_request_diffs.id = merge_request_diff_commits.merge_request_diff_id - )'.squish - - MergeRequestDiff.where(id: start_id..stop_id).where(commits_count: nil).update_all(update) - end - end - end -end diff --git a/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb b/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb deleted file mode 100644 index 2148e96f6b4..00000000000 --- a/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Compare all current rules to project rules - class AddModifiedToApprovalMergeRequestRule - # Stubbed class to access the Group table - class Group < ActiveRecord::Base - self.table_name = 'namespaces' - self.inheritance_column = :_type_disabled - end - - # Stubbed class to access the ApprovalMergeRequestRule table - class ApprovalMergeRequestRule < ActiveRecord::Base - self.table_name = 'approval_merge_request_rules' - - has_one :approval_merge_request_rule_source, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRuleSource' - has_one :approval_project_rule, through: :approval_merge_request_rule_source - has_and_belongs_to_many :groups, - class_name: 'AddModifiedToApprovalMergeRequestRule::Group', join_table: "#{self.table_name}_groups" - end - - # Stubbed class to access the ApprovalProjectRule table - class ApprovalProjectRule < ActiveRecord::Base - self.table_name = 'approval_project_rules' - - has_many :approval_merge_request_rule_sources, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRuleSource' - has_and_belongs_to_many :groups, - class_name: 'AddModifiedToApprovalMergeRequestRule::Group', join_table: "#{self.table_name}_groups" - end - - # Stubbed class to access the ApprovalMergeRequestRuleSource table - class ApprovalMergeRequestRuleSource < ActiveRecord::Base - self.table_name = 'approval_merge_request_rule_sources' - - belongs_to :approval_merge_request_rule, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRule' - belongs_to :approval_project_rule, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalProjectRule' - end - - def perform(start_id, stop_id) - approval_merge_requests_rules = ApprovalMergeRequestRule - .joins(:groups, :approval_merge_request_rule_source) - .where(id: start_id..stop_id) - .pluck( - 'approval_merge_request_rule_sources.id as ars_id', - 'approval_merge_request_rules_groups.id as amrg_id' - ) - - approval_project_rules = ApprovalProjectRule - .joins(:groups, approval_merge_request_rule_sources: :approval_merge_request_rule) - .where(approval_merge_request_rules: { id: start_id..stop_id }) - .pluck( - 'approval_merge_request_rule_sources.id as ars_id', - 'approval_project_rules_groups.id as apg_id' - ) - - different_names_or_approval_sources = ApprovalMergeRequestRule.joins(:approval_project_rule, :approval_merge_request_rule_source) - .where(id: start_id..stop_id) - .where('approval_merge_request_rules.name != approval_project_rules.name OR ' \ - 'approval_merge_request_rules.approvals_required != approval_project_rules.approvals_required') - .pluck('approval_merge_request_rule_sources.id as ars_id') - - intersected_set = approval_merge_requests_rules.to_set ^ approval_project_rules.to_set - source_ids = intersected_set.collect { |rule| rule[0] }.uniq - - rule_sources = ApprovalMergeRequestRuleSource.where(id: source_ids + different_names_or_approval_sources) - changed_merge_request_rules = ApprovalMergeRequestRule.where(id: rule_sources.select(:approval_merge_request_rule_id)) - - changed_merge_request_rules.update_all(modified_from_project_rule: true) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments.rb b/lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments.rb deleted file mode 100644 index 9778f360e87..00000000000 --- a/lib/gitlab/background_migration/backfill_deployment_clusters_from_deployments.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill deployment_clusters for a range of deployments - class BackfillDeploymentClustersFromDeployments - def perform(start_id, end_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO deployment_clusters (deployment_id, cluster_id) - SELECT deployments.id, deployments.cluster_id - FROM deployments - WHERE deployments.cluster_id IS NOT NULL - AND deployments.id BETWEEN #{start_id} AND #{end_id} - ON CONFLICT DO NOTHING - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb b/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb deleted file mode 100644 index 4fd3b81fda3..00000000000 --- a/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # BackfillEnvironmentIdDeploymentMergeRequests deletes duplicates - # from deployment_merge_requests table and backfills environment_id - class BackfillEnvironmentIdDeploymentMergeRequests - def perform(_start_mr_id, _stop_mr_id) - # no-op - - # Background migration removed due to - # https://gitlab.com/gitlab-org/gitlab/-/issues/217191 - end - - def backfill_range(start_mr_id, stop_mr_id) - start_mr_id = Integer(start_mr_id) - stop_mr_id = Integer(stop_mr_id) - - ActiveRecord::Base.connection.execute(<<~SQL) - DELETE FROM deployment_merge_requests - WHERE (deployment_id, merge_request_id) in ( - SELECT t.deployment_id, t.merge_request_id FROM ( - SELECT mrd.merge_request_id, mrd.deployment_id, ROW_NUMBER() OVER w AS rnum - FROM deployment_merge_requests as mrd - INNER JOIN "deployments" ON "deployments"."id" = "mrd"."deployment_id" - WHERE mrd.merge_request_id BETWEEN #{start_mr_id} AND #{stop_mr_id} - WINDOW w AS ( - PARTITION BY merge_request_id, deployments.environment_id - ORDER BY deployments.id - ) - ) t - WHERE t.rnum > 1 - ); - SQL - - ActiveRecord::Base.connection.execute(<<~SQL) - UPDATE deployment_merge_requests - SET environment_id = deployments.environment_id - FROM deployments - WHERE deployments.id = "deployment_merge_requests".deployment_id - AND "deployment_merge_requests".environment_id IS NULL - AND "deployment_merge_requests".merge_request_id BETWEEN #{start_mr_id} AND #{stop_mr_id} - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb b/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb deleted file mode 100644 index 8a58cf9b302..00000000000 --- a/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill merge request cleanup schedules of closed/merged merge requests - # without any corresponding records. - class BackfillMergeRequestCleanupSchedules - # Model used for migration added in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46782. - class MergeRequest < ActiveRecord::Base - include EachBatch - - self.table_name = 'merge_requests' - - def self.eligible - where('merge_requests.state_id IN (2, 3)') - end - end - - def perform(start_id, end_id) - eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id) - scheduled_at_column = "COALESCE(metrics.merged_at, COALESCE(metrics.latest_closed_at, merge_requests.updated_at)) + interval '14 days'" - query = - eligible_mrs - .select("merge_requests.id, #{scheduled_at_column}, NOW(), NOW()") - .joins('LEFT JOIN merge_request_metrics metrics ON metrics.merge_request_id = merge_requests.id') - - result = ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO merge_request_cleanup_schedules (merge_request_id, scheduled_at, created_at, updated_at) - #{query.to_sql} - ON CONFLICT (merge_request_id) DO NOTHING; - SQL - - ::Gitlab::BackgroundMigration::Logger.info( - message: 'Backfilled merge_request_cleanup_schedules records', - count: result.cmd_tuples - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_namespace_settings.rb b/lib/gitlab/background_migration/backfill_namespace_settings.rb deleted file mode 100644 index a391d5f4ebe..00000000000 --- a/lib/gitlab/background_migration/backfill_namespace_settings.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfillnamespace_settings for a range of namespaces - class BackfillNamespaceSettings - def perform(start_id, end_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO namespace_settings (namespace_id, created_at, updated_at) - SELECT namespaces.id, now(), now() - FROM namespaces - WHERE namespaces.id BETWEEN #{start_id} AND #{end_id} - ON CONFLICT (namespace_id) DO NOTHING; - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_project_settings.rb b/lib/gitlab/background_migration/backfill_project_settings.rb deleted file mode 100644 index 7d7f19e1fda..00000000000 --- a/lib/gitlab/background_migration/backfill_project_settings.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill project_settings for a range of projects - class BackfillProjectSettings - def perform(start_id, end_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO project_settings (project_id, created_at, updated_at) - SELECT projects.id, now(), now() - FROM projects - WHERE projects.id BETWEEN #{start_id} AND #{end_id} - ON CONFLICT (project_id) DO NOTHING; - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_push_rules_id_in_projects.rb b/lib/gitlab/background_migration/backfill_push_rules_id_in_projects.rb deleted file mode 100644 index 9b9ef70424a..00000000000 --- a/lib/gitlab/background_migration/backfill_push_rules_id_in_projects.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class that will insert record into project_push_rules - # for each existing push_rule - class BackfillPushRulesIdInProjects - # Temporary AR table for push rules - class ProjectSetting < ActiveRecord::Base - self.table_name = 'project_settings' - end - - def perform(start_id, stop_id) - ProjectSetting.connection.execute(<<~SQL) - UPDATE project_settings ps1 - SET push_rule_id = pr.id - FROM project_settings ps2 - INNER JOIN push_rules pr - ON ps2.project_id = pr.project_id - WHERE pr.is_sample = false - AND pr.id BETWEEN #{start_id} AND #{stop_id} - AND ps1.project_id = ps2.project_id - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/drop_invalid_remediations.rb b/lib/gitlab/background_migration/drop_invalid_remediations.rb new file mode 100644 index 00000000000..f0a0de586f5 --- /dev/null +++ b/lib/gitlab/background_migration/drop_invalid_remediations.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class DropInvalidRemediations + def perform(start_id, stop_id) + end + end + # rubocop: enable Style/Documentation + end +end + +Gitlab::BackgroundMigration::DropInvalidRemediations.prepend_mod_with('Gitlab::BackgroundMigration::DropInvalidRemediations') diff --git a/lib/gitlab/background_migration/drop_invalid_security_findings.rb b/lib/gitlab/background_migration/drop_invalid_security_findings.rb new file mode 100644 index 00000000000..87551bb1b1e --- /dev/null +++ b/lib/gitlab/background_migration/drop_invalid_security_findings.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true +module Gitlab + module BackgroundMigration + # Drop rows from security_findings where the uuid is NULL + class DropInvalidSecurityFindings + # rubocop:disable Style/Documentation + class SecurityFinding < ActiveRecord::Base + include ::EachBatch + self.table_name = 'security_findings' + scope :no_uuid, -> { where(uuid: nil) } + end + # rubocop:enable Style/Documentation + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + ranged_query = SecurityFinding + .where(id: start_id..end_id) + .no_uuid + + ranged_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + + # The query need to be reconstructed because .each_batch modifies the default scope + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 + SecurityFinding.unscoped + .where(id: first..last) + .no_uuid + .delete_all + + sleep PAUSE_SECONDS + end + + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids.rb b/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids.rb deleted file mode 100644 index 1a80ccdee92..00000000000 --- a/lib/gitlab/background_migration/fix_promoted_epics_discussion_ids.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This migration updates discussion ids for epics that were promoted from issue so that the discussion id on epics - # is different from discussion id on issue, which was causing problems when replying to epic discussions as it would - # identify the discussion as related to an issue and complaint about missing project_id - class FixPromotedEpicsDiscussionIds - # notes model to iterate through the notes to be updated - class Note < ActiveRecord::Base - self.table_name = 'notes' - self.inheritance_column = :_type_disabled - end - - def perform(discussion_ids) - Note.where(noteable_type: 'Epic') - .where(discussion_id: discussion_ids) - .update_all("discussion_id=MD5(discussion_id)||substring(discussion_id from 1 for 8)") - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_user_namespace_names.rb b/lib/gitlab/background_migration/fix_user_namespace_names.rb deleted file mode 100644 index cd5b4ab103d..00000000000 --- a/lib/gitlab/background_migration/fix_user_namespace_names.rb +++ /dev/null @@ -1,68 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This migration fixes the namespaces.name for all user-namespaces that have names - # that aren't equal to the users name. - # Then it uses the updated names of the namespaces to update the associated routes - # For more info see https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/23272 - class FixUserNamespaceNames - def perform(from_id, to_id) - fix_namespace_names(from_id, to_id) - fix_namespace_route_names(from_id, to_id) - end - - def fix_namespace_names(from_id, to_id) - ActiveRecord::Base.connection.execute <<~UPDATE_NAMESPACES - WITH namespaces_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT - namespaces.id, - users.name AS correct_name - FROM - namespaces - INNER JOIN users ON namespaces.owner_id = users.id - WHERE - namespaces.type IS NULL - AND namespaces.id BETWEEN #{from_id} AND #{to_id} - AND namespaces.name != users.name - ) - UPDATE - namespaces - SET - name = correct_name - FROM - namespaces_to_update - WHERE - namespaces.id = namespaces_to_update.id - UPDATE_NAMESPACES - end - - def fix_namespace_route_names(from_id, to_id) - ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT - routes.id, - users.name AS correct_name - FROM - routes - INNER JOIN namespaces ON routes.source_id = namespaces.id - INNER JOIN users ON namespaces.owner_id = users.id - WHERE - namespaces.type IS NULL - AND routes.source_type = 'Namespace' - AND namespaces.id BETWEEN #{from_id} AND #{to_id} - AND (routes.name != users.name OR routes.name IS NULL) - ) - UPDATE - routes - SET - name = correct_name - FROM - routes_to_update - WHERE - routes_to_update.id = routes.id - ROUTES_UPDATE - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_user_project_route_names.rb b/lib/gitlab/background_migration/fix_user_project_route_names.rb deleted file mode 100644 index e534f2449aa..00000000000 --- a/lib/gitlab/background_migration/fix_user_project_route_names.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This migration fixes the routes.name for all user-projects that have names - # that don't start with the users name. - # For more info see https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/23272 - class FixUserProjectRouteNames - def perform(from_id, to_id) - ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT - routes.id, - users.name || ' / ' || projects.name AS correct_name - FROM - routes - INNER JOIN projects ON routes.source_id = projects.id - INNER JOIN namespaces ON projects.namespace_id = namespaces.id - INNER JOIN users ON namespaces.owner_id = users.id - WHERE - routes.source_type = 'Project' - AND routes.id BETWEEN #{from_id} AND #{to_id} - AND namespaces.type IS NULL - AND (routes.name NOT LIKE users.name || '%' OR routes.name IS NULL) - ) - UPDATE - routes - SET - name = routes_to_update.correct_name - FROM - routes_to_update - WHERE - routes_to_update.id = routes.id - ROUTES_UPDATE - end - end - end -end diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb index 1c8819eaa62..cfbe7167677 100644 --- a/lib/gitlab/background_migration/job_coordinator.rb +++ b/lib/gitlab/background_migration/job_coordinator.rb @@ -8,24 +8,33 @@ module Gitlab # convention of how the queues and worker classes are setup for each database. # # Also provides a database connection to the correct tracking database. - class JobCoordinator - VALID_DATABASES = %i[main].freeze - WORKER_CLASS_NAME = 'BackgroundMigrationWorker' - - def self.for_database(database) - database = database.to_sym + class JobCoordinator # rubocop:disable Metrics/ClassLength + class << self + def worker_classes + @worker_classes ||= [ + BackgroundMigrationWorker + ].freeze + end - unless VALID_DATABASES.include?(database) - raise ArgumentError, "database must be one of [#{VALID_DATABASES.join(', ')}], got '#{database}'" + def worker_for_tracking_database + @worker_for_tracking_database ||= worker_classes + .index_by(&:tracking_database) + .with_indifferent_access + .freeze end - namespace = database.to_s.capitalize unless database == :main - namespaced_worker_class = [namespace, WORKER_CLASS_NAME].compact.join('::') + def for_tracking_database(tracking_database) + worker_class = worker_for_tracking_database[tracking_database] - new(database, "::#{namespaced_worker_class}".constantize) + if worker_class.nil? + raise ArgumentError, "tracking_database must be one of [#{worker_for_tracking_database.keys.join(', ')}]" + end + + new(worker_class) + end end - attr_reader :database, :worker_class + attr_reader :worker_class def queue @queue ||= worker_class.sidekiq_options['queue'] @@ -118,15 +127,14 @@ module Gitlab private - def initialize(database, worker_class) - @database = database + def initialize(worker_class) @worker_class = worker_class end def connection @connection ||= Gitlab::Database .database_base_models - .fetch(database, Gitlab::Database::PRIMARY_DATABASE_NAME) + .fetch(worker_class.tracking_database, Gitlab::Database::PRIMARY_DATABASE_NAME) .connection end end diff --git a/lib/gitlab/background_migration/link_lfs_objects_projects.rb b/lib/gitlab/background_migration/link_lfs_objects_projects.rb deleted file mode 100644 index 983470c5121..00000000000 --- a/lib/gitlab/background_migration/link_lfs_objects_projects.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Create missing LfsObjectsProject records for forks - class LinkLfsObjectsProjects - # Model specifically used for migration. - class LfsObjectsProject < ActiveRecord::Base - include EachBatch - - self.table_name = 'lfs_objects_projects' - - def self.linkable - where( - <<~SQL - lfs_objects_projects.project_id IN ( - SELECT fork_network_members.forked_from_project_id - FROM fork_network_members - WHERE fork_network_members.forked_from_project_id IS NOT NULL - ) - SQL - ) - end - end - - # Model specifically used for migration. - class ForkNetworkMember < ActiveRecord::Base - include EachBatch - - self.table_name = 'fork_network_members' - - def self.without_lfs_object(lfs_object_id) - where( - <<~SQL - fork_network_members.project_id NOT IN ( - SELECT lop.project_id - FROM lfs_objects_projects lop - WHERE lop.lfs_object_id = #{lfs_object_id} - ) - SQL - ) - end - end - - BATCH_SIZE = 1000 - - def perform(start_id, end_id) - lfs_objects_projects = - Gitlab::BackgroundMigration::LinkLfsObjectsProjects::LfsObjectsProject - .linkable - .where(id: start_id..end_id) - - return if lfs_objects_projects.empty? - - lfs_objects_projects.find_each do |lop| - ForkNetworkMember - .select("#{lop.lfs_object_id}, fork_network_members.project_id, NOW(), NOW()") - .without_lfs_object(lop.lfs_object_id) - .where(forked_from_project_id: lop.project_id) - .each_batch(of: BATCH_SIZE) do |batch, index| - execute <<~SQL - INSERT INTO lfs_objects_projects (lfs_object_id, project_id, created_at, updated_at) - #{batch.to_sql} - SQL - - logger.info(message: "LinkLfsObjectsProjects: created missing LfsObjectsProject records for LfsObject #{lop.lfs_object_id}") - end - end - end - - private - - def execute(sql) - ::ActiveRecord::Base.connection.execute(sql) - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb deleted file mode 100644 index 36a339c6b80..00000000000 --- a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class is responsible to update all sha256 fingerprints within the keys table - class MigrateFingerprintSha256WithinKeys - # Temporary AR table for keys - class Key < ActiveRecord::Base - include EachBatch - - self.table_name = 'keys' - self.inheritance_column = :_type_disabled - end - - TEMP_TABLE = 'tmp_fingerprint_sha256_migration' - - def perform(start_id, stop_id) - ActiveRecord::Base.transaction do - execute(<<~SQL) - CREATE TEMPORARY TABLE #{TEMP_TABLE} - (id bigint primary key, fingerprint_sha256 bytea not null) - ON COMMIT DROP - SQL - - fingerprints = [] - Key.where(id: start_id..stop_id, fingerprint_sha256: nil).find_each do |regular_key| - if fingerprint = generate_ssh_public_key(regular_key.key) - bytea = ActiveRecord::Base.connection.escape_bytea(Base64.decode64(fingerprint)) - - fingerprints << { - id: regular_key.id, - fingerprint_sha256: bytea - } - end - end - - ApplicationRecord.legacy_bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert - - execute("ANALYZE #{TEMP_TABLE}") - - execute(<<~SQL) - UPDATE keys - SET fingerprint_sha256 = t.fingerprint_sha256 - FROM #{TEMP_TABLE} t - WHERE keys.id = t.id - SQL - end - end - - private - - def generate_ssh_public_key(regular_key) - Gitlab::SSHPublicKey.new(regular_key).fingerprint("SHA256")&.gsub("SHA256:", "") - end - - def execute(query) - ActiveRecord::Base.connection.execute(query) - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_pages_metadata.rb b/lib/gitlab/background_migration/migrate_pages_metadata.rb deleted file mode 100644 index 68fd0c17d29..00000000000 --- a/lib/gitlab/background_migration/migrate_pages_metadata.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class that will insert record into project_pages_metadata - # for each existing project - class MigratePagesMetadata - def perform(start_id, stop_id) - perform_on_relation(Project.where(id: start_id..stop_id)) - end - - def perform_on_relation(relation) - successful_pages_deploy = <<~SQL - SELECT TRUE - FROM ci_builds - WHERE ci_builds.type = 'GenericCommitStatus' - AND ci_builds.status = 'success' - AND ci_builds.stage = 'deploy' - AND ci_builds.name = 'pages:deploy' - AND ci_builds.project_id = projects.id - LIMIT 1 - SQL - - select_from = relation - .select("projects.id", "COALESCE((#{successful_pages_deploy}), FALSE)") - .to_sql - - ActiveRecord::Base.connection_pool.with_connection do |connection| - connection.execute <<~SQL - INSERT INTO project_pages_metadata (project_id, deployed) - #{select_from} - ON CONFLICT (project_id) DO NOTHING - SQL - end - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_to_hashed_storage.rb b/lib/gitlab/background_migration/migrate_to_hashed_storage.rb deleted file mode 100644 index 4054db4fb87..00000000000 --- a/lib/gitlab/background_migration/migrate_to_hashed_storage.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Background migration to move any legacy project to Hashed Storage - class MigrateToHashedStorage - def perform - batch_size = helper.batch_size - legacy_projects_count = Project.with_unmigrated_storage.count - - if storage_migrator.rollback_pending? - logger.warn( - migrator: 'MigrateToHashedStorage', - message: 'Aborting an storage rollback operation currently in progress' - ) - - storage_migrator.abort_rollback! - end - - if legacy_projects_count == 0 - logger.info( - migrator: 'MigrateToHashedStorage', - message: 'There are no projects requiring migration to Hashed Storage' - ) - - return - end - - logger.info( - migrator: 'MigrateToHashedStorage', - message: "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}" - ) - - helper.project_id_batches_migration do |start, finish| - storage_migrator.bulk_schedule_migration(start: start, finish: finish) - - logger.info( - migrator: 'MigrateToHashedStorage', - message: "Enqueuing migration of projects in batches of #{batch_size} from ID=#{start} to ID=#{finish}", - batch_from: start, - batch_to: finish - ) - end - end - - private - - def helper - Gitlab::HashedStorage::RakeHelper - end - - def storage_migrator - @storage_migrator ||= Gitlab::HashedStorage::Migrator.new - end - - def logger - @logger ||= ::Gitlab::BackgroundMigration::Logger.build - end - end - end -end diff --git a/lib/gitlab/background_migration/move_epic_issues_after_epics.rb b/lib/gitlab/background_migration/move_epic_issues_after_epics.rb deleted file mode 100644 index 174994c7862..00000000000 --- a/lib/gitlab/background_migration/move_epic_issues_after_epics.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class MoveEpicIssuesAfterEpics - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics.prepend_mod_with('Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb deleted file mode 100644 index 890a43800c9..00000000000 --- a/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration creates any approver rule records according - # to the given merge request IDs range. A _single_ INSERT is issued for the given range. - class PopulateAnyApprovalRuleForMergeRequests - def perform(from_id, to_id) - end - end - end -end - -Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests.prepend_mod_with('Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb deleted file mode 100644 index ac7ed18ba14..00000000000 --- a/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration creates any approver rule records according - # to the given project IDs range. A _single_ INSERT is issued for the given range. - class PopulateAnyApprovalRuleForProjects - def perform(from_id, to_id) - end - end - end -end - -Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects.prepend_mod_with('Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects') diff --git a/lib/gitlab/background_migration/populate_canonical_emails.rb b/lib/gitlab/background_migration/populate_canonical_emails.rb deleted file mode 100644 index 052e75c5655..00000000000 --- a/lib/gitlab/background_migration/populate_canonical_emails.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class to populate new rows of UserCanonicalEmail based on existing email addresses - class PopulateCanonicalEmails - def perform(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO - user_canonical_emails ( - user_id, - canonical_email, - created_at, - updated_at - ) - SELECT users.id AS user_id, - concat(translate(split_part(split_part(users.email, '@', 1), '+', 1), '.', ''), '@gmail.com') AS canonical_email, - NOW() AS created_at, - NOW() AS updated_at - FROM users - WHERE users.email ILIKE '%@gmail.com' - AND users.id BETWEEN #{start_id} AND #{stop_id} - ON CONFLICT DO NOTHING; - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_dismissed_state_for_vulnerabilities.rb b/lib/gitlab/background_migration/populate_dismissed_state_for_vulnerabilities.rb deleted file mode 100644 index 68c91650d93..00000000000 --- a/lib/gitlab/background_migration/populate_dismissed_state_for_vulnerabilities.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class updates vulnerabilities entities with state dismissed - class PopulateDismissedStateForVulnerabilities - class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'vulnerabilities' - end - - def perform(*vulnerability_ids) - Vulnerability.where(id: vulnerability_ids).update_all(state: 2) - PopulateMissingVulnerabilityDismissalInformation.new.perform(*vulnerability_ids) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb deleted file mode 100644 index 28ff2070209..00000000000 --- a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class populates missing dismissal information for - # vulnerability entries. - class PopulateHasVulnerabilities - class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'project_settings' - - def self.upsert_for(project_ids) - connection.execute(upsert_sql % { project_ids: project_ids.join(', ') }) - end - - def self.upsert_sql - <<~SQL - WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) - ) - INSERT INTO project_settings - (project_id, has_vulnerabilities, created_at, updated_at) - (SELECT * FROM upsert_data) - ON CONFLICT (project_id) - DO UPDATE SET - has_vulnerabilities = true, - updated_at = EXCLUDED.updated_at - SQL - end - end - - class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'vulnerabilities' - end - - def perform(*project_ids) - ProjectSetting.upsert_for(project_ids) - rescue StandardError => e - log_error(e, project_ids) - ensure - log_info(project_ids) - end - - private - - def log_error(error, project_ids) - ::Gitlab::BackgroundMigration::Logger.error( - migrator: self.class.name, - message: error.message, - project_ids: project_ids - ) - end - - def log_info(project_ids) - ::Gitlab::BackgroundMigration::Logger.info( - migrator: self.class.name, - message: 'Projects has been processed to populate `has_vulnerabilities` information', - count: project_ids.length - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb deleted file mode 100644 index 28cc4a5e3fa..00000000000 --- a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration creates records on merge_request_assignees according - # to the given merge request IDs range. A _single_ INSERT is issued for the given range. - # This is required for supporting multiple assignees on merge requests. - class PopulateMergeRequestAssigneesTable - def perform(from_id, to_id) - select_sql = - MergeRequest - .where(merge_request_assignees_not_exists_clause) - .where(id: from_id..to_id) - .where.not(assignee_id: nil) - .select(:id, :assignee_id) - .to_sql - - execute("INSERT INTO merge_request_assignees (merge_request_id, user_id) #{select_sql}") - end - - def perform_all_sync(batch_size:) - MergeRequest.each_batch(of: batch_size) do |batch| - range = batch.pluck('MIN(id)', 'MAX(id)').first - - perform(*range) - end - end - - private - - def merge_request_assignees_not_exists_clause - <<~SQL - NOT EXISTS (SELECT 1 FROM merge_request_assignees - WHERE merge_request_assignees.merge_request_id = merge_requests.id) - SQL - end - - def execute(sql) - @connection ||= ActiveRecord::Base.connection - @connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb deleted file mode 100644 index 04342fdabd4..00000000000 --- a/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class populates missing dismissal information for - # vulnerability entries. - class PopulateMissingVulnerabilityDismissalInformation - class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'vulnerabilities' - - has_one :finding, class_name: '::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation::Finding' - - scope :broken, -> { where('state = 2 AND (dismissed_at IS NULL OR dismissed_by_id IS NULL)') } - - def copy_dismissal_information - return unless finding&.dismissal_feedback - - update_columns( - dismissed_at: finding.dismissal_feedback.created_at, - dismissed_by_id: finding.dismissal_feedback.author_id - ) - end - end - - class Finding < ActiveRecord::Base # rubocop:disable Style/Documentation - include ShaAttribute - include ::Gitlab::Utils::StrongMemoize - - self.table_name = 'vulnerability_occurrences' - - sha_attribute :project_fingerprint - - def dismissal_feedback - strong_memoize(:dismissal_feedback) do - Feedback.dismissal.where(category: report_type, project_fingerprint: project_fingerprint, project_id: project_id).first - end - end - end - - class Feedback < ActiveRecord::Base # rubocop:disable Style/Documentation - DISMISSAL_TYPE = 0 - - self.table_name = 'vulnerability_feedback' - - scope :dismissal, -> { where(feedback_type: DISMISSAL_TYPE) } - end - - def perform(*vulnerability_ids) - Vulnerability.includes(:finding).where(id: vulnerability_ids).each { |vulnerability| populate_for(vulnerability) } - - log_info(vulnerability_ids) - end - - private - - def populate_for(vulnerability) - log_warning(vulnerability) unless vulnerability.copy_dismissal_information - rescue StandardError => error - log_error(error, vulnerability) - end - - def log_info(vulnerability_ids) - ::Gitlab::BackgroundMigration::Logger.info( - migrator: self.class.name, - message: 'Dismissal information has been copied', - count: vulnerability_ids.length - ) - end - - def log_warning(vulnerability) - ::Gitlab::BackgroundMigration::Logger.warn( - migrator: self.class.name, - message: 'Could not update vulnerability!', - vulnerability_id: vulnerability.id - ) - end - - def log_error(error, vulnerability) - ::Gitlab::BackgroundMigration::Logger.error( - migrator: self.class.name, - message: error.message, - vulnerability_id: vulnerability.id - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb deleted file mode 100644 index ed7ffce8018..00000000000 --- a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class creates/updates those personal snippets statistics - # that haven't been created nor initialized. - # It also updates the related root storage namespace stats - class PopulatePersonalSnippetStatistics - def perform(snippet_ids) - personal_snippets(snippet_ids).group_by(&:author).each do |author, author_snippets| - upsert_snippet_statistics(author_snippets) - update_namespace_statistics(author.namespace) - end - end - - private - - def personal_snippets(snippet_ids) - PersonalSnippet - .where(id: snippet_ids) - .includes(author: :namespace) - .includes(:statistics) - .includes(snippet_repository: :shard) - end - - def upsert_snippet_statistics(snippets) - snippets.each do |snippet| - response = Snippets::UpdateStatisticsService.new(snippet).execute - - error_message("#{response.message} snippet: #{snippet.id}") if response.error? - end - end - - def update_namespace_statistics(namespace) - Namespaces::StatisticsRefresherService.new.execute(namespace) - rescue StandardError => e - error_message("Error updating statistics for namespace #{namespace.id}: #{e.message}") - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - - def error_message(message) - logger.error(message: "Snippet Statistics Migration: #{message}") - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_project_snippet_statistics.rb b/lib/gitlab/background_migration/populate_project_snippet_statistics.rb deleted file mode 100644 index 37af320f044..00000000000 --- a/lib/gitlab/background_migration/populate_project_snippet_statistics.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class creates/updates those project snippets statistics - # that haven't been created nor initialized. - # It also updates the related project statistics and its root storage namespace stats - class PopulateProjectSnippetStatistics - def perform(snippet_ids) - project_snippets(snippet_ids).group_by(&:namespace_id).each do |namespace_id, namespace_snippets| - namespace_snippets.group_by(&:project).each do |project, snippets| - upsert_snippet_statistics(snippets) - update_project_statistics(project) - rescue StandardError - error_message("Error updating statistics for project #{project.id}") - end - - update_namespace_statistics(namespace_snippets.first.project.root_namespace) - rescue StandardError => e - error_message("Error updating statistics for namespace #{namespace_id}: #{e.message}") - end - end - - private - - def project_snippets(snippet_ids) - ProjectSnippet - .select('snippets.*, projects.namespace_id') - .where(id: snippet_ids) - .joins(:project) - .includes(:statistics) - .includes(snippet_repository: :shard) - .includes(project: [:route, :statistics, :namespace]) - end - - def upsert_snippet_statistics(snippets) - snippets.each do |snippet| - response = Snippets::UpdateStatisticsService.new(snippet).execute - - error_message("#{response.message} snippet: #{snippet.id}") if response.error? - end - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - - def error_message(message) - logger.error(message: "Snippet Statistics Migration: #{message}") - end - - def update_project_statistics(project) - project.statistics&.refresh!(only: [:snippets_size]) - end - - def update_namespace_statistics(namespace) - Namespaces::StatisticsRefresherService.new.execute(namespace) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb deleted file mode 100644 index 8241fea66db..00000000000 --- a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class updates vulnerability feedback entities with no pipeline id assigned. - class PopulateVulnerabilityFeedbackPipelineId - def perform(project_ids) - end - end - end -end - -Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_mod_with('Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId') diff --git a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb deleted file mode 100644 index 9a9f23e29ea..00000000000 --- a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class creates/updates those project historical vulnerability statistics - # that haven't been created nor initialized. It should only be executed in EE. - class PopulateVulnerabilityHistoricalStatistics - def perform(project_ids, retention_period = 90) - end - end - end -end - -Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics') diff --git a/lib/gitlab/background_migration/prune_orphaned_geo_events.rb b/lib/gitlab/background_migration/prune_orphaned_geo_events.rb deleted file mode 100644 index 0efbe72775c..00000000000 --- a/lib/gitlab/background_migration/prune_orphaned_geo_events.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true -# -# rubocop:disable Style/Documentation - -# This job is added to fix https://gitlab.com/gitlab-org/gitlab/issues/30229 -# It's not used anywhere else. -# Can be removed in GitLab 13.* -module Gitlab - module BackgroundMigration - class PruneOrphanedGeoEvents - def perform(table_name) - end - end - end -end - -Gitlab::BackgroundMigration::PruneOrphanedGeoEvents.prepend_mod_with('Gitlab::BackgroundMigration::PruneOrphanedGeoEvents') diff --git a/lib/gitlab/background_migration/recalculate_project_authorizations_with_min_max_user_id.rb b/lib/gitlab/background_migration/recalculate_project_authorizations_with_min_max_user_id.rb deleted file mode 100644 index b66fdfd5c65..00000000000 --- a/lib/gitlab/background_migration/recalculate_project_authorizations_with_min_max_user_id.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop:disable Style/Documentation - class RecalculateProjectAuthorizationsWithMinMaxUserId - def perform(min_user_id, max_user_id) - User.where(id: min_user_id..max_user_id).find_each do |user| - service = Users::RefreshAuthorizedProjectsService.new( - user, - incorrect_auth_found_callback: - ->(project_id, access_level) do - logger.info(message: 'Removing ProjectAuthorizations', - user_id: user.id, - project_id: project_id, - access_level: access_level) - end, - missing_auth_found_callback: - ->(project_id, access_level) do - logger.info(message: 'Creating ProjectAuthorizations', - user_id: user.id, - project_id: project_id, - access_level: access_level) - end - ) - - service.execute - end - end - - private - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_security_scans.rb b/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb index 0ae984f2dbc..20200a1d508 100644 --- a/lib/gitlab/background_migration/migrate_security_scans.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb @@ -3,11 +3,11 @@ module Gitlab module BackgroundMigration # rubocop: disable Style/Documentation - class MigrateSecurityScans + class RecalculateVulnerabilityFindingSignaturesForFindings def perform(start_id, stop_id) end end end end -Gitlab::BackgroundMigration::MigrateSecurityScans.prepend_mod_with('Gitlab::BackgroundMigration::MigrateSecurityScans') +Gitlab::BackgroundMigration::RecalculateVulnerabilityFindingSignaturesForFindings.prepend_mod diff --git a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb deleted file mode 100644 index 17ef6dec4c0..00000000000 --- a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class RemoveDuplicateCsFindings - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::RemoveDuplicateCsFindings.prepend_mod_with('Gitlab::BackgroundMigration::RemoveDuplicateCsFindings') diff --git a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb deleted file mode 100644 index e5772fc7375..00000000000 --- a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class RemoveDuplicatedCsFindingsWithoutVulnerabilityId - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_mod_with('Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId') diff --git a/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb b/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb deleted file mode 100644 index cb6a600a525..00000000000 --- a/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop:disable Style/Documentation - class RemoveInaccessibleEpicTodos - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos.prepend_mod_with('Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos') diff --git a/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb new file mode 100644 index 00000000000..7fe5a427d10 --- /dev/null +++ b/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +# This migration will look for Vulnerabilities::Finding objects that would have a duplicate UUIDv5 if the UUID was +# recalculated. Then it removes Vulnerabilities::FindingPipeline objects associated with those Findings. +# We can't just drop those Findings directly since the cascade drop will timeout if any given Finding has too many +# associated FindingPipelines +class Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicateVulnerabilitiesFindings + # rubocop:disable Gitlab/NamespacedClass, Style/Documentation + class VulnerabilitiesFinding < ActiveRecord::Base + self.table_name = "vulnerability_occurrences" + end + + class VulnerabilitiesFindingPipeline < ActiveRecord::Base + include EachBatch + self.table_name = "vulnerability_occurrence_pipelines" + end + # rubocop:enable Gitlab/NamespacedClass, Style/Documentation + + def perform(start_id, end_id) + ids_to_look_for = findings_that_would_produce_duplicate_uuids(start_id, end_id) + + ids_to_look_for.each do |finding_id| + VulnerabilitiesFindingPipeline.where(occurrence_id: finding_id).each_batch(of: 1000) do |pipelines| + pipelines.delete_all + end + end + + VulnerabilitiesFinding.where(id: ids_to_look_for).delete_all + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def findings_that_would_produce_duplicate_uuids(start_id, end_id) + VulnerabilitiesFinding + .from("vulnerability_occurrences to_delete") + .where("to_delete.id BETWEEN ? AND ?", start_id, end_id) + .where( + "EXISTS ( + SELECT 1 + FROM vulnerability_occurrences duplicates + WHERE duplicates.report_type = to_delete.report_type + AND duplicates.location_fingerprint = to_delete.location_fingerprint + AND duplicates.primary_identifier_id = to_delete.primary_identifier_id + AND duplicates.project_id = to_delete.project_id + AND duplicates.id > to_delete.id + )" + ) + .pluck("to_delete.id") + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end +end diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb deleted file mode 100644 index 4be61bfb689..00000000000 --- a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class RemoveUndefinedVulnerabilityConfidenceLevel - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel') diff --git a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb new file mode 100644 index 00000000000..31fb5e97c5d --- /dev/null +++ b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Remove vulnerability finding link records + # The records will be repopulated from the `raw_metadata` + # column of `vulnerability_occurrences` once the unique + # index is in place. + class RemoveVulnerabilityFindingLinks + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, stop_id) + define_batchable_model('vulnerability_finding_links').where(id: start_id..stop_id).delete_all + end + end + end +end diff --git a/lib/gitlab/background_migration/replace_blocked_by_links.rb b/lib/gitlab/background_migration/replace_blocked_by_links.rb deleted file mode 100644 index 0c29887bb00..00000000000 --- a/lib/gitlab/background_migration/replace_blocked_by_links.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class ReplaceBlockedByLinks - class IssueLink < ActiveRecord::Base - self.table_name = 'issue_links' - end - - def perform(start_id, stop_id) - blocked_by_links = IssueLink.where(id: start_id..stop_id).where(link_type: 2) - - ActiveRecord::Base.transaction do - # There could be two edge cases: - # 1) issue1 is blocked by issue2 AND issue2 blocks issue1 (type 1) - # 2) issue1 is blocked by issue2 AND issue2 is related to issue1 (type 0) - # In both cases cases we couldn't convert blocked by relation to - # `issue2 blocks issue` because there is already a link with the same - # source/target id. To avoid these conflicts, we first delete any - # "opposite" links before we update `blocked by` relation. This - # should be rare as we have a pre-create check which checks if issues - # are already linked - opposite_ids = blocked_by_links - .select('opposite_links.id') - .joins('INNER JOIN issue_links as opposite_links ON issue_links.source_id = opposite_links.target_id AND issue_links.target_id = opposite_links.source_id') - IssueLink.where(id: opposite_ids).delete_all - - blocked_by_links.update_all('source_id=target_id,target_id=source_id,link_type=1') - end - end - end - end -end diff --git a/lib/gitlab/background_migration/reset_merge_status.rb b/lib/gitlab/background_migration/reset_merge_status.rb deleted file mode 100644 index d040b4931be..00000000000 --- a/lib/gitlab/background_migration/reset_merge_status.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Updates the range of given MRs to merge_status "unchecked", if they're opened - # and mergeable. - class ResetMergeStatus - def perform(from_id, to_id) - relation = MergeRequest.where(id: from_id..to_id, - state_id: 1, # opened - merge_status: 'can_be_merged') - - relation.update_all(merge_status: 'unchecked') - end - end - end -end diff --git a/lib/gitlab/background_migration/reset_shared_runners_for_transferred_projects.rb b/lib/gitlab/background_migration/reset_shared_runners_for_transferred_projects.rb deleted file mode 100644 index 0053cafb4ac..00000000000 --- a/lib/gitlab/background_migration/reset_shared_runners_for_transferred_projects.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Resets inconsistent state of shared_runners_enabled for projects that have been transferred - class ResetSharedRunnersForTransferredProjects - # Model specifically used for migration. - class Namespace < ActiveRecord::Base - include EachBatch - - self.table_name = 'namespaces' - end - - # Model specifically used for migration. - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - - def perform(start_id, stop_id) - Project.reset_column_information - - Namespace.where(id: start_id..stop_id).each_batch(of: 1_000) do |relation| - ids = relation.where(shared_runners_enabled: false, allow_descendants_override_disabled_shared_runners: false).select(:id) - - Project.where(namespace_id: ids).update_all(shared_runners_enabled: false) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb deleted file mode 100644 index 527dd2a0a83..00000000000 --- a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Sets the MergeRequestDiff#files_count value for old rows - class SetMergeRequestDiffFilesCount - # Some historic data has a *lot* of files. Apply a sentinel to these cases - FILES_COUNT_SENTINEL = 2**15 - 1 - - def self.count_subquery - <<~SQL - files_count = ( - SELECT LEAST(#{FILES_COUNT_SENTINEL}, count(*)) - FROM merge_request_diff_files - WHERE merge_request_diff_files.merge_request_diff_id = merge_request_diffs.id - ) - SQL - end - - class MergeRequestDiff < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'merge_request_diffs' - end - - def perform(start_id, end_id) - MergeRequestDiff.where(id: start_id..end_id).each_batch do |relation| - relation.update_all(self.class.count_subquery) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/update_existing_subgroup_to_match_visibility_level_of_parent.rb b/lib/gitlab/background_migration/update_existing_subgroup_to_match_visibility_level_of_parent.rb deleted file mode 100644 index 9e330f7c008..00000000000 --- a/lib/gitlab/background_migration/update_existing_subgroup_to_match_visibility_level_of_parent.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration updates children of group to match visibility of a parent - class UpdateExistingSubgroupToMatchVisibilityLevelOfParent - def perform(parents_groups_ids, level) - groups_ids = Gitlab::ObjectHierarchy.new(Group.where(id: parents_groups_ids)) - .base_and_descendants - .where("visibility_level > ?", level) - .select(:id) - - return if groups_ids.empty? - - Group - .where(id: groups_ids) - .update_all(visibility_level: level) - end - end - end -end diff --git a/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb b/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb deleted file mode 100644 index d97765cd398..00000000000 --- a/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class UpdateExistingUsersThatRequireTwoFactorAuth # rubocop:disable Metrics/ClassLength - def perform(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - UPDATE - users - SET - require_two_factor_authentication_from_group = FALSE - WHERE - users.id BETWEEN #{start_id} - AND #{stop_id} - AND users.require_two_factor_authentication_from_group = TRUE - AND users.id NOT IN ( SELECT DISTINCT - users_groups_query.user_id - FROM ( - SELECT - users.id AS user_id, - members.source_id AS group_ids - FROM - users - LEFT JOIN members ON members.source_type = 'Namespace' - AND members.requested_at IS NULL - AND members.user_id = users.id - AND members.type = 'GroupMember' - WHERE - users.require_two_factor_authentication_from_group = TRUE - AND users.id BETWEEN #{start_id} - AND #{stop_id}) AS users_groups_query - INNER JOIN LATERAL ( WITH RECURSIVE "base_and_ancestors" AS ( - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = users_groups_query.group_ids) - UNION ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces", - "base_and_ancestors" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = "base_and_ancestors"."parent_id")), - "base_and_descendants" AS ( - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = users_groups_query.group_ids) - UNION ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces", - "base_and_descendants" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."parent_id" = "base_and_descendants"."id")) - SELECT - "namespaces".* - FROM (( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "base_and_ancestors" AS "namespaces" - WHERE - "namespaces"."type" = 'Group') - UNION ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "base_and_descendants" AS "namespaces" - WHERE - "namespaces"."type" = 'Group')) namespaces - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."require_two_factor_authentication" = TRUE) AS hierarchy_tree ON TRUE); - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb deleted file mode 100644 index 054b918dade..00000000000 --- a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class UpdateLocationFingerprintForContainerScanningFindings - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_mod_with('Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings') diff --git a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb new file mode 100644 index 00000000000..c95ef9f5515 --- /dev/null +++ b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class to populate spent_at for timelogs + class UpdateTimelogsNullSpentAt + include Gitlab::Database::DynamicModelHelpers + + BATCH_SIZE = 100 + + def perform(start_id, stop_id) + define_batchable_model('timelogs').where(spent_at: nil, id: start_id..stop_id).each_batch(of: 100) do |subbatch| + batch_start, batch_end = subbatch.pluck('min(id), max(id)').first + + update_timelogs(batch_start, batch_end) + end + end + + def update_timelogs(batch_start, batch_stop) + execute(<<~SQL) + UPDATE timelogs + SET spent_at = created_at + WHERE spent_at IS NULL + AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop}; + SQL + end + + def execute(sql) + @connection ||= ::ActiveRecord::Base.connection + @connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb b/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb deleted file mode 100644 index 1cc03f061fb..00000000000 --- a/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class UpdateVulnerabilitiesFromDismissalFeedback - def perform(project_id) - end - end - end -end - -Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback') diff --git a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb deleted file mode 100644 index b3876018553..00000000000 --- a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - class CreateResourceUserMention - # Resources that have mentions to be migrated: - # issue, merge_request, epic, commit, snippet, design - - BULK_INSERT_SIZE = 1_000 - ISOLATION_MODULE = 'Gitlab::BackgroundMigration::UserMentions::Models' - - def perform(resource_model, join, conditions, with_notes, start_id, end_id) - return unless Feature.enabled?(:migrate_user_mentions, default_enabled: true) - - resource_model = "#{ISOLATION_MODULE}::#{resource_model}".constantize if resource_model.is_a?(String) - model = with_notes ? Gitlab::BackgroundMigration::UserMentions::Models::Note : resource_model - resource_user_mention_model = resource_model.user_mention_model - - records = model.joins(join).where(conditions).where(id: start_id..end_id) - - records.each_batch(of: BULK_INSERT_SIZE) do |records| - mentions = [] - records.each do |record| - mention_record = record.build_mention_values(resource_user_mention_model.resource_foreign_key) - mentions << mention_record unless mention_record.blank? - end - - resource_user_mention_model.insert_all(mentions) unless mentions.empty? - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb deleted file mode 100644 index 3def5eb3369..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Banzai - # isolated Banzai::ReferenceParser - module ReferenceParser - # Returns the reference parser class for the given type - # - # Example: - # - # Banzai::ReferenceParser['isolated_mentioned_group'] - # - # This would return the `::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::IsolatedMentionedGroupParser` class. - def self.[](name) - const_get("::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::#{name.to_s.camelize}Parser", false) - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb deleted file mode 100644 index d3d032ba433..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Banzai - module ReferenceParser - # isolated Banzai::ReferenceParser::MentionedGroupParser - class IsolatedMentionedGroupParser < ::Banzai::ReferenceParser::MentionedGroupParser - extend ::Gitlab::Utils::Override - - self.reference_type = :user - - override :references_relation - def references_relation - ::Gitlab::BackgroundMigration::UserMentions::Models::Group - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb deleted file mode 100644 index 5930d65bc2c..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Banzai - module ReferenceParser - # isolated Banzai::ReferenceParser::MentionedGroupParser - class IsolatedMentionedProjectParser < ::Banzai::ReferenceParser::MentionedProjectParser - extend ::Gitlab::Utils::Override - - self.reference_type = :user - - override :references_relation - def references_relation - ::Gitlab::BackgroundMigration::UserMentions::Models::Project - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb deleted file mode 100644 index f5f98517433..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Banzai - module ReferenceParser - # isolated Banzai::ReferenceParser::MentionedGroupParser - class IsolatedMentionedUserParser < ::Banzai::ReferenceParser::MentionedUserParser - extend ::Gitlab::Utils::Override - - self.reference_type = :user - - override :references_relation - def references_relation - ::Gitlab::BackgroundMigration::UserMentions::Models::User - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb deleted file mode 100644 index 8610129533d..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Gitlab - # Extract possible GFM references from an arbitrary String for further processing. - class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor - REFERABLES = %i(isolated_mentioned_group isolated_mentioned_user isolated_mentioned_project).freeze - - REFERABLES.each do |type| - define_method("#{type}s") do - @references[type] ||= isolated_references(type) - end - end - - def isolated_references(type) - context = ::Banzai::RenderContext.new(project, current_user) - processor = ::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser[type].new(context) - - refs = processor.process(html_documents) - refs[:visible] - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb deleted file mode 100644 index 0334ea1dd08..00000000000 --- a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Lib - module Gitlab - # Gitlab::IsolatedVisibilityLevel module - # - # Define allowed public modes that can be used for - # GitLab projects to determine project public mode - # - module IsolatedVisibilityLevel - extend ::ActiveSupport::Concern - - included do - scope :public_to_user, -> (user = nil) do - where(visibility_level: IsolatedVisibilityLevel.levels_for_user(user)) - end - end - - PRIVATE = 0 unless const_defined?(:PRIVATE) - INTERNAL = 10 unless const_defined?(:INTERNAL) - PUBLIC = 20 unless const_defined?(:PUBLIC) - - class << self - def levels_for_user(user = nil) - return [PUBLIC] unless user - - if user.can_read_all_resources? - [PRIVATE, INTERNAL, PUBLIC] - elsif user.external? - [PUBLIC] - else - [INTERNAL, PUBLIC] - end - end - end - - def private? - visibility_level_value == PRIVATE - end - - def internal? - visibility_level_value == INTERNAL - end - - def public? - visibility_level_value == PUBLIC - end - - def visibility_level_value - self[visibility_level_field] - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/commit.rb b/lib/gitlab/background_migration/user_mentions/models/commit.rb deleted file mode 100644 index 65f4a7a25b6..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/commit.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class Commit - include EachBatch - include Concerns::IsolatedMentionable - include Concerns::MentionableMigrationMethods - - def self.user_mention_model - Gitlab::BackgroundMigration::UserMentions::Models::CommitUserMention - end - - def user_mention_model - self.class.user_mention_model - end - - def user_mention_resource_id - id - end - - def user_mention_note_id - 'NULL' - end - - def self.no_quote_columns - [:note_id] - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb deleted file mode 100644 index f4cc96c8bc0..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class CommitUserMention < ActiveRecord::Base - self.table_name = 'commit_user_mentions' - self.inheritance_column = :_type_disabled - - def self.resource_foreign_key - :commit_id - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb deleted file mode 100644 index ba6b783f9f1..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - module Concerns - # isolated FeatureGate module - module IsolatedFeatureGate - def flipper_id - return if new_record? - - "#{self.class.name}:#{id}" - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb deleted file mode 100644 index f684f789ea9..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb +++ /dev/null @@ -1,104 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - module Concerns - # == IsolatedMentionable concern - # - # Shortcutted for isolation version of Mentionable to be used in mentions migrations - # - module IsolatedMentionable - extend ::ActiveSupport::Concern - - class_methods do - # Indicate which attributes of the Mentionable to search for GFM references. - def attr_mentionable(attr, options = {}) - attr = attr.to_s - mentionable_attrs << [attr, options] - end - end - - included do - # Accessor for attributes marked mentionable. - cattr_accessor :mentionable_attrs, instance_accessor: false do - [] - end - - if self < Participable - participant -> (user, ext) { all_references(user, extractor: ext) } - end - end - - def all_references(current_user = nil, extractor: nil) - # Use custom extractor if it's passed in the function parameters. - if extractor - extractors[current_user] = extractor - else - extractor = extractors[current_user] ||= - Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedReferenceExtractor.new(project, current_user) - - extractor.reset_memoized_values - end - - self.class.mentionable_attrs.each do |attr, options| - text = __send__(attr) # rubocop:disable GitlabSecurity/PublicSend - options = options.merge( - cache_key: [self, attr], - author: author, - skip_project_check: skip_project_check? - ).merge(mentionable_params) - - cached_html = self.try(:updated_cached_html_for, attr.to_sym) - options[:rendered] = cached_html if cached_html - - extractor.analyze(text, options) - end - - extractor - end - - def extractors - @extractors ||= {} - end - - def skip_project_check? - false - end - - def build_mention_values(resource_foreign_key) - refs = all_references(author) - - mentioned_users_ids = array_to_sql(refs.isolated_mentioned_users.pluck(:id)) - mentioned_projects_ids = array_to_sql(refs.isolated_mentioned_projects.pluck(:id)) - mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id)) - - return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank? - - { - "#{resource_foreign_key}": user_mention_resource_id, - note_id: user_mention_note_id, - mentioned_users_ids: mentioned_users_ids, - mentioned_projects_ids: mentioned_projects_ids, - mentioned_groups_ids: mentioned_groups_ids - } - end - - def array_to_sql(ids_array) - return unless ids_array.present? - - '{' + ids_array.join(", ") + '}' - end - - private - - def mentionable_params - {} - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/mentionable_migration_methods.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/mentionable_migration_methods.rb deleted file mode 100644 index efb08d44100..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/mentionable_migration_methods.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - module Concerns - # Extract common no_quote_columns method used in determining the columns that do not need - # to be quoted for corresponding models - module MentionableMigrationMethods - extend ::ActiveSupport::Concern - - class_methods do - def no_quote_columns - [ - :note_id, - user_mention_model.resource_foreign_key - ] - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb deleted file mode 100644 index 75759ed0111..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - module Concerns - module Namespace - # isolate recursive traversal code for namespace hierarchy - module RecursiveTraversal - extend ActiveSupport::Concern - - def root_ancestor - return self if persisted? && parent_id.nil? - - strong_memoize(:root_ancestor) do - Gitlab::ObjectHierarchy - .new(self.class.where(id: id)) - .base_and_ancestors - .reorder(nil) - .find_by(parent_id: nil) - end - end - - # Returns all ancestors, self, and descendants of the current namespace. - def self_and_hierarchy - Gitlab::ObjectHierarchy - .new(self.class.where(id: id)) - .all_objects - end - - # Returns all the ancestors of the current namespaces. - def ancestors - return self.class.none unless parent_id - - Gitlab::ObjectHierarchy - .new(self.class.where(id: parent_id)) - .base_and_ancestors - end - - # returns all ancestors upto but excluding the given namespace - # when no namespace is given, all ancestors upto the top are returned - def ancestors_upto(top = nil, hierarchy_order: nil) - Gitlab::ObjectHierarchy.new(self.class.where(id: id)) - .ancestors(upto: top, hierarchy_order: hierarchy_order) - end - - def self_and_ancestors(hierarchy_order: nil) - return self.class.where(id: id) unless parent_id - - Gitlab::ObjectHierarchy - .new(self.class.where(id: id)) - .base_and_ancestors(hierarchy_order: hierarchy_order) - end - - # Returns all the descendants of the current namespace. - def descendants - Gitlab::ObjectHierarchy - .new(self.class.where(parent_id: id)) - .base_and_descendants - end - - def self_and_descendants - Gitlab::ObjectHierarchy - .new(self.class.where(id: id)) - .base_and_descendants - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb deleted file mode 100644 index d010d68600d..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - module DesignManagement - class Design < ActiveRecord::Base - include EachBatch - include Concerns::MentionableMigrationMethods - - self.table_name = 'design_management_designs' - self.inheritance_column = :_type_disabled - - def self.user_mention_model - Gitlab::BackgroundMigration::UserMentions::Models::DesignUserMention - end - - def user_mention_model - self.class.user_mention_model - end - - def user_mention_resource_id - id - end - - def user_mention_note_id - 'NULL' - end - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb deleted file mode 100644 index eb00f6cfa3f..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class DesignUserMention < ActiveRecord::Base - self.table_name = 'design_user_mentions' - self.inheritance_column = :_type_disabled - - def self.resource_foreign_key - :design_id - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb deleted file mode 100644 index cfd9a4faa9b..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/epic.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class Epic < ActiveRecord::Base - include EachBatch - include Concerns::IsolatedMentionable - include Concerns::MentionableMigrationMethods - include CacheMarkdownField - - attr_mentionable :title, pipeline: :single_line - attr_mentionable :description - cache_markdown_field :title, pipeline: :single_line - cache_markdown_field :description, issuable_state_filter_enabled: true - - self.table_name = 'epics' - self.inheritance_column = :_type_disabled - - belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" - belongs_to :group, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" - - def self.user_mention_model - Gitlab::BackgroundMigration::UserMentions::Models::EpicUserMention - end - - def user_mention_model - self.class.user_mention_model - end - - def project - nil - end - - def mentionable_params - { group: group, label_url_method: :group_epics_url } - end - - def user_mention_resource_id - id - end - - def user_mention_note_id - 'NULL' - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb deleted file mode 100644 index 579e4d99612..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class EpicUserMention < ActiveRecord::Base - self.table_name = 'epic_user_mentions' - self.inheritance_column = :_type_disabled - - def self.resource_foreign_key - :epic_id - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb deleted file mode 100644 index 310723570c2..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/group.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - # isolated Group model - class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace - self.store_full_sti_class = false - self.inheritance_column = :_type_disabled - - has_one :saml_provider - - def root_saml_provider - root_ancestor.saml_provider - end - - def self.declarative_policy_class - "GroupPolicy" - end - - def max_member_access_for_user(user) - return GroupMember::NO_ACCESS unless user - - return GroupMember::OWNER if user.admin? - - max_member_access = members_with_parents.where(user_id: user) - .reorder(access_level: :desc) - .first - &.access_level - - max_member_access || GroupMember::NO_ACCESS - end - - def members_with_parents - # Avoids an unnecessary SELECT when the group has no parents - source_ids = - if has_parent? - self_and_ancestors.reorder(nil).select(:id) - else - id - end - - group_hierarchy_members = GroupMember.active_without_invites_and_requests - .where(source_id: source_ids) - - GroupMember.from_union([group_hierarchy_members, - members_from_self_and_ancestor_group_shares]) - end - - # rubocop: disable Metrics/AbcSize - def members_from_self_and_ancestor_group_shares - group_group_link_table = GroupGroupLink.arel_table - group_member_table = GroupMember.arel_table - - source_ids = - if has_parent? - self_and_ancestors.reorder(nil).select(:id) - else - id - end - - group_group_links_query = GroupGroupLink.where(shared_group_id: source_ids) - cte = Gitlab::SQL::CTE.new(:group_group_links_cte, group_group_links_query) - cte_alias = cte.table.alias(GroupGroupLink.table_name) - - # Instead of members.access_level, we need to maximize that access_level at - # the respective group_group_links.group_access. - member_columns = GroupMember.attribute_names.map do |column_name| - if column_name == 'access_level' - smallest_value_arel([cte_alias[:group_access], group_member_table[:access_level]], - 'access_level') - else - group_member_table[column_name] - end - end - - GroupMember - .with(cte.to_arel) - .select(*member_columns) - .from([group_member_table, cte.alias_to(group_group_link_table)]) - .where(group_member_table[:requested_at].eq(nil)) - .where(group_member_table[:source_id].eq(group_group_link_table[:shared_with_group_id])) - .where(group_member_table[:source_type].eq('Namespace')) - end - # rubocop: enable Metrics/AbcSize - - def smallest_value_arel(args, column_alias) - Arel::Nodes::As.new( - Arel::Nodes::NamedFunction.new('LEAST', args), - Arel::Nodes::SqlLiteral.new(column_alias)) - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb deleted file mode 100644 index 13addcc3c55..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class MergeRequest < ActiveRecord::Base - include EachBatch - include Concerns::IsolatedMentionable - include CacheMarkdownField - include Concerns::MentionableMigrationMethods - - attr_mentionable :title, pipeline: :single_line - attr_mentionable :description - cache_markdown_field :title, pipeline: :single_line - cache_markdown_field :description, issuable_state_filter_enabled: true - - self.table_name = 'merge_requests' - self.inheritance_column = :_type_disabled - - belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" - belongs_to :target_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" - belongs_to :source_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" - - alias_attribute :project, :target_project - - def self.user_mention_model - Gitlab::BackgroundMigration::UserMentions::Models::MergeRequestUserMention - end - - def user_mention_model - self.class.user_mention_model - end - - def user_mention_resource_id - id - end - - def user_mention_note_id - 'NULL' - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb deleted file mode 100644 index 4a85892d7b8..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class MergeRequestUserMention < ActiveRecord::Base - self.table_name = 'merge_request_user_mentions' - self.inheritance_column = :_type_disabled - - def self.resource_foreign_key - :merge_request_id - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb deleted file mode 100644 index d76d06606ee..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - # isolated Namespace model - class Namespace < ActiveRecord::Base - self.inheritance_column = :_type_disabled - - include Concerns::IsolatedFeatureGate - include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel - include ::Gitlab::Utils::StrongMemoize - include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal - - belongs_to :parent, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace" - - def visibility_level_field - :visibility_level - end - - def has_parent? - parent_id.present? || parent.present? - end - - # Deprecated, use #licensed_feature_available? instead. Remove once Namespace#feature_available? isn't used anymore. - def feature_available?(feature) - licensed_feature_available?(feature) - end - - # Overridden in EE::Namespace - def licensed_feature_available?(_feature) - false - end - end - end - end - end -end - -Namespace.prepend_mod_with('Namespace') diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb deleted file mode 100644 index 4026a91903f..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/note.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - class Note < ActiveRecord::Base - include EachBatch - include Concerns::IsolatedMentionable - include CacheMarkdownField - - self.table_name = 'notes' - self.inheritance_column = :_type_disabled - - attr_mentionable :note, pipeline: :note - cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true - - belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" - belongs_to :noteable, polymorphic: true - belongs_to :project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" - - def for_personal_snippet? - noteable && noteable.instance_of?(PersonalSnippet) - end - - def for_project_noteable? - !for_personal_snippet? && !for_epic? - end - - def skip_project_check? - !for_project_noteable? - end - - def for_epic? - noteable && noteable_type == 'Epic' - end - - def user_mention_resource_id - noteable_id || commit_id - end - - def user_mention_note_id - id - end - - def noteable - super unless for_commit? - end - - def for_commit? - noteable_type == "Commit" - end - - private - - def mentionable_params - return super unless for_epic? - - super.merge(banzai_context_params) - end - - def banzai_context_params - return {} unless noteable - - { group: noteable.group, label_url_method: :group_epics_url } - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/project.rb b/lib/gitlab/background_migration/user_mentions/models/project.rb deleted file mode 100644 index 4e02bf97d12..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/project.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - # isolated Namespace model - class Project < ActiveRecord::Base - include Concerns::IsolatedFeatureGate - include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel - - self.table_name = 'projects' - self.inheritance_column = :_type_disabled - - belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id', class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" - belongs_to :namespace, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace" - alias_method :parent, :namespace - - # Returns a collection of projects that is either public or visible to the - # logged in user. - def self.public_or_visible_to_user(user = nil, min_access_level = nil) - min_access_level = nil if user&.can_read_all_resources? - - return public_to_user unless user - - if user.is_a?(::Gitlab::BackgroundMigration::UserMentions::Models::User) - where('EXISTS (?) OR projects.visibility_level IN (?)', - user.authorizations_for_projects(min_access_level: min_access_level), - levels_for_user(user)) - end - end - - def grafana_integration - nil - end - - def default_issues_tracker? - true # we do not care of the issue tracker type(internal or external) when parsing mentions - end - - def visibility_level_field - :visibility_level - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/models/user.rb b/lib/gitlab/background_migration/user_mentions/models/user.rb deleted file mode 100644 index a30220b6934..00000000000 --- a/lib/gitlab/background_migration/user_mentions/models/user.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module UserMentions - module Models - # isolated Namespace model - class User < ActiveRecord::Base - include Concerns::IsolatedFeatureGate - - self.table_name = 'users' - self.inheritance_column = :_type_disabled - - has_many :project_authorizations, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent - - def authorizations_for_projects(min_access_level: nil, related_project_column: 'projects.id') - authorizations = project_authorizations - .select(1) - .where("project_authorizations.project_id = #{related_project_column}") - - return authorizations unless min_access_level.present? - - authorizations.where('project_authorizations.access_level >= ?', min_access_level) - end - - def can_read_all_resources? - can?(:read_all_resources) - end - - def can?(action, subject = :global) - Ability.allowed?(self, action, subject) - end - end - end - end - end -end diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 899e2e6c1c5..242979da367 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -19,7 +19,8 @@ module Gitlab end def self.refmap - [:heads, :tags, '+refs/pull-requests/*/to:refs/merge-requests/*/head'] + # We omit :heads and :tags since these are fetched in the import_repository + ['+refs/pull-requests/*/to:refs/merge-requests/*/head'] end # Unlike GitHub, you can't grab the commit SHAs for pull requests that @@ -140,11 +141,11 @@ module Gitlab def import_repository log_info(stage: 'import_repository', message: 'starting import') - project.ensure_repository + project.repository.import_repository(project.import_url) project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap) log_info(stage: 'import_repository', message: 'finished import') - rescue Gitlab::Shell::Error => e + rescue ::Gitlab::Git::CommandError => e Gitlab::ErrorTracking.log_exception( e, stage: 'import_repository', message: 'failed import', error: e.message diff --git a/lib/gitlab/chat/output.rb b/lib/gitlab/chat/output.rb index 4a55b81a9eb..b4ce05ca06d 100644 --- a/lib/gitlab/chat/output.rb +++ b/lib/gitlab/chat/output.rb @@ -48,10 +48,10 @@ module Gitlab # follows it will produce a nil. For example: # # "\n".split("\n") # => [] - # "\n".split("\n")[1..-1] # => nil + # "\n".split("\n")[1..] # => nil # # To work around this we only "join" if we're given an Array. - if (converted = output.split("\n")[1..-1]) + if (converted = output.split("\n")[1..]) converted.join("\n") else '' diff --git a/lib/gitlab/ci/badge/coverage/report.rb b/lib/gitlab/ci/badge/coverage/report.rb index 78b51dbdaf0..4d1193176ad 100644 --- a/lib/gitlab/ci/badge/coverage/report.rb +++ b/lib/gitlab/ci/badge/coverage/report.rb @@ -30,7 +30,7 @@ module Gitlab::Ci @coverage ||= raw_coverage return unless @coverage - @coverage.to_f.round(2) + @coverage.round(2) end def metadata diff --git a/lib/gitlab/ci/badge/metadata.rb b/lib/gitlab/ci/badge/metadata.rb index eec9fedfaa9..244e3aff851 100644 --- a/lib/gitlab/ci/badge/metadata.rb +++ b/lib/gitlab/ci/badge/metadata.rb @@ -8,14 +8,13 @@ module Gitlab::Ci class Metadata include Gitlab::Routing include ActionView::Helpers::AssetTagHelper - include ActionView::Helpers::UrlHelper def initialize(badge) @badge = badge end def to_html - link_to(image_tag(image_url, alt: title), link_url) + ApplicationController.helpers.link_to(image_tag(image_url, alt: title), link_url) end def to_markdown diff --git a/lib/gitlab/ci/build/context/base.rb b/lib/gitlab/ci/build/context/base.rb index c7ea7c78e2f..81f96e822f4 100644 --- a/lib/gitlab/ci/build/context/base.rb +++ b/lib/gitlab/ci/build/context/base.rb @@ -17,6 +17,12 @@ module Gitlab raise NotImplementedError end + def variables_hash + strong_memoize(:variables_hash) do + variables.to_hash + end + end + def project pipeline.project end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb index 7b1ce6330f0..810523052ae 100644 --- a/lib/gitlab/ci/build/policy/variables.rb +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -10,7 +10,7 @@ module Gitlab end def satisfied_by?(pipeline, context) - variables = context.variables + variables = context.variables_hash statements = @expressions.map do |statement| ::Gitlab::Ci::Pipeline::Expression::Statement diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 9c2f6eea1dd..82a59fdb4e1 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -23,7 +23,7 @@ module Gitlab return @globs unless context @globs.map do |glob| - ExpandVariables.expand_existing(glob, context.variables) + ExpandVariables.expand_existing(glob, -> { context.variables_hash }) end end end diff --git a/lib/gitlab/ci/build/rules/rule/clause/if.rb b/lib/gitlab/ci/build/rules/rule/clause/if.rb index 6143a736ca6..499a265a1e2 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/if.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/if.rb @@ -10,7 +10,7 @@ module Gitlab def satisfied_by?(pipeline, context) ::Gitlab::Ci::Pipeline::Expression::Statement.new( - @expression, context.variables).truthful? + @expression, context.variables_hash).truthful? end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 6f149385969..42b487fdf81 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -17,21 +17,27 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root, :context, :source_ref_path, :source + attr_reader :root, :context, :source_ref_path, :source, :logger - def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil) + def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, logger: nil) + @logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project) @source_ref_path = pipeline&.source_ref_path - @context = build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) + @context = self.logger.instrument(:config_build_context) do + build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) + end + @context.set_deadline(TIMEOUT_SECONDS) @source = source - @config = expand_config(config) - - @root = Entry::Root.new(@config) - @root.compose! + @config = self.logger.instrument(:config_expand) do + expand_config(config) + end + @root = self.logger.instrument(:config_compose) do + Entry::Root.new(@config).tap(&:compose!) + end rescue *rescue_errors => e raise Config::ConfigError, e.message end @@ -94,11 +100,25 @@ module Gitlab end def build_config(config) - initial_config = Config::Yaml.load!(config) - initial_config = Config::External::Processor.new(initial_config, @context).perform - initial_config = Config::Extendable.new(initial_config).to_hash - initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash - Config::EdgeStagesInjector.new(initial_config).to_hash + initial_config = logger.instrument(:config_yaml_load) do + Config::Yaml.load!(config) + end + + initial_config = logger.instrument(:config_external_process) do + Config::External::Processor.new(initial_config, @context).perform + end + + initial_config = logger.instrument(:config_yaml_extend) do + Config::Extendable.new(initial_config).to_hash + end + + initial_config = logger.instrument(:config_tags_resolve) do + Config::Yaml::Tags::Resolver.new(initial_config).to_hash + end + + logger.instrument(:config_stages_inject) do + Config::EdgeStagesInjector.new(initial_config).to_hash + end end def find_sha(project) @@ -115,10 +135,20 @@ module Gitlab sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, - variables: build_variables(project: project, pipeline: pipeline)) + variables: build_variables(project: project, pipeline: pipeline), + logger: logger) end def build_variables(project:, pipeline:) + logger.instrument(:config_build_variables) do + build_variables_without_instrumentation( + project: project, + pipeline: pipeline + ) + end + end + + def build_variables_without_instrumentation(project:, pipeline:) Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless project diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 520b1ce6119..43475742214 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -26,7 +26,7 @@ module Gitlab validates :name, length: { maximum: 255 }, if: -> { ::Feature.enabled?(:ci_validate_job_length, default_enabled: :yaml) } validates :config, disallowed_keys: { - in: %i[only except when start_in], + in: %i[only except start_in], message: 'key may not be used with `rules`' }, if: :has_rules? diff --git a/lib/gitlab/ci/config/entry/tags.rb b/lib/gitlab/ci/config/entry/tags.rb index ca3b48372e2..6044cfddbdc 100644 --- a/lib/gitlab/ci/config/entry/tags.rb +++ b/lib/gitlab/ci/config/entry/tags.rb @@ -16,8 +16,6 @@ module Gitlab validates :config, array_of_strings: true validate do - next unless ::Feature.enabled?(:ci_build_tags_limit, default_enabled: :yaml) - if config.is_a?(Array) && config.size >= TAGS_LIMIT errors.add(:config, _("must be less than the limit of %{tag_limit} tags") % { tag_limit: TAGS_LIMIT }) end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 51624dc30ea..308414af47d 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -9,17 +9,22 @@ module Gitlab TimeoutError = Class.new(StandardError) + include ::Gitlab::Utils::StrongMemoize + attr_reader :project, :sha, :user, :parent_pipeline, :variables - attr_reader :expandset, :execution_deadline + attr_reader :expandset, :execution_deadline, :logger + + delegate :instrument, to: :logger - def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: []) + def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, logger: nil) @project = project @sha = sha @user = user @parent_pipeline = parent_pipeline - @variables = variables + @variables = variables || Ci::Variables::Collection.new @expandset = Set.new @execution_deadline = 0 + @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) yield self if block_given? end @@ -36,10 +41,17 @@ module Gitlab end end + def variables_hash + strong_memoize(:variables_hash) do + variables.to_hash + end + end + def mutate(attrs = {}) self.class.new(**attrs) do |ctx| ctx.expandset = expandset ctx.execution_deadline = execution_deadline + ctx.logger = logger end end @@ -60,7 +72,7 @@ module Gitlab protected - attr_writer :expandset, :execution_deadline + attr_writer :expandset, :execution_deadline, :logger private diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 95f1a842c50..a5bf066c81f 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -30,6 +30,18 @@ module Gitlab def process return [] if locations.empty? + logger.instrument(:config_mapper_process) do + process_without_instrumentation + end + end + + private + + attr_reader :locations, :context + + delegate :expandset, :logger, to: :context + + def process_without_instrumentation locations .compact .map(&method(:normalize_location)) @@ -41,14 +53,14 @@ module Gitlab .map(&method(:select_first_matching)) end - private - - attr_reader :locations, :context - - delegate :expandset, to: :context + def normalize_location(location) + logger.instrument(:config_mapper_normalize) do + normalize_location_without_instrumentation(location) + end + end # convert location if String to canonical form - def normalize_location(location) + def normalize_location_without_instrumentation(location) if location.is_a?(String) expanded_location = expand_variables(location) normalize_location_string(expanded_location) @@ -58,6 +70,12 @@ module Gitlab end def verify_rules(location) + logger.instrument(:config_mapper_rules) do + verify_rules_without_instrumentation(location) + end + end + + def verify_rules_without_instrumentation(location) return unless Rules.new(location[:rules]).evaluate(context).pass? location @@ -72,6 +90,12 @@ module Gitlab end def expand_wildcard_paths(location) + logger.instrument(:config_mapper_wildcards) do + expand_wildcard_paths_without_instrumentation(location) + end + end + + def expand_wildcard_paths_without_instrumentation(location) # We only support local files for wildcard paths return location unless location[:local] && location[:local].include?('*') @@ -89,6 +113,12 @@ module Gitlab end def verify_duplicates!(location) + logger.instrument(:config_mapper_verify) do + verify_duplicates_without_instrumentation!(location) + end + end + + def verify_duplicates_without_instrumentation!(location) if expandset.count >= MAX_INCLUDES raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" end @@ -106,6 +136,12 @@ module Gitlab end def select_first_matching(location) + logger.instrument(:config_mapper_select) do + select_first_matching_without_instrumentation(location) + end + end + + def select_first_matching_without_instrumentation(location) matching = FILE_CLASSES.map do |file_class| file_class.new(location, context) end.select(&:matching?) @@ -116,6 +152,12 @@ module Gitlab end def expand_variables(data) + logger.instrument(:config_mapper_variables) do + expand_variables_without_instrumentation(data) + end + end + + def expand_variables_without_instrumentation(data) if data.is_a?(String) expand(data) else @@ -137,7 +179,7 @@ module Gitlab end def expand(data) - ExpandVariables.expand(data, context.variables) + ExpandVariables.expand(data, -> { context.variables_hash }) end end end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index de69a1b1e8f..6a4aee26d80 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -7,10 +7,13 @@ module Gitlab class Processor IncludeError = Class.new(StandardError) + attr_reader :context, :logger + def initialize(values, context) @values = values @external_files = External::Mapper.new(values, context).process @content = {} + @logger = context.logger rescue External::Mapper::Error, OpenSSL::SSL::SSLError => e raise IncludeError, e.message @@ -29,13 +32,17 @@ module Gitlab def validate_external_files! @external_files.each do |file| - raise IncludeError, file.error_message unless file.valid? + logger.instrument(:config_external_verify) do + raise IncludeError, file.error_message unless file.valid? + end end end def merge_external_files! @external_files.each do |file| - @content.deep_merge!(file.to_hash) + logger.instrument(:config_external_merge) do + @content.deep_merge!(file.to_hash) + end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb deleted file mode 100644 index 51051b0490f..00000000000 --- a/lib/gitlab/ci/features.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - ## - # Deprecated: Ci::Features is a class that aggregates all CI/CD feature flags in one place. - # - module Features - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` - # is a safe switch to disable the feature for a particular project when something went wrong, - # therefore it's not supposed to be enabled by default. - def self.disallow_to_create_merge_request_pipelines_in_target_project?(target_project) - ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) - end - - def self.accept_trace?(project) - ::Feature.enabled?(:ci_enable_live_trace, project) && - ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: true) - end - - def self.log_invalid_trace_chunks?(project) - ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) - end - - def self.gldropdown_tags_enabled? - ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) - end - end - end -end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 73cfa02ce4b..651ed23eb25 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -34,7 +34,7 @@ module Gitlab end def file_name - "#{report_type.to_s.dasherize}-report-format.json" + report_type == :api_fuzzing ? "dast-report-format.json" : "#{report_type.to_s.dasherize}-report-format.json" end end diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb index f9afa58f915..041d73cd914 100644 --- a/lib/gitlab/ci/parsers/terraform/tfplan.rb +++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb @@ -34,7 +34,7 @@ module Gitlab def job_details(job) { 'job_id' => job.id.to_s, - 'job_name' => job.options.dig(:artifacts, :name).to_s, + 'job_name' => job.name, 'job_path' => Gitlab::Routing.url_helpers.project_job_path(job.project, job) } end diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index 9b494f3a7ec..28567437719 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -7,7 +7,7 @@ module Gitlab class Base attr_reader :pipeline, :command, :config - delegate :project, :current_user, :parent_pipeline, to: :command + delegate :project, :current_user, :parent_pipeline, :logger, to: :command def initialize(pipeline, command) @pipeline = pipeline diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 6feb693221b..bbdc6b65b96 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -21,6 +21,10 @@ module Gitlab merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, locked: @command.project.default_pipeline_lock) + + # Initialize the feature flag at the beginning of the pipeline creation process + # so that the flag references in the latter chains return the same value. + @pipeline.create_deployment_in_separate_transaction? end def break? diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index beb8801096b..c466b8b36d0 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -11,7 +11,7 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, + :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, :logger, # These attributes are set by Chains during processing: :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed ) do @@ -88,7 +88,14 @@ module Gitlab @metrics ||= ::Gitlab::Ci::Pipeline::Metrics end + def logger + self[:logger] ||= ::Gitlab::Ci::Pipeline::Logger.new(project: project) + end + def observe_step_duration(step_class, duration) + step = step_class.name.underscore.parameterize(separator: '_') + logger.observe("pipeline_step_#{step}_duration_s", duration) + if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops, default_enabled: :yaml) metrics.pipeline_creation_step_duration_histogram .observe({ step: step_class.name }, duration.seconds) @@ -96,11 +103,15 @@ module Gitlab end def observe_creation_duration(duration) + logger.observe(:pipeline_creation_duration_s, duration) + metrics.pipeline_creation_duration_histogram .observe({}, duration.seconds) end def observe_pipeline_size(pipeline) + logger.observe(:pipeline_size_count, pipeline.total_size) + metrics.pipeline_size_histogram .observe({ source: pipeline.source.to_s }, pipeline.total_size) end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index f3c937ddd28..64d1b001e3c 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -11,16 +11,21 @@ module Gitlab def perform! raise ArgumentError, 'missing config content' unless @command.config_content - result = ::Gitlab::Ci::YamlProcessor.new( - @command.config_content, { - project: project, - pipeline: @pipeline, - sha: @pipeline.sha, - source: @pipeline.source, - user: current_user, - parent_pipeline: parent_pipeline - } - ).execute + result = logger.instrument(:pipeline_config_process) do + processor = ::Gitlab::Ci::YamlProcessor.new( + @command.config_content, { + project: project, + pipeline: @pipeline, + sha: @pipeline.sha, + source: @pipeline.source, + user: current_user, + parent_pipeline: parent_pipeline, + logger: logger + } + ) + + processor.execute + end add_warnings_to_pipeline(result.warnings) diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 81ef3bb074d..15b0ff3c04d 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -6,10 +6,18 @@ module Gitlab module Chain class Create < Chain::Base include Chain::Helpers + include Gitlab::Utils::StrongMemoize def perform! - BulkInsertableAssociations.with_bulk_insert do - pipeline.save! + logger.instrument(:pipeline_save) do + BulkInsertableAssociations.with_bulk_insert do + tags = extract_tag_list_by_status + + pipeline.transaction do + pipeline.save! + CommitStatus.bulk_insert_tags!(statuses, tags) if bulk_insert_tags? + end + end end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") @@ -18,6 +26,37 @@ module Gitlab def break? !pipeline.persisted? end + + private + + def statuses + strong_memoize(:statuses) do + pipeline.stages.flat_map(&:statuses) + end + end + + # We call `job.tag_list=` to assign tags to the jobs from the + # Chain::Seed step which uses the `@tag_list` instance variable to + # store them on the record. We remove them here because we want to + # bulk insert them, otherwise they would be inserted and assigned one + # by one with callbacks. We must use `remove_instance_variable` + # because having the instance variable defined would still run the callbacks + def extract_tag_list_by_status + return {} unless bulk_insert_tags? + + statuses.each.with_object({}) do |job, acc| + tag_list = job.clear_memoization(:tag_list) + next unless tag_list + + acc[job.name] = tag_list + end + end + + def bulk_insert_tags? + strong_memoize(:bulk_insert_tags) do + ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb new file mode 100644 index 00000000000..b92aa89d62d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class CreateDeployments < Chain::Base + DeploymentCreationError = Class.new(StandardError) + + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + create_deployments! + end + + def break? + false + end + + private + + def create_deployments! + pipeline.stages.map(&:statuses).flatten.map(&method(:create_deployment)) + end + + def create_deployment(build) + return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present? + + deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment + .new(build, build.persisted_environment).to_resource + + return unless deployment + + deployment.deployable = build + deployment.save! + rescue ActiveRecord::RecordInvalid => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + DeploymentCreationError.new(e.message), build_id: build.id) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb new file mode 100644 index 00000000000..424e1d87fb4 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class EnsureEnvironments < Chain::Base + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_environment)) + end + + def break? + false + end + + private + + def ensure_environment(build) + return unless build.instance_of?(::Ci::Build) && build.has_environment? + + environment = ::Gitlab::Ci::Pipeline::Seed::Environment.new(build).to_resource + + if environment.persisted? + build.persisted_environment = environment + build.assign_attributes(metadata_attributes: { expanded_environment_name: environment.name }) + else + build.assign_attributes(status: :failed, failure_reason: :environment_creation_failure) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb b/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb new file mode 100644 index 00000000000..f4e5e6e467a --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class EnsureResourceGroups < Chain::Base + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_resource_group)) + end + + def break? + false + end + + private + + def ensure_resource_group(processable) + return unless processable.is_a?(::Ci::Processable) + + key = processable.options.delete(:resource_group_key) + + resource_group = ::Gitlab::Ci::Pipeline::Seed::Processable::ResourceGroup + .new(processable, key).to_resource + + processable.resource_group = resource_group + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index ef7447fa83d..356eeb76908 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -13,8 +13,10 @@ module Gitlab raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result # Allocate next IID. This operation must be outside of transactions of pipeline creations. - pipeline.ensure_project_iid! - pipeline.ensure_ci_ref! + logger.instrument(:pipeline_allocate_seed_attributes) do + pipeline.ensure_project_iid! + pipeline.ensure_ci_ref! + end # Protect the pipeline. This is assigned in Populate instead of # Build to prevent erroring out on ambiguous refs. @@ -23,8 +25,12 @@ module Gitlab ## # Gather all runtime build/stage errors # - if pipeline_seed.errors - return error(pipeline_seed.errors.join("\n"), config_error: true) + seed_errors = logger.instrument(:pipeline_seed_evaluation) do + pipeline_seed.errors + end + + if seed_errors + return error(seed_errors.join("\n"), config_error: true) end @command.pipeline_seed = pipeline_seed @@ -38,8 +44,11 @@ module Gitlab def pipeline_seed strong_memoize(:pipeline_seed) do - stages_attributes = @command.yaml_processor_result.stages_attributes - Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) + logger.instrument(:pipeline_seed_initialization) do + stages_attributes = @command.yaml_processor_result.stages_attributes + + Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) + end end end @@ -48,9 +57,11 @@ module Gitlab end def root_variables - ::Gitlab::Ci::Variables::Helpers.merge_variables( - @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables - ) + logger.instrument(:pipeline_seed_merge_variables) do + ::Gitlab::Ci::Variables::Helpers.merge_variables( + @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + ) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 845eb6c7a42..de147914850 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -9,30 +9,36 @@ module Gitlab @pipeline = pipeline @command = command @sequence = sequence - @start = Time.now + @start = current_monotonic_time end def build! @sequence.each do |step_class| - step_start = ::Gitlab::Metrics::System.monotonic_time + step_start = current_monotonic_time step = step_class.new(@pipeline, @command) step.perform! @command.observe_step_duration( step_class, - ::Gitlab::Metrics::System.monotonic_time - step_start + current_monotonic_time - step_start ) break if step.break? end - @command.observe_creation_duration(Time.now - @start) + @command.observe_creation_duration(current_monotonic_time - @start) @command.observe_pipeline_size(@pipeline) @command.observe_jobs_count_in_alive_pipelines @pipeline end + + private + + def current_monotonic_time + ::Gitlab::Metrics::System.monotonic_time + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 28ba1cd4d47..85bd5f0a7c1 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -113,7 +113,7 @@ module Gitlab name: build[:name], stage: build[:stage], image: build.dig(:options, :image, :name), - services: build.dig(:options, :services)&.map { |service| service[:name] }, + services: service_names(build), script: [ build.dig(:options, :before_script), build.dig(:options, :script), @@ -122,6 +122,14 @@ module Gitlab } end + def service_names(build) + services = build.dig(:options, :services) + + return unless services + + services.compact.map { |service| service[:name] } + end + def stages_attributes command.yaml_processor_result.stages_attributes end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index 11d2010909f..6da88fd287e 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -9,7 +9,11 @@ module Gitlab PATTERN = /\$(?<name>\w+)/.freeze def evaluate(variables = {}) - variables.with_indifferent_access.fetch(@value, nil) + unless variables.is_a?(ActiveSupport::HashWithIndifferentAccess) + variables = variables.with_indifferent_access + end + + variables.fetch(@value, nil) end def inspect diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 5f3310dd668..4b13cae792e 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -9,7 +9,7 @@ module Gitlab def initialize(statement, variables = nil) @lexer = Expression::Lexer.new(statement) - @variables = variables&.to_hash + @variables = variables || {} end def parse_tree @@ -19,7 +19,7 @@ module Gitlab end def evaluate - parse_tree.evaluate(@variables.to_h) + parse_tree.evaluate(@variables) end def truthful? diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb new file mode 100644 index 00000000000..97f7dddd09a --- /dev/null +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + class Logger + include ::Gitlab::Utils::StrongMemoize + + def self.current_monotonic_time + ::Gitlab::Metrics::System.monotonic_time + end + + def initialize(project:, destination: Gitlab::AppJsonLogger) + @started_at = current_monotonic_time + @project = project + @destination = destination + @log_conditions = [] + + yield(self) if block_given? + end + + def log_when(&block) + log_conditions.push(block) + end + + def instrument(operation) + return yield unless enabled? + + raise ArgumentError, 'block not given' unless block_given? + + op_started_at = current_monotonic_time + + result = yield + + observe("#{operation}_duration_s", current_monotonic_time - op_started_at) + + result + end + + def observe(operation, value) + return unless enabled? + + observations[operation.to_s].push(value) + end + + def commit(pipeline:, caller:) + return unless log? + + attributes = { + class: self.class.name.to_s, + pipeline_creation_caller: caller, + project_id: project.id, + pipeline_id: pipeline.id, + pipeline_persisted: pipeline.persisted?, + pipeline_source: pipeline.source, + pipeline_creation_service_duration_s: age + }.stringify_keys.merge(observations_hash) + + destination.info(attributes) + end + + def observations_hash + observations.transform_values do |values| + next if values.empty? + + { + 'count' => values.size, + 'min' => values.min, + 'max' => values.max, + 'avg' => values.sum / values.size + } + end.compact + end + + private + + attr_reader :project, :destination, :started_at, :log_conditions + delegate :current_monotonic_time, to: :class + + def age + current_monotonic_time - started_at + end + + def log? + return false unless enabled? + return true if log_conditions.empty? + + log_conditions.any? { |cond| cond.call(observations) } + end + + def enabled? + strong_memoize(:enabled) do + ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops, default_enabled: :yaml) + end + end + + def observations + @observations ||= Hash.new { |hash, key| hash[key] = [] } + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 72837b8ec22..762292f0fa3 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -7,8 +7,6 @@ module Gitlab class Build < Seed::Base include Gitlab::Utils::StrongMemoize - EnvironmentCreationFailure = Class.new(StandardError) - delegate :dig, to: :@seed_attributes def initialize(context, attributes, stages_for_needs_lookup = []) @@ -30,7 +28,7 @@ module Gitlab @except = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:except)) @rules = Gitlab::Ci::Build::Rules - .new(attributes.delete(:rules), default_when: 'on_success') + .new(attributes.delete(:rules), default_when: attributes[:when]) @cache = Gitlab::Ci::Build::Cache .new(attributes.delete(:cache), @pipeline) @@ -80,7 +78,7 @@ module Gitlab def to_resource strong_memoize(:resource) do processable = initialize_processable - assign_resource_group(processable) + assign_resource_group(processable) unless @pipeline.create_deployment_in_separate_transaction? processable end end @@ -90,7 +88,9 @@ module Gitlab ::Ci::Bridge.new(attributes) else ::Ci::Build.new(attributes).tap do |build| - build.assign_attributes(self.class.deployment_attributes_for(build)) + unless @pipeline.create_deployment_in_separate_transaction? + build.assign_attributes(self.class.deployment_attributes_for(build)) + end end end end @@ -107,20 +107,7 @@ module Gitlab environment = Seed::Environment.new(build).to_resource if environment.nil? unless environment.persisted? - if Feature.enabled?(:surface_environment_creation_failure, build.project, default_enabled: :yaml) && - Feature.disabled?(:surface_environment_creation_failure_override, build.project) - return { status: :failed, failure_reason: :environment_creation_failure } - end - - # If there is a validation error on environment creation, such as - # the name contains invalid character, the build falls back to a - # non-environment job. - Gitlab::ErrorTracking.track_exception( - EnvironmentCreationFailure.new, - project_id: build.project_id, - reason: environment.errors.full_messages.to_sentence) - - return { environment: nil } + return { status: :failed, failure_reason: :environment_creation_failure } end build.persisted_environment = environment @@ -215,12 +202,14 @@ module Gitlab end def runner_tags - { tag_list: evaluate_runner_tags }.compact + strong_memoize(:runner_tags) do + { tag_list: evaluate_runner_tags }.compact + end end def evaluate_runner_tags - @seed_attributes[:tag_list]&.map do |tag| - ExpandVariables.expand_existing(tag, evaluate_context.variables) + @seed_attributes.delete(:tag_list)&.map do |tag| + ExpandVariables.expand_existing(tag, -> { evaluate_context.variables_hash }) end end diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index 417319cb5be..3e4a44a2e70 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -51,7 +51,7 @@ module Gitlab def replace_with!(other) instance_variables.each do |ivar| - instance_variable_set(ivar, other.public_send(ivar.to_s[1..-1])) # rubocop:disable GitlabSecurity/PublicSend + instance_variable_set(ivar, other.public_send(ivar.to_s[1..])) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index d66d4b20bba..eaa87157716 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -16,7 +16,11 @@ module Gitlab def details_path return unless can?(user, :read_pipeline, downstream_pipeline) - project_pipeline_path(downstream_project, downstream_pipeline) + if Feature.enabled?(:ci_retry_downstream_pipeline, subject.project, default_enabled: :yaml) + project_job_path(subject.project, subject) + else + project_pipeline_path(downstream_project, downstream_pipeline) + end end def has_action? diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index b0f12ff7517..5dd28157b1f 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -34,7 +34,8 @@ module Gitlab no_matching_runner: 'no matching runner available', trace_size_exceeded: 'log size limit exceeded', builds_disabled: 'project builds are disabled', - environment_creation_failure: 'environment creation failure' + environment_creation_failure: 'environment creation failure', + deployment_rejected: 'deployment rejected' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb new file mode 100644 index 00000000000..a299df7e2d9 --- /dev/null +++ b/lib/gitlab/ci/tags/bulk_insert.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Tags + class BulkInsert + TAGGINGS_BATCH_SIZE = 1000 + TAGS_BATCH_SIZE = 500 + + def initialize(statuses, tag_list_by_status) + @statuses = statuses + @tag_list_by_status = tag_list_by_status + end + + def insert! + return false if tag_list_by_status.empty? + + persist_build_tags! + end + + private + + attr_reader :statuses, :tag_list_by_status + + def persist_build_tags! + all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?) + tag_records_by_name = create_tags(all_tags).index_by(&:name) + taggings = build_taggings_attributes(tag_records_by_name) + + return false if taggings.empty? + + taggings.each_slice(TAGGINGS_BATCH_SIZE) do |taggings_slice| + ActsAsTaggableOn::Tagging.insert_all!(taggings) + end + + true + end + + # rubocop: disable CodeReuse/ActiveRecord + def create_tags(tags) + existing_tag_records = ActsAsTaggableOn::Tag.where(name: tags).to_a + missing_tags = detect_missing_tags(tags, existing_tag_records) + return existing_tag_records if missing_tags.empty? + + missing_tags + .map { |tag| { name: tag } } + .each_slice(TAGS_BATCH_SIZE) do |tags_attributes| + ActsAsTaggableOn::Tag.insert_all!(tags_attributes) + end + + ActsAsTaggableOn::Tag.where(name: tags).to_a + end + # rubocop: enable CodeReuse/ActiveRecord + + def build_taggings_attributes(tag_records_by_name) + taggings = statuses.flat_map do |status| + tag_list = tag_list_by_status[status.name] + next unless tag_list + + tags = tag_records_by_name.values_at(*tag_list) + taggings_for(tags, status) + end + + taggings.compact! + taggings + end + + def taggings_for(tags, status) + tags.map do |tag| + { + tag_id: tag.id, + taggable_type: CommitStatus.name, + taggable_id: status.id, + created_at: Time.current, + context: 'tags' + } + end + end + + def detect_missing_tags(tags, tag_records) + if tags.size != tag_records.size + tags - tag_records.map(&:name) + else + [] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 89fd59d98f4..fddcc1492a8 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -53,6 +53,9 @@ variables: # KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. # KUBE_INGRESS_BASE_DOMAIN: domain.example.com + # Allows Container-Scanning to correctly correlate image names when using Jobs/Build.gitlab-ci.yml + CS_DEFAULT_BRANCH_IMAGE: $CI_REGISTRY_IMAGE/$CI_DEFAULT_BRANCH:$CI_COMMIT_SHA + POSTGRES_USER: user POSTGRES_PASSWORD: testing-password POSTGRES_ENABLED: "true" diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index b763705857e..fa7f6ffa2b7 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -24,7 +24,7 @@ kics-iac-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 0 + SAST_ANALYZER_IMAGE_TAG: 1 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 17ed1d2e87f..d32444833fb 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -9,6 +9,7 @@ pages: script: - mkdir .public - cp -r * .public + - rm -rf public - mv .public public artifacts: paths: diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index aec41c137a4..4917abf6ae9 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -23,7 +23,7 @@ cache: - venv/ before_script: - - python -V # Print out python version for debugging + - python --version # For debugging - pip install virtualenv - virtualenv venv - source venv/bin/activate diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index ff8f9601189..de54d64dc42 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -13,8 +13,10 @@ before_script: - apt-get update -yqq - apt-get install apt-transport-https -yqq # Add keyserver for SBT - - echo "deb http://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list - - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list + - mkdir -p /root/.gnupg + - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg # Install SBT - apt-get update -yqq - apt-get install sbt -yqq diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 544774d3b06..01041f4f056 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -11,11 +11,11 @@ variables: FUZZAPI_VERSION: "1" SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} + FUZZAPI_IMAGE: api-fuzzing apifuzzer_fuzz: stage: fuzz - image: $FUZZAPI_IMAGE + image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION allow_failure: true rules: - if: $API_FUZZING_DISABLED diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 89e6743b0e4..65a2b20d5c0 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -38,7 +38,8 @@ container_scanning: artifacts: reports: container_scanning: gl-container-scanning-report.json - paths: [gl-container-scanning-report.json] + dependency_scanning: gl-dependency-scanning-report.json + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] dependencies: [] script: - gtcs scan diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml new file mode 100644 index 00000000000..57f1993921d --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -0,0 +1,52 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dast-API.gitlab-ci.yml + +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: DAST-API.latest.gitlab-ci.yml +# +# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST API: +# +# stages: +# - build +# - test +# - deploy +# - dast + +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html + +# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables + +variables: + # Setting this variable affects all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # + DAST_API_VERSION: "1" + DAST_API_IMAGE: api-fuzzing + +dast_api: + stage: dast + image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION + allow_failure: true + rules: + - if: $DAST_API_DISABLED + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH + script: + - /peach/analyzer-dast-api + artifacts: + when: always + paths: + - gl-assets + - gl-dast-api-report.json + - gl-*.log + reports: + dast: gl-dast-api-report.json diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 4f63ff93d4d..8f4a836441d 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -13,7 +13,7 @@ stages: a11y: stage: accessibility - image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.0.1 + image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.1.1 script: /gitlab-accessibility.sh $a11y_urls allow_failure: true artifacts: diff --git a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml index edd0fb0ba07..09fce67db2d 100644 --- a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml @@ -1,4 +1,4 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: +# To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.yml @@ -14,10 +14,10 @@ # The 'latest' tag targets the latest available version of .NET Core SDK image. # If preferred, you can explicitly specify version of .NET Core (e.g. using '2.2-sdk' tag). # -# See other available tags for .NET Core: https://hub.docker.com/r/microsoft/dotnet +# See other available tags for .NET Core: https://hub.docker.com/_/microsoft-dotnet # Learn more about Docker tags: https://docs.docker.com/glossary/?term=tag # and the Docker itself: https://opensource.com/resources/what-docker -image: microsoft/dotnet:latest +image: mcr.microsoft.com/dotnet/sdk:latest # ### Define variables # diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index f4c5a06af97..3e2c2c7fc1a 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -12,7 +12,7 @@ module Gitlab def scoped_variables(job, environment:, dependencies:) Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.concat(predefined_variables(job)) if pipeline.predefined_vars_in_builder_enabled? + variables.concat(predefined_variables(job)) end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 1aa3dbc5e47..296b0cfded2 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -29,10 +29,8 @@ module Gitlab run_logical_validations! Result.new(ci_config: @ci_config, warnings: @ci_config&.warnings) - rescue Gitlab::Ci::Config::ConfigError => e Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) - rescue ValidationError => e Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 6215ba40ebe..f14279dca2d 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -92,6 +92,7 @@ module Gitlab script: job[:script], after_script: job[:after_script], environment: job[:environment], + resource_group_key: job[:resource_group], retry: job[:retry], parallel: job[:parallel], instance: job[:instance], diff --git a/lib/gitlab/config/entry/undefined.rb b/lib/gitlab/config/entry/undefined.rb index 5f708abc80c..55393890693 100644 --- a/lib/gitlab/config/entry/undefined.rb +++ b/lib/gitlab/config/entry/undefined.rb @@ -31,6 +31,10 @@ module Gitlab false end + def type + nil + end + def inspect "#<#{self.class.name}>" end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index bdae59e7e3c..87bc2ace204 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -36,6 +36,7 @@ module Gitlab if Rails.env.development? allow_webpack_dev_server(directives) allow_letter_opener(directives) + allow_snowplow_micro(directives) if Gitlab::Tracking.snowplow_micro_enabled? allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present? end @@ -138,13 +139,15 @@ module Gitlab append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, '/rails/letter_opener/')) end + def self.allow_snowplow_micro(directives) + url = URI.join(Gitlab::Tracking::Destinations::SnowplowMicro.new.uri, '/').to_s + append_to_directive(directives, 'connect_src', url) + end + # Using 'self' in the CSP introduces several CSP bypass opportunities # for this reason we list the URLs where GitLab frames itself instead def self.allow_framed_gitlab_paths(directives) - # We need the version without trailing / for the sidekiq page itself - # and we also need the version with trailing / for "deeper" pages - # like /admin/sidekiq/busy - ['/admin/sidekiq', '/admin/sidekiq/', '/-/speedscope/index.html'].map do |path| + ['/admin/', '/assets/', '/-/speedscope/index.html'].map do |path| append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, path)) end end diff --git a/lib/gitlab/content_security_policy/directives.rb b/lib/gitlab/content_security_policy/directives.rb index 30f3c16247d..3b958f8c92e 100644 --- a/lib/gitlab/content_security_policy/directives.rb +++ b/lib/gitlab/content_security_policy/directives.rb @@ -8,7 +8,7 @@ module Gitlab module ContentSecurityPolicy module Directives def self.frame_src - "https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com" + "https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com https://www.googletagmanager.com/ns.html" end def self.script_src diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index deaaab953aa..f48ba27888c 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -10,7 +10,7 @@ module Gitlab def initialize(contributor, current_user = nil) @contributor = contributor - @contributor_time_instance = local_time_instance(contributor.timezone) + @contributor_time_instance = local_timezone_instance(contributor.timezone).now @current_user = current_user @projects = if @contributor.include_private_contributions? ContributedProjectsFinder.new(@contributor).execute(@contributor) @@ -21,27 +21,33 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def activity_dates + return {} if @projects.empty? return @activity_dates if @activity_dates.present? + start_time = @contributor_time_instance.years_ago(1).beginning_of_day + end_time = @contributor_time_instance.end_of_day + + date_interval = "INTERVAL '#{@contributor_time_instance.utc_offset} seconds'" + # Can't use Event.contributions here because we need to check 3 different # project_features for the (currently) 3 different contribution types - date_from = @contributor_time_instance.now.years_ago(1) - repo_events = event_counts(date_from, :repository) - .having(action: :pushed) - issue_events = event_counts(date_from, :issues) - .having(action: [:created, :closed], target_type: "Issue") - mr_events = event_counts(date_from, :merge_requests) - .having(action: [:merged, :created, :closed], target_type: "MergeRequest") - note_events = event_counts(date_from, :merge_requests) - .having(action: :commented) + repo_events = events_created_between(start_time, end_time, :repository) + .where(action: :pushed) + issue_events = events_created_between(start_time, end_time, :issues) + .where(action: [:created, :closed], target_type: "Issue") + mr_events = events_created_between(start_time, end_time, :merge_requests) + .where(action: [:merged, :created, :closed], target_type: "MergeRequest") + note_events = events_created_between(start_time, end_time, :merge_requests) + .where(action: :commented) events = Event - .select(:project_id, :target_type, :action, :date, :total_amount) - .from_union([repo_events, issue_events, mr_events, note_events]) + .select("date(created_at + #{date_interval}) AS date", 'COUNT(*) AS num_events') + .from_union([repo_events, issue_events, mr_events, note_events], remove_duplicates: false) + .group(:date) .map(&:attributes) @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| - activities[event["date"]] += event["total_amount"] + activities[event["date"]] += event["num_events"] end end # rubocop: enable CodeReuse/ActiveRecord @@ -50,7 +56,7 @@ module Gitlab def events_by_date(date) return Event.none unless can_read_cross_project? - date_in_time_zone = date.in_time_zone(@contributor_time_instance) + date_in_time_zone = date.in_time_zone(@contributor_time_instance.time_zone) Event.contributions.where(author_id: contributor.id) .where(created_at: date_in_time_zone.beginning_of_day..date_in_time_zone.end_of_day) @@ -60,11 +66,11 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def starting_year - @contributor_time_instance.now.years_ago(1).year + @contributor_time_instance.years_ago(1).year end def starting_month - @contributor_time_instance.today.month + @contributor_time_instance.month end private @@ -74,29 +80,31 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def event_counts(date_from, feature) - t = Event.arel_table - + def events_created_between(start_time, end_time, feature) # re-running the contributed projects query in each union is expensive, so # use IN(project_ids...) instead. It's the intersection of two users so # the list will be (relatively) short @contributed_project_ids ||= projects.distinct.pluck(:id) - authed_projects = Project.where(id: @contributed_project_ids) - .with_feature_available_for_user(feature, current_user) - .reorder(nil) - .select(:id) - - conditions = t[:created_at].gteq(date_from.beginning_of_day) - .and(t[:created_at].lteq(@contributor_time_instance.today.end_of_day)) - .and(t[:author_id].eq(contributor.id)) - date_interval = "INTERVAL '#{@contributor_time_instance.now.utc_offset} seconds'" + # no need to check feature access of current user, if the contributor opted-in + # to show all private events anyway - otherwise they would get filtered out again + authed_projects = if @contributor.include_private_contributions? + @contributed_project_ids + else + ProjectFeature + .with_feature_available_for_user(feature, current_user) + .where(project_id: @contributed_project_ids) + .reorder(nil) + .select(:project_id) + end Event.reorder(nil) - .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount') - .group(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval})") - .where(conditions) - .where("events.project_id in (#{authed_projects.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection + .select(:created_at) + .where( + author_id: contributor.id, + created_at: start_time..end_time, + events: { project_id: authed_projects } + ) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index ddb9d907640..058fe1c8139 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -2,16 +2,16 @@ module Gitlab class Daemon - def self.initialize_instance(*args) + def self.initialize_instance(...) raise "#{name} singleton instance already initialized" if @instance - @instance = new(*args) + @instance = new(...) Kernel.at_exit(&@instance.method(:stop)) @instance end - def self.instance(*args) - @instance ||= initialize_instance(*args) + def self.instance(...) + @instance ||= initialize_instance(...) end attr_reader :thread @@ -20,7 +20,8 @@ module Gitlab !thread.nil? end - def initialize + def initialize(**options) + @synchronous = options[:synchronous] @mutex = Mutex.new end @@ -43,6 +44,10 @@ module Gitlab Thread.current.name = thread_name run_thread end + + @thread.join if @synchronous + + @thread end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 9c74e5d2ca8..f9c346a272f 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -63,6 +63,19 @@ module Gitlab }.compact.with_indifferent_access.freeze end + # This returns a list of base models with connection associated for a given gitlab_schema + def self.schemas_to_base_models + @schemas_to_base_models ||= { + gitlab_main: [self.database_base_models.fetch(:main)], + gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main + gitlab_shared: self.database_base_models.values # all models + }.with_indifferent_access.freeze + end + + def self.all_database_names + DATABASE_NAMES + end + # We configure the database connection pool size automatically based on the # configured concurrency. We also add some headroom, to make sure we don't # run out of connections when more threads besides the 'user-facing' ones diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb index 994a1deba57..2fb4cc8f675 100644 --- a/lib/gitlab/database/async_indexes/index_creator.rb +++ b/lib/gitlab/database/async_indexes/index_creator.rb @@ -47,6 +47,10 @@ module Gitlab TIMEOUT_PER_ACTION end + def lease_key + [super, async_index.connection_db_config.name].join('/') + end + def set_statement_timeout connection.execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) yield diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 32765cb6a56..503172dd750 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -21,6 +21,7 @@ module Gitlab from_union([failed_jobs, self.stuck]) } + scope :except_succeeded, -> { where(status: self.statuses.except(:succeeded).values) } enum status: { pending: 0, diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index d9fc2ea48f6..2844cbe4a74 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -18,6 +18,8 @@ module Gitlab scope: [:job_class_name, :table_name, :column_name] } + validate :validate_batched_jobs_status, if: -> { status_changed? && finished? } + scope :queue_order, -> { order(id: :asc) } scope :queued, -> { where(status: [:active, :paused]) } scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do @@ -92,11 +94,11 @@ module Gitlab end def job_class_name=(class_name) - write_attribute(:job_class_name, class_name.demodulize) + write_attribute(:job_class_name, class_name.delete_prefix("::")) end def batch_class_name=(class_name) - write_attribute(:batch_class_name, class_name.demodulize) + write_attribute(:batch_class_name, class_name.delete_prefix("::")) end def migrated_tuple_count @@ -133,6 +135,12 @@ module Gitlab def optimize! BatchOptimizer.new(self).optimize! end + + private + + def validate_batched_jobs_status + errors.add(:batched_jobs, 'jobs need to be succeeded') if batched_jobs.except_succeeded.exists? + end end end end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 870cf25984b..68a0c15480a 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -32,12 +32,12 @@ module Gitlab # Models using single-type inheritance (STI) don't work with # reltuple count estimates. We just have to ignore them and # use another strategy to compute them. - def non_sti_models + def non_sti_models(models) models.reject { |model| sti_model?(model) } end - def non_sti_table_names - non_sti_models.map(&:table_name) + def non_sti_table_names(models) + non_sti_models(models).map(&:table_name) end def sti_model?(model) @@ -45,21 +45,34 @@ module Gitlab model.base_class != model end - def table_names - models.map(&:table_name) + def table_to_model_mapping + @table_to_model_mapping ||= models.each_with_object({}) { |model, h| h[model.table_name] = model } + end + + def table_to_model(table_name) + table_to_model_mapping[table_name] end def size_estimates(check_statistics: true) - table_to_model = models.each_with_object({}) { |model, h| h[model.table_name] = model } - - # Querying tuple stats only works on the primary. Due to load balancing, the - # easiest way to do this is to start a transaction. - ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases - get_statistics(non_sti_table_names, check_statistics: check_statistics).each_with_object({}) do |row, data| - model = table_to_model[row.table_name] - data[model] = row.estimate + results = {} + + models.group_by { |model| model.connection_db_config.name }.map do |db_name, models_for_db| + base_model = Gitlab::Database.database_base_models[db_name] + tables = non_sti_table_names(models_for_db) + + # Querying tuple stats only works on the primary. Due to load balancing, the + # easiest way to do this is to start a transaction. + base_model.transaction do + Gitlab::Database::SharedModel.using_connection(base_model.connection) do + get_statistics(tables, check_statistics: check_statistics).each do |row| + model = table_to_model(row.table_name) + results[model] = row.estimate + end + end end end + + results end # Generates the PostgreSQL query to return the tuples for tables diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index 489bc0aacea..92c8de9aeac 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -61,7 +61,7 @@ module Gitlab #{where_clause(model)} SQL - rows = ActiveRecord::Base.connection.select_all(query) # rubocop: disable Database/MultipleDatabases + rows = model.connection.select_all(query) Integer(rows.first['count']) end diff --git a/lib/gitlab/database/gitlab_loose_foreign_keys.yml b/lib/gitlab/database/gitlab_loose_foreign_keys.yml new file mode 100644 index 00000000000..0343c054f23 --- /dev/null +++ b/lib/gitlab/database/gitlab_loose_foreign_keys.yml @@ -0,0 +1,69 @@ +ci_pipeline_chat_data: + - table: chat_names + column: chat_name_id + on_delete: async_delete +dast_scanner_profiles_builds: + - table: ci_builds + column: ci_build_id + on_delete: async_delete +dast_scanner_profiles_builds: + - table: ci_builds + column: ci_build_id + on_delete: async_delete +dast_profiles_pipelines: + - table: ci_pipelines + column: ci_pipeline_id + on_delete: async_delete +clusters_applications_runners: + - table: ci_runners + column: runner_id + on_delete: async_nullify +ci_namespace_mirrors: + - table: namespaces + column: namespace_id + on_delete: async_delete +ci_builds: + - table: users + column: user_id + on_delete: async_nullify +ci_pipelines: + - table: merge_requests + column: merge_request_id + on_delete: async_delete + - table: external_pull_requests + column: external_pull_request_id + on_delete: async_nullify + - table: users + column: user_id + on_delete: async_nullify +ci_project_mirrors: + - table: projects + column: project_id + on_delete: async_delete + - table: namespaces + column: namespace_id + on_delete: async_delete +packages_build_infos: + - table: ci_pipelines + column: pipeline_id + on_delete: async_nullify +packages_package_file_build_infos: + - table: ci_pipelines + column: pipeline_id + on_delete: async_nullify +pages_deployments: + - table: ci_builds + column: ci_build_id + on_delete: async_nullify +terraform_state_versions: + - table: ci_builds + column: ci_build_id + on_delete: async_nullify +merge_request_metrics: + - table: ci_pipelines + column: pipeline_id + on_delete: async_delete +project_pages_metadata: + - table: ci_job_artifacts + column: artifacts_archive_id + on_delete: async_nullify diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 66157e998a0..24c2d634780 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -1,4 +1,5 @@ abuse_reports: :gitlab_main +agent_activity_events: :gitlab_main agent_group_authorizations: :gitlab_main agent_project_authorizations: :gitlab_main alert_management_alert_assignees: :gitlab_main @@ -85,6 +86,7 @@ ci_job_token_project_scope_links: :gitlab_ci ci_job_variables: :gitlab_ci ci_minutes_additional_packs: :gitlab_ci ci_namespace_monthly_usages: :gitlab_ci +ci_namespace_mirrors: :gitlab_ci ci_pending_builds: :gitlab_ci ci_pipeline_artifacts: :gitlab_ci ci_pipeline_chat_data: :gitlab_ci @@ -96,6 +98,7 @@ ci_pipelines: :gitlab_ci ci_pipeline_variables: :gitlab_ci ci_platform_metrics: :gitlab_ci ci_project_monthly_usages: :gitlab_ci +ci_project_mirrors: :gitlab_ci ci_refs: :gitlab_ci ci_resource_groups: :gitlab_ci ci_resources: :gitlab_ci @@ -161,6 +164,7 @@ dependency_proxy_group_settings: :gitlab_main dependency_proxy_image_ttl_group_policies: :gitlab_main dependency_proxy_manifests: :gitlab_main deploy_keys_projects: :gitlab_main +deployment_approvals: :gitlab_main deployment_clusters: :gitlab_main deployment_merge_requests: :gitlab_main deployments: :gitlab_main @@ -249,6 +253,7 @@ incident_management_oncall_schedules: :gitlab_main incident_management_oncall_shifts: :gitlab_main incident_management_pending_alert_escalations: :gitlab_main incident_management_pending_issue_escalations: :gitlab_main +incident_management_timeline_events: :gitlab_main index_statuses: :gitlab_main in_product_marketing_emails: :gitlab_main insights: :gitlab_main @@ -260,6 +265,7 @@ issuable_severities: :gitlab_main issuable_slas: :gitlab_main issue_assignees: :gitlab_main issue_customer_relations_contacts: :gitlab_main +issue_emails: :gitlab_main issue_email_participants: :gitlab_main issue_links: :gitlab_main issue_metrics: :gitlab_main @@ -281,6 +287,7 @@ ldap_group_links: :gitlab_main lfs_file_locks: :gitlab_main lfs_objects: :gitlab_main lfs_objects_projects: :gitlab_main +lfs_object_states: :gitlab_main licenses: :gitlab_main lists: :gitlab_main list_user_preferences: :gitlab_main @@ -290,6 +297,7 @@ members: :gitlab_main merge_request_assignees: :gitlab_main merge_request_blocks: :gitlab_main merge_request_cleanup_schedules: :gitlab_main +merge_requests_compliance_violations: :gitlab_main merge_request_context_commit_diff_files: :gitlab_main merge_request_context_commits: :gitlab_main merge_request_diff_commits: :gitlab_main @@ -314,6 +322,7 @@ namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main namespace_settings: :gitlab_main namespaces: :gitlab_main +namespaces_sync_events: :gitlab_main namespace_statistics: :gitlab_main note_diff_files: :gitlab_main notes: :gitlab_main @@ -363,6 +372,7 @@ packages_pypi_metadata: :gitlab_main packages_rubygems_metadata: :gitlab_main packages_tags: :gitlab_main pages_deployments: :gitlab_main +pages_deployment_states: :gitlab_main pages_domain_acme_orders: :gitlab_main pages_domains: :gitlab_main partitioned_foreign_keys: :gitlab_main @@ -408,6 +418,7 @@ project_repository_storage_moves: :gitlab_main project_security_settings: :gitlab_main project_settings: :gitlab_main projects: :gitlab_main +projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main project_tracing_settings: :gitlab_main @@ -485,6 +496,7 @@ trending_projects: :gitlab_main u2f_registrations: :gitlab_main upcoming_reconciliations: :gitlab_main uploads: :gitlab_main +upload_states: :gitlab_main user_agent_details: :gitlab_main user_callouts: :gitlab_main user_canonical_emails: :gitlab_main @@ -526,6 +538,7 @@ vulnerability_issue_links: :gitlab_main vulnerability_occurrence_identifiers: :gitlab_main vulnerability_occurrence_pipelines: :gitlab_main vulnerability_occurrences: :gitlab_main +vulnerability_reads: :gitlab_main vulnerability_remediations: :gitlab_main vulnerability_scanners: :gitlab_main vulnerability_statistics: :gitlab_main diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index 52eb0764ae3..e16db5af8ce 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -30,6 +30,10 @@ module Gitlab end end + def self.primary_only? + each_load_balancer.all?(&:primary_only?) + end + def self.release_hosts each_load_balancer(&:release_host) end diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index da313361073..e769cb5c35c 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -107,7 +107,11 @@ module Gitlab hosts.any? || service_discovery_enabled? end + # This is disabled for Rake tasks to ensure e.g. database migrations + # always produce consistent results. def service_discovery_enabled? + return false if Gitlab::Runtime.rake? + service_discovery[:record].present? end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index b9acc36b4cc..5d91292b8de 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -6,6 +6,8 @@ module Gitlab class SidekiqServerMiddleware JobReplicaNotUpToDate = Class.new(StandardError) + MINIMUM_DELAY_INTERVAL_SECONDS = 0.8 + def call(worker, job, _queue) worker_class = worker.class strategy = select_load_balancing_strategy(worker_class, job) @@ -42,11 +44,15 @@ module Gitlab wal_locations = get_wal_locations(job) - return :primary_no_wal unless wal_locations + return :primary_no_wal if wal_locations.blank? + + # Happy case: we can read from a replica. + return replica_strategy(worker_class, job) if databases_in_sync?(wal_locations) + + sleep_if_needed(job) if databases_in_sync?(wal_locations) - # Happy case: we can read from a replica. - retried_before?(worker_class, job) ? :replica_retried : :replica + replica_strategy(worker_class, job) elsif can_retry?(worker_class, job) # Optimistic case: The worker allows retries and we have retries left. :retry @@ -56,17 +62,14 @@ module Gitlab end end - def get_wal_locations(job) - job['dedup_wal_locations'] || job['wal_locations'] || legacy_wal_location(job) - end + def sleep_if_needed(job) + remaining_delay = MINIMUM_DELAY_INTERVAL_SECONDS - (Time.current.to_f - job['created_at'].to_f) - # Already scheduled jobs could still contain legacy database write location. - # TODO: remove this in the next iteration - # https://gitlab.com/gitlab-org/gitlab/-/issues/338213 - def legacy_wal_location(job) - wal_location = job['database_write_location'] || job['database_replica_location'] + sleep remaining_delay if remaining_delay > 0 && remaining_delay < MINIMUM_DELAY_INTERVAL_SECONDS + end - { ::Gitlab::Database::MAIN_DATABASE_NAME.to_sym => wal_location } if wal_location + def get_wal_locations(job) + job['dedup_wal_locations'] || job['wal_locations'] end def load_balancing_available?(worker_class) @@ -79,6 +82,10 @@ module Gitlab worker_class.get_data_consistency == :delayed && not_yet_retried?(job) end + def replica_strategy(worker_class, job) + retried_before?(worker_class, job) ? :replica_retried : :replica + end + def retried_before?(worker_class, job) worker_class.get_data_consistency == :delayed && !not_yet_retried?(job) end diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb index 834e9c6d3c6..8e5dc98e96e 100644 --- a/lib/gitlab/database/load_balancing/sticking.rb +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -123,21 +123,18 @@ module Gitlab def unstick(namespace, id) Gitlab::Redis::SharedState.with do |redis| redis.del(redis_key_for(namespace, id)) - redis.del(old_redis_key_for(namespace, id)) end end def set_write_location_for(namespace, id, location) Gitlab::Redis::SharedState.with do |redis| redis.set(redis_key_for(namespace, id), location, ex: EXPIRATION) - redis.set(old_redis_key_for(namespace, id), location, ex: EXPIRATION) end end def last_write_location_for(namespace, id) Gitlab::Redis::SharedState.with do |redis| - redis.get(redis_key_for(namespace, id)) || - redis.get(old_redis_key_for(namespace, id)) + redis.get(redis_key_for(namespace, id)) end end @@ -146,10 +143,6 @@ module Gitlab "database-load-balancing/write-location/#{name}/#{namespace}/#{id}" end - - def old_redis_key_for(namespace, id) - "database-load-balancing/write-location/#{namespace}/#{id}" - end end end end diff --git a/lib/gitlab/database/loose_foreign_keys.rb b/lib/gitlab/database/loose_foreign_keys.rb new file mode 100644 index 00000000000..1ecfb5ce47f --- /dev/null +++ b/lib/gitlab/database/loose_foreign_keys.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LooseForeignKeys + def self.definitions_by_table + @definitions_by_table ||= definitions.group_by(&:to_table).with_indifferent_access.freeze + end + + def self.definitions + @definitions ||= loose_foreign_keys_yaml.flat_map do |child_table_name, configs| + configs.map { |config| build_definition(child_table_name, config) } + end.freeze + end + + def self.build_definition(child_table_name, config) + parent_table_name = config.fetch('table') + + ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new( + child_table_name, + parent_table_name, + { + column: config.fetch('column'), + on_delete: config.fetch('on_delete').to_sym, + gitlab_schema: GitlabSchema.table_schema(child_table_name) + } + ) + end + + def self.loose_foreign_keys_yaml + @loose_foreign_keys_yaml ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_loose_foreign_keys.yml')) + end + + private_class_method :build_definition + private_class_method :loose_foreign_keys_yaml + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 7dce4fa0ce2..4245dd80714 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -4,6 +4,7 @@ module Gitlab module Database module MigrationHelpers include Migrations::BackgroundMigrationHelpers + include Migrations::BatchedBackgroundMigrationHelpers include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index bdaf0d35a83..8c33c41ce77 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -5,59 +5,7 @@ module Gitlab module Migrations module BackgroundMigrationHelpers BATCH_SIZE = 1_000 # Number of rows to process per job - SUB_BATCH_SIZE = 100 # Number of rows to process per sub-batch JOB_BUFFER_SIZE = 1_000 # Number of jobs to bulk queue at a time - BATCH_CLASS_NAME = 'PrimaryKeyBatchingStrategy' # Default batch class for batched migrations - BATCH_MIN_VALUE = 1 # Default minimum value for batched migrations - BATCH_MIN_DELAY = 2.minutes.freeze # Minimum delay between batched migrations - - # Bulk queues background migration jobs for an entire table, batched by ID range. - # "Bulk" meaning many jobs will be pushed at a time for efficiency. - # If you need a delay interval per job, then use `queue_background_migration_jobs_by_range_at_intervals`. - # - # model_class - The table being iterated over - # job_class_name - The background migration job class as a string - # batch_size - The maximum number of rows per job - # - # Example: - # - # class Route < ActiveRecord::Base - # include EachBatch - # self.table_name = 'routes' - # end - # - # bulk_queue_background_migration_jobs_by_range(Route, 'ProcessRoutes') - # - # Where the model_class includes EachBatch, and the background migration exists: - # - # class Gitlab::BackgroundMigration::ProcessRoutes - # def perform(start_id, end_id) - # # do something - # end - # end - def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BATCH_SIZE) - raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') - - jobs = [] - table_name = model_class.quoted_table_name - - model_class.each_batch(of: batch_size) do |relation| - start_id, end_id = relation.pluck("MIN(#{table_name}.id)", "MAX(#{table_name}.id)").first - - if jobs.length >= JOB_BUFFER_SIZE - # Note: This code path generally only helps with many millions of rows - # We push multiple jobs at a time to reduce the time spent in - # Sidekiq/Redis operations. We're using this buffer based approach so we - # don't need to run additional queries for every range. - bulk_migrate_async(jobs) - jobs.clear - end - - jobs << [job_class_name, [start_id, end_id]] - end - - bulk_migrate_async(jobs) unless jobs.empty? - end # Queues background migration jobs for an entire table in batches. # The default batching column used is the standard primary key `id`. @@ -137,6 +85,7 @@ module Gitlab # Requeue pending jobs previously queued with #queue_background_migration_jobs_by_range_at_intervals # # This method is useful to schedule jobs that had previously failed. + # It can only be used if the previous background migration used job tracking like the queue_background_migration_jobs_by_range_at_intervals helper. # # job_class_name - The background migration job class as a string # delay_interval - The duration between each job's scheduled time @@ -170,100 +119,6 @@ module Gitlab duration end - # Creates a batched background migration for the given table. A batched migration runs one job - # at a time, computing the bounds of the next batch based on the current migration settings and the previous - # batch bounds. Each job's execution status is tracked in the database as the migration runs. The given job - # class must be present in the Gitlab::BackgroundMigration module, and the batch class (if specified) must be - # present in the Gitlab::BackgroundMigration::BatchingStrategies module. - # - # If migration with same job_class_name, table_name, column_name, and job_aruments already exists, this helper - # will log an warning and not create a new one. - # - # job_class_name - The background migration job class as a string - # batch_table_name - The name of the table the migration will batch over - # batch_column_name - The name of the column the migration will batch over - # job_arguments - Extra arguments to pass to the job instance when the migration runs - # job_interval - The pause interval between each job's execution, minimum of 2 minutes - # batch_min_value - The value in the column the batching will begin at - # batch_max_value - The value in the column the batching will end at, defaults to `SELECT MAX(batch_column)` - # batch_class_name - The name of the class that will be called to find the range of each next batch - # batch_size - The maximum number of rows per job - # sub_batch_size - The maximum number of rows processed per "iteration" within the job - # - # - # *Returns the created BatchedMigration record* - # - # Example: - # - # queue_batched_background_migration( - # 'CopyColumnUsingBackgroundMigrationJob', - # :events, - # :id, - # job_interval: 2.minutes, - # other_job_arguments: ['column1', 'column2']) - # - # Where the the background migration exists: - # - # class Gitlab::BackgroundMigration::CopyColumnUsingBackgroundMigrationJob - # def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, *other_args) - # # do something - # end - # end - def queue_batched_background_migration( # rubocop:disable Metrics/ParameterLists - job_class_name, - batch_table_name, - batch_column_name, - *job_arguments, - job_interval:, - batch_min_value: BATCH_MIN_VALUE, - batch_max_value: nil, - batch_class_name: BATCH_CLASS_NAME, - batch_size: BATCH_SIZE, - sub_batch_size: SUB_BATCH_SIZE - ) - - if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration(job_class_name, batch_table_name, batch_column_name, job_arguments).exists? - Gitlab::AppLogger.warn "Batched background migration not enqueued because it already exists: " \ - "job_class_name: #{job_class_name}, table_name: #{batch_table_name}, column_name: #{batch_column_name}, " \ - "job_arguments: #{job_arguments.inspect}" - return - end - - job_interval = BATCH_MIN_DELAY if job_interval < BATCH_MIN_DELAY - - batch_max_value ||= connection.select_value(<<~SQL) - SELECT MAX(#{connection.quote_column_name(batch_column_name)}) - FROM #{connection.quote_table_name(batch_table_name)} - SQL - - migration_status = batch_max_value.nil? ? :finished : :active - batch_max_value ||= batch_min_value - - migration = Gitlab::Database::BackgroundMigration::BatchedMigration.create!( - job_class_name: job_class_name, - table_name: batch_table_name, - column_name: batch_column_name, - job_arguments: job_arguments, - interval: job_interval, - min_value: batch_min_value, - max_value: batch_max_value, - batch_class_name: batch_class_name, - batch_size: batch_size, - sub_batch_size: sub_batch_size, - status: migration_status) - - # This guard is necessary since #total_tuple_count was only introduced schema-wise, - # after this migration helper had been used for the first time. - return migration unless migration.respond_to?(:total_tuple_count) - - # We keep track of the estimated number of tuples to reason later - # about the overall progress of a migration. - migration.total_tuple_count = Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate - migration.save! - - migration - end - # Force a background migration to complete. # # WARNING: This method will block the caller and move the background migration from an @@ -275,6 +130,7 @@ module Gitlab # 4. Optionally remove job tracking information. # # This method does not garauntee that all jobs completed successfully. + # It can only be used if the previous background migration used the queue_background_migration_jobs_by_range_at_intervals helper. def finalize_background_migration(class_name, delete_tracking_jobs: ['succeeded']) # Empty the sidekiq queue. Gitlab::BackgroundMigration.steal(class_name) diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb new file mode 100644 index 00000000000..dcaf7fad05f --- /dev/null +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + # BatchedBackgroundMigrations are a new approach to scheduling and executing background migrations, which uses + # persistent state in the database to track each migration. This avoids having to batch over an entire table and + # schedule a large number of sidekiq jobs upfront. It also provides for more flexibility as the migration runs, + # as it can be paused and restarted, and have configuration values like the batch size updated dynamically as the + # migration runs. + # + # For now, these migrations are not considered ready for general use, for more information see the tracking epic: + # https://gitlab.com/groups/gitlab-org/-/epics/6751 + module BatchedBackgroundMigrationHelpers + BATCH_SIZE = 1_000 # Number of rows to process per job + SUB_BATCH_SIZE = 100 # Number of rows to process per sub-batch + BATCH_CLASS_NAME = 'PrimaryKeyBatchingStrategy' # Default batch class for batched migrations + BATCH_MIN_VALUE = 1 # Default minimum value for batched migrations + BATCH_MIN_DELAY = 2.minutes.freeze # Minimum delay between batched migrations + + # Creates a batched background migration for the given table. A batched migration runs one job + # at a time, computing the bounds of the next batch based on the current migration settings and the previous + # batch bounds. Each job's execution status is tracked in the database as the migration runs. The given job + # class must be present in the Gitlab::BackgroundMigration module, and the batch class (if specified) must be + # present in the Gitlab::BackgroundMigration::BatchingStrategies module. + # + # If migration with same job_class_name, table_name, column_name, and job_aruments already exists, this helper + # will log an warning and not create a new one. + # + # job_class_name - The background migration job class as a string + # batch_table_name - The name of the table the migration will batch over + # batch_column_name - The name of the column the migration will batch over + # job_arguments - Extra arguments to pass to the job instance when the migration runs + # job_interval - The pause interval between each job's execution, minimum of 2 minutes + # batch_min_value - The value in the column the batching will begin at + # batch_max_value - The value in the column the batching will end at, defaults to `SELECT MAX(batch_column)` + # batch_class_name - The name of the class that will be called to find the range of each next batch + # batch_size - The maximum number of rows per job + # sub_batch_size - The maximum number of rows processed per "iteration" within the job + # + # *Returns the created BatchedMigration record* + # + # Example: + # + # queue_batched_background_migration( + # 'CopyColumnUsingBackgroundMigrationJob', + # :events, + # :id, + # job_interval: 2.minutes, + # other_job_arguments: ['column1', 'column2']) + # + # Where the the background migration exists: + # + # class Gitlab::BackgroundMigration::CopyColumnUsingBackgroundMigrationJob + # def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, *other_args) + # # do something + # end + # end + def queue_batched_background_migration( # rubocop:disable Metrics/ParameterLists + job_class_name, + batch_table_name, + batch_column_name, + *job_arguments, + job_interval:, + batch_min_value: BATCH_MIN_VALUE, + batch_max_value: nil, + batch_class_name: BATCH_CLASS_NAME, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + + if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration(job_class_name, batch_table_name, batch_column_name, job_arguments).exists? + Gitlab::AppLogger.warn "Batched background migration not enqueued because it already exists: " \ + "job_class_name: #{job_class_name}, table_name: #{batch_table_name}, column_name: #{batch_column_name}, " \ + "job_arguments: #{job_arguments.inspect}" + return + end + + job_interval = BATCH_MIN_DELAY if job_interval < BATCH_MIN_DELAY + + batch_max_value ||= connection.select_value(<<~SQL) + SELECT MAX(#{connection.quote_column_name(batch_column_name)}) + FROM #{connection.quote_table_name(batch_table_name)} + SQL + + migration_status = batch_max_value.nil? ? :finished : :active + batch_max_value ||= batch_min_value + + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.create!( + job_class_name: job_class_name, + table_name: batch_table_name, + column_name: batch_column_name, + job_arguments: job_arguments, + interval: job_interval, + min_value: batch_min_value, + max_value: batch_max_value, + batch_class_name: batch_class_name, + batch_size: batch_size, + sub_batch_size: sub_batch_size, + status: migration_status) + + # This guard is necessary since #total_tuple_count was only introduced schema-wise, + # after this migration helper had been used for the first time. + return migration unless migration.respond_to?(:total_tuple_count) + + # We keep track of the estimated number of tuples to reason later + # about the overall progress of a migration. + migration.total_tuple_count = Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate + end + migration.save! + + migration + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 6e5ffb74411..1f7e81cae84 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -14,11 +14,11 @@ module Gitlab @result_dir = result_dir end - def observe(version:, name:, &block) + def observe(version:, name:, connection:, &block) observation = Observation.new(version, name) observation.success = true - observers = observer_classes.map { |c| c.new(observation, @result_dir) } + observers = observer_classes.map { |c| c.new(observation, @result_dir, connection) } exception = nil diff --git a/lib/gitlab/database/migrations/observers/migration_observer.rb b/lib/gitlab/database/migrations/observers/migration_observer.rb index 106f8f1f829..0006af73f6c 100644 --- a/lib/gitlab/database/migrations/observers/migration_observer.rb +++ b/lib/gitlab/database/migrations/observers/migration_observer.rb @@ -7,8 +7,8 @@ module Gitlab class MigrationObserver attr_reader :connection, :observation, :output_dir - def initialize(observation, output_dir) - @connection = ActiveRecord::Base.connection + def initialize(observation, output_dir, connection) + @connection = connection @observation = observation @output_dir = output_dir end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index b267a64256b..f0bac594119 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -69,7 +69,7 @@ module Gitlab instrumentation = Instrumentation.new(result_dir: result_dir) sorted_migrations.each do |migration| - instrumentation.observe(version: migration.version, name: migration.name) do + instrumentation.observe(version: migration.version, name: migration.name, connection: ActiveRecord::Migration.connection) do ActiveRecord::Migrator.new(direction, migration_context.migrations, migration_context.schema_migration, migration.version).run end end diff --git a/lib/gitlab/database/partitioning/detached_partition_dropper.rb b/lib/gitlab/database/partitioning/detached_partition_dropper.rb index 593824384b5..5e32ecad4ca 100644 --- a/lib/gitlab/database/partitioning/detached_partition_dropper.rb +++ b/lib/gitlab/database/partitioning/detached_partition_dropper.rb @@ -4,8 +4,6 @@ module Gitlab module Partitioning class DetachedPartitionDropper def perform - return unless Feature.enabled?(:drop_detached_partitions, default_enabled: :yaml) - Gitlab::AppLogger.info(message: "Checking for previously detached partitions to drop") Postgresql::DetachedPartition.ready_to_drop.find_each do |detached_partition| diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index c93e775d7ed..9c8cccb3dc6 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -36,6 +36,10 @@ module Gitlab partitions end + def after_adding_partitions + # No-op, required by the partition manager + end + private def desired_partitions diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index 8742c0ff166..aa824dfbd2f 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -25,10 +25,8 @@ module Gitlab partitions_to_create = missing_partitions create(partitions_to_create) unless partitions_to_create.empty? - if Feature.enabled?(:partition_pruning, default_enabled: :yaml) - partitions_to_detach = extra_partitions - detach(partitions_to_detach) unless partitions_to_detach.empty? - end + partitions_to_detach = extra_partitions + detach(partitions_to_detach) unless partitions_to_detach.empty? end rescue StandardError => e Gitlab::AppLogger.error(message: "Failed to create / detach partition(s)", @@ -73,6 +71,8 @@ module Gitlab partition_name: partition.partition_name, table_name: partition.table) end + + model.partitioning_strategy.after_adding_partitions end end end diff --git a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb new file mode 100644 index 00000000000..23ac73a0e53 --- /dev/null +++ b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class SingleNumericListPartition + include Comparable + + def self.from_sql(table, partition_name, definition) + # A list partition can support multiple values, but we only support a single number + matches = definition.match(/FOR VALUES IN \('(?<value>\d+)'\)/) + + raise ArgumentError, 'Unknown partition definition' unless matches + + value = Integer(matches[:value]) + + new(table, value, partition_name: partition_name) + end + + attr_reader :table, :value + + def initialize(table, value, partition_name: nil ) + @table = table + @value = value + @partition_name = partition_name + end + + def partition_name + @partition_name || "#{table}_#{value}" + end + + def to_sql + <<~SQL + CREATE TABLE IF NOT EXISTS #{fully_qualified_partition} + PARTITION OF #{conn.quote_table_name(table)} + FOR VALUES IN (#{conn.quote(value)}) + SQL + end + + def to_detach_sql + <<~SQL + ALTER TABLE #{conn.quote_table_name(table)} + DETACH PARTITION #{fully_qualified_partition} + SQL + end + + def ==(other) + table == other.table && + partition_name == other.partition_name && + value == other.value + end + alias_method :eql?, :== + + def hash + [table, partition_name, value].hash + end + + def <=>(other) + return if table != other.table + + value <=> other.value + end + + private + + def fully_qualified_partition + "%s.%s" % [conn.quote_table_name(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA), conn.quote_table_name(partition_name)] + end + + def conn + @conn ||= Gitlab::Database::SharedModel.connection + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/sliding_list_strategy.rb b/lib/gitlab/database/partitioning/sliding_list_strategy.rb new file mode 100644 index 00000000000..21b86b43ae7 --- /dev/null +++ b/lib/gitlab/database/partitioning/sliding_list_strategy.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class SlidingListStrategy + attr_reader :model, :partitioning_key, :next_partition_if, :detach_partition_if + + delegate :table_name, to: :model + + def initialize(model, partitioning_key, next_partition_if:, detach_partition_if:) + @model = model + @partitioning_key = partitioning_key + @next_partition_if = next_partition_if + @detach_partition_if = detach_partition_if + + ensure_partitioning_column_ignored! + end + + def current_partitions + Gitlab::Database::PostgresPartition.for_parent_table(table_name).map do |partition| + SingleNumericListPartition.from_sql(table_name, partition.name, partition.condition) + end.sort + end + + def missing_partitions + if no_partitions_exist? + [initial_partition] + elsif next_partition_if.call(active_partition.value) + [next_partition] + else + [] + end + end + + def initial_partition + SingleNumericListPartition.new(table_name, 1) + end + + def next_partition + SingleNumericListPartition.new(table_name, active_partition.value + 1) + end + + def extra_partitions + possibly_extra = current_partitions[0...-1] # Never consider the most recent partition + + possibly_extra.take_while { |p| detach_partition_if.call(p.value) } + end + + def after_adding_partitions + active_value = active_partition.value + model.connection.change_column_default(model.table_name, partitioning_key, active_value) + end + + def active_partition + # The current partitions list is sorted, so the last partition has the highest value + # This is the only partition that receives inserts. + current_partitions.last + end + + def no_partitions_exist? + current_partitions.empty? + end + + private + + def ensure_partitioning_column_ignored! + unless model.ignored_columns.include?(partitioning_key.to_s) + raise "Add #{partitioning_key} to #{model.name}.ignored_columns to use it with SlidingListStrategy" + end + end + end + end + end +end diff --git a/lib/gitlab/database/pg_class.rb b/lib/gitlab/database/pg_class.rb index 0ce9eebc14c..bd582d903c6 100644 --- a/lib/gitlab/database/pg_class.rb +++ b/lib/gitlab/database/pg_class.rb @@ -2,7 +2,7 @@ module Gitlab module Database - class PgClass < ActiveRecord::Base + class PgClass < SharedModel self.table_name = 'pg_class' def self.for_table(relname) diff --git a/lib/gitlab/database/postgres_hll/buckets.rb b/lib/gitlab/database/postgres_hll/buckets.rb index 429e823379f..76818bbf340 100644 --- a/lib/gitlab/database/postgres_hll/buckets.rb +++ b/lib/gitlab/database/postgres_hll/buckets.rb @@ -65,8 +65,7 @@ module Gitlab ).to_i if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS - ((0.7213 / (1 + 1.079 / TOTAL_BUCKETS)) * (TOTAL_BUCKETS * - Math.log2(TOTAL_BUCKETS.to_f / num_zero_buckets))) + TOTAL_BUCKETS * Math.log(TOTAL_BUCKETS.to_f / num_zero_buckets) else num_uniques end diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb index 0f285688876..2736f9d18dc 100644 --- a/lib/gitlab/database/query_analyzer.rb +++ b/lib/gitlab/database/query_analyzer.rb @@ -58,17 +58,15 @@ module Gitlab return unless parsed analyzers.each do |analyzer| - next if analyzer.suppressed? + next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) analyzer.analyze(parsed) - rescue StandardError => e + rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e # We catch all standard errors to prevent validation errors to introduce fatal errors in production Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end end - private - # Enable query analyzers def begin! analyzers = all_analyzers.select do |analyzer| @@ -77,7 +75,7 @@ module Gitlab true end - rescue StandardError => e + rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) false @@ -90,13 +88,15 @@ module Gitlab def end! enabled_analyzers.select do |analyzer| analyzer.end! - rescue StandardError => e + rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end Thread.current[:query_analyzer_enabled_analyzers] = nil end + private + def enabled_analyzers Thread.current[:query_analyzer_enabled_analyzers] end diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb index e8066f7a706..0802d3c8013 100644 --- a/lib/gitlab/database/query_analyzers/base.rb +++ b/lib/gitlab/database/query_analyzers/base.rb @@ -4,10 +4,17 @@ module Gitlab module Database module QueryAnalyzers class Base + # `Exception` to ensure that is not easily rescued when running in test env + QueryAnalyzerError = Class.new(Exception) # rubocop:disable Lint/InheritException + def self.suppressed? Thread.current[self.suppress_key] end + def self.requires_tracking?(parsed) + false + end + def self.suppress=(value) Thread.current[self.suppress_key] = value end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index 2233f3c4646..2e3db2a5c6e 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -4,7 +4,7 @@ module Gitlab module Database module QueryAnalyzers class PreventCrossDatabaseModification < Database::QueryAnalyzers::Base - CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(StandardError) + CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(QueryAnalyzerError) # This method will allow cross database modifications within the block # Example: @@ -36,29 +36,36 @@ module Gitlab Feature.enabled?(:detect_cross_database_modification, default_enabled: :yaml) end + def self.requires_tracking?(parsed) + # The transaction boundaries always needs to be tracked regardless of suppress behavior + self.transaction_begin?(parsed) || self.transaction_end?(parsed) + end + # rubocop:disable Metrics/AbcSize def self.analyze(parsed) - return if in_factory_bot_create? - database = ::Gitlab::Database.db_config_name(parsed.connection) sql = parsed.sql # We ignore BEGIN in tests as this is the outer transaction for # DatabaseCleaner - if sql.start_with?('SAVEPOINT') || (!Rails.env.test? && sql.start_with?('BEGIN')) + if self.transaction_begin?(parsed) context[:transaction_depth_by_db][database] += 1 return - elsif sql.start_with?('RELEASE SAVEPOINT', 'ROLLBACK TO SAVEPOINT') || (!Rails.env.test? && sql.start_with?('ROLLBACK', 'COMMIT')) + elsif self.transaction_end?(parsed) context[:transaction_depth_by_db][database] -= 1 - if context[:transaction_depth_by_db][database] <= 0 + if context[:transaction_depth_by_db][database] == 0 context[:modified_tables_by_db][database].clear + elsif context[:transaction_depth_by_db][database] < 0 + context[:transaction_depth_by_db][database] = 0 + raise CrossDatabaseModificationAcrossUnsupportedTablesError, "Misaligned cross-DB transactions discovered at query #{sql}. This could be a bug in #{self.class} or a valid issue to investigate. Read more at https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions ." end return end - return if context[:transaction_depth_by_db].values.all?(&:zero?) + return unless self.in_transaction? + return if in_factory_bot_create? # PgQuery might fail in some cases due to limited nesting: # https://github.com/pganalyze/pg_query/issues/209 @@ -97,6 +104,42 @@ module Gitlab end # rubocop:enable Metrics/AbcSize + def self.transaction_begin?(parsed) + # We ignore BEGIN or START in tests + unless Rails.env.test? + return true if transaction_stmt?(parsed, :TRANS_STMT_BEGIN) + return true if transaction_stmt?(parsed, :TRANS_STMT_START) + end + + # SAVEPOINT + return true if transaction_stmt?(parsed, :TRANS_STMT_SAVEPOINT) + + false + end + + def self.transaction_end?(parsed) + # We ignore ROLLBACK or COMMIT in tests + unless Rails.env.test? + return true if transaction_stmt?(parsed, :TRANS_STMT_COMMIT) + return true if transaction_stmt?(parsed, :TRANS_STMT_COMMIT_PREPARED) + return true if transaction_stmt?(parsed, :TRANS_STMT_ROLLBACK) + return true if transaction_stmt?(parsed, :TRANS_STMT_ROLLBACK_PREPARED) + end + + # RELEASE (SAVEPOINT) or ROLLBACK TO (SAVEPOINT) + return true if transaction_stmt?(parsed, :TRANS_STMT_RELEASE) + return true if transaction_stmt?(parsed, :TRANS_STMT_ROLLBACK_TO) + + false + end + + # Known kinds: https://github.com/pganalyze/pg_query/blob/f6588703deb9d7a94b87b34b7c3bab240087fbc4/ext/pg_query/include/nodes/parsenodes.h#L3050 + def self.transaction_stmt?(parsed, kind) + parsed.pg.tree.stmts.map(&:stmt).any? do |stmt| + stmt.node == :transaction_stmt && stmt.transaction_stmt.kind == kind + end + end + # We only raise in tests for now otherwise some features will be broken # in development. For now we've mostly only added allowlist based on # spec names. Until we have allowed all the violations inline we don't @@ -105,13 +148,21 @@ module Gitlab Rails.env.test? end + def self.in_transaction? + context[:transaction_depth_by_db].values.any?(&:positive?) + end + # We ignore execution in the #create method from FactoryBot # because it is not representative of real code we run in # production. There are far too many false positives caused # by instantiating objects in different `gitlab_schema` in a # FactoryBot `create`. def self.in_factory_bot_create? - Rails.env.test? && caller_locations.any? { |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' } + Rails.env.test? && caller_locations.any? do |l| + l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' || + l.path.end_with?('lib/factory_bot/strategy/create.rb') || + l.path.end_with?('shoulda/matchers/active_record/validate_uniqueness_of_matcher.rb') && l.label == 'create_existing_record' + end end end end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 7a22e324bdb..6ffe14249f0 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -15,6 +15,26 @@ module Gitlab # on e.g. vacuum. REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 + def self.enabled? + Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml) + end + + def self.invoke(database = nil) + Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| + next if database && database.to_s != connection_name.to_s + + Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) + + # Hack: Before we do actual reindexing work, create async indexes + Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) + + automatic_reindexing + end + rescue StandardError => e + Gitlab::AppLogger.error(e) + raise + end + # Performs automatic reindexing for a limited number of indexes per call # 1. Consume from the explicit reindexing queue # 2. Apply bloat heuristic to find most bloated indexes and reindex those diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index 13298f67ca9..3e4a83aa2e7 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -53,6 +53,10 @@ module Gitlab def lease_timeout TIMEOUT_PER_ACTION end + + def lease_key + [super, index.connection_db_config.name].join('/') + end end end end diff --git a/lib/gitlab/database/schema_cache_with_renamed_table.rb b/lib/gitlab/database/schema_cache_with_renamed_table.rb index 28123edd708..74900dc0d26 100644 --- a/lib/gitlab/database/schema_cache_with_renamed_table.rb +++ b/lib/gitlab/database/schema_cache_with_renamed_table.rb @@ -42,7 +42,7 @@ module Gitlab def renamed_tables_cache @renamed_tables ||= begin Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| - ActiveRecord::Base.connection.view_exists?(old_name) + connection.view_exists?(old_name) end end end diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index 3d929c62933..9ddc5391689 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -25,6 +25,7 @@ module Gitlab CREATE TRIGGER #{name} #{fires} ON #{table_name} FOR EACH ROW + #{yield if block_given?} EXECUTE FUNCTION #{function_name}() SQL end diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb index f31dbc01907..17d7886e8c8 100644 --- a/lib/gitlab/database/shared_model.rb +++ b/lib/gitlab/database/shared_model.rb @@ -39,6 +39,10 @@ module Gitlab Thread.current[:overriding_connection] = connection end end + + def connection_db_config + self.class.connection_db_config + end end end end diff --git a/lib/gitlab/database/type/json_pg_safe.rb b/lib/gitlab/database/type/json_pg_safe.rb new file mode 100644 index 00000000000..bbc207bd0d9 --- /dev/null +++ b/lib/gitlab/database/type/json_pg_safe.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Type + # Extends Rails' ActiveRecord::Type::Json data type to remove JSON + # encooded nullbytes `\u0000` to prevent PostgreSQL errors like + # `PG::UntranslatableCharacter: ERROR: unsupported Unicode escape + # sequence`. + # + # Example: + # + # class SomeModel < ApplicationRecord + # # some_model.a_field is of type `jsonb` + # attribute :a_field, Gitlab::Database::Type::JsonPgSafe.new + # end + class JsonPgSafe < ActiveRecord::Type::Json + def serialize(value) + super&.gsub('\u0000', '') + end + end + end + end +end diff --git a/lib/gitlab/diff/custom_diff.rb b/lib/gitlab/diff/custom_diff.rb new file mode 100644 index 00000000000..3928ece9281 --- /dev/null +++ b/lib/gitlab/diff/custom_diff.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true +module Gitlab + module Diff + module CustomDiff + class << self + def preprocess_before_diff(path, old_blob, new_blob) + return unless path.ends_with? '.ipynb' + + transformed_diff(old_blob&.data, new_blob&.data)&.tap do + transformed_for_diff(new_blob, old_blob) + Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' }) + end + rescue IpynbDiff::InvalidNotebookError => e + Gitlab::ErrorTracking.log_exception(e) + nil + end + + def transformed_diff(before, after) + transformed_diff = IpynbDiff.diff(before, after, + diff_opts: { context: 5, include_diff_info: true }, + transform_options: { cell_decorator: :percent }, + raise_if_invalid_notebook: true) + strip_diff_frontmatter(transformed_diff) + end + + def transformed_blob_language(blob) + 'md' if transformed_for_diff?(blob) + end + + def transformed_blob_data(blob) + if transformed_for_diff?(blob) + IpynbDiff.transform(blob.data, + raise_errors: true, + options: { include_metadata: false, cell_decorator: :percent }) + end + end + + def strip_diff_frontmatter(diff_content) + diff_content.scan(/.*\n/)[2..]&.join('') if diff_content.present? + end + + def blobs_with_transformed_diffs + @blobs_with_transformed_diffs ||= {} + end + + def transformed_for_diff?(blob) + blobs_with_transformed_diffs[blob] + end + + def transformed_for_diff(*blobs) + blobs.each do |b| + blobs_with_transformed_diffs[b] = true if b + end + end + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 83f242ff902..d9860d9fb86 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -44,7 +44,11 @@ module Gitlab new_blob_lazy old_blob_lazy - preprocess_before_diff(diff) if Feature.enabled?(:jupyter_clean_diffs, repository.project, default_enabled: true) + diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff if use_custom_diff? + end + + def use_custom_diff? + strong_memoize(:_custom_diff_enabled) { Feature.enabled?(:jupyter_clean_diffs, repository.project, default_enabled: true) } end def position(position_marker, position_type: :text) @@ -450,33 +454,6 @@ module Gitlab find_renderable_viewer_class(classes) end - def preprocess_before_diff(diff) - return unless diff.new_path.ends_with? '.ipynb' - - from = old_blob_lazy&.data - to = new_blob_lazy&.data - - transformed_diff = IpynbDiff.diff(from, to, - diff_opts: { context: 5, include_diff_info: true }, - transform_options: { cell_decorator: :percent }, - raise_if_invalid_notebook: true) - new_diff = strip_diff_frontmatter(transformed_diff) - - if new_diff - diff.diff = new_diff - new_blob_lazy.transformed_for_diff = true if new_blob_lazy - old_blob_lazy.transformed_for_diff = true if old_blob_lazy - end - - Gitlab::AppLogger.info({ message: new_diff ? 'IPYNB_DIFF_GENERATED' : 'IPYNB_DIFF_NIL' }) - rescue IpynbDiff::InvalidNotebookError => e - Gitlab::ErrorTracking.log_exception(e) - end - - def strip_diff_frontmatter(diff_content) - diff_content.scan(/.*\n/)[2..-1]&.join('') if diff_content.present? - end - def alternate_viewer_class return unless viewer.instance_of?(DiffViewer::Renamed) diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index 692186fc323..b459e3f6619 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -13,7 +13,7 @@ module Gitlab super(merge_request_diff, project: merge_request_diff.project, - diff_options: merged_diff_options(diff_options), + diff_options: diff_options, diff_refs: merge_request_diff.diff_refs, fallback_diff_refs: merge_request_diff.fallback_diff_refs) end @@ -68,13 +68,6 @@ module Gitlab diff_stats_cache.read || super end end - - def merged_diff_options(diff_options) - project = @merge_request_diff.project - max_diff_options = ::Commit.max_diff_options(project: project).merge(project: project) - - diff_options.present? ? diff_options.merge(max_diff_options) : max_diff_options - end end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index aedcfe3cb40..47f3324752d 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -153,13 +153,10 @@ module Gitlab blob.load_all_data! - return blob.present.highlight_transformed.lines if Feature.enabled?(:jupyter_clean_diffs, @project, default_enabled: true) - blob.present.highlight.lines end def blobs_too_large? - return false unless Feature.enabled?(:limited_diff_highlighting, project, default_enabled: :yaml) return true if Gitlab::Highlight.too_large?(diff_file.old_blob&.size) Gitlab::Highlight.too_large?(diff_file.new_blob&.size) diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index f70618195d0..802da50cfc6 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -6,8 +6,8 @@ module Gitlab attr_accessor :old_line, :new_line, :offset def initialize(old_line, new_line, offset: 0) - @old_line = old_line[offset..-1] - @new_line = new_line[offset..-1] + @old_line = old_line[offset..] + @new_line = new_line[offset..] @offset = offset end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 4b490ae0d26..7b31dd9926b 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -60,7 +60,7 @@ module Gitlab current_user: author, params: { title: mail.subject, - description: message_including_reply + description: message_including_reply_or_only_quotes }, spam_params: nil ).execute diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index a717509e24d..43cbc49b9d2 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -35,6 +35,10 @@ module Gitlab @message_with_reply ||= process_message(trim_reply: false) end + def message_including_reply_or_only_quotes + @message_including_reply_or_only_quotes ||= process_message(trim_reply: false, allow_only_quotes: true) + end + def message_with_appended_reply @message_with_appended_reply ||= process_message(append_reply: true) end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 8d73aa842be..71b1d4ed8f9 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -32,11 +32,11 @@ module Gitlab def execute raise ProjectNotFound if project.nil? - create_issue! + create_issue_or_note if from_address add_email_participant - send_thank_you_email + send_thank_you_email unless reply_email? end end @@ -82,6 +82,14 @@ module Gitlab project.present? && slug == project.full_path_slug end + def create_issue_or_note + if reply_email? + create_note_from_reply_email + else + create_issue! + end + end + def create_issue! @issue = ::Issues::CreateService.new( project: project, @@ -97,18 +105,42 @@ module Gitlab raise InvalidIssueError unless @issue.persisted? + begin + ::Issue::Email.create!(issue: @issue, email_message_id: mail.message_id) + rescue StandardError => e + Gitlab::ErrorTracking.log_exception(e) + end + if service_desk_setting&.issue_template_missing? - create_template_not_found_note(@issue) + create_template_not_found_note + end + end + + def issue_from_reply_to + strong_memoize(:issue_from_reply_to) do + next unless mail.in_reply_to + + Issue::Email.find_by_email_message_id(mail.in_reply_to)&.issue end end + def reply_email? + issue_from_reply_to.present? + end + + def create_note_from_reply_email + @issue = issue_from_reply_to + + create_note(message_including_reply) + end + def send_thank_you_email Notify.service_desk_thank_you_email(@issue.id).deliver_later Gitlab::Metrics::BackgroundTransaction.current&.add_event(:service_desk_thank_you_email) end def message_including_template - description = process_message(trim_reply: false, allow_only_quotes: true) + description = message_including_reply_or_only_quotes template_content = service_desk_setting&.issue_template_content if template_content.present? @@ -124,7 +156,7 @@ module Gitlab end end - def create_template_not_found_note(issue) + def create_template_not_found_note issue_template_key = service_desk_setting&.issue_template_key warning_note = <<-MD.strip_heredoc @@ -132,15 +164,15 @@ module Gitlab Please check service desk settings and update the file to be used. MD - note_params = { - noteable: issue, - note: warning_note - } + create_note(warning_note) + end + def create_note(note) ::Notes::CreateService.new( project, User.support_bot, - note_params + noteable: @issue, + note: note ).execute end @@ -157,6 +189,8 @@ module Gitlab end def add_email_participant + return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) + @issue.issue_email_participants.create(email: from_address) end end diff --git a/lib/gitlab/email/hook/smime_signature_interceptor.rb b/lib/gitlab/email/hook/smime_signature_interceptor.rb index 0b092b3e41e..ed1bf3e12bf 100644 --- a/lib/gitlab/email/hook/smime_signature_interceptor.rb +++ b/lib/gitlab/email/hook/smime_signature_interceptor.rb @@ -45,7 +45,6 @@ module Gitlab end def overwrite_headers(message, signed_email) - message.content_disposition = signed_email.content_disposition message.content_transfer_encoding = signed_email.content_transfer_encoding message.content_type = signed_email.content_type end diff --git a/lib/gitlab/email/message/in_product_marketing/admin_verify.rb b/lib/gitlab/email/message/in_product_marketing/admin_verify.rb index 19d9cf99cdb..888f84cde23 100644 --- a/lib/gitlab/email/message/in_product_marketing/admin_verify.rb +++ b/lib/gitlab/email/message/in_product_marketing/admin_verify.rb @@ -38,7 +38,7 @@ module Gitlab end def invite_members? - invite_members_for_task_experiment_enabled? + user.can?(:admin_group_member, group) end end end diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb index 7cd54390b9f..9b50d86de58 100644 --- a/lib/gitlab/email/message/in_product_marketing/base.rb +++ b/lib/gitlab/email/message/in_product_marketing/base.rb @@ -88,29 +88,6 @@ module Gitlab end end - def address - s_('InProductMarketing|%{strong_start}GitLab Inc.%{strong_end} 268 Bush Street, #350, San Francisco, CA 94104, USA').html_safe % strong_options - end - - def footer_links - links = [ - [s_('InProductMarketing|Blog'), 'https://about.gitlab.com/blog'], - [s_('InProductMarketing|Twitter'), 'https://twitter.com/gitlab'], - [s_('InProductMarketing|Facebook'), 'https://www.facebook.com/gitlab'], - [s_('InProductMarketing|YouTube'), 'https://www.youtube.com/channel/UCnMGQ8QHMAnVIsI3xJrihhg'] - ] - case format - when :html - links.map do |text, link| - ActionController::Base.helpers.link_to(text, link) - end - else - '| ' + links.map do |text, link| - [text, link].join(' ') - end.join("\n| ") - end - end - def logo_path ["mailers/in_product_marketing", "#{track}-#{series}.png"].join('/') end @@ -166,16 +143,6 @@ module Gitlab link(s_('InProductMarketing|update your preferences'), preference_link) end - def invite_members_for_task_experiment_enabled? - return unless user.can?(:admin_group_member, group) - - experiment(:invite_members_for_task, namespace: group) do |e| - e.candidate { true } - e.record! - e.run - end - end - def validate_series! raise ArgumentError, "Only #{total_series} series available for this track." unless @series.between?(0, total_series - 1) end diff --git a/lib/gitlab/email/message/in_product_marketing/create.rb b/lib/gitlab/email/message/in_product_marketing/create.rb index 2c396775374..6b01c83b8e7 100644 --- a/lib/gitlab/email/message/in_product_marketing/create.rb +++ b/lib/gitlab/email/message/in_product_marketing/create.rb @@ -62,7 +62,7 @@ module Gitlab end def invite_members? - invite_members_for_task_experiment_enabled? + user.can?(:admin_group_member, group) end private diff --git a/lib/gitlab/email/message/in_product_marketing/helper.rb b/lib/gitlab/email/message/in_product_marketing/helper.rb index bffa90ed4ec..329cace9e9d 100644 --- a/lib/gitlab/email/message/in_product_marketing/helper.rb +++ b/lib/gitlab/email/message/in_product_marketing/helper.rb @@ -8,6 +8,29 @@ module Gitlab include ActionView::Context include ActionView::Helpers::TagHelper + def footer_links + links = [ + [s_('InProductMarketing|Blog'), 'https://about.gitlab.com/blog'], + [s_('InProductMarketing|Twitter'), 'https://twitter.com/gitlab'], + [s_('InProductMarketing|Facebook'), 'https://www.facebook.com/gitlab'], + [s_('InProductMarketing|YouTube'), 'https://www.youtube.com/channel/UCnMGQ8QHMAnVIsI3xJrihhg'] + ] + case format + when :html + links.map do |text, link| + ActionController::Base.helpers.link_to(text, link) + end + else + '| ' + links.map do |text, link| + [text, link].join(' ') + end.join("\n| ") + end + end + + def address + s_('InProductMarketing|%{strong_start}GitLab Inc.%{strong_end} 268 Bush Street, #350, San Francisco, CA 94104, USA').html_safe % strong_options + end + private def list(array) diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb index daf0c969f2b..d2a78b53e1f 100644 --- a/lib/gitlab/email/message/in_product_marketing/verify.rb +++ b/lib/gitlab/email/message/in_product_marketing/verify.rb @@ -66,7 +66,7 @@ module Gitlab end def invite_members? - invite_members_for_task_experiment_enabled? + user.can?(:admin_group_member, group) end private diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 526f1188065..5b2bbfbe66b 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -73,7 +73,7 @@ module Gitlab def key_from_to_header mail.to.find do |address| - key = Gitlab::IncomingEmail.key_from_address(address) + key = email_class.key_from_address(address) break key if key end end @@ -100,7 +100,7 @@ module Gitlab def find_key_from_references ensure_references_array(mail.references).find do |mail_id| - key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) + key = email_class.key_from_fallback_message_id(mail_id) break key if key end end @@ -119,21 +119,21 @@ module Gitlab def find_key_from_delivered_to_header delivered_to.find do |header| - key = Gitlab::IncomingEmail.key_from_address(header.value) + key = email_class.key_from_address(header.value) break key if key end end def find_key_from_envelope_to_header envelope_to.find do |header| - key = Gitlab::IncomingEmail.key_from_address(header.value) + key = email_class.key_from_address(header.value) break key if key end end def find_key_from_x_envelope_to_header x_envelope_to.find do |header| - key = Gitlab::IncomingEmail.key_from_address(header.value) + key = email_class.key_from_address(header.value) break key if key end end @@ -158,6 +158,10 @@ module Gitlab autoreply && autoreply == 'yes' end + + def email_class + Gitlab::IncomingEmail + end end end end diff --git a/lib/gitlab/email/service_desk_receiver.rb b/lib/gitlab/email/service_desk_receiver.rb index 133c4ee4b45..6c6eb3b0a65 100644 --- a/lib/gitlab/email/service_desk_receiver.rb +++ b/lib/gitlab/email/service_desk_receiver.rb @@ -6,22 +6,13 @@ module Gitlab private def find_handler - return unless service_desk_key + return unless mail_key - Gitlab::Email::Handler::ServiceDeskHandler.new(mail, nil, service_desk_key: service_desk_key) + Gitlab::Email::Handler::ServiceDeskHandler.new(mail, nil, service_desk_key: mail_key) end - def service_desk_key - strong_memoize(:service_desk_key) do - find_service_desk_key - end - end - - def find_service_desk_key - mail.to.find do |address| - key = ::Gitlab::ServiceDeskEmail.key_from_address(address) - break key if key - end + def email_class + ::Gitlab::ServiceDeskEmail end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index 519b1d94bf5..3c5d223b106 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -6,7 +6,7 @@ module Gitlab # When updating emoji assets increase the version below # and update the version number in `app/assets/javascripts/emoji/index.js` - EMOJI_VERSION = 1 + EMOJI_VERSION = 2 # Return a Pathname to emoji's current versioned folder # diff --git a/lib/gitlab/empty_search_results.rb b/lib/gitlab/empty_search_results.rb new file mode 100644 index 00000000000..71a78cb297b --- /dev/null +++ b/lib/gitlab/empty_search_results.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + # This class has the same interface as SearchResults except + # it is empty and does not do any work. + # + # We use this when responding to abusive search requests. + class EmptySearchResults + def initialize(*) + end + + def objects(*) + Kaminari.paginate_array([]) + end + + def formatted_count(*) + '0' + end + + def highlight_map(*) + {} + end + + def aggregations(*) + [] + end + end +end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 38ac5d9af74..6a637306225 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -140,12 +140,9 @@ module Gitlab end def inject_context_for_exception(event, ex) - case ex - when ActiveRecord::StatementInvalid - event.extra[:sql] = PgQuery.normalize(ex.sql.to_s) - else - inject_context_for_exception(event, ex.cause) if ex.cause.present? - end + sql = Gitlab::ExceptionLogFormatter.find_sql(ex) + + event.extra[:sql] = sql if sql end end end diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb index 2b8639b9411..1f56670ee7f 100644 --- a/lib/gitlab/etag_caching/router/graphql.rb +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -17,6 +17,11 @@ module Gitlab %r(\Apipelines/sha/\w{7,40}\z), 'ci_editor', 'pipeline_authoring' + ], + [ + %r(\Aon_demand_scan/counts/), + 'on_demand_scans', + 'dynamic_application_security_testing' ] ].map(&method(:build_route)).freeze diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index d0d790a7c72..44c6984c09b 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -12,14 +12,18 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } end - def touch(key, only_if_missing: false) - etag = generate_etag + def touch(*keys, only_if_missing: false) + etags = keys.map { generate_etag } Gitlab::Redis::SharedState.with do |redis| - redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + redis.pipelined do + keys.each_with_index do |key, i| + redis.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing) + end + end end - etag + keys.size > 1 ? etags : etags.first end private diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb index 9898651c9e3..315574fed31 100644 --- a/lib/gitlab/exception_log_formatter.rb +++ b/lib/gitlab/exception_log_formatter.rb @@ -2,18 +2,41 @@ module Gitlab module ExceptionLogFormatter - def self.format!(exception, payload) - return unless exception + class << self + def format!(exception, payload) + return unless exception - # Elasticsearch/Fluentd don't handle nested structures well. - # Use periods to flatten the fields. - payload.merge!( - 'exception.class' => exception.class.name, - 'exception.message' => exception.message - ) + # Elasticsearch/Fluentd don't handle nested structures well. + # Use periods to flatten the fields. + payload.merge!( + 'exception.class' => exception.class.name, + 'exception.message' => exception.message + ) - if exception.backtrace - payload['exception.backtrace'] = Rails.backtrace_cleaner.clean(exception.backtrace) + if exception.backtrace + payload['exception.backtrace'] = Rails.backtrace_cleaner.clean(exception.backtrace) + end + + if sql = find_sql(exception) + payload['exception.sql'] = sql + end + end + + def find_sql(exception) + if exception.is_a?(ActiveRecord::StatementInvalid) + # StatementInvalid may be caused by a statement timeout or a bad query + normalize_query(exception.sql.to_s) + elsif exception.cause.present? + find_sql(exception.cause) + end + end + + private + + def normalize_query(sql) + PgQuery.normalize(sql) + rescue PgQuery::ParseError + sql end end end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index c2009628c56..4cc653bec43 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -37,9 +37,6 @@ module Gitlab remove_known_trial_form_fields_noneditable: { tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFieldsNoneditable', rollout_strategy: :user - }, - invite_members_new_dropdown: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' } }.freeze diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index 7cc29cde45c..303d952381f 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -16,7 +16,7 @@ module Gitlab included do before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :record_experiment_group, :tracking_label + helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :record_experiment_group end def set_experimentation_subject_id_cookie diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 4d82acd9d87..5d0a638f97a 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -57,7 +57,7 @@ module Gitlab def unfold_reference(reference, match, target_parent) before = @text[0...match.begin(0)] - after = @text[match.end(0)..-1] + after = @text[match.end(0)..] referable = find_referable(reference) return reference unless referable diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index b0d194f309a..f72217dedde 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -24,7 +24,7 @@ module Gitlab LFS_POINTER_MIN_SIZE = 120.bytes LFS_POINTER_MAX_SIZE = 200.bytes - attr_accessor :size, :mode, :id, :commit_id, :loaded_size, :binary, :transformed_for_diff + attr_accessor :size, :mode, :id, :commit_id, :loaded_size, :binary attr_writer :name, :path, :data def self.gitlab_blob_truncated_true @@ -127,7 +127,6 @@ module Gitlab # Retain the actual size before it is encoded @loaded_size = @data.bytesize if @data @loaded_all_data = @loaded_size == size - @transformed_for_diff = false record_metric_blob_size record_metric_truncated(truncated?) diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 631624c068c..24b67424f28 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -11,13 +11,13 @@ module Gitlab delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits - def self.default_limits(project: nil) - { max_files: ::Commit.diff_safe_max_files(project: project), max_lines: ::Commit.diff_safe_max_lines(project: project) } + def self.default_limits + { max_files: ::Commit.diff_safe_max_files, max_lines: ::Commit.diff_safe_max_lines } end def self.limits(options = {}) limits = {} - defaults = default_limits(project: options[:project]) + defaults = default_limits limits[:max_files] = options.fetch(:max_files, defaults[:max_files]) limits[:max_lines] = options.fetch(:max_lines, defaults[:max_lines]) limits[:max_bytes] = limits[:max_files] * 5.kilobytes # Average 5 KB per file diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb index e30ec836a49..6a689f55478 100644 --- a/lib/gitlab/git/diff_stats_collection.rb +++ b/lib/gitlab/git/diff_stats_collection.rb @@ -22,8 +22,8 @@ module Gitlab @collection.map(&:path) end - def real_size(project: nil) - max_files = ::Commit.max_diff_options(project: project)[:max_files] + def real_size + max_files = ::Commit.diff_max_files if paths.size > max_files "#{max_files}+" else diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 5afdcc0bd4c..240a701aba9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -160,6 +160,8 @@ module Gitlab wrapped_gitaly_errors do gitaly_repository_client.remove end + rescue NoRepository + nil end def replicate(source_repository) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 7c688044e9c..c2b4182f609 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -204,19 +204,6 @@ module Gitlab Gitlab::Git::Commit.new(@repository, gitaly_commit) end - def between(from, to) - return list_commits(["^" + from, to], reverse: true) if Feature.enabled?(:between_commits_via_list_commits) - - request = Gitaly::CommitsBetweenRequest.new( - repository: @gitaly_repo, - from: from, - to: to - ) - - response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request, timeout: GitalyClient.medium_timeout) - consume_commits_response(response) - end - def diff_stats(left_commit_sha, right_commit_sha) request = Gitaly::DiffStatsRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index efa816c5eb0..d2495b32800 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -21,6 +21,7 @@ module Gitlab SEARCH_MAX_REQUESTS_PER_MINUTE = 30 DEFAULT_PER_PAGE = 100 LOWER_PER_PAGE = 50 + CLIENT_CONNECTION_ERROR = ::Faraday::ConnectionFailed # used/set in sawyer agent which octokit uses # A single page of data and the corresponding page number. Page = Struct.new(:objects, :number) @@ -148,14 +149,14 @@ module Gitlab # whether we are running in parallel mode or not. For more information see # `#rate_or_wait_for_rate_limit`. def with_rate_limit - return yield unless rate_limiting_enabled? + return with_retry { yield } unless rate_limiting_enabled? request_count_counter.increment raise_or_wait_for_rate_limit unless requests_remaining? begin - yield + with_retry { yield } rescue ::Octokit::TooManyRequests raise_or_wait_for_rate_limit @@ -166,7 +167,7 @@ module Gitlab end def search_repos_by_name(name, options = {}) - octokit.search_repositories(search_query(str: name, type: :name), options) + with_retry { octokit.search_repositories(search_query(str: name, type: :name), options) } end def search_query(str:, type:, include_collaborations: true, include_orgs: true) @@ -270,6 +271,25 @@ module Gitlab .map { |org| "org:#{org.login}" } .join(' ') end + + def with_retry + Retriable.retriable(on: CLIENT_CONNECTION_ERROR, on_retry: on_retry) do + yield + end + end + + def on_retry + proc do |exception, try, elapsed_time, next_interval| + Gitlab::Import::Logger.info( + message: "GitHub connection retry triggered", + 'error.class': exception.class, + 'error.message': exception.message, + try_count: try, + elapsed_time_s: elapsed_time, + wait_to_retry_s: next_interval + ) + end + end end end end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 0aa0896aa57..8a8d23401c1 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -31,6 +31,10 @@ module Gitlab else import_with_legacy_diff_note end + rescue ::DiffNote::NoteDiffFileCreationError => e + Logger.warn(message: e.message, 'error.class': e.class.name) + + import_with_legacy_diff_note rescue ActiveRecord::InvalidForeignKey => e # It's possible the project and the issue have been deleted since # scheduling this job. In this case we'll just skip creating the note diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 2cc3a82dd9b..673f56b5753 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -29,6 +29,7 @@ module Gitlab project_id: project.id, author_id: author_id, note: note_body, + discussion_id: note.discussion_id, system: false, created_at: note.created_at, updated_at: note.updated_at diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index f72e595e8e9..a71590c02f8 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -35,7 +35,9 @@ module Gitlab def execute Gitlab::Import::SetAsyncJid.set_jid(project.import_state) + # We need to track this job's status for use by Gitlab::GithubImport::RefreshImportJidWorker. Stage::ImportRepositoryWorker + .with_status .perform_async(project.id) true diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index fecff0644c2..04f53accfeb 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -4,6 +4,7 @@ module Gitlab module GithubImport module Representation class DiffNote + include Gitlab::Utils::StrongMemoize include ToHash include ExposeAttribute @@ -127,15 +128,17 @@ module Gitlab end def discussion_id - if in_reply_to_id.present? - current_discussion_id - else - Discussion.discussion_id( - Struct - .new(:noteable_id, :noteable_type) - .new(merge_request.id, NOTEABLE_TYPE) - ).tap do |discussion_id| - cache_discussion_id(discussion_id) + strong_memoize(:discussion_id) do + if in_reply_to_id.present? + current_discussion_id + else + Discussion.discussion_id( + Struct + .new(:noteable_id, :noteable_type) + .new(merge_request.id, NOTEABLE_TYPE) + ).tap do |discussion_id| + cache_discussion_id(discussion_id) + end end end end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index bcdb1a5459b..bbf20b7e9e6 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -63,6 +63,14 @@ module Gitlab @attributes = attributes end + def discussion_id + Discussion.discussion_id( + Struct + .new(:noteable_id, :noteable_type) + .new(noteable_id, noteable_type) + ) + end + alias_method :issuable_type, :noteable_type def github_identifiers diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9ad902efb3a..bb3ba1129fc 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -56,7 +56,6 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix, default_enabled: false) push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) - push_frontend_feature_flag(:configure_iac_scanning_via_mr, current_user, default_enabled: :yaml) push_frontend_feature_flag(:bootstrap_confirmation_modals, default_enabled: :yaml) end diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 3d9b06855ff..f55afd90ac0 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -48,7 +48,7 @@ module Gitlab raw_keys.each_with_object({}) do |raw_key, grouped_subkeys| primary_subkey_id = raw_key.primary_subkey.keyid - grouped_subkeys[primary_subkey_id] = raw_key.subkeys[1..-1].map do |s| + grouped_subkeys[primary_subkey_id] = raw_key.subkeys[1..].map do |s| { keyid: s.keyid, fingerprint: s.fingerprint } end end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 9a6317e2b76..59882e8d4f8 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -25,7 +25,7 @@ module Gitlab def lazy_signature BatchLoader.for(@commit.sha).batch do |shas, loader| - GpgSignature.by_commit_sha(shas).each do |signature| + CommitSignatures::GpgSignature.by_commit_sha(shas).each do |signature| loader.call(signature.commit_sha, signature) end end @@ -62,9 +62,9 @@ module Gitlab def create_cached_signature! using_keychain do |gpg_key| attributes = attributes(gpg_key) - break GpgSignature.new(attributes) if Gitlab::Database.read_only? + break CommitSignatures::GpgSignature.new(attributes) if Gitlab::Database.read_only? - GpgSignature.safe_create!(attributes) + CommitSignatures::GpgSignature.safe_create!(attributes) end end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index d892d27a917..0eb105143ea 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -9,9 +9,9 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def run - GpgSignature + CommitSignatures::GpgSignature .select(:id, :commit_sha, :project_id) - .where('gpg_key_id IS NULL OR verification_status <> ?', GpgSignature.verification_statuses[:verified]) + .where('gpg_key_id IS NULL OR verification_status <> ?', CommitSignatures::GpgSignature.verification_statuses[:verified]) .where(gpg_key_primary_keyid: @gpg_key.keyids) .find_each { |sig| sig.gpg_commit&.update_signature!(sig) } end diff --git a/lib/gitlab/graphql/tracers/logger_tracer.rb b/lib/gitlab/graphql/tracers/logger_tracer.rb index c7ba56824db..3302b2bae3f 100644 --- a/lib/gitlab/graphql/tracers/logger_tracer.rb +++ b/lib/gitlab/graphql/tracers/logger_tracer.rb @@ -11,19 +11,20 @@ module Gitlab end def trace(key, data) - result = yield - + yield + rescue StandardError => e + data[:exception] = e + raise e + ensure case key when "execute_query" log_execute_query(**data) end - - result end private - def log_execute_query(query: nil, duration_s: 0) + def log_execute_query(query: nil, duration_s: 0, exception: nil) # execute_query should always have :query, but we're just being defensive return unless query @@ -39,6 +40,8 @@ module Gitlab query_string: query.query_string } + Gitlab::ExceptionLogFormatter.format!(exception, info) + info.merge!(::Gitlab::ApplicationContext.current) info.merge!(analysis_info) if analysis_info diff --git a/lib/gitlab/graphql/tracers/timer_tracer.rb b/lib/gitlab/graphql/tracers/timer_tracer.rb index 326620a22bc..8e058621110 100644 --- a/lib/gitlab/graphql/tracers/timer_tracer.rb +++ b/lib/gitlab/graphql/tracers/timer_tracer.rb @@ -17,13 +17,9 @@ module Gitlab def trace(key, data) start_time = Gitlab::Metrics::System.monotonic_time - result = yield - - duration_s = Gitlab::Metrics::System.monotonic_time - start_time - - data[:duration_s] = duration_s - - result + yield + ensure + data[:duration_s] = Gitlab::Metrics::System.monotonic_time - start_time end end end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index db807a3c557..0e787a77a25 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -7,6 +7,7 @@ module Gitlab %i[ assignee_id author_id + blocking_discussions_resolved created_at description head_pipeline_id @@ -57,7 +58,8 @@ module Gitlab human_time_estimate: merge_request.human_time_estimate, assignee_ids: merge_request.assignee_ids, assignee_id: merge_request.assignee_ids.first, # This key is deprecated - state: merge_request.state # This key is deprecated + state: merge_request.state, # This key is deprecated + blocking_discussions_resolved: merge_request.mergeable_discussions_state? } merge_request.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 251bc34d462..12203cab8c8 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,27 +43,27 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 52, + 'da_DK' => 51, 'de' => 15, 'en' => 100, 'eo' => 0, - 'es' => 40, + 'es' => 39, 'fil_PH' => 0, - 'fr' => 11, + 'fr' => 12, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 2, - 'ja' => 36, + 'ja' => 35, 'ko' => 11, - 'nb_NO' => 34, + 'nb_NO' => 33, 'nl_NL' => 0, 'pl_PL' => 5, 'pt_BR' => 49, - 'ro_RO' => 24, - 'ru' => 26, + 'ro_RO' => 23, + 'ru' => 25, 'tr_TR' => 15, - 'uk' => 39, - 'zh_CN' => 97, + 'uk' => 45, + 'zh_CN' => 95, 'zh_HK' => 2, 'zh_TW' => 3 }.freeze diff --git a/lib/gitlab/import/import_failure_service.rb b/lib/gitlab/import/import_failure_service.rb index 142c00f7a6b..bebd64b29a9 100644 --- a/lib/gitlab/import/import_failure_service.rb +++ b/lib/gitlab/import/import_failure_service.rb @@ -15,11 +15,21 @@ module Gitlab exception: exception, import_state: import_state, project_id: project_id, - error_source: error_source - ).execute(fail_import: fail_import, metrics: metrics) + error_source: error_source, + fail_import: fail_import, + metrics: metrics + ).execute end - def initialize(exception:, import_state: nil, project_id: nil, error_source: nil) + def initialize( + exception:, + import_state: nil, + project_id: nil, + error_source: nil, + fail_import: false, + metrics: false + ) + if import_state.blank? && project_id.blank? raise ArgumentError, 'import_state OR project_id must be provided' end @@ -34,9 +44,11 @@ module Gitlab @exception = exception @error_source = error_source + @fail_import = fail_import + @metrics = metrics end - def execute(fail_import:, metrics:) + def execute track_exception persist_failure @@ -46,7 +58,7 @@ module Gitlab private - attr_reader :exception, :import_state, :project, :error_source + attr_reader :exception, :import_state, :project, :error_source, :fail_import, :metrics def track_exception attributes = { @@ -65,12 +77,15 @@ module Gitlab Gitlab::ErrorTracking.track_exception(exception, attributes) end + # Failures with `retry_count: 0` are considered "hard_failures" and those + # are exposed on the REST API projects/:id/import def persist_failure project.import_failures.create( source: error_source, exception_class: exception.class.to_s, exception_message: exception.message.truncate(255), - correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, + retry_count: fail_import ? 0 : nil ) end diff --git a/lib/gitlab/import/set_async_jid.rb b/lib/gitlab/import/set_async_jid.rb index 527d84477fe..054fcdb433f 100644 --- a/lib/gitlab/import/set_async_jid.rb +++ b/lib/gitlab/import/set_async_jid.rb @@ -13,7 +13,7 @@ module Gitlab def self.set_jid(import_state) jid = generate_jid(import_state) - Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION) + Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION, value: 2) import_state.update_column(:jid, jid) end diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index f3c392b8c20..cbc8ee9e18b 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -106,12 +106,7 @@ module Gitlab def update_params! params = @importable_attributes.except(*relations.keys.map(&:to_s)) params = params.merge(present_override_params) - - # Cleaning all imported and overridden params - params = Gitlab::ImportExport::AttributeCleaner.clean( - relation_hash: params, - relation_class: importable_class, - excluded_keys: excluded_keys_for_relation(importable_class_sym)) + params = filter_attributes(params) @importable.assign_attributes(params) @@ -122,6 +117,25 @@ module Gitlab end end + def filter_attributes(params) + if use_attributes_permitter? && attributes_permitter.permitted_attributes_defined?(importable_class_sym) + attributes_permitter.permit(importable_class_sym, params) + else + Gitlab::ImportExport::AttributeCleaner.clean( + relation_hash: params, + relation_class: importable_class, + excluded_keys: excluded_keys_for_relation(importable_class_sym)) + end + end + + def attributes_permitter + @attributes_permitter ||= Gitlab::ImportExport::AttributesPermitter.new + end + + def use_attributes_permitter? + Feature.enabled?(:permitted_attributes_for_import_export, default_enabled: :yaml) + end + def present_override_params # we filter out the empty strings from the overrides # keeping the default values configured diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 9ab8fa68d0e..fb8d6e7d89b 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -40,6 +40,13 @@ module Gitlab end end + def serialize_root(exportable_path = @exportable_path) + attributes = exportable.as_json( + relations_schema.merge(include: nil, preloads: nil)) + + json_writer.write_attributes(exportable_path, attributes) + end + def serialize_relation(definition) raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash) raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one? @@ -60,12 +67,6 @@ module Gitlab attr_reader :json_writer, :relations_schema, :exportable - def serialize_root - attributes = exportable.as_json( - relations_schema.merge(include: nil, preloads: nil)) - json_writer.write_attributes(@exportable_path, attributes) - end - def serialize_many_relations(key, records, options) enumerator = Enumerator.new do |items| key_preloads = preloads&.dig(key) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index d815dd284ba..ef146359da9 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -126,7 +126,6 @@ included_attributes: - :project_id project_badges: - :created_at - - :group_id - :image_url - :link_url - :name @@ -414,7 +413,6 @@ included_attributes: - :b_mode - :too_large - :binary - - :diff metrics: - :created_at - :updated_at @@ -572,7 +570,6 @@ included_attributes: - :updated_at actions: - :event - - :image_v432x230 design: &design_definition - :iid - :project_id @@ -632,6 +629,74 @@ included_attributes: - :expires_at - :ldap - :override + project: + - :approvals_before_merge + - :archived + - :auto_cancel_pending_pipelines + - :autoclose_referenced_issues + - :build_allow_git_fetch + - :build_coverage_regex + - :build_timeout + - :ci_config_path + - :delete_error + - :description + - :disable_overriding_approvers_per_merge_request + - :external_authorization_classification_label + - :has_external_issue_tracker + - :has_external_wiki + - :issues_template + - :jobs_cache_index + - :last_repository_check_failed + - :merge_requests_author_approval + - :merge_requests_disable_committers_approval + - :merge_requests_ff_only_enabled + - :merge_requests_rebase_enabled + - :merge_requests_template + - :only_allow_merge_if_all_discussions_are_resolved + - :only_allow_merge_if_pipeline_succeeds + - :pages_https_only + - :pending_delete + - :printing_merge_request_link_enabled + - :public_builds + - :remove_source_branch_after_merge + - :request_access_enabled + - :require_password_to_approve + - :reset_approvals_on_push + - :resolve_outdated_diff_discussions + - :service_desk_enabled + - :shared_runners_enabled + - :suggestion_commit_message + - :visibility_level + - :hooks + - :issues_access_level + - :forking_access_level + - :merge_requests_access_level + - :wiki_access_level + - :snippets_access_level + - :builds_access_level + - :repository_access_level + - :pages_access_level + - :metrics_dashboard_access_level + - :analytics_access_level + - :operations_access_level + - :security_and_compliance_access_level + - :container_registry_access_level + - :allow_merge_on_skipped_pipeline + - :auto_devops_deploy_strategy + - :auto_devops_enabled + - :container_registry_enabled + - :issues_enabled + - :jobs_enabled + - :merge_method + - :merge_requests_enabled + - :snippets_enabled + - :squash_option + - :topics + - :visibility + - :wiki_enabled + - :build_git_strategy + - :build_enabled + - :security_and_compliance_enabled # Do not include the following attributes for the models specified. excluded_attributes: @@ -747,6 +812,7 @@ excluded_attributes: - :service_desk_reply_to - :upvotes_count - :work_item_type_id + - :email_message_id merge_request: &merge_request_excluded_definition - :milestone_id - :sprint_id @@ -889,8 +955,6 @@ excluded_attributes: system_note_metadata: - :description_version_id - :note_id - pipeline_schedules: - - :active methods: notes: - :type @@ -1021,3 +1085,6 @@ ee: - :auto_fix_dast - :auto_fix_dependency_scanning - :auto_fix_sast + project: + - :requirements_enabled + - :requirements_access_level diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index d84db92fe69..c391f86b47b 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -131,7 +131,9 @@ module Gitlab end def setup_diff - @relation_hash['diff'] = @relation_hash.delete('utf8_diff') + diff = @relation_hash.delete('utf8_diff') + + parsed_relation_hash['diff'] = diff end def setup_pipeline diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index e6c9ba0773c..84ead5119d5 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -32,6 +32,10 @@ module Gitlab ::Gitlab::ExceptionLogFormatter.format!(exception, payload) + if Feature.enabled?(:feature_flag_state_logs, type: :ops) + payload[:feature_flag_states] = Feature.logged_states.map { |key, state| "#{key}:#{state ? 1 : 0}" } + end + payload end end diff --git a/lib/gitlab/merge_requests/merge_commit_message.rb b/lib/gitlab/merge_requests/commit_message_generator.rb index 2a6a7859b33..0e9ec6f5cb3 100644 --- a/lib/gitlab/merge_requests/merge_commit_message.rb +++ b/lib/gitlab/merge_requests/commit_message_generator.rb @@ -1,31 +1,21 @@ # frozen_string_literal: true module Gitlab module MergeRequests - class MergeCommitMessage + class CommitMessageGenerator def initialize(merge_request:) @merge_request = merge_request end - def message + def merge_message return unless @merge_request.target_project.merge_commit_template.present? - message = @merge_request.target_project.merge_commit_template - message = message.delete("\r") + replace_placeholders(@merge_request.target_project.merge_commit_template) + end - # Remove placeholders that correspond to empty values and are the last word in the line - # along with all whitespace characters preceding them. - # This allows us to recreate previous default merge commit message behaviour - we skipped new line character - # before empty description and before closed issues when none were present. - PLACEHOLDERS.each do |key, value| - unless value.call(merge_request).present? - message = message.gsub(BLANK_PLACEHOLDERS_REGEXES[key], '') - end - end + def squash_message + return unless @merge_request.target_project.squash_commit_template.present? - Gitlab::StringPlaceholderReplacer - .replace_string_placeholders(message, PLACEHOLDERS_REGEX) do |key| - PLACEHOLDERS[key].call(merge_request) - end + replace_placeholders(@merge_request.target_project.squash_commit_template) end private @@ -45,7 +35,9 @@ module Gitlab "Closes #{closes_issues_references.to_sentence}" end, 'description' => ->(merge_request) { merge_request.description.presence || '' }, - 'reference' => ->(merge_request) { merge_request.to_reference(full: true) } + 'reference' => ->(merge_request) { merge_request.to_reference(full: true) }, + 'first_commit' => -> (merge_request) { merge_request.first_commit&.safe_message&.strip.presence || '' }, + 'first_multiline_commit' => -> (merge_request) { merge_request.first_multiline_commit&.safe_message&.strip.presence || merge_request.title } }.freeze PLACEHOLDERS_REGEX = Regexp.union(PLACEHOLDERS.keys.map do |key| @@ -55,6 +47,26 @@ module Gitlab BLANK_PLACEHOLDERS_REGEXES = (PLACEHOLDERS.map do |key, value| [key, Regexp.new("[\n\r]+%{#{Regexp.escape(key)}}$")] end).to_h.freeze + + def replace_placeholders(message) + # convert CRLF to LF + message = message.delete("\r") + + # Remove placeholders that correspond to empty values and are the last word in the line + # along with all whitespace characters preceding them. + # This allows us to recreate previous default merge commit message behaviour - we skipped new line character + # before empty description and before closed issues when none were present. + PLACEHOLDERS.each do |key, value| + unless value.call(merge_request).present? + message = message.gsub(BLANK_PLACEHOLDERS_REGEXES[key], '') + end + end + + Gitlab::StringPlaceholderReplacer + .replace_string_placeholders(message, PLACEHOLDERS_REGEX) do |key| + PLACEHOLDERS[key].call(merge_request) + end + end end end end diff --git a/lib/gitlab/merge_requests/mergeability/redis_interface.rb b/lib/gitlab/merge_requests/mergeability/redis_interface.rb index 081ccfca360..b0e739f91ff 100644 --- a/lib/gitlab/merge_requests/mergeability/redis_interface.rb +++ b/lib/gitlab/merge_requests/mergeability/redis_interface.rb @@ -7,13 +7,13 @@ module Gitlab VERSION = 1 def save_check(merge_check:, result_hash:) - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.set(merge_check.cache_key + ":#{VERSION}", result_hash.to_json, ex: EXPIRATION) end end def retrieve_check(merge_check:) - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Cache.with do |redis| Gitlab::Json.parse(redis.get(merge_check.cache_key + ":#{VERSION}")) end end diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index ff8b8bf2237..47c862c0232 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -11,12 +11,14 @@ module Gitlab attr_accessor :readiness_checks - def enabled? - settings.enabled + def initialize(settings, **options) + super(**options) + + @settings = settings end - def settings - raise NotImplementedError + def enabled? + settings.enabled end def log_filename @@ -25,6 +27,8 @@ module Gitlab private + attr_reader :settings + def start_working logger = WEBrick::Log.new(log_filename) logger.time_format = "[%Y-%m-%dT%H:%M:%S.%L%z]" diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb index 4d38d9e67bf..eea71fda6a0 100644 --- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -4,10 +4,6 @@ module Gitlab module Metrics module Exporter class SidekiqExporter < BaseExporter - def settings - Settings.monitoring.sidekiq_exporter - end - def log_filename if settings['log_enabled'] File.join(Rails.root, 'log', 'sidekiq_exporter.log') @@ -15,29 +11,6 @@ module Gitlab File::NULL end end - - private - - # Sidekiq Exporter does not work properly in sidekiq-cluster - # mode. It tries to start the service on the same port for - # each of the cluster workers, this results in failure - # due to duplicate binding. - # - # For now we ignore this error, as metrics are still "kind of" - # valid as they are rendered from shared directory. - # - # Issue: https://gitlab.com/gitlab-org/gitlab/issues/5714 - def start_working - super - rescue Errno::EADDRINUSE => e - Sidekiq.logger.error( - class: self.class.to_s, - message: 'Cannot start sidekiq_exporter', - 'exception.message' => e.message - ) - - false - end end end end diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index c5fa1e545d7..d41484aaaa7 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -27,7 +27,7 @@ module Gitlab # This exporter is always run on master process def initialize - super + super(Settings.monitoring.web_exporter) # DEPRECATED: # these `readiness_checks` are deprecated @@ -39,10 +39,6 @@ module Gitlab ] end - def settings - Gitlab.config.monitoring.web_exporter - end - def log_filename File.join(Rails.root, 'log', 'web_exporter.log') end diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index 5d7f434b660..965d85e20e5 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -38,16 +38,42 @@ module Gitlab end def host_stats - return [] unless ActiveRecord::Base.connected? + connection_class_stats + replica_host_stats + end + + def connection_class_stats + Gitlab::Database.database_base_models.each_value.with_object([]) do |base_model, stats| + next unless base_model.connected? + + stats << { labels: labels_for_class(base_model), stats: base_model.connection_pool.stat } + end + end + + def replica_host_stats + Gitlab::Database::LoadBalancing.each_load_balancer.with_object([]) do |load_balancer, stats| + next if load_balancer.primary_only? - [{ labels: labels_for_class(ActiveRecord::Base), stats: ActiveRecord::Base.connection_pool.stat }] + load_balancer.host_list.hosts.each do |host| + stats << { labels: labels_for_replica_host(load_balancer, host), stats: host.connection.pool.stat } + end + end end def labels_for_class(klass) { host: klass.connection_db_config.host, port: klass.connection_db_config.configuration_hash[:port], - class: klass.to_s + class: klass.to_s, + db_config_name: klass.connection_db_config.name + } + end + + def labels_for_replica_host(load_balancer, host) + { + host: host.host, + port: host.port, + class: load_balancer.configuration.primary_connection_specification_name, + db_config_name: Gitlab::Database.db_config_name(host.connection) } end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index df0582149a9..715dd86d93c 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -5,6 +5,8 @@ module Gitlab module Subscribers # Class for tracking the total query duration of a transaction. class ActiveRecord < ActiveSupport::Subscriber + extend Gitlab::Utils::StrongMemoize + attach_to :active_record IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze @@ -107,7 +109,7 @@ module Gitlab # Per database metrics db_config_name = db_config_name(event.payload) - duration_key = compose_metric_key(:duration_s, db_role, db_config_name) + duration_key = compose_metric_key(:duration_s, nil, db_config_name) ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration end @@ -144,7 +146,7 @@ module Gitlab # when we are also logging the db_role. Otherwise it will be hard to # tell if the log key is referring to a db_role OR a db_config_name. if db_role.present? && db_config_name.present? - log_key = compose_metric_key(counter, db_role, db_config_name) + log_key = compose_metric_key(counter, nil, db_config_name) Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 end end @@ -172,26 +174,34 @@ module Gitlab end def self.load_balancing_metric_counter_keys - load_balancing_metric_keys(DB_LOAD_BALANCING_COUNTERS) + strong_memoize(:load_balancing_metric_counter_keys) do + load_balancing_metric_keys(DB_LOAD_BALANCING_COUNTERS) + end end def self.load_balancing_metric_duration_keys - load_balancing_metric_keys(DB_LOAD_BALANCING_DURATIONS) + strong_memoize(:load_balancing_metric_duration_keys) do + load_balancing_metric_keys(DB_LOAD_BALANCING_DURATIONS) + end end def self.load_balancing_metric_keys(metrics) - [].tap do |counters| + counters = [] + + metrics.each do |metric| DB_LOAD_BALANCING_ROLES.each do |role| - metrics.each do |metric| - counters << compose_metric_key(metric, role) - next unless ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] + counters << compose_metric_key(metric, role) + end - ::Gitlab::Database.db_config_names.each do |config_name| - counters << compose_metric_key(metric, role, config_name) - end + if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] + ::Gitlab::Database.db_config_names.each do |config_name| + counters << compose_metric_key(metric, nil, config_name) # main + counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica end end end + + counters end def compose_metric_key(metric, db_role = nil, db_config_name = nil) diff --git a/lib/gitlab/pagination/keyset/iterator.rb b/lib/gitlab/pagination/keyset/iterator.rb index bcd17fd0d34..33587f9d6f4 100644 --- a/lib/gitlab/pagination/keyset/iterator.rb +++ b/lib/gitlab/pagination/keyset/iterator.rb @@ -9,7 +9,7 @@ module Gitlab raise(UnsupportedScopeOrder) unless success @cursor = cursor - @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) + @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(@scope) @use_union_optimization = in_operator_optimization_options ? false : use_union_optimization @in_operator_optimization_options = in_operator_optimization_options end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index 80726fc8efd..1a00692bdbe 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -155,7 +155,7 @@ module Gitlab def apply_cursor_conditions(scope, values = {}, options = { use_union_optimization: false, in_operator_optimization_options: nil }) values ||= {} transformed_values = values.with_indifferent_access - scope = apply_custom_projections(scope.dup) + scope = apply_custom_projections(scope) where_values = build_where_values(transformed_values) diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 7b5013f137b..4f8a6ffb2cc 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -29,7 +29,7 @@ module Gitlab return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops) limited_total_count = pagination_data.total_count_with_limit - if limited_total_count > max_limit + if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT # The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?` # We need to call `reset` because `without_count` relies on `@arel` being unmemoized pagination_data.reset.without_count @@ -38,14 +38,6 @@ module Gitlab end end - def max_limit - if Feature.enabled?(:lower_relation_max_count_limit, type: :ops) - Kaminari::ActiveRecordRelationMethods::MAX_COUNT_NEW_LOWER_LIMIT - else - Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT - end - end - def needs_pagination?(relation) return true unless relation.respond_to?(:current_page) return true if params[:page].present? && relation.current_page != params[:page].to_i diff --git a/lib/gitlab/patch/legacy_database_config.rb b/lib/gitlab/patch/legacy_database_config.rb index a7d4fdf7490..6040f737c75 100644 --- a/lib/gitlab/patch/legacy_database_config.rb +++ b/lib/gitlab/patch/legacy_database_config.rb @@ -35,6 +35,40 @@ module Gitlab attr_reader :uses_legacy_database_config end + def load_database_yaml + return super unless Gitlab.ee? + + super.deep_merge(load_geo_database_yaml) + end + + # This method is taken from Rails to load a database YAML file without + # evaluating ERB. This allows us to create the rake tasks for the Geo + # tracking database without filling in the configuration values or + # loading the environment. To be removed when we start configure Geo + # tracking database in database.yml instead of custom database_geo.yml + # + # https://github.com/rails/rails/blob/v6.1.4/railties/lib/rails/application/configuration.rb#L255 + def load_geo_database_yaml + path = Rails.root.join("config/database_geo.yml") + return {} unless File.exist?(path) + + require "rails/application/dummy_erb_compiler" + + yaml = DummyERB.new(Pathname.new(path).read).result + config = YAML.load(yaml) || {} # rubocop:disable Security/YAMLLoad + + config.to_h do |env, configs| + # This check is taken from Rails where the transformation + # of a flat database.yml is done into `primary:` + # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/database_configurations.rb#L169 + if configs.is_a?(Hash) && !configs.all? { |_, v| v.is_a?(Hash) } + configs = { "geo" => configs } + end + + [env, configs] + end + end + def database_configuration @uses_legacy_database_config = false # rubocop:disable Gitlab/ModuleWithInstanceVariables @@ -48,6 +82,16 @@ module Gitlab @uses_legacy_database_config = true # rubocop:disable Gitlab/ModuleWithInstanceVariables end + if Gitlab.ee? && File.exist?(Rails.root.join("config/database_geo.yml")) + migrations_paths = ["ee/db/geo/migrate"] + migrations_paths << "ee/db/geo/post_migrate" unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] + + configs["geo"] = + Rails.application.config_for(:database_geo) + .merge(migrations_paths: migrations_paths, schema_migrations_path: "ee/db/geo/schema_migrations") + .stringify_keys + end + [env, configs] end end diff --git a/lib/gitlab/patch/sidekiq_client.rb b/lib/gitlab/patch/sidekiq_client.rb deleted file mode 100644 index 2de13560cce..00000000000 --- a/lib/gitlab/patch/sidekiq_client.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Patch - module SidekiqClient - private - - # This is a copy of https://github.com/mperham/sidekiq/blob/v6.2.2/lib/sidekiq/client.rb#L187-L194 - # but using `conn.pipelined` instead of `conn.multi`. The multi call isn't needed here because in - # the case of scheduled jobs, only one Redis call is made. For other jobs, we don't really need - # the commands to be atomic. - def raw_push(payloads) - @redis_pool.with do |conn| # rubocop:disable Gitlab/ModuleWithInstanceVariables - conn.pipelined do - atomic_push(conn, payloads) - end - end - true - end - end - end -end diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_poller.rb index 56ca24c68f5..d4264cec1ab 100644 --- a/lib/gitlab/patch/sidekiq_cron_poller.rb +++ b/lib/gitlab/patch/sidekiq_poller.rb @@ -2,7 +2,7 @@ module Gitlab module Patch - module SidekiqCronPoller + module SidekiqPoller def enqueue Rails.application.reloader.wrap do ::Gitlab::WithRequestStore.with_request_store do diff --git a/lib/gitlab/process_management.rb b/lib/gitlab/process_management.rb new file mode 100644 index 00000000000..25a198e4a6a --- /dev/null +++ b/lib/gitlab/process_management.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Gitlab + module ProcessManagement + # Traps the given signals and yields the block whenever these signals are + # received. + # + # The block is passed the name of the signal. + # + # Example: + # + # trap_signals(%i(HUP TERM)) do |signal| + # ... + # end + def self.trap_signals(signals) + signals.each do |signal| + trap(signal) do + yield signal + end + end + end + + # Traps the given signals with the given command. + # + # Example: + # + # modify_signals(%i(HUP TERM), 'DEFAULT') + def self.modify_signals(signals, command) + signals.each { |signal| trap(signal, command) } + end + + def self.signal(pid, signal) + Process.kill(signal, pid) + true + rescue Errno::ESRCH + false + end + + def self.signal_processes(pids, signal) + pids.each { |pid| signal(pid, signal) } + end + + # Waits for the given process to complete using a separate thread. + def self.wait_async(pid) + Thread.new do + Process.wait(pid) rescue Errno::ECHILD + end + end + + # Returns true if all the processes are alive. + def self.all_alive?(pids) + pids.each do |pid| + return false unless process_alive?(pid) + end + + true + end + + def self.any_alive?(pids) + pids_alive(pids).any? + end + + def self.pids_alive(pids) + pids.select { |pid| process_alive?(pid) } + end + + def self.process_alive?(pid) + return false if pid.nil? + + # Signal 0 tests whether the process exists and we have access to send signals + # but is otherwise a noop (doesn't actually send a signal to the process) + signal(pid, 0) + end + + def self.process_died?(pid) + !process_alive?(pid) + end + + def self.write_pid(path) + File.open(path, 'w') do |handle| + handle.write(Process.pid.to_s) + end + end + end +end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index a55ead519e2..5fd422db722 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -206,7 +206,7 @@ module Gitlab end desc _('Add Zoom meeting') - explanation _('Adds a Zoom meeting') + explanation _('Adds a Zoom meeting.') params '<Zoom URL>' types Issue condition do @@ -223,7 +223,7 @@ module Gitlab end desc _('Remove Zoom meeting') - explanation _('Remove Zoom meeting') + explanation _('Remove Zoom meeting.') execution_message _('Zoom meeting removed') types Issue condition do @@ -236,7 +236,7 @@ module Gitlab end desc _('Add email participant(s)') - explanation _('Adds email participant(s)') + explanation _('Adds email participant(s).') params 'email1@example.com email2@example.com (up to 6 emails)' types Issue condition do @@ -285,6 +285,34 @@ module Gitlab end end + desc _('Add customer relation contacts') + explanation _('Add customer relation contact(s).') + params 'contact@example.com person@example.org' + types Issue + condition do + current_user.can?(:set_issue_crm_contacts, quick_action_target) + end + execution_message do + _('One or more contacts were successfully added.') + end + command :add_contacts do |contact_emails| + @updates[:add_contacts] = contact_emails.split(' ') + end + + desc _('Remove customer relation contacts') + explanation _('Remove customer relation contact(s).') + params 'contact@example.com person@example.org' + types Issue + condition do + current_user.can?(:set_issue_crm_contacts, quick_action_target) + end + execution_message do + _('One or more contacts were successfully removed.') + end + command :remove_contacts do |contact_emails| + @updates[:remove_contacts] = contact_emails.split(' ') + end + private def zoom_link_service diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index dbc77c9f9d7..94ae29af3d0 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -139,14 +139,12 @@ module Gitlab def throttle_unauthenticated_files_api? files_api_path? && - Feature.enabled?(:files_api_throttling, default_enabled: :yaml) && Gitlab::Throttle.settings.throttle_unauthenticated_files_api_enabled && unauthenticated? end def throttle_authenticated_files_api? files_api_path? && - Feature.enabled?(:files_api_throttling, default_enabled: :yaml) && Gitlab::Throttle.settings.throttle_authenticated_files_api_enabled end diff --git a/lib/gitlab/rate_limit_helpers.rb b/lib/gitlab/rate_limit_helpers.rb deleted file mode 100644 index 653410a40a5..00000000000 --- a/lib/gitlab/rate_limit_helpers.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RateLimitHelpers - ARCHIVE_RATE_LIMIT_REACHED_MESSAGE = 'This archive has been requested too many times. Try again later.' - ARCHIVE_RATE_ANONYMOUS_THRESHOLD = 100 # Allow 100 requests/min for anonymous users - ARCHIVE_RATE_THROTTLE_KEY = :project_repositories_archive - - def archive_rate_limit_reached?(user, project) - return false unless Feature.enabled?(:archive_rate_limit) - - key = ARCHIVE_RATE_THROTTLE_KEY - - if rate_limiter.throttled?(key, scope: [project, user], threshold: archive_rate_threshold_by_user(user)) - rate_limiter.log_request(request, "#{key}_request_limit".to_sym, user) - - return true - end - - false - end - - def archive_rate_threshold_by_user(user) - if user - nil # Use the defaults - else - ARCHIVE_RATE_ANONYMOUS_THRESHOLD - end - end - - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end - end -end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index f930a0040bc..500b62bf0e8 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -21,6 +21,8 @@ module Gitlab FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis primary_store.' FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' + SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i(info).freeze + READ_COMMANDS = %i( get mget @@ -39,41 +41,42 @@ module Gitlab flushdb ).freeze - def initialize(primary_store, secondary_store, instance_name = nil) + def initialize(primary_store, secondary_store, instance_name) @primary_store = primary_store @secondary_store = secondary_store @instance_name = instance_name validate_stores! end - + # rubocop:disable GitlabSecurity/PublicSend READ_COMMANDS.each do |name| define_method(name) do |*args, &block| - if multi_store_enabled? + if use_primary_and_secondary_stores? read_command(name, *args, &block) else - secondary_store.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + default_store.send(name, *args, &block) end end end WRITE_COMMANDS.each do |name| define_method(name) do |*args, &block| - if multi_store_enabled? + if use_primary_and_secondary_stores? write_command(name, *args, &block) else - secondary_store.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + default_store.send(name, *args, &block) end end end def method_missing(...) - return @instance.send(...) if @instance # rubocop:disable GitlabSecurity/PublicSend + return @instance.send(...) if @instance log_method_missing(...) - secondary_store.send(...) # rubocop:disable GitlabSecurity/PublicSend + default_store.send(...) end + # rubocop:enable GitlabSecurity/PublicSend def respond_to_missing?(command_name, include_private = false) true @@ -83,23 +86,37 @@ module Gitlab # https://github.com/redis-store/redis-rack/blob/a833086ba494083b6a384a1a4e58b36573a9165d/lib/redis/rack/connection.rb#L15 # Done similarly in https://github.com/lsegal/yard/blob/main/lib/yard/templates/template.rb#L122 def is_a?(klass) - return true if klass == secondary_store.class + return true if klass == default_store.class super(klass) end alias_method :kind_of?, :is_a? def to_s - if multi_store_enabled? - primary_store.to_s - else - secondary_store.to_s - end + use_primary_and_secondary_stores? ? primary_store.to_s : default_store.to_s + end + + def use_primary_and_secondary_stores? + feature_flags_available? && + Feature.enabled?("use_primary_and_secondary_stores_for_#{instance_name.underscore}", default_enabled: :yaml) && + !same_redis_store? + end + + def use_primary_store_as_default? + feature_flags_available? && + Feature.enabled?("use_primary_store_as_default_for_#{instance_name.underscore}", default_enabled: :yaml) && + !same_redis_store? end private + def default_store + use_primary_store_as_default? ? primary_store : secondary_store + end + def log_method_missing(command_name, *_args) + return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name) + log_error(MethodMissingError.new, command_name) increment_method_missing_count(command_name) end @@ -155,10 +172,6 @@ module Gitlab send_command(secondary_store, command_name, *args, &block) end - def multi_store_enabled? - Feature.enabled?(:use_multi_store, default_enabled: :yaml) && !same_redis_store? - end - def same_redis_store? strong_memoize(:same_redis_store) do # <Redis client v4.4.0 for redis:///path_to/redis/redis.socket/5>" @@ -194,12 +207,13 @@ module Gitlab def increment_method_missing_count(command_name) @method_missing_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_method_missing_total, 'Client side Redis MultiStore method missing') - @method_missing_counter.increment(command: command_name, innamece_name: instance_name) + @method_missing_counter.increment(command: command_name, instance_name: instance_name) end def validate_stores! raise ArgumentError, 'primary_store is required' unless primary_store raise ArgumentError, 'secondary_store is required' unless secondary_store + raise ArgumentError, 'instance_name is required' unless instance_name raise ArgumentError, 'invalid primary_store' unless primary_store.is_a?(::Redis) raise ArgumentError, 'invalid secondary_store' unless secondary_store.is_a?(::Redis) end diff --git a/lib/gitlab/redis/sessions.rb b/lib/gitlab/redis/sessions.rb index 3bf1eb6211d..c547828d907 100644 --- a/lib/gitlab/redis/sessions.rb +++ b/lib/gitlab/redis/sessions.rb @@ -3,9 +3,45 @@ module Gitlab module Redis class Sessions < ::Gitlab::Redis::Wrapper - # The data we store on Sessions used to be stored on SharedState. - def self.config_fallback - SharedState + SESSION_NAMESPACE = 'session:gitlab' + USER_SESSIONS_NAMESPACE = 'session:user:gitlab' + USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab' + IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' + OTP_SESSIONS_NAMESPACE = 'session:otp' + + class << self + # The data we store on Sessions used to be stored on SharedState. + def config_fallback + SharedState + end + + private + + def redis + # Don't use multistore if redis.sessions configuration is not provided + return super if config_fallback? + + primary_store = ::Redis.new(params) + secondary_store = ::Redis.new(config_fallback.params) + + MultiStore.new(primary_store, secondary_store, store_name) + end + end + + def store(extras = {}) + # Don't use multistore if redis.sessions configuration is not provided + return super if self.class.config_fallback? + + primary_store = create_redis_store(redis_store_options, extras) + secondary_store = create_redis_store(self.class.config_fallback.params, extras) + + MultiStore.new(primary_store, secondary_store, self.class.store_name) + end + + private + + def create_redis_store(options, extras) + ::Redis::Store.new(options.merge(extras)) end end end diff --git a/lib/gitlab/redis/sessions_store_helper.rb b/lib/gitlab/redis/sessions_store_helper.rb new file mode 100644 index 00000000000..c80442847f1 --- /dev/null +++ b/lib/gitlab/redis/sessions_store_helper.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + module SessionsStoreHelper + extend ActiveSupport::Concern + + module StoreMethods + def redis_store_class + use_redis_session_store? ? Gitlab::Redis::Sessions : Gitlab::Redis::SharedState + end + + private + + def use_redis_session_store? + Gitlab::Utils.to_boolean(ENV['GITLAB_USE_REDIS_SESSIONS_STORE'], default: true) + end + end + + include StoreMethods + + included do + extend StoreMethods + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 1250eabb041..fb3a143121b 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -3,10 +3,6 @@ module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper - SESSION_NAMESPACE = 'session:gitlab' - USER_SESSIONS_NAMESPACE = 'session:user:gitlab' - USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab' - IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 985c8dc619c..75dbccb965d 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -28,7 +28,7 @@ module Gitlab end def pool - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool ||= ConnectionPool.new(size: pool_size) { redis } end def pool_size @@ -67,6 +67,10 @@ module Gitlab File.expand_path('../../..', __dir__) end + def config_fallback? + config_file_name == config_fallback&.config_file_name + end + def config_file_name [ # Instance specific config sources: @@ -100,6 +104,12 @@ module Gitlab "::Gitlab::Instrumentation::Redis::#{store_name}".constantize end + + private + + def redis + ::Redis.new(params) + end end def initialize(rails_env = nil) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 904fc744c6b..8e139ae0709 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -16,8 +16,13 @@ module Gitlab @conan_revision_regex ||= %r{\A0\z}.freeze end + def conan_recipe_user_channel_regex + %r{\A(_|#{conan_name_regex})\z}.freeze + end + def conan_recipe_component_regex - @conan_recipe_component_regex ||= %r{\A[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}\z}.freeze + # https://docs.conan.io/en/latest/reference/conanfile/attributes.html#name + @conan_recipe_component_regex ||= %r{\A#{conan_name_regex}\z}.freeze end def composer_package_version_regex @@ -211,6 +216,12 @@ module Gitlab def generic_package_file_name_regex generic_package_name_regex end + + private + + def conan_name_regex + @conan_name_regex ||= %r{[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}}.freeze + end end extend self @@ -413,7 +424,11 @@ module Gitlab end def issue - @issue ||= /(?<issue>\d+\b)/ + @issue ||= /(?<issue>\d+)(?<format>\+)?(?=\W|\z)/ + end + + def merge_request + @merge_request ||= /(?<merge_request>\d+)(?<format>\+)?/ end def base64_regex @@ -430,3 +445,5 @@ module Gitlab end end end + +Gitlab::Regex.prepend_mod diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb index 1e738aef9b0..98e52e8e767 100644 --- a/lib/gitlab/relative_positioning/item_context.rb +++ b/lib/gitlab/relative_positioning/item_context.rb @@ -66,19 +66,11 @@ module Gitlab end def lhs_neighbour - scoped_items - .where('relative_position < ?', relative_position) - .reorder(relative_position: :desc) - .first - .then { |x| neighbour(x) } + neighbour(object.next_object_by_relative_position(ignoring: ignoring, order: :desc)) end def rhs_neighbour - scoped_items - .where('relative_position > ?', relative_position) - .reorder(relative_position: :asc) - .first - .then { |x| neighbour(x) } + neighbour(object.next_object_by_relative_position(ignoring: ignoring, order: :asc)) end def neighbour(item) @@ -87,12 +79,6 @@ module Gitlab self.class.new(item, range, ignoring: ignoring) end - def scoped_items - r = model_class.relative_positioning_query_base(object) - r = object.exclude_self(r, excluded: ignoring) if ignoring.present? - r - end - def calculate_relative_position(calculation) # When calculating across projects, this is much more efficient than # MAX(relative_position) without the GROUP BY, due to index usage: @@ -186,6 +172,10 @@ module Gitlab Gap.new(gap.first, gap.second || default_end) end + def scoped_items + object.relative_positioning_scoped_items(ignoring: ignoring) + end + def relative_position object.relative_position end diff --git a/lib/gitlab/repository_archive_rate_limiter.rb b/lib/gitlab/repository_archive_rate_limiter.rb new file mode 100644 index 00000000000..31a3dc34bf6 --- /dev/null +++ b/lib/gitlab/repository_archive_rate_limiter.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module RepositoryArchiveRateLimiter + def check_archive_rate_limit!(current_user, project, &block) + return unless Feature.enabled?(:archive_rate_limit) + + threshold = current_user ? nil : 100 + + check_rate_limit!(:project_repositories_archive, scope: [project, current_user], threshold: threshold, &block) + end + end +end diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb index 1e00bd4cbfc..577e33fd700 100644 --- a/lib/gitlab/saas.rb +++ b/lib/gitlab/saas.rb @@ -13,6 +13,10 @@ module Gitlab 'https://staging.gitlab.com' end + def self.canary_toggle_com_url + 'https://next.gitlab.com' + end + def self.subdomain_regex %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}.freeze end diff --git a/lib/gitlab/search/abuse_detection.rb b/lib/gitlab/search/abuse_detection.rb new file mode 100644 index 00000000000..7b5377bce88 --- /dev/null +++ b/lib/gitlab/search/abuse_detection.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module Search + class AbuseDetection + include ActiveModel::Validations + include AbuseValidators + + ABUSIVE_TERM_SIZE = 100 + ALLOWED_CHARS_REGEX = %r{\A[[:alnum:]_\-\/\.!]+\z}.freeze + MINIMUM_SEARCH_CHARS = 2 + + ALLOWED_SCOPES = %w( + blobs + code + commits + epics + issues + merge_requests + milestones + notes + projects + snippet_titles + users + wiki_blobs + ).freeze + + READABLE_PARAMS = %i( + group_id + project_id + project_ref + query_string + repository_ref + scope + ).freeze + + STOP_WORDS = %w( + a an and are as at be but by for if in into is it no not of on or such that the their then there these they this to was will with + ).freeze + + validates :project_id, :group_id, + numericality: { only_integer: true, message: "abusive ID detected" }, allow_blank: true + + validates :scope, inclusion: { in: ALLOWED_SCOPES, message: 'abusive scope detected' }, allow_blank: true + + validates :repository_ref, :project_ref, + format: { with: ALLOWED_CHARS_REGEX, message: "abusive characters detected" }, allow_blank: true + + validates :query_string, + exclusion: { in: STOP_WORDS, message: 'stopword only abusive search detected' }, allow_blank: true + + validates :query_string, + length: { minimum: MINIMUM_SEARCH_CHARS, message: 'abusive tiny search detected' }, unless: :skip_tiny_search_validation?, allow_blank: true + + validates :query_string, + no_abusive_term_length: { maximum: ABUSIVE_TERM_SIZE, maximum_for_url: ABUSIVE_TERM_SIZE * 2 } + + validates :query_string, :repository_ref, :project_ref, no_abusive_coercion_from_string: true + + attr_reader(*READABLE_PARAMS) + + def initialize(params) + READABLE_PARAMS.each { |p| instance_variable_set("@#{p}", params[p]) } + end + + private + + def skip_tiny_search_validation? + wildcard_search? || stop_word_search? + end + + def wildcard_search? + query_string == '*' + end + + def stop_word_search? + STOP_WORDS.include? query_string + end + end + end +end diff --git a/lib/gitlab/search/abuse_validators/no_abusive_coercion_from_string_validator.rb b/lib/gitlab/search/abuse_validators/no_abusive_coercion_from_string_validator.rb new file mode 100644 index 00000000000..06464980afd --- /dev/null +++ b/lib/gitlab/search/abuse_validators/no_abusive_coercion_from_string_validator.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Search + module AbuseValidators + class NoAbusiveCoercionFromStringValidator < ActiveModel::EachValidator + def validate_each(instance, attribute, value) + if value.present? && !value.is_a?(String) + instance.errors.add attribute, "abusive coercion from string detected" + end + end + end + end + end +end diff --git a/lib/gitlab/search/abuse_validators/no_abusive_term_length_validator.rb b/lib/gitlab/search/abuse_validators/no_abusive_term_length_validator.rb new file mode 100644 index 00000000000..8a94520d8fd --- /dev/null +++ b/lib/gitlab/search/abuse_validators/no_abusive_term_length_validator.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Search + module AbuseValidators + class NoAbusiveTermLengthValidator < ActiveModel::EachValidator + def validate_each(instance, attribute, value) + return unless value.is_a?(String) + + if value.split.any? { |term| term_too_long?(term) } + instance.errors.add attribute, 'abusive term length detected' + end + end + + private + + def term_too_long?(term) + char_limit = url_detected?(term) ? maximum_for_url : maximum + term.length >= char_limit + end + + def url_detected?(uri_str) + URI::DEFAULT_PARSER.regexp[:ABS_URI].match? uri_str + end + + def maximum_for_url + options.fetch(:maximum_for_url, maximum) + end + + def maximum + options.fetch(:maximum) + end + end + end + end +end diff --git a/lib/gitlab/search/params.rb b/lib/gitlab/search/params.rb new file mode 100644 index 00000000000..e6a1305a82a --- /dev/null +++ b/lib/gitlab/search/params.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +module Gitlab + module Search + class Params + include ActiveModel::Validations + + SEARCH_CHAR_LIMIT = 4096 + SEARCH_TERM_LIMIT = 64 + + # Generic validation + validates :query_string, length: { maximum: SEARCH_CHAR_LIMIT } + validate :not_too_many_terms + + attr_reader :raw_params, :query_string, :abuse_detection + alias_method :search, :query_string + alias_method :term, :query_string + + def initialize(params, detect_abuse: true) + @raw_params = params.is_a?(Hash) ? params.with_indifferent_access : params.dup + @query_string = strip_surrounding_whitespace(@raw_params[:search] || @raw_params[:term]) + @detect_abuse = detect_abuse + @abuse_detection = AbuseDetection.new(self) if @detect_abuse + + validate + end + + def [](key) + if respond_to? key + # We have this logic here to support reading custom attributes + # like @query_string + # + # This takes precedence over values in @raw_params + public_send(key) # rubocop:disable GitlabSecurity/PublicSend + else + raw_params[key] + end + end + + def abusive? + detect_abuse? && abuse_detection.errors.any? + end + + def valid_query_length? + return true unless errors.has_key? :query_string + + errors[:query_string].none? { |msg| msg.include? SEARCH_CHAR_LIMIT.to_s } + end + + def valid_terms_count? + return true unless errors.has_key? :query_string + + errors[:query_string].none? { |msg| msg.include? SEARCH_TERM_LIMIT.to_s } + end + + def validate + if detect_abuse? + abuse_detection.validate + end + + super + end + + def valid? + if detect_abuse? + abuse_detection.valid? && super + else + super + end + end + + private + + def detect_abuse? + @detect_abuse + end + + def not_too_many_terms + if query_string.split.count { |word| word.length >= 3 } > SEARCH_TERM_LIMIT + errors.add :query_string, "has too many search terms (maximum is #{SEARCH_TERM_LIMIT})" + end + end + + def strip_surrounding_whitespace(obj) + obj.to_s.strip + end + end + end +end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb index c0420126ada..97ee7c7817d 100644 --- a/lib/gitlab/search/query.rb +++ b/lib/gitlab/search/query.rb @@ -44,7 +44,7 @@ module Gitlab next unless match - input = match.split(':')[1..-1].join + input = match.split(':')[1..].join next if input.empty? filter[:negated] = match.start_with?("-") diff --git a/lib/gitlab/security/scan_configuration.rb b/lib/gitlab/security/scan_configuration.rb new file mode 100644 index 00000000000..eaccbb3be7e --- /dev/null +++ b/lib/gitlab/security/scan_configuration.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Security + class ScanConfiguration + include ::Gitlab::Utils::StrongMemoize + include Gitlab::Routing.url_helpers + + attr_reader :type + + def initialize(project:, type:, configured: false) + @project = project + @type = type + @configured = configured + end + + def available? + # SAST and Secret Detection are always available, but this isn't + # reflected by our license model yet. + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/333113 + %i[sast secret_detection].include?(type) + end + + def configured? + configured + end + + def configuration_path + configurable_scans[type] + end + + private + + attr_reader :project, :configured + + def configurable_scans + strong_memoize(:configurable_scans) do + { + sast: project_security_configuration_sast_path(project) + } + end + end + end + end +end + +Gitlab::Security::ScanConfiguration.prepend_mod_with('Gitlab::Security::ScanConfiguration') diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb index 52d88f074b7..aa25eb5a571 100644 --- a/lib/gitlab/sherlock/line_profiler.rb +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -70,7 +70,7 @@ module Gitlab next if total_duration <= MINIMUM_DURATION - stats[1..-1].each_with_index do |data, index| + stats[1..].each_with_index do |data, index| next unless source_lines[index] duration = microsec_to_millisec(data[0]) diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 07ddac209f8..3eef41a2ca2 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -8,6 +8,7 @@ module Gitlab EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml' JH_QUEUE_CONFIG_PATH = 'jh/app/workers/all_queues.yml' SIDEKIQ_QUEUES_PATH = 'config/sidekiq_queues.yml' + JH_SIDEKIQ_QUEUES_PATH = 'jh/config/sidekiq_queues.yml' QUEUE_CONFIG_PATHS = [ FOSS_QUEUE_CONFIG_PATH, @@ -100,18 +101,24 @@ module Gitlab def queues_for_sidekiq_queues_yml namespaces_with_equal_weights = workers + .reject { |worker| worker.jh? } .group_by(&:queue_namespace) .map(&:last) .select { |workers| workers.map(&:get_weight).uniq.count == 1 } .map(&:first) namespaces = namespaces_with_equal_weights.map(&:queue_namespace).to_set - remaining_queues = workers.reject { |worker| namespaces.include?(worker.queue_namespace) } + remaining_queues = workers.reject { |worker| worker.jh? }.reject { |worker| namespaces.include?(worker.queue_namespace) } (namespaces_with_equal_weights.map(&:namespace_and_weight) + remaining_queues.map(&:queue_and_weight)).sort end + # Override in JH repo + def jh_queues_for_sidekiq_queues_yml + [] + end + # YAML.load_file is OK here as we control the file contents def sidekiq_queues_yml_outdated? config_queues = YAML.load_file(SIDEKIQ_QUEUES_PATH)[:queues] @@ -154,3 +161,5 @@ module Gitlab end end end + +Gitlab::SidekiqConfig.prepend_mod diff --git a/lib/gitlab/sidekiq_enq.rb b/lib/gitlab/sidekiq_enq.rb deleted file mode 100644 index de0c00fe561..00000000000 --- a/lib/gitlab/sidekiq_enq.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class SidekiqEnq - LUA_ZPOPBYSCORE = <<~EOS - local key, now = KEYS[1], ARGV[1] - local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1) - if jobs[1] then - redis.call("zrem", key, jobs[1]) - return jobs[1] - end - EOS - - LUA_ZPOPBYSCORE_SHA = Digest::SHA1.hexdigest(LUA_ZPOPBYSCORE) - - def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = Sidekiq::Scheduled::SETS) - Rails.application.reloader.wrap do - ::Gitlab::WithRequestStore.with_request_store do - if Feature.enabled?(:atomic_sidekiq_scheduler, default_enabled: :yaml) - atomic_find_jobs_and_enqueue(now, sorted_sets) - else - find_jobs_and_enqueue(now, sorted_sets) - end - - ensure - ::Gitlab::Database::LoadBalancing.release_hosts - end - end - end - - private - - # This is a copy of https://github.com/mperham/sidekiq/blob/32c55e31659a1e6bd42f98334cca5eef2863de8d/lib/sidekiq/scheduled.rb#L11-L34 - # - # It effectively reverts - # https://github.com/mperham/sidekiq/commit/9b75467b33759888753191413eddbc15c37a219e - # because we observe that the extra ZREMs caused by this change can lead to high - # CPU usage on Redis at peak times: - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1179 - # - def find_jobs_and_enqueue(now, sorted_sets) - # A job's "score" in Redis is the time at which it should be processed. - # Just check Redis for the set of jobs with a timestamp before now. - Sidekiq.redis do |conn| - sorted_sets.each do |sorted_set| - start_time = ::Gitlab::Metrics::System.monotonic_time - jobs = redundant_jobs = 0 - - Sidekiq.logger.info(message: 'Enqueuing scheduled jobs', status: 'start', sorted_set: sorted_set) - - # Get the next item in the queue if it's score (time to execute) is <= now. - # We need to go through the list one at a time to reduce the risk of something - # going wrong between the time jobs are popped from the scheduled queue and when - # they are pushed onto a work queue and losing the jobs. - while job = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 1]).first - # Pop item off the queue and add it to the work queue. If the job can't be popped from - # the queue, it's because another process already popped it so we can move on to the - # next one. - if conn.zrem(sorted_set, job) - jobs += 1 - Sidekiq::Client.push(Sidekiq.load_json(job)) - else - redundant_jobs += 1 - end - end - - end_time = ::Gitlab::Metrics::System.monotonic_time - Sidekiq.logger.info(message: 'Enqueuing scheduled jobs', - status: 'done', - sorted_set: sorted_set, - jobs_count: jobs, - redundant_jobs_count: redundant_jobs, - duration_s: end_time - start_time) - end - end - end - - def atomic_find_jobs_and_enqueue(now, sorted_sets) - Sidekiq.redis do |conn| - sorted_sets.each do |sorted_set| - start_time = ::Gitlab::Metrics::System.monotonic_time - jobs = 0 - - Sidekiq.logger.info(message: 'Atomically enqueuing scheduled jobs', status: 'start', sorted_set: sorted_set) - - while job = redis_eval_lua(conn, LUA_ZPOPBYSCORE, LUA_ZPOPBYSCORE_SHA, keys: [sorted_set], argv: [now]) - jobs += 1 - Sidekiq::Client.push(Sidekiq.load_json(job)) - end - - end_time = ::Gitlab::Metrics::System.monotonic_time - Sidekiq.logger.info(message: 'Atomically enqueuing scheduled jobs', - status: 'done', - sorted_set: sorted_set, - jobs_count: jobs, - duration_s: end_time - start_time) - end - end - end - - def redis_eval_lua(conn, script, sha, keys: nil, argv: nil) - conn.evalsha(sha, keys: keys, argv: argv) - rescue ::Redis::CommandError => e - if e.message.start_with?('NOSCRIPT') - conn.eval(script, keys: keys, argv: argv) - else - raise - end - end - end -end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index fbf2718d718..120d18f63f2 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -29,13 +29,16 @@ module Gitlab # for most jobs. DEFAULT_EXPIRATION = 30.minutes.to_i + DEFAULT_VALUE = 1 + DEFAULT_VALUE_MESSAGE = 'Keys using the default value for SidekiqStatus detected' + # Starts tracking of the given job. # # jid - The Sidekiq job ID # expire - The expiration time of the Redis key. - def self.set(jid, expire = DEFAULT_EXPIRATION) + def self.set(jid, expire = DEFAULT_EXPIRATION, value: DEFAULT_VALUE) Sidekiq.redis do |redis| - redis.set(key_for(jid), 1, ex: expire) + redis.set(key_for(jid), value, ex: expire) end end @@ -88,13 +91,20 @@ module Gitlab # true = job is still running or enqueued # false = job completed def self.job_status(job_ids) + return [] if job_ids.empty? + keys = job_ids.map { |jid| key_for(jid) } + results = Sidekiq.redis { |redis| redis.mget(*keys) } - Sidekiq.redis do |redis| - redis.pipelined do - keys.each { |key| redis.exists(key) } - end + if Feature.enabled?(:log_implicit_sidekiq_status_calls, default_enabled: :yaml) + to_log = keys.zip(results).select do |_key, result| + result == DEFAULT_VALUE.to_s + end.map(&:first) + + Sidekiq.logger.info(message: DEFAULT_VALUE_MESSAGE, keys: to_log) if to_log.any? end + + results.map { |result| !result.nil? } end # Returns the JIDs that are completed diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb index bfd5038557d..cee7270f2fb 100644 --- a/lib/gitlab/sidekiq_status/client_middleware.rb +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -5,8 +5,9 @@ module Gitlab class ClientMiddleware def call(_, job, _, _) status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION + value = job['status_expiration'] ? 2 : Gitlab::SidekiqStatus::DEFAULT_VALUE - Gitlab::SidekiqStatus.set(job['jid'], status_expiration) + Gitlab::SidekiqStatus.set(job['jid'], status_expiration, value: value) yield end end diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 925ca44dfc9..40b01552244 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -21,14 +21,16 @@ module Gitlab update: ::Spamcheck::Action::UPDATE }.freeze + URL_SCHEME_REGEX = %r{^grpc://|^tls://}.freeze + def initialize @endpoint_url = Gitlab::CurrentSettings.current_application_settings.spam_check_endpoint_url - # remove the `grpc://` as it's only useful to ensure we're expecting to - # connect with Spamcheck - @endpoint_url = @endpoint_url.gsub(%r(^grpc:\/\/), '') + @creds = client_creds(@endpoint_url) - @creds = stub_creds + # remove the `grpc://` or 'tls://' as it's only useful to ensure we're expecting to + # connect with Spamcheck + @endpoint_url = @endpoint_url.sub(URL_SCHEME_REGEX, '') end def issue_spam?(spam_issue:, user:, context: {}) @@ -96,11 +98,11 @@ module Gitlab nanos: ar_timestamp.to_time.nsec) end - def stub_creds - if Rails.env.development? || Rails.env.test? - :this_channel_is_insecure + def client_creds(url) + if URI(url).scheme == 'tls' + GRPC::Core::ChannelCredentials.new(::Gitlab::X509::Certificate.ca_certs_bundle) else - GRPC::Core::ChannelCredentials.new ::Gitlab::X509::Certificate.ca_certs_bundle + :this_channel_is_insecure end end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index 5ddc88edf50..292a9d07e6a 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -99,7 +99,7 @@ module Gitlab start = prev = positions[0] range = MarkerRange.new(start, prev, mode: mode) - positions[1..-1].each do |pos| + positions[1..].each do |pos| if pos == prev + 1 range = MarkerRange.new(start, pos, mode: mode) prev = pos diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index 4f6d25097e2..d987247fdc4 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -18,6 +18,10 @@ module Gitlab "#{self.subscriptions_url}/payment_forms/cc_validation" end + def self.registration_validation_form_url + "#{self.subscriptions_url}/payment_forms/cc_registration_validation" + end + def self.subscriptions_comparison_url 'https://about.gitlab.com/pricing/gitlab-com/feature-comparison' end @@ -62,6 +66,10 @@ module Gitlab "#{self.subscriptions_url}/gitlab/namespaces/#{group_id}/renew" end + def self.edit_account_url + "#{self.subscriptions_url}/customers/edit" + end + def self.subscription_portal_admin_email ENV.fetch('SUBSCRIPTION_PORTAL_ADMIN_EMAIL', 'gl_com_api@gitlab.com') end @@ -69,9 +77,15 @@ module Gitlab def self.subscription_portal_admin_token ENV.fetch('SUBSCRIPTION_PORTAL_ADMIN_TOKEN', 'customer_admin_token') end + + def self.renewal_service_email + 'renewals-support@gitlab.com' + end end end Gitlab::SubscriptionPortal.prepend_mod Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze Gitlab::SubscriptionPortal::PAYMENT_FORM_URL = Gitlab::SubscriptionPortal.payment_form_url.freeze +Gitlab::SubscriptionPortal::RENEWAL_SERVICE_EMAIL = Gitlab::SubscriptionPortal.renewal_service_email.freeze +Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_URL = Gitlab::SubscriptionPortal.registration_validation_form_url.freeze diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 227962fc0f7..6a98fa12903 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -190,6 +190,9 @@ module Gitlab end def checkout_version(version, target_dir) + # Explicitly setting the git protocol version to v2 allows older Git binaries + # to do have a shallow clone obtain objects by object ID. + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} config protocol.version 2]) run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet origin #{version}]) run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout -f --quiet FETCH_HEAD --]) end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index ec032cf2d3c..a58b4beb0df 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -2,11 +2,9 @@ module Gitlab module Tracking - SNOWPLOW_NAMESPACE = 'gl' - class << self def enabled? - snowplow_micro_enabled? || Gitlab::CurrentSettings.snowplow_enabled? + snowplow.enabled? end def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists @@ -25,6 +23,10 @@ module Gitlab snowplow.hostname end + def snowplow_micro_enabled? + Rails.env.development? && Gitlab::Utils.to_boolean(ENV['SNOWPLOW_MICRO_ENABLE']) + end + private def snowplow @@ -34,10 +36,6 @@ module Gitlab Gitlab::Tracking::Destinations::Snowplow.new end end - - def snowplow_micro_enabled? - Rails.env.development? && Gitlab::Utils.to_boolean(ENV['SNOWPLOW_MICRO_ENABLE']) - end end end end diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb index 5596e9acd30..ddcd4693738 100644 --- a/lib/gitlab/tracking/destinations/snowplow.rb +++ b/lib/gitlab/tracking/destinations/snowplow.rb @@ -8,6 +8,8 @@ module Gitlab class Snowplow < Base extend ::Gitlab::Utils::Override + SNOWPLOW_NAMESPACE = 'gl' + override :event def event(category, action, label: nil, property: nil, value: nil, context: nil) return unless enabled? @@ -19,7 +21,7 @@ module Gitlab def options(group) additional_features = Feature.enabled?(:additional_snowplow_tracking, group, type: :ops) { - namespace: Gitlab::Tracking::SNOWPLOW_NAMESPACE, + namespace: SNOWPLOW_NAMESPACE, hostname: hostname, cookie_domain: cookie_domain, app_id: app_id, @@ -28,16 +30,16 @@ module Gitlab }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym } end + def enabled? + Gitlab::CurrentSettings.snowplow_enabled? + end + def hostname Gitlab::CurrentSettings.snowplow_collector_hostname end private - def enabled? - Gitlab::Tracking.enabled? - end - def app_id Gitlab::CurrentSettings.snowplow_app_id end @@ -54,7 +56,7 @@ module Gitlab @tracker ||= SnowplowTracker::Tracker.new( emitter, SnowplowTracker::Subject.new, - Gitlab::Tracking::SNOWPLOW_NAMESPACE, + SNOWPLOW_NAMESPACE, app_id ) end diff --git a/lib/gitlab/tracking/destinations/snowplow_micro.rb b/lib/gitlab/tracking/destinations/snowplow_micro.rb index b818d349a6d..3553efba1e1 100644 --- a/lib/gitlab/tracking/destinations/snowplow_micro.rb +++ b/lib/gitlab/tracking/destinations/snowplow_micro.rb @@ -15,7 +15,12 @@ module Gitlab protocol: uri.scheme, port: uri.port, force_secure_tracker: false - ) + ).transform_keys! { |key| key.to_s.camelize(:lower).to_sym } + end + + override :enabled? + def enabled? + true end override :hostname @@ -23,8 +28,6 @@ module Gitlab "#{uri.host}:#{uri.port}" end - private - def uri strong_memoize(:snowplow_uri) do uri = URI(ENV['SNOWPLOW_MICRO_URI'] || DEFAULT_URI) @@ -33,6 +36,8 @@ module Gitlab end end + private + override :cookie_domain def cookie_domain '.gitlab.com' diff --git a/lib/gitlab/usage/metrics/instrumentations/snowplow_configured_to_gitlab_collector_metric.rb b/lib/gitlab/usage/metrics/instrumentations/snowplow_configured_to_gitlab_collector_metric.rb new file mode 100644 index 00000000000..3a92525303b --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/snowplow_configured_to_gitlab_collector_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class SnowplowConfiguredToGitlabCollectorMetric < GenericMetric + GITLAB_SNOWPLOW_COLLECTOR_HOSTNAME = 'snowplow.trx.gitlab.net' + + def value + Gitlab::CurrentSettings.snowplow_collector_hostname == GITLAB_SNOWPLOW_COLLECTOR_HOSTNAME + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/snowplow_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/snowplow_enabled_metric.rb new file mode 100644 index 00000000000..5d504c70e73 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/snowplow_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class SnowplowEnabledMetric < GenericMetric + def value + Gitlab::CurrentSettings.snowplow_enabled? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb index 6dcbe5f5fe5..d1a7bb65cc3 100644 --- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb +++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb @@ -18,10 +18,6 @@ module Gitlab private - def instrumentation_metrics - ::Gitlab::UsageDataMetrics.suggested_names - end - def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) Gitlab::Usage::Metrics::NameSuggestion.for(:count, column: column, relation: relation) end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 20e526aeefa..917c273d3f6 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -10,6 +10,12 @@ # alt_usage_data { Gitlab::VERSION } # redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) # redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } + +# NOTE: +# Implementing metrics direct in `usage_data.rb` is deprecated, +# please add new instrumentation class and use add_metric method. +# For more information, see https://docs.gitlab.com/ee/development/service_ping/metrics_instrumentation.html + module Gitlab class UsageData DEPRECATED_VALUE = -1000 @@ -45,10 +51,7 @@ module Gitlab clear_memoized with_finished_at(:recording_ce_finished_at) do - usage_data = usage_data_metrics - usage_data = usage_data.with_indifferent_access.deep_merge(instrumentation_metrics.with_indifferent_access) if Feature.enabled?(:usage_data_instrumentation) - - usage_data + usage_data_metrics end end @@ -225,7 +228,9 @@ module Gitlab operating_system: alt_usage_data(fallback: nil) { operating_system }, gitaly_apdex: alt_usage_data { gitaly_apdex }, collected_data_categories: add_metric('CollectedDataCategoriesMetric', time_frame: 'none'), - service_ping_features_enabled: add_metric('ServicePingFeaturesMetric', time_frame: 'none') + service_ping_features_enabled: add_metric('ServicePingFeaturesMetric', time_frame: 'none'), + snowplow_enabled: add_metric('SnowplowEnabledMetric', time_frame: 'none'), + snowplow_configured_to_gitlab_collector: add_metric('SnowplowConfiguredToGitlabCollectorMetric', time_frame: 'none') } } end @@ -401,7 +406,8 @@ module Gitlab results[:projects_jira_cloud_active] = jira_integration_data_hash[:projects_jira_cloud_active] results - rescue ActiveRecord::StatementInvalid + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK } end # rubocop: enable CodeReuse/ActiveRecord @@ -663,8 +669,6 @@ module Gitlab end def redis_hll_counters - return {} unless Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml) - { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } end @@ -726,7 +730,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord # rubocop: disable UsageData/LargeTable start = ::Event.where(time_period).select(:id).order(created_at: :asc).first&.id - finish = ::Event.where(time_period).select(:id).order(created_at: :asc).first&.id + finish = ::Event.where(time_period).select(:id).order(created_at: :desc).first&.id estimate_batch_distinct_count(::Event.where(time_period), :author_id, start: start, finish: finish) # rubocop: enable UsageData/LargeTable # rubocop: enable CodeReuse/ActiveRecord @@ -753,10 +757,6 @@ module Gitlab .deep_merge(aggregated_metrics_data) end - def instrumentation_metrics - Gitlab::UsageDataMetrics.uncached_data # rubocop:disable UsageData/LargeTable - end - def metric_time_period(time_period) time_period.present? ? '28d' : 'none' end diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml index 261bdeb9bfa..5a1e7f03278 100644 --- a/lib/gitlab/usage_data_counters/known_events/analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -66,3 +66,15 @@ category: analytics redis_slot: analytics aggregation: weekly +- name: p_analytics_ci_cd_pipelines + category: analytics + redis_slot: analytics + aggregation: weekly +- name: p_analytics_ci_cd_deployment_frequency + category: analytics + redis_slot: analytics + aggregation: weekly +- name: p_analytics_ci_cd_lead_time + category: analytics + redis_slot: analytics + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index 40922433635..d90960b344c 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -119,6 +119,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_security_dast_api_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_security_container_scanning category: ci_templates redis_slot: ci_templates @@ -551,6 +555,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_security_dast_api_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_security_container_scanning category: ci_templates redis_slot: ci_templates diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index feebc7f395a..bb98a0b262a 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -355,6 +355,19 @@ category: testing aggregation: weekly feature_flag: users_expanding_widgets_usage_data +- name: users_expanding_testing_license_compliance_report + redis_slot: testing + category: testing + aggregation: weekly + feature_flag: users_expanding_widgets_usage_data +- name: users_visiting_testing_license_compliance_full_report + redis_slot: testing + category: testing + aggregation: weekly +- name: users_visiting_testing_manage_license_compliance + redis_slot: testing + category: testing + aggregation: weekly # Container Security - Network Policies - name: clusters_using_network_policies_ui redis_slot: network_policies diff --git a/lib/gitlab/usage_data_counters/known_events/importer_events.yml b/lib/gitlab/usage_data_counters/known_events/importer_events.yml index 79bbac229bc..c84d756a013 100644 --- a/lib/gitlab/usage_data_counters/known_events/importer_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/importer_events.yml @@ -4,14 +4,11 @@ category: importer redis_slot: import aggregation: weekly - feature_flag: track_importer_activity - name: github_import_project_success category: importer redis_slot: import aggregation: weekly - feature_flag: track_importer_activity - name: github_import_project_failure category: importer redis_slot: import aggregation: weekly - feature_flag: track_importer_activity diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index dff2c4f8d03..d831ac02dd1 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -279,3 +279,11 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_add_contacts + category: quickactions + redis_slot: quickactions + aggregation: weekly +- name: i_quickactions_remove_contacts + category: quickactions + redis_slot: quickactions + aggregation: weekly diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb index be5a571fb82..1661a1b6987 100644 --- a/lib/gitlab/usage_data_non_sql_metrics.rb +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -5,13 +5,6 @@ module Gitlab SQL_METRIC_DEFAULT = -3 class << self - def uncached_data - # instrumentation_metrics is already included with feature flag enabled - return super if Feature.enabled?(:usage_data_instrumentation) - - super.with_indifferent_access.deep_merge(instrumentation_metrics.with_indifferent_access) - end - def add_metric(metric, time_frame: 'none', options: {}) metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize @@ -50,12 +43,6 @@ module Gitlab projects_jira_cloud_active: 0 } end - - private - - def instrumentation_metrics - ::Gitlab::Usage::Metric.all.map(&:with_instrumentation).reduce({}, :deep_merge) - end end end end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index f543b29e43f..c6490ba7374 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -5,13 +5,6 @@ module Gitlab # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091 class UsageDataQueries < UsageData class << self - def uncached_data - # instrumentation_metrics is already included with feature flag enabled - return super if Feature.enabled?(:usage_data_instrumentation) - - super.with_indifferent_access.deep_merge(instrumentation_metrics.with_indifferent_access) - end - def add_metric(metric, time_frame: 'none', options: {}) metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize @@ -71,12 +64,6 @@ module Gitlab def epics_deepest_relationship_level { epics_deepest_relationship_level: 0 } end - - private - - def instrumentation_metrics - ::Gitlab::Usage::Metric.all.map(&:with_instrumentation).reduce({}, :deep_merge) - end end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index cb34ed69a9c..96cff024371 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -120,18 +120,14 @@ module Gitlab Random.rand(Float::MAX.to_i).to_s(36) end - # See: http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby - # Cross-platform way of finding an executable in the $PATH. + # Behaves like `which` on Linux machines: given PATH, try to resolve the given + # executable name to an absolute path, or return nil. # # which('ruby') #=> /usr/bin/ruby - def which(cmd, env = ENV) - exts = env['PATHEXT'] ? env['PATHEXT'].split(';') : [''] - - env['PATH'].split(File::PATH_SEPARATOR).each do |path| - exts.each do |ext| - exe = File.join(path, "#{cmd}#{ext}") - return exe if File.executable?(exe) && !File.directory?(exe) - end + def which(filename) + ENV['PATH']&.split(File::PATH_SEPARATOR)&.each do |path| + full_path = File.join(path, filename) + return full_path if File.executable?(full_path) end nil diff --git a/lib/gitlab/utils/nokogiri.rb b/lib/gitlab/utils/nokogiri.rb index 4b37bb7e5ea..5113553c584 100644 --- a/lib/gitlab/utils/nokogiri.rb +++ b/lib/gitlab/utils/nokogiri.rb @@ -16,7 +16,7 @@ module Gitlab # we remove the leading `//` and add `descendant-or-self::` # in order to ensure we're searching from this node and all # descendants. - xpath.map { |t| "descendant-or-self::#{t[2..-1]}" }.join('|') + xpath.map { |t| "descendant-or-self::#{t[2..]}" }.join('|') end end end diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index 483bfe12c68..255fa0169bf 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_dependency 'gitlab/utils' - module Gitlab module Utils module StrongMemoize diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 77f04929661..e347168f419 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -43,13 +43,8 @@ module Gitlab HISTOGRAM_FALLBACK = { '-1' => -1 }.freeze DISTRIBUTED_HLL_FALLBACK = -2 MAX_BUCKET_SIZE = 100 - INSTRUMENTATION_CLASS_FALLBACK = -100 def add_metric(metric, time_frame: 'none', options: {}) - # Results of this method should be overwritten by instrumentation class values - # -100 indicates the metric was not properly merged. - return INSTRUMENTATION_CLASS_FALLBACK if Feature.enabled?(:usage_data_instrumentation) - metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize metric_class.new(time_frame: time_frame, options: options).value @@ -61,7 +56,8 @@ module Gitlab else relation.count end - rescue ActiveRecord::StatementInvalid + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK end @@ -71,7 +67,8 @@ module Gitlab else relation.distinct_count_by(column) end - rescue ActiveRecord::StatementInvalid + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK end @@ -83,7 +80,8 @@ module Gitlab yield buckets if block_given? buckets.estimated_distinct_count - rescue ActiveRecord::StatementInvalid + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK # catch all rescue should be removed as a part of feature flag rollout issue # https://gitlab.com/gitlab-org/gitlab/-/issues/285485 @@ -94,7 +92,8 @@ module Gitlab def sum(relation, column, batch_size: nil, start: nil, finish: nil) Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish) - rescue ActiveRecord::StatementInvalid + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) FALLBACK end @@ -160,7 +159,8 @@ module Gitlab query: query.to_sql, message: e.message ) - + # Raises error for dev env + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) HISTOGRAM_FALLBACK end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/x509/commit.rb b/lib/gitlab/x509/commit.rb index 91951a3e505..c7f4b7cbdf5 100644 --- a/lib/gitlab/x509/commit.rb +++ b/lib/gitlab/x509/commit.rb @@ -25,7 +25,7 @@ module Gitlab def lazy_signature BatchLoader.for(@commit.sha).batch do |shas, loader| - X509CommitSignature.by_commit_sha(shas).each do |signature| + CommitSignatures::X509CommitSignature.by_commit_sha(shas).each do |signature| loader.call(signature.commit_sha, signature) end end @@ -49,9 +49,9 @@ module Gitlab def create_cached_signature! return if attributes.nil? - return X509CommitSignature.new(attributes) if Gitlab::Database.read_only? + return CommitSignatures::X509CommitSignature.new(attributes) if Gitlab::Database.read_only? - X509CommitSignature.safe_create!(attributes) + CommitSignatures::X509CommitSignature.safe_create!(attributes) end end end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index c917debd3d9..9bd2309d2b7 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -1,10 +1,12 @@ # frozen_string_literal: true +require 'securerandom' require 'google/apis/compute_v1' require 'google/apis/container_v1' require 'google/apis/container_v1beta1' require 'google/apis/cloudbilling_v1' require 'google/apis/cloudresourcemanager_v1' +require 'google/apis/iam_v1' module GoogleApi module CloudPlatform @@ -83,6 +85,51 @@ module GoogleApi m[1] if m end + def list_projects + result = [] + + service = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new + service.authorization = access_token + + response = service.fetch_all(items: :projects) do |token| + service.list_projects + end + + # Google API results are paged by default, so we need to iterate through + response.each do |project| + result.append(project) + end + + result + end + + def create_service_account(gcp_project_id, display_name, description) + name = "projects/#{gcp_project_id}" + + # initialize google iam service + service = Google::Apis::IamV1::IamService.new + service.authorization = access_token + + # generate account id + random_account_id = "gitlab-" + SecureRandom.hex(11) + + body_params = { account_id: random_account_id, + service_account: { display_name: display_name, + description: description } } + + request_body = Google::Apis::IamV1::CreateServiceAccountRequest.new(**body_params) + service.create_service_account(name, request_body) + end + + def create_service_account_key(gcp_project_id, service_account_id) + service = Google::Apis::IamV1::IamService.new + service.authorization = access_token + + name = "projects/#{gcp_project_id}/serviceAccounts/#{service_account_id}" + request_body = Google::Apis::IamV1::CreateServiceAccountKeyRequest.new + service.create_service_account_key(name, request_body) + end + private def make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac, enable_addons) diff --git a/lib/safe_zip/extract.rb b/lib/safe_zip/extract.rb index ac33f0b3c2c..74df7895afe 100644 --- a/lib/safe_zip/extract.rb +++ b/lib/safe_zip/extract.rb @@ -25,7 +25,7 @@ module SafeZip private def extract_with_ruby_zip(params) - ::Zip::File.open(archive_path) do |zip_archive| + ::Zip::File.open(archive_path) do |zip_archive| # rubocop:disable Performance/Rubyzip # Extract all files in the following order: # 1. Directories first, # 2. Files next, diff --git a/lib/sidebars/concerns/container_with_html_options.rb b/lib/sidebars/concerns/container_with_html_options.rb index 79dddd309b5..796b7cbe275 100644 --- a/lib/sidebars/concerns/container_with_html_options.rb +++ b/lib/sidebars/concerns/container_with_html_options.rb @@ -3,6 +3,8 @@ module Sidebars module Concerns module ContainerWithHtmlOptions + include LinkWithHtmlOptions + # The attributes returned from this method # will be applied to helper methods like # `link_to` or the div containing the container. diff --git a/lib/sidebars/concerns/link_with_html_options.rb b/lib/sidebars/concerns/link_with_html_options.rb new file mode 100644 index 00000000000..4ca748e0a71 --- /dev/null +++ b/lib/sidebars/concerns/link_with_html_options.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Sidebars + module Concerns + module LinkWithHtmlOptions + # add on specific items as the pertain to `link_to` objects specifically + def link_html_options + container_html_options.tap do |html_options| + html_options[:class] = [*html_options[:class], 'gl-link'].join(' ') + end + end + end + end +end diff --git a/lib/sidebars/concerns/positionable_list.rb b/lib/sidebars/concerns/positionable_list.rb index 0bbe1d918e5..459af466690 100644 --- a/lib/sidebars/concerns/positionable_list.rb +++ b/lib/sidebars/concerns/positionable_list.rb @@ -44,6 +44,14 @@ module Sidebars list[index] = new_element end + def remove_element(list, element_to_remove) + index = index_of(list, element_to_remove) + + return unless index + + list.slice!(index) + end + private # Classes including this method will have to define diff --git a/lib/sidebars/groups/menus/customer_relations_menu.rb b/lib/sidebars/groups/menus/customer_relations_menu.rb index fdbbd662ad6..002197965d1 100644 --- a/lib/sidebars/groups/menus/customer_relations_menu.rb +++ b/lib/sidebars/groups/menus/customer_relations_menu.rb @@ -32,7 +32,7 @@ module Sidebars def contacts_menu_item ::Sidebars::MenuItem.new( title: _('Contacts'), - link: contacts_group_crm_index_path(context.group), + link: group_crm_contacts_path(context.group), active_routes: { path: 'groups/crm#contacts' }, item_id: :crm_contacts ) @@ -41,7 +41,7 @@ module Sidebars def organizations_menu_item ::Sidebars::MenuItem.new( title: _('Organizations'), - link: organizations_group_crm_index_path(context.group), + link: group_crm_organizations_path(context.group), active_routes: { path: 'groups/crm#organizations' }, item_id: :crm_organizations ) diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb index 46fcec9f7b8..60d91c8fd10 100644 --- a/lib/sidebars/groups/menus/packages_registries_menu.rb +++ b/lib/sidebars/groups/menus/packages_registries_menu.rb @@ -26,9 +26,7 @@ module Sidebars private def packages_registry_menu_item - unless context.group.packages_feature_enabled? - return ::Sidebars::NilMenuItem.new(item_id: :packages_registry) - end + return nil_menu_item(:packages_registry) unless context.group.packages_feature_enabled? ::Sidebars::MenuItem.new( title: _('Package Registry'), @@ -40,7 +38,7 @@ module Sidebars def container_registry_menu_item if !::Gitlab.config.registry.enabled || !can?(context.current_user, :read_container_image, context.group) - return ::Sidebars::NilMenuItem.new(item_id: :container_registry) + return nil_menu_item(:container_registry) end ::Sidebars::MenuItem.new( @@ -52,9 +50,11 @@ module Sidebars end def dependency_proxy_menu_item - unless can?(context.current_user, :read_dependency_proxy, context.group) - return ::Sidebars::NilMenuItem.new(item_id: :dependency_proxy) - end + setting_does_not_exist_or_is_enabled = !context.group.dependency_proxy_setting || + context.group.dependency_proxy_setting.enabled + + return nil_menu_item(:dependency_proxy) unless can?(context.current_user, :read_dependency_proxy, context.group) + return nil_menu_item(:dependency_proxy) unless setting_does_not_exist_or_is_enabled ::Sidebars::MenuItem.new( title: _('Dependency Proxy'), @@ -63,6 +63,10 @@ module Sidebars item_id: :dependency_proxy ) end + + def nil_menu_item(item_id) + ::Sidebars::NilMenuItem.new(item_id: item_id) + end end end end diff --git a/lib/sidebars/menu_item.rb b/lib/sidebars/menu_item.rb index 1375f9fffca..7ed3e47ae34 100644 --- a/lib/sidebars/menu_item.rb +++ b/lib/sidebars/menu_item.rb @@ -2,6 +2,8 @@ module Sidebars class MenuItem + include ::Sidebars::Concerns::LinkWithHtmlOptions + attr_reader :title, :link, :active_routes, :item_id, :container_html_options, :sprite_icon, :sprite_icon_html_options, :hint_html_options def initialize(title:, link:, active_routes:, item_id: nil, container_html_options: {}, sprite_icon: nil, sprite_icon_html_options: {}, hint_html_options: {}) diff --git a/lib/sidebars/panel.rb b/lib/sidebars/panel.rb index e8c02a2d707..2a172cfffe3 100644 --- a/lib/sidebars/panel.rb +++ b/lib/sidebars/panel.rb @@ -37,6 +37,10 @@ module Sidebars replace_element(@menus, menu_to_replace, new_menu) end + def remove_menu(menu_to_remove) + remove_element(@menus, menu_to_remove) + end + def set_scope_menu(scope_menu) @scope_menu = scope_menu end diff --git a/lib/sidebars/projects/menus/analytics_menu.rb b/lib/sidebars/projects/menus/analytics_menu.rb index 2a89dc66219..b9bcc3267d6 100644 --- a/lib/sidebars/projects/menus/analytics_menu.rb +++ b/lib/sidebars/projects/menus/analytics_menu.rb @@ -10,9 +10,9 @@ module Sidebars def configure_menu_items return false unless can?(context.current_user, :read_analytics, context.project) + add_item(cycle_analytics_menu_item) add_item(ci_cd_analytics_menu_item) add_item(repository_analytics_menu_item) - add_item(cycle_analytics_menu_item) true end diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index ccc4787601a..1018bdd545b 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -57,9 +57,9 @@ module Sidebars data: { trigger: 'manual', container: 'body', placement: 'right', - highlight: UserCalloutsHelper::GKE_CLUSTER_INTEGRATION, - highlight_priority: UserCallout.feature_names[:GKE_CLUSTER_INTEGRATION], - dismiss_endpoint: user_callouts_path, + highlight: Users::CalloutsHelper::GKE_CLUSTER_INTEGRATION, + highlight_priority: Users::Callout.feature_names[:GKE_CLUSTER_INTEGRATION], + dismiss_endpoint: callouts_path, auto_devops_help_path: help_page_path('topics/autodevops/index.md') } } end @@ -90,7 +90,7 @@ module Sidebars end def google_cloud_menu_item - feature_is_enabled = Feature.enabled?(:incubation_5mp_google_cloud) + feature_is_enabled = Feature.enabled?(:incubation_5mp_google_cloud, context.project) user_has_permissions = can?(context.current_user, :admin_project_google_cloud, context.project) unless feature_is_enabled && user_has_permissions @@ -100,7 +100,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Google Cloud'), link: project_google_cloud_index_path(context.project), - active_routes: { controller: :google_cloud }, + active_routes: { controller: [:google_cloud, :service_accounts] }, item_id: :google_cloud ) end diff --git a/lib/sidebars/projects/menus/shimo_menu.rb b/lib/sidebars/projects/menus/shimo_menu.rb new file mode 100644 index 00000000000..c93c4f6a0a4 --- /dev/null +++ b/lib/sidebars/projects/menus/shimo_menu.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class ShimoMenu < ::Sidebars::Menu + override :link + def link + project_integrations_shimo_path(context.project) + end + + override :title + def title + s_('Shimo|Shimo') + end + + override :image_path + def image_path + 'logos/shimo.svg' + end + + override :image_html_options + def image_html_options + { + size: 16 + } + end + + override :render? + def render? + context.project.has_shimo? + end + + override :active_routes + def active_routes + { controller: :shimo } + end + end + end + end +end diff --git a/lib/sidebars/projects/panel.rb b/lib/sidebars/projects/panel.rb index 8fbd71c1543..6bb4fb52e2a 100644 --- a/lib/sidebars/projects/panel.rb +++ b/lib/sidebars/projects/panel.rb @@ -32,8 +32,7 @@ module Sidebars add_menu(Sidebars::Projects::Menus::InfrastructureMenu.new(context)) add_menu(Sidebars::Projects::Menus::PackagesRegistriesMenu.new(context)) add_menu(Sidebars::Projects::Menus::AnalyticsMenu.new(context)) - add_menu(confluence_or_wiki_menu) - add_menu(Sidebars::Projects::Menus::ExternalWikiMenu.new(context)) + add_wiki_menus add_menu(Sidebars::Projects::Menus::SnippetsMenu.new(context)) add_menu(Sidebars::Projects::Menus::SettingsMenu.new(context)) add_invite_members_menu @@ -46,10 +45,16 @@ module Sidebars end end - def confluence_or_wiki_menu - confluence_menu = ::Sidebars::Projects::Menus::ConfluenceMenu.new(context) + def add_wiki_menus + add_menu((third_party_wiki_menu || Sidebars::Projects::Menus::WikiMenu).new(context)) + add_menu(Sidebars::Projects::Menus::ExternalWikiMenu.new(context)) + end + + def third_party_wiki_menu + wiki_menu_list = [::Sidebars::Projects::Menus::ConfluenceMenu] + wiki_menu_list << ::Sidebars::Projects::Menus::ShimoMenu if Feature.enabled?(:shimo_integration, context.project) - confluence_menu.render? ? confluence_menu : Sidebars::Projects::Menus::WikiMenu.new(context) + wiki_menu_list.find { |wiki_menu| wiki_menu.new(context).render? } end end end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index e03c78d5a40..17f9414ad52 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -58,6 +58,7 @@ namespace :gettext do task lint: :environment do require 'simple_po_parser' require 'gitlab/utils' + require 'parallel' FastGettext.silence_errors files = Dir.glob(Rails.root.join('locale/*/gitlab.po')) @@ -70,7 +71,9 @@ namespace :gettext do linters.unshift(Gitlab::I18n::PoLinter.new(po_path: pot_file_path)) - failed_linters = linters.select { |linter| linter.errors.any? } + failed_linters = Parallel + .map(linters, progress: 'Linting po files') { |linter| linter if linter.errors.any? } + .compact if failed_linters.empty? puts 'All PO files are valid.' @@ -129,14 +132,6 @@ namespace :gettext do ) end - # Disallow HTML from translatable strings - # See: https://docs.gitlab.com/ee/development/i18n/externalization.html#html - def html_todolist - return @html_todolist if defined?(@html_todolist) - - @html_todolist = YAML.safe_load(File.read(Rails.root.join('lib/gitlab/i18n/html_todo.yml'))) - end - def report_errors_for_file(file, errors_for_file) puts "Errors in `#{file}`:" diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake index c978a2807ca..c7f3d003f9f 100644 --- a/lib/tasks/gitlab/background_migrations.rake +++ b/lib/tasks/gitlab/background_migrations.rake @@ -2,6 +2,7 @@ namespace :gitlab do namespace :background_migrations do + desc 'Synchronously finish executing a batched background migration' task :finalize, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args| [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument| unless args[argument] @@ -19,5 +20,23 @@ namespace :gitlab do puts "Done.".color(:green) end + + desc 'Display the status of batched background migrations' + task status: :environment do + statuses = Gitlab::Database::BackgroundMigration::BatchedMigration.statuses + max_status_length = statuses.keys.map(&:length).max + format_string = "%-#{max_status_length}s | %s\n" + + Gitlab::Database::BackgroundMigration::BatchedMigration.find_each(batch_size: 100) do |migration| + identification_fields = [ + migration.job_class_name, + migration.table_name, + migration.column_name, + migration.job_arguments.to_json + ].join(',') + + printf(format_string, migration.status, identification_fields) + end + end end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 0cd4ab354c9..8f033a41e3d 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -100,13 +100,15 @@ namespace :gitlab do namespace :sessions do desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys" task active_sessions_lookup_keys: :gitlab_environment do - session_key_pattern = "#{Gitlab::Redis::SharedState::USER_SESSIONS_LOOKUP_NAMESPACE}:*" + use_redis_session_store = Gitlab::Utils.to_boolean(ENV['GITLAB_USE_REDIS_SESSIONS_STORE'], default: true) + redis_store_class = use_redis_session_store ? Gitlab::Redis::Sessions : Gitlab::Redis::SharedState + session_key_pattern = "#{Gitlab::Redis::Sessions::USER_SESSIONS_LOOKUP_NAMESPACE}:*" last_save_check = Time.at(0) wait_time = 10.seconds cursor = 0 total_users_scanned = 0 - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| begin cursor, keys = redis.scan(cursor, match: session_key_pattern) total_users_scanned += keys.count @@ -119,27 +121,16 @@ namespace :gitlab do last_save_check = Time.now end + user = Struct.new(:id) + keys.each do |key| user_id = key.split(':').last - lookup_key_count = redis.scard(key) - - session_ids = ActiveSession.session_ids_for_user(user_id) - entries = ActiveSession.raw_active_session_entries(redis, session_ids, user_id) - session_ids_and_entries = session_ids.zip(entries) - - inactive_session_ids = session_ids_and_entries.map do |session_id, session| - session_id if session.nil? - end.compact - - redis.pipelined do |conn| - inactive_session_ids.each do |session_id| - conn.srem(key, session_id) - end - end + removed = [] + active = ActiveSession.cleaned_up_lookup_entries(redis, user.new(user_id), removed) - if inactive_session_ids - puts "deleted #{inactive_session_ids.count} out of #{lookup_key_count} lookup keys for User ##{user_id}" + if removed.any? + puts "deleted #{removed.count} out of #{active.count + removed.count} lookup keys for User ##{user_id}" end end end while cursor.to_i != 0 diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index e83c4cbdb39..9e733fc3a0f 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -1,5 +1,7 @@ # frozen_string_literal: true +databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml + namespace :gitlab do namespace :db do desc 'GitLab | DB | Manually insert schema migration version' @@ -49,7 +51,7 @@ namespace :gitlab do # Drop all extra schema objects GitLab owns Gitlab::Database::EXTRA_SCHEMAS.each do |schema| - connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)}") + connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)} CASCADE") end end @@ -83,7 +85,7 @@ namespace :gitlab do desc 'GitLab | DB | Sets up EE specific database functionality' if Gitlab.ee? - task setup_ee: %w[geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate] + task setup_ee: %w[db:drop:geo db:create:geo db:schema:load:geo db:migrate:geo] else task :setup_ee end @@ -116,6 +118,19 @@ namespace :gitlab do Rake::Task['gitlab:db:clean_structure_sql'].invoke end + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + # Inform Rake that custom tasks should be run every time rake db:structure:dump is run + # + # Rails 6.1 deprecates db:structure:dump in favor of db:schema:dump + Rake::Task["db:structure:dump:#{name}"].enhance do + Rake::Task['gitlab:db:clean_structure_sql'].invoke + end + + Rake::Task["db:schema:dump:#{name}"].enhance do + Rake::Task['gitlab:db:clean_structure_sql'].invoke + end + end + desc 'Create missing dynamic database partitions' task create_dynamic_partitions: :environment do Gitlab::Database::Partitioning.sync_partitions @@ -160,24 +175,30 @@ namespace :gitlab do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end - desc 'execute reindexing without downtime to eliminate bloat' + desc "Reindex database without downtime to eliminate bloat" task reindex: :environment do - unless Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml) + unless Gitlab::Database::Reindexing.enabled? puts "This feature (database_reindexing) is currently disabled.".color(:yellow) exit end - Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| - Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) + Gitlab::Database::Reindexing.invoke + end + + namespace :reindex do + databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml - # Hack: Before we do actual reindexing work, create async indexes - Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database_name| + desc "Reindex #{database_name} database without downtime to eliminate bloat" + task database_name => :environment do + unless Gitlab::Database::Reindexing.enabled? + puts "This feature (database_reindexing) is currently disabled.".color(:yellow) + exit + end - Gitlab::Database::Reindexing.automatic_reindexing + Gitlab::Database::Reindexing.invoke(database_name) + end end - rescue StandardError => e - Gitlab::AppLogger.error(e) - raise end desc 'Enqueue an index for reindexing' @@ -243,7 +264,9 @@ namespace :gitlab do # Only for development environments, # we execute pending data migrations inline for convenience. Rake::Task['db:migrate'].enhance do - Rake::Task['gitlab:db:execute_batched_migrations'].invoke if Rails.env.development? + if Rails.env.development? && Gitlab::Database::BackgroundMigration::BatchedMigration.table_exists? + Rake::Task['gitlab:db:execute_batched_migrations'].invoke + end end end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index eabbb8652f1..b01a7902bf2 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -37,8 +37,8 @@ namespace :gitlab do raise end - desc 'GitLab | Gitaly | Install or upgrade gitaly' - task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| + desc 'GitLab | Gitaly | Clone and checkout gitaly' + task :clone, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab unless args.dir.present? && args.storage_path.present? @@ -51,6 +51,11 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") version = Gitlab::GitalyClient.expected_server_version checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1]) + end + + desc 'GitLab | Gitaly | Install or upgrade gitaly' + task :install, [:dir, :storage_path, :repo] => [:gitlab_environment, 'gitlab:gitaly:clone'] do |t, args| + warn_user_is_not_gitlab storage_paths = { 'default' => args.storage_path } Gitlab::SetupHelper::Gitaly.create_configuration(args.dir, storage_paths) diff --git a/lib/tasks/gitlab/seed/group_seed.rake b/lib/tasks/gitlab/seed/group_seed.rake index bc705c94422..a9a350fb6c3 100644 --- a/lib/tasks/gitlab/seed/group_seed.rake +++ b/lib/tasks/gitlab/seed/group_seed.rake @@ -184,7 +184,7 @@ class GroupSeeder group = Group.find(group_id) @resource_count.times do |i| - _, project_path = PROJECT_URL.split('/')[-2..-1] + _, project_path = PROJECT_URL.split('/')[-2..] project_path.gsub!('.git', '') diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 2e383065b64..10492e183c5 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -100,6 +100,10 @@ namespace :gitlab do queues_and_weights = Gitlab::SidekiqConfig.queues_for_sidekiq_queues_yml write_yaml(Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH, banner, queues: queues_and_weights) + + if Gitlab.jh? + write_yaml(Gitlab::SidekiqConfig::JH_SIDEKIQ_QUEUES_PATH, banner, queues: Gitlab::SidekiqConfig.jh_queues_for_sidekiq_queues_yml) + end end desc 'GitLab | Sidekiq | Validate that sidekiq_queues.yml matches worker definitions' @@ -113,6 +117,7 @@ namespace :gitlab do Then commit and push the changes from: - #{Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH} + #{"- " + Gitlab::SidekiqConfig::JH_SIDEKIQ_QUEUES_PATH if Gitlab.jh?} MSG end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index e3a4e7f50b8..247897bed0b 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -32,7 +32,7 @@ namespace :gitlab do tmp_namespace_path = "tmp-project-import-#{Time.now.to_i}" puts "Creating temporary namespace #{tmp_namespace_path}" - tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path) + tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path, type: Namespaces::UserNamespace.sti_name) templates = if template_names.empty? Gitlab::ProjectTemplate.all diff --git a/lib/tasks/gitlab/x509/update.rake b/lib/tasks/gitlab/x509/update.rake index d3c63fa8514..7b7d15479bf 100644 --- a/lib/tasks/gitlab/x509/update.rake +++ b/lib/tasks/gitlab/x509/update.rake @@ -12,14 +12,14 @@ namespace :gitlab do def update_certificates logger = Logger.new($stdout) - unless X509CommitSignature.exists? + unless CommitSignatures::X509CommitSignature.exists? logger.info("Unable to find any x509 commit signatures. Exiting.") return end logger.info("Start to update x509 commit signatures") - X509CommitSignature.find_each do |sig| + CommitSignatures::X509CommitSignature.find_each do |sig| sig.x509_commit&.update_signature!(sig) end diff --git a/lib/version_check.rb b/lib/version_check.rb index a8b7c7371ca..e5a4c244c7a 100644 --- a/lib/version_check.rb +++ b/lib/version_check.rb @@ -2,22 +2,60 @@ require "base64" -# This class is used to build image URL to -# check if it is a new version for update class VersionCheck + include ReactiveCaching + + self.reactive_cache_work_type = :external_dependency + self.reactive_cache_worker_finder = ->(_id, *args) { from_cache } + def self.data { version: Gitlab::VERSION } end - def self.url + def self.headers + { "REFERER": Gitlab.config.gitlab.url } + end + + # This is temporary and will be removed when the new UI is hooked up + # to the version_check.json endpoint. + def self.image_url encoded_data = Base64.urlsafe_encode64(data.to_json) "#{host}/check.svg?gitlab_info=#{encoded_data}" end + def self.url + encoded_data = Base64.urlsafe_encode64(data.to_json) + + "#{host}/check.json?gitlab_info=#{encoded_data}" + end + def self.host 'https://version.gitlab.com' end + + def self.from_cache(*) + new + end + + def id + Gitlab::VERSION + end + + def calculate_reactive_cache(*) + response = Gitlab::HTTP.try_get(self.class.url, headers: self.class.headers) + + case response&.code + when 200 + response.body + end + end + + def response + with_reactive_cache do |data| + Gitlab::Json.parse(data) if data + end + end end VersionCheck.prepend_mod |