diff options
Diffstat (limited to 'lib')
374 files changed, 7290 insertions, 3279 deletions
diff --git a/lib/api/admin/sidekiq.rb b/lib/api/admin/sidekiq.rb new file mode 100644 index 00000000000..a700bea0fd7 --- /dev/null +++ b/lib/api/admin/sidekiq.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module API + module Admin + class Sidekiq < Grape::API + before { authenticated_as_admin! } + + namespace 'admin' do + namespace 'sidekiq' do + namespace 'queues' do + desc 'Drop jobs matching the given metadata from the Sidekiq queue' + params do + Labkit::Context::KNOWN_KEYS.each do |key| + optional key, type: String, allow_blank: false + end + + at_least_one_of(*Labkit::Context::KNOWN_KEYS) + end + delete ':queue_name' do + result = + Gitlab::SidekiqQueue + .new(params[:queue_name]) + .drop_jobs!(declared_params, timeout: 30) + + present result + rescue Gitlab::SidekiqQueue::NoMetadataError + render_api_error!("Invalid metadata: #{declared_params}", 400) + rescue Gitlab::SidekiqQueue::InvalidQueueError + not_found!(params[:queue_name]) + end + end + end + end + end + end +end diff --git a/lib/api/api.rb b/lib/api/api.rb index 9a1e0e3f8e9..02b3fe7e03e 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -45,7 +45,7 @@ module API before do Gitlab::ApplicationContext.push( - user: -> { current_user }, + user: -> { @current_user }, project: -> { @project }, namespace: -> { @group }, caller_id: route.origin @@ -110,6 +110,7 @@ module API # Keep in alphabetical order mount ::API::AccessRequests + mount ::API::Admin::Sidekiq mount ::API::Appearance mount ::API::Applications mount ::API::Avatar @@ -121,6 +122,7 @@ module API mount ::API::Commits mount ::API::CommitStatuses mount ::API::DeployKeys + mount ::API::DeployTokens mount ::API::Deployments mount ::API::Environments mount ::API::ErrorTracking diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 0769e464d26..5cab13f001e 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -50,17 +50,13 @@ module API user = find_user_from_sources return unless user + # Sessions are enforced to be unavailable for API calls, so ignore them for admin mode + Gitlab::Auth::CurrentUserMode.bypass_session!(user.id) if Feature.enabled?(:user_mode_in_session) + unless api_access_allowed?(user) forbidden!(api_access_denied_message(user)) end - # Set admin mode for API requests (if admin) - if Feature.enabled?(:user_mode_in_session) - current_user_mode = Gitlab::Auth::CurrentUserMode.new(user) - - current_user_mode.enable_sessionless_admin_mode! - end - user end @@ -154,19 +150,13 @@ module API end class AdminModeMiddleware < ::Grape::Middleware::Base - def initialize(app, **options) - super - end + def after + # Use a Grape middleware since the Grape `after` blocks might run + # before we are finished rendering the `Grape::Entity` classes + Gitlab::Auth::CurrentUserMode.reset_bypass_session! if Feature.enabled?(:user_mode_in_session) - def call(env) - if Feature.enabled?(:user_mode_in_session) - session = {} - Gitlab::Session.with_session(session) do - app.call(env) - end - else - app.call(env) - end + # Explicit nil is needed or the api call return value will be overwritten + nil end end end diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index af7c69f857e..42e7dc751f0 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -36,6 +36,7 @@ module API optional :font, type: String, desc: 'Foreground color' optional :target_path, type: String, desc: 'Target path' optional :broadcast_type, type: String, values: BroadcastMessage.broadcast_types.keys, desc: 'Broadcast type. Defaults to banner', default: -> { 'banner' } + optional :dismissable, type: Boolean, desc: 'Is dismissable' end post do authenticated_as_admin! @@ -75,6 +76,7 @@ module API optional :font, type: String, desc: 'Foreground color' optional :target_path, type: String, desc: 'Target path' optional :broadcast_type, type: String, values: BroadcastMessage.broadcast_types.keys, desc: 'Broadcast Type' + optional :dismissable, type: Boolean, desc: 'Is dismissable' end put ':id' do authenticated_as_admin! diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb new file mode 100644 index 00000000000..2b1c485785b --- /dev/null +++ b/lib/api/deploy_tokens.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +module API + class DeployTokens < Grape::API + include PaginationParams + + helpers do + def scope_params + scopes = params.delete(:scopes) + + result_hash = {} + result_hash[:read_registry] = scopes.include?('read_registry') + result_hash[:read_repository] = scopes.include?('read_repository') + result_hash + end + end + + desc 'Return all deploy tokens' do + detail 'This feature was introduced in GitLab 12.9.' + success Entities::DeployToken + end + params do + use :pagination + end + get 'deploy_tokens' do + service_unavailable! unless Feature.enabled?(:deploy_tokens_api, default_enabled: true) + + authenticated_as_admin! + + present paginate(DeployToken.all), with: Entities::DeployToken + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + before do + service_unavailable! unless Feature.enabled?(:deploy_tokens_api, user_project, default_enabled: true) + end + + params do + use :pagination + end + desc 'List deploy tokens for a project' do + detail 'This feature was introduced in GitLab 12.9' + success Entities::DeployToken + end + get ':id/deploy_tokens' do + authorize!(:read_deploy_token, user_project) + + present paginate(user_project.deploy_tokens), with: Entities::DeployToken + end + + params do + requires :name, type: String, desc: "New deploy token's name" + requires :expires_at, type: DateTime, desc: 'Expiration date for the deploy token. Does not expire if no value is provided.' + requires :username, type: String, desc: 'Username for deploy token. Default is `gitlab+deploy-token-{n}`' + requires :scopes, type: Array[String], values: ::DeployToken::AVAILABLE_SCOPES.map(&:to_s), + desc: 'Indicates the deploy token scopes. Must be at least one of "read_repository" or "read_registry".' + end + desc 'Create a project deploy token' do + detail 'This feature was introduced in GitLab 12.9' + success Entities::DeployTokenWithToken + end + post ':id/deploy_tokens' do + authorize!(:create_deploy_token, user_project) + + deploy_token = ::Projects::DeployTokens::CreateService.new( + user_project, current_user, scope_params.merge(declared(params, include_missing: false, include_parent_namespaces: false)) + ).execute + + present deploy_token, with: Entities::DeployTokenWithToken + end + + desc 'Delete a project deploy token' do + detail 'This feature was introduced in GitLab 12.9' + end + params do + requires :token_id, type: Integer, desc: 'The deploy token ID' + end + delete ':id/deploy_tokens/:token_id' do + authorize!(:destroy_deploy_token, user_project) + + deploy_token = user_project.project_deploy_tokens + .find_by_deploy_token_id(params[:token_id]) + + not_found!('Deploy Token') unless deploy_token + + deploy_token.destroy + no_content! + end + end + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + before do + service_unavailable! unless Feature.enabled?(:deploy_tokens_api, user_group, default_enabled: true) + end + + params do + use :pagination + end + desc 'List deploy tokens for a group' do + detail 'This feature was introduced in GitLab 12.9' + success Entities::DeployToken + end + get ':id/deploy_tokens' do + authorize!(:read_deploy_token, user_group) + + present paginate(user_group.deploy_tokens), with: Entities::DeployToken + end + + params do + requires :name, type: String, desc: 'The name of the deploy token' + requires :expires_at, type: DateTime, desc: 'Expiration date for the deploy token. Does not expire if no value is provided.' + requires :username, type: String, desc: 'Username for deploy token. Default is `gitlab+deploy-token-{n}`' + requires :scopes, type: Array[String], values: ::DeployToken::AVAILABLE_SCOPES.map(&:to_s), + desc: 'Indicates the deploy token scopes. Must be at least one of "read_repository" or "read_registry".' + end + desc 'Create a group deploy token' do + detail 'This feature was introduced in GitLab 12.9' + success Entities::DeployTokenWithToken + end + post ':id/deploy_tokens' do + authorize!(:create_deploy_token, user_group) + + deploy_token = ::Groups::DeployTokens::CreateService.new( + user_group, current_user, scope_params.merge(declared(params, include_missing: false, include_parent_namespaces: false)) + ).execute + + present deploy_token, with: Entities::DeployTokenWithToken + end + + desc 'Delete a group deploy token' do + detail 'This feature was introduced in GitLab 12.9' + end + delete ':id/deploy_tokens/:token_id' do + authorize!(:destroy_deploy_token, user_group) + + deploy_token = user_group.group_deploy_tokens + .find_by_deploy_token_id!(params[:token_id]) + + destroy_conditionally!(deploy_token) + end + end + end +end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 487d4e37a56..cb1dca11e87 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -143,6 +143,7 @@ module API success Entities::MergeRequestBasic end params do + use :pagination requires :deployment_id, type: Integer, desc: 'The deployment ID' use :merge_requests_base_params end @@ -153,7 +154,7 @@ module API mr_params = declared_params.merge(deployment_id: params[:deployment_id]) merge_requests = MergeRequestsFinder.new(current_user, mr_params).execute - present merge_requests, { with: Entities::MergeRequestBasic, current_user: current_user } + present paginate(merge_requests), { with: Entities::MergeRequestBasic, current_user: current_user } end end end diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 25d38615c7f..a1cec148aeb 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -230,7 +230,7 @@ module API .fresh # Without RendersActions#prepare_notes_for_rendering, - # Note#cross_reference_not_visible_for? will attempt to render + # Note#system_note_with_references_visible_for? will attempt to render # Markdown references mentioned in the note to see whether they # should be redacted. For notes that reference a commit, this # would also incur a Gitaly call to verify the commit exists. @@ -239,7 +239,7 @@ module API # because notes are redacted if they point to projects that # cannot be accessed by the user. notes = prepare_notes_for_rendering(notes) - notes.select { |n| n.visible_for?(current_user) } + notes.select { |n| n.readable_by?(current_user) } end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/api/entities/broadcast_message.rb b/lib/api/entities/broadcast_message.rb index 403677aa300..e42b110adbe 100644 --- a/lib/api/entities/broadcast_message.rb +++ b/lib/api/entities/broadcast_message.rb @@ -3,7 +3,7 @@ module API module Entities class BroadcastMessage < Grape::Entity - expose :id, :message, :starts_at, :ends_at, :color, :font, :target_path, :broadcast_type + expose :id, :message, :starts_at, :ends_at, :color, :font, :target_path, :broadcast_type, :dismissable expose :active?, as: :active end end diff --git a/lib/api/entities/commit.rb b/lib/api/entities/commit.rb index 7ce97c2c3d8..3eaf896f1ac 100644 --- a/lib/api/entities/commit.rb +++ b/lib/api/entities/commit.rb @@ -9,6 +9,10 @@ module API expose :safe_message, as: :message expose :author_name, :author_email, :authored_date expose :committer_name, :committer_email, :committed_date + + expose :web_url do |commit, _options| + Gitlab::UrlBuilder.build(commit) + end end end end diff --git a/lib/api/entities/deploy_token.rb b/lib/api/entities/deploy_token.rb new file mode 100644 index 00000000000..9c5bf54e299 --- /dev/null +++ b/lib/api/entities/deploy_token.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module API + module Entities + class DeployToken < Grape::Entity + # exposing :token is a security risk and should be avoided + expose :id, :name, :username, :expires_at, :scopes + end + end +end diff --git a/lib/api/entities/deploy_token_with_token.rb b/lib/api/entities/deploy_token_with_token.rb new file mode 100644 index 00000000000..11efe3720fa --- /dev/null +++ b/lib/api/entities/deploy_token_with_token.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module API + module Entities + class DeployTokenWithToken < Entities::DeployToken + expose :token + end + end +end diff --git a/lib/api/entities/discussion.rb b/lib/api/entities/discussion.rb index dd1dd40da23..0740de97897 100644 --- a/lib/api/entities/discussion.rb +++ b/lib/api/entities/discussion.rb @@ -5,7 +5,7 @@ module API class Discussion < Grape::Entity expose :id expose :individual_note?, as: :individual_note - expose :notes, using: Entities::Note + expose :notes, using: Entities::NoteWithGitlabEmployeeBadge end end end diff --git a/lib/api/entities/gpg_key.rb b/lib/api/entities/gpg_key.rb index a97e704a5dd..50b72680cc8 100644 --- a/lib/api/entities/gpg_key.rb +++ b/lib/api/entities/gpg_key.rb @@ -2,7 +2,7 @@ module API module Entities - class GPGKey < Grape::Entity + class GpgKey < Grape::Entity expose :id, :key, :created_at end end diff --git a/lib/api/entities/group.rb b/lib/api/entities/group.rb index ae5ee4784ed..10e10e52d9f 100644 --- a/lib/api/entities/group.rb +++ b/lib/api/entities/group.rb @@ -13,6 +13,7 @@ module API expose :emails_disabled expose :mentions_disabled expose :lfs_enabled?, as: :lfs_enabled + expose :default_branch_protection expose :avatar_url do |group, options| group.avatar_url(only_path: false) end diff --git a/lib/api/entities/internal.rb b/lib/api/entities/internal/pages/lookup_path.rb index 8f79bd14833..1bf94f74fb4 100644 --- a/lib/api/entities/internal.rb +++ b/lib/api/entities/internal/pages/lookup_path.rb @@ -8,11 +8,6 @@ module API expose :project_id, :access_control, :source, :https_only, :prefix end - - class VirtualDomain < Grape::Entity - expose :certificate, :key - expose :lookup_paths, using: LookupPath - end end end end diff --git a/lib/api/entities/internal/pages/virtual_domain.rb b/lib/api/entities/internal/pages/virtual_domain.rb new file mode 100644 index 00000000000..27eb7571368 --- /dev/null +++ b/lib/api/entities/internal/pages/virtual_domain.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Internal + module Pages + class VirtualDomain < Grape::Entity + expose :certificate, :key + expose :lookup_paths, using: LookupPath + end + end + end + end +end diff --git a/lib/api/entities/internal/serverless/lookup_path.rb b/lib/api/entities/internal/serverless/lookup_path.rb new file mode 100644 index 00000000000..8ca40b4f128 --- /dev/null +++ b/lib/api/entities/internal/serverless/lookup_path.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + module Internal + module Serverless + class LookupPath < Grape::Entity + expose :source + end + end + end + end +end diff --git a/lib/api/entities/internal/serverless/virtual_domain.rb b/lib/api/entities/internal/serverless/virtual_domain.rb new file mode 100644 index 00000000000..8b53aa51bf5 --- /dev/null +++ b/lib/api/entities/internal/serverless/virtual_domain.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Internal + module Serverless + class VirtualDomain < Grape::Entity + expose :certificate, :key + expose :lookup_paths, using: LookupPath + end + end + end + end +end diff --git a/lib/api/entities/milestone_with_stats.rb b/lib/api/entities/milestone_with_stats.rb new file mode 100644 index 00000000000..33fa322573b --- /dev/null +++ b/lib/api/entities/milestone_with_stats.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module API + module Entities + class MilestoneWithStats < Entities::Milestone + expose :issue_stats do + expose :total_issues_count, as: :total + expose :closed_issues_count, as: :closed + end + end + end +end diff --git a/lib/api/entities/note_with_gitlab_employee_badge.rb b/lib/api/entities/note_with_gitlab_employee_badge.rb new file mode 100644 index 00000000000..2ea300ffeb6 --- /dev/null +++ b/lib/api/entities/note_with_gitlab_employee_badge.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module API + module Entities + class NoteWithGitlabEmployeeBadge < Note + expose :author, using: Entities::UserWithGitlabEmployeeBadge + expose :resolved_by, using: Entities::UserWithGitlabEmployeeBadge, if: ->(note, options) { note.resolvable? } + end + end +end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 6ed2ed34360..85a00273192 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -106,6 +106,9 @@ module API project.auto_devops.nil? ? 'continuous' : project.auto_devops.deploy_strategy end expose :autoclose_referenced_issues + expose :repository_storage, if: ->(project, options) { + Ability.allowed?(options[:current_user], :change_repository_storage, project) + } # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) diff --git a/lib/api/entities/project_upload.rb b/lib/api/entities/project_upload.rb new file mode 100644 index 00000000000..f38f8d74f7b --- /dev/null +++ b/lib/api/entities/project_upload.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module API + module Entities + class ProjectUpload < Grape::Entity + include Gitlab::Routing + + expose :markdown_name, as: :alt + expose :secure_url, as: :url + expose :full_path do |uploader| + show_project_uploads_path( + uploader.model, + uploader.secret, + uploader.filename + ) + end + + expose :markdown_link, as: :markdown + end + end +end diff --git a/lib/api/entities/release.rb b/lib/api/entities/release.rb index dc4b91e594e..c70982a9ece 100644 --- a/lib/api/entities/release.rb +++ b/lib/api/entities/release.rb @@ -11,14 +11,14 @@ module API expose :tag, as: :tag_name, if: ->(_, _) { can_download_code? } expose :description expose :description_html do |entity| - MarkupHelper.markdown_field(entity, :description) + MarkupHelper.markdown_field(entity, :description, current_user: options[:current_user]) end expose :created_at expose :released_at expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? } expose :commit, using: Entities::Commit, if: ->(_, _) { can_download_code? } expose :upcoming_release?, as: :upcoming_release - expose :milestones, using: Entities::Milestone, if: -> (release, _) { release.milestones.present? && can_read_milestone? } + expose :milestones, using: Entities::MilestoneWithStats, if: -> (release, _) { release.milestones.present? && can_read_milestone? } expose :commit_path, expose_nil: false expose :tag_path, expose_nil: false expose :evidence_sha, expose_nil: false, if: ->(_, _) { can_download_code? } diff --git a/lib/api/entities/releases/link.rb b/lib/api/entities/releases/link.rb index 6cc01e0e981..f4edb83bd58 100644 --- a/lib/api/entities/releases/link.rb +++ b/lib/api/entities/releases/link.rb @@ -7,7 +7,17 @@ module API expose :id expose :name expose :url + expose :direct_asset_url expose :external?, as: :external + + def direct_asset_url + return object.url unless object.filepath + + release = object.release + project = release.project + + Gitlab::Routing.url_helpers.project_release_url(project, release) << object.filepath + end end end end diff --git a/lib/api/entities/remote_mirror.rb b/lib/api/entities/remote_mirror.rb index dde3e9dea99..18d51726bab 100644 --- a/lib/api/entities/remote_mirror.rb +++ b/lib/api/entities/remote_mirror.rb @@ -12,6 +12,9 @@ module API expose :last_successful_update_at expose :last_error expose :only_protected_branches + expose :keep_divergent_refs, if: -> (mirror, _options) do + ::Feature.enabled?(:keep_divergent_refs, mirror.project) + end end end end diff --git a/lib/api/entities/ssh_key.rb b/lib/api/entities/ssh_key.rb index 0e2f6ebae8c..aae216173c7 100644 --- a/lib/api/entities/ssh_key.rb +++ b/lib/api/entities/ssh_key.rb @@ -3,7 +3,7 @@ module API module Entities class SSHKey < Grape::Entity - expose :id, :title, :key, :created_at + expose :id, :title, :key, :created_at, :expires_at end end end diff --git a/lib/api/entities/user.rb b/lib/api/entities/user.rb index 15e4619cdb8..4a1f570c3f0 100644 --- a/lib/api/entities/user.rb +++ b/lib/api/entities/user.rb @@ -4,7 +4,7 @@ module API module Entities class User < UserBasic expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } - expose :bio, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization + expose :bio, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization, :job_title end end end diff --git a/lib/api/entities/user_details_with_admin.rb b/lib/api/entities/user_details_with_admin.rb index 9ea5c583437..22a842983e2 100644 --- a/lib/api/entities/user_details_with_admin.rb +++ b/lib/api/entities/user_details_with_admin.rb @@ -9,3 +9,5 @@ module API end end end + +API::Entities::UserDetailsWithAdmin.prepend_if_ee('EE::API::Entities::UserDetailsWithAdmin') diff --git a/lib/api/entities/user_with_gitlab_employee_badge.rb b/lib/api/entities/user_with_gitlab_employee_badge.rb new file mode 100644 index 00000000000..36b9f633132 --- /dev/null +++ b/lib/api/entities/user_with_gitlab_employee_badge.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module API + module Entities + class UserWithGitlabEmployeeBadge < UserBasic + expose :gitlab_employee?, as: :is_gitlab_employee, if: ->(user, options) { ::Feature.enabled?(:gitlab_employee_badge) && user.gitlab_employee? } + end + end +end diff --git a/lib/api/files.rb b/lib/api/files.rb index feed22d188c..76ab9a2190b 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -61,7 +61,7 @@ module API end params :simple_file_params do - requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false requires :commit_message, type: String, allow_blank: false, desc: 'Commit message' optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' @@ -85,7 +85,7 @@ module API desc 'Get blame file metadata from repository' params do - requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end head ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do @@ -96,7 +96,7 @@ module API desc 'Get blame file from the repository' params do - requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end get ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do @@ -110,7 +110,7 @@ module API desc 'Get raw file metadata from repository' params do - requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do @@ -121,7 +121,7 @@ module API desc 'Get raw file contents from the repository' params do - requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag commit', allow_blank: false end get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do @@ -135,7 +135,7 @@ module API desc 'Get file metadata from repository' params do - requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end head ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do @@ -146,7 +146,7 @@ module API desc 'Get a file from the repository' params do - requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 47fcbabb4d4..916f89649a5 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -47,6 +47,7 @@ module API requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' + optional :masked, type: String, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' end post ':id/variables' do @@ -68,6 +69,7 @@ module API optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' + optional :masked, type: String, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 001fb92ec52..c3b5654e217 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -142,6 +142,12 @@ module API end end + def check_namespace_access(namespace) + return namespace if can?(current_user, :read_namespace, namespace) + + not_found!('Namespace') + end + # rubocop: disable CodeReuse/ActiveRecord def find_namespace(id) if id.to_s =~ /^\d+$/ @@ -153,13 +159,15 @@ module API # rubocop: enable CodeReuse/ActiveRecord def find_namespace!(id) - namespace = find_namespace(id) + check_namespace_access(find_namespace(id)) + end - if can?(current_user, :read_namespace, namespace) - namespace - else - not_found!('Namespace') - end + def find_namespace_by_path(path) + Namespace.find_by_full_path(path) + end + + def find_namespace_by_path!(path) + check_namespace_access(find_namespace_by_path(path)) end def find_branch!(branch_name) @@ -359,6 +367,10 @@ module API render_api_error!('405 Method Not Allowed', 405) end + def service_unavailable! + render_api_error!('503 Service Unavailable', 503) + end + def conflict!(message = nil) render_api_error!(message || '409 Conflict', 409) end diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb index c86eae6f2da..4c15c1d01cd 100644 --- a/lib/api/helpers/custom_validators.rb +++ b/lib/api/helpers/custom_validators.rb @@ -3,6 +3,28 @@ module API module Helpers module CustomValidators + class FilePath < Grape::Validations::Base + def validate_param!(attr_name, params) + path = params[attr_name] + + Gitlab::Utils.check_path_traversal!(path) + rescue StandardError + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], + message: "should be a valid file path" + end + end + + class GitSha < Grape::Validations::Base + def validate_param!(attr_name, params) + sha = params[attr_name] + + return if Commit::EXACT_COMMIT_SHA_PATTERN.match?(sha) + + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], + message: "should be a valid sha" + end + end + class Absence < Grape::Validations::Base def validate_param!(attr_name, params) return if params.respond_to?(:key?) && !params.key?(attr_name) @@ -38,6 +60,8 @@ module API end end +Grape::Validations.register_validator(:file_path, ::API::Helpers::CustomValidators::FilePath) +Grape::Validations.register_validator(:git_sha, ::API::Helpers::CustomValidators::GitSha) Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence) Grape::Validations.register_validator(:integer_none_any, ::API::Helpers::CustomValidators::IntegerNoneAny) Grape::Validations.register_validator(:array_none_any, ::API::Helpers::CustomValidators::ArrayNoneAny) diff --git a/lib/api/helpers/file_upload_helpers.rb b/lib/api/helpers/file_upload_helpers.rb index c5fb291a2b7..dd551ec2976 100644 --- a/lib/api/helpers/file_upload_helpers.rb +++ b/lib/api/helpers/file_upload_helpers.rb @@ -4,11 +4,12 @@ module API module Helpers module FileUploadHelpers def file_is_valid? - params[:file] && params[:file]['tempfile'].respond_to?(:read) + filename = params[:file]&.original_filename + filename && ImportExportUploader::EXTENSION_WHITELIST.include?(File.extname(filename).delete('.')) end def validate_file! - render_api_error!('Uploaded file is invalid', 400) unless file_is_valid? + render_api_error!({ error: _('You need to upload a GitLab project export archive (ending in .gz).') }, 422) unless file_is_valid? end end end diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index e0fea4c7c96..f3dfc093926 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -11,6 +11,8 @@ module API optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group' + # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 + optional :avatar, type: File, desc: 'Avatar image for the group' # rubocop:disable Scalability/FileUploads optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users in this group to setup Two-factor authentication' optional :two_factor_grace_period, type: Integer, desc: 'Time before Two-factor authentication is enforced' @@ -21,6 +23,7 @@ module API optional :mentions_disabled, type: Boolean, desc: 'Disable a group from getting mentioned' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to master' end params :optional_params_ee do diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index ab43096a1de..f7aabc8ce4f 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -3,7 +3,7 @@ module API module Helpers module InternalHelpers - attr_reader :redirected_path + attr_reader :redirected_path, :container delegate :wiki?, to: :repo_type @@ -22,10 +22,10 @@ module API end def access_checker_for(actor, protocol) - access_checker_klass.new(actor.key_or_user, project, protocol, + access_checker_klass.new(actor.key_or_user, container, protocol, authentication_abilities: ssh_authentication_abilities, namespace_path: namespace_path, - project_path: project_path, + repository_path: project_path, redirected_path: redirected_path) end @@ -80,7 +80,7 @@ module API # rubocop:disable Gitlab/ModuleWithInstanceVariables def set_project - @project, @repo_type, @redirected_path = + @container, @project, @repo_type, @redirected_path = if params[:gl_repository] Gitlab::GlRepository.parse(params[:gl_repository]) elsif params[:project] @@ -92,17 +92,17 @@ module API # Project id to pass between components that don't share/don't have # access to the same filesystem mounts def gl_repository - repo_type.identifier_for_container(project) + repo_type.identifier_for_container(container) end - def gl_project_path + def gl_repository_path repository.full_path end # Return the repository depending on whether we want the wiki or the # regular repository def repository - @repository ||= repo_type.repository_for(project) + @repository ||= repo_type.repository_for(container) end # Return the Gitaly Address if it is enabled @@ -111,8 +111,8 @@ module API { repository: repository.gitaly_repository, - address: Gitlab::GitalyClient.address(project.repository_storage), - token: Gitlab::GitalyClient.token(project.repository_storage), + address: Gitlab::GitalyClient.address(container.repository_storage), + token: Gitlab::GitalyClient.token(container.repository_storage), features: Feature::Gitaly.server_feature_flags } end diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 3c453953e37..bed0345a608 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -62,7 +62,7 @@ module API def get_note(noteable, note_id) note = noteable.notes.with_metadata.find(note_id) - can_read_note = note.visible_for?(current_user) + can_read_note = note.readable_by?(current_user) if can_read_note present note, with: Entities::Note diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index c7c9f3ba077..85ed8a4d636 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -54,6 +54,7 @@ module API optional :auto_devops_enabled, type: Boolean, desc: 'Flag indication if Auto DevOps is enabled' optional :auto_devops_deploy_strategy, type: String, values: %w(continuous manual timed_incremental), desc: 'Auto Deploy strategy' optional :autoclose_referenced_issues, type: Boolean, desc: 'Flag indication if referenced issues auto-closing is enabled' + optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins' end params :optional_project_params_ee do @@ -125,6 +126,7 @@ module API :wiki_access_level, :avatar, :suggestion_commit_message, + :repository_storage, # TODO: remove in API v5, replaced by *_access_level :issues_enabled, diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 382bbeb66de..9c37b610cca 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -40,7 +40,7 @@ module API # Stores some Git-specific env thread-safely env = parse_env - Gitlab::Git::HookEnv.set(gl_repository, env) if project + Gitlab::Git::HookEnv.set(gl_repository, env) if container actor.update_last_used_at! access_checker = access_checker_for(actor, params[:protocol]) @@ -49,7 +49,11 @@ module API result = access_checker.check(params[:action], params[:changes]) @project ||= access_checker.project result - rescue Gitlab::GitAccess::UnauthorizedError => e + rescue Gitlab::GitAccess::ForbiddenError => e + # The return code needs to be 401. If we return 403 + # the custom message we return won't be shown to the user + # and, instead, the default message 'GitLab: API is not accessible' + # will be displayed return response_with_status(code: 401, success: false, message: e.message) rescue Gitlab::GitAccess::TimeoutError => e return response_with_status(code: 503, success: false, message: e.message) @@ -63,7 +67,7 @@ module API when ::Gitlab::GitAccessResult::Success payload = { gl_repository: gl_repository, - gl_project_path: gl_project_path, + gl_project_path: gl_repository_path, gl_id: Gitlab::GlId.gl_id(actor.user), gl_username: actor.username, git_config_options: [], @@ -104,6 +108,10 @@ module API # check_ip - optional, only in EE version, may limit access to # group resources based on its IP restrictions post "/allowed" do + if repo_type.snippet? && Feature.disabled?(:version_snippets, actor.user) + break response_with_status(code: 404, success: false, message: 'The project you were looking for could not be found.') + end + # It was moved to a separate method so that EE can alter its behaviour more # easily. check_allowed(params) @@ -212,7 +220,7 @@ module API post '/post_receive' do status 200 - response = PostReceiveService.new(actor.user, project, params).execute + response = PostReceiveService.new(actor.user, repository, project, params).execute ee_post_receive_response_hook(response) diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb index a2fe3e09df8..4339d2ef490 100644 --- a/lib/api/internal/pages.rb +++ b/lib/api/internal/pages.rb @@ -24,13 +24,26 @@ module API requires :host, type: String, desc: 'The host to query for' end get "/" do - host = Namespace.find_by_pages_host(params[:host]) || PagesDomain.find_by_domain(params[:host]) - no_content! unless host + serverless_domain_finder = ServerlessDomainFinder.new(params[:host]) + if serverless_domain_finder.serverless? + # Handle Serverless domains + serverless_domain = serverless_domain_finder.execute + no_content! unless serverless_domain - virtual_domain = host.pages_virtual_domain - no_content! unless virtual_domain + virtual_domain = Serverless::VirtualDomain.new(serverless_domain) + no_content! unless virtual_domain - present virtual_domain, with: Entities::Internal::Pages::VirtualDomain + present virtual_domain, with: Entities::Internal::Serverless::VirtualDomain + else + # Handle Pages domains + host = Namespace.find_by_pages_host(params[:host]) || PagesDomain.find_by_domain_case_insensitive(params[:host]) + no_content! unless host + + virtual_domain = host.pages_virtual_domain + no_content! unless virtual_domain + + present virtual_domain, with: Entities::Internal::Pages::VirtualDomain + end end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index e5bfca13d66..d34c205281a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -247,6 +247,7 @@ module API requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' optional :title, type: String, desc: 'The title of an issue' optional :updated_at, type: DateTime, + allow_blank: false, desc: 'Date time when the issue was updated. Available only for admins and project owners.' optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' use :issue_params diff --git a/lib/api/lsif_data.rb b/lib/api/lsif_data.rb index 63e6eb3ab2d..a673ccb4af0 100644 --- a/lib/api/lsif_data.rb +++ b/lib/api/lsif_data.rb @@ -15,22 +15,24 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do segment ':id/commits/:commit_id' do params do - requires :path, type: String, desc: 'The path of a file' + requires :paths, type: Array, desc: 'The paths of the files' end get 'lsif/info' do authorize! :download_code, user_project artifact = - @project.job_artifacts + Ci::JobArtifact .with_file_types(['lsif']) - .for_sha(params[:commit_id]) + .for_sha(params[:commit_id], @project.id) .last not_found! unless artifact authorize! :read_pipeline, artifact.job.pipeline file_too_large! if artifact.file.cached_size > MAX_FILE_SIZE - ::Projects::LsifDataService.new(artifact.file, @project, params).execute + service = ::Projects::LsifDataService.new(artifact.file, @project, params[:commit_id]) + + params[:paths].to_h { |path| [path, service.execute(path)] } end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 35eda481a4f..7237fa24bab 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -45,7 +45,7 @@ module API # array returned, but this is really a edge-case. notes = paginate(raw_notes) notes = prepare_notes_for_rendering(notes) - notes = notes.select { |note| note.visible_for?(current_user) } + notes = notes.select { |note| note.readable_by?(current_user) } present notes, with: Entities::Note end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 445a37a70c0..edc99590cdb 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -22,7 +22,7 @@ module API get ':id/pipeline_schedules' do authorize! :read_pipeline_schedule, user_project - schedules = PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope]) + schedules = Ci::PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope]) .preload([:owner, :last_pipeline]) present paginate(schedules), with: Entities::PipelineSchedule end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 66a183173af..06f8920b37c 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -27,7 +27,7 @@ module API optional :username, type: String, desc: 'The username of the user who triggered pipelines' optional :updated_before, type: DateTime, desc: 'Return pipelines updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' optional :updated_after, type: DateTime, desc: 'Return pipelines updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' - optional :order_by, type: String, values: PipelinesFinder::ALLOWED_INDEXED_COLUMNS, default: 'id', + optional :order_by, type: String, values: Ci::PipelinesFinder::ALLOWED_INDEXED_COLUMNS, default: 'id', desc: 'Order pipelines' optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Sort pipelines' @@ -36,7 +36,7 @@ module API authorize! :read_pipeline, user_project authorize! :read_build, user_project - pipelines = PipelinesFinder.new(user_project, current_user, params).execute + pipelines = Ci::PipelinesFinder.new(user_project, current_user, params).execute present paginate(pipelines), with: Entities::PipelineBasic end diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index 70c913bea98..555fd98b451 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -69,7 +69,11 @@ module API end params do requires :repository_id, type: Integer, desc: 'The ID of the repository' - requires :name_regex, type: String, desc: 'The tag name regexp to delete, specify .* to delete all' + optional :name_regex_delete, type: String, desc: 'The tag name regexp to delete, specify .* to delete all' + optional :name_regex, type: String, desc: 'The tag name regexp to delete, specify .* to delete all' + # require either name_regex (deprecated) or name_regex_delete, it is ok to have both + at_least_one_of :name_regex, :name_regex_delete + optional :name_regex_keep, type: String, desc: 'The tag name regexp to retain' optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name' optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month' end diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index ea793a09f6c..ffa9dd13754 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -4,6 +4,8 @@ module API class ProjectImport < Grape::API include PaginationParams + MAXIMUM_FILE_SIZE = 50.megabytes + helpers Helpers::ProjectsHelpers helpers Helpers::FileUploadHelpers @@ -26,10 +28,21 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Workhorse authorize the project import upload' do + detail 'This feature was introduced in GitLab 12.9' + end + post 'import/authorize' do + require_gitlab_workhorse! + + status 200 + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + + ImportExportUploader.workhorse_authorize(has_length: false, maximum_size: MAXIMUM_FILE_SIZE) + end + params do requires :path, type: String, desc: 'The new project path and name' - # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 - requires :file, type: File, desc: 'The project export file to be imported' # rubocop:disable Scalability/FileUploads + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The project export file to be imported' optional :name, type: String, desc: 'The name of the project to be imported. Defaults to the path of the project if not provided.' optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace." optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it' @@ -38,12 +51,24 @@ module API desc: 'New project params to override values in the export' do use :optional_project_params end + optional 'file.path', type: String, desc: 'Path to locally stored body (generated by Workhorse)' + optional 'file.name', type: String, desc: 'Real filename as send in Content-Disposition (generated by Workhorse)' + optional 'file.type', type: String, desc: 'Real content type as send in Content-Type (generated by Workhorse)' + optional 'file.size', type: Integer, desc: 'Real size of file (generated by Workhorse)' + optional 'file.md5', type: String, desc: 'MD5 checksum of the file (generated by Workhorse)' + optional 'file.sha1', type: String, desc: 'SHA1 checksum of the file (generated by Workhorse)' + optional 'file.sha256', type: String, desc: 'SHA256 checksum of the file (generated by Workhorse)' + optional 'file.etag', type: String, desc: 'Etag of the file (generated by Workhorse)' + optional 'file.remote_id', type: String, desc: 'Remote_id of the file (generated by Workhorse)' + optional 'file.remote_url', type: String, desc: 'Remote_url of the file (generated by Workhorse)' end desc 'Create a new project import' do detail 'This feature was introduced in GitLab 10.6.' success Entities::ProjectImportStatus end post 'import' do + require_gitlab_workhorse! + key = "project_import".to_sym if throttled?(key, [current_user, key]) @@ -52,10 +77,10 @@ module API render_api_error!({ error: _('This endpoint has been requested too many times. Try again later.') }, 429) end - validate_file! - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42437') + validate_file! + namespace = if import_params[:namespace] find_namespace!(import_params[:namespace]) else @@ -66,7 +91,7 @@ module API path: import_params[:path], namespace_id: namespace.id, name: import_params[:name], - file: import_params[:file]['tempfile'], + file: import_params[:file], overwrite: import_params[:overwrite] } diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 3040c3c27c6..e8234a9285c 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -5,12 +5,17 @@ module API include PaginationParams before { authenticate! } + before { check_snippets_enabled } params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do + def check_snippets_enabled + forbidden! unless user_project.feature_available?(:snippets, current_user) + end + def handle_project_member_errors(errors) if errors[:project_access].any? error!(errors[:project_access], 422) diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 2271131ced3..3717e25d997 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -25,6 +25,7 @@ module API end def verify_update_project_attrs!(project, attrs) + attrs.delete(:repository_storage) unless can?(current_user, :change_repository_storage, project) end def delete_project(user_project) @@ -176,6 +177,7 @@ module API use :create_params end post do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/21139') attrs = declared_params(include_missing: false) attrs = translate_params_for_compatibility(attrs) filter_attributes_using_license!(attrs) @@ -208,6 +210,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord post "user/:user_id" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/21139') authenticated_as_admin! user = User.find_by(id: params.delete(:user_id)) not_found!('User') unless user @@ -260,32 +263,40 @@ module API success Entities::Project end params do - optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' + optional :namespace, type: String, desc: '(deprecated) The ID or name of the namespace that the project will be forked into' + optional :namespace_id, type: Integer, desc: 'The ID of the namespace that the project will be forked into' + optional :namespace_path, type: String, desc: 'The path of the namespace that the project will be forked into' optional :path, type: String, desc: 'The path that will be assigned to the fork' optional :name, type: String, desc: 'The name that will be assigned to the fork' end post ':id/fork' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42284') - fork_params = declared_params(include_missing: false) - namespace_id = fork_params[:namespace] + not_found! unless can?(current_user, :fork_project, user_project) - if namespace_id.present? - fork_params[:namespace] = find_namespace(namespace_id) + fork_params = declared_params(include_missing: false) - unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) - not_found!('Target Namespace') + fork_params[:namespace] = + if fork_params[:namespace_id].present? + find_namespace!(fork_params[:namespace_id]) + elsif fork_params[:namespace_path].present? + find_namespace_by_path!(fork_params[:namespace_path]) + elsif fork_params[:namespace].present? + find_namespace!(fork_params[:namespace]) end - end - forked_project = ::Projects::ForkService.new(user_project, current_user, fork_params).execute + service = ::Projects::ForkService.new(user_project, current_user, fork_params) + + not_found!('Target Namespace') unless service.valid_fork_target? + + forked_project = service.execute if forked_project.errors.any? conflict!(forked_project.errors.messages) else present forked_project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, forked_project), - current_user: current_user + user_can_admin_project: can?(current_user, :admin_project, forked_project), + current_user: current_user end end @@ -496,7 +507,9 @@ module API requires :file, type: File, desc: 'The file to be uploaded' # rubocop:disable Scalability/FileUploads end post ":id/uploads" do - UploadService.new(user_project, params[:file]).execute.to_h + upload = UploadService.new(user_project, params[:file]).execute + + present upload, with: Entities::ProjectUpload end desc 'Get the users list of a project' do diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb index def36dc8529..f72230c084c 100644 --- a/lib/api/release/links.rb +++ b/lib/api/release/links.rb @@ -39,6 +39,7 @@ module API params do requires :name, type: String, desc: 'The name of the link' requires :url, type: String, desc: 'The URL of the link' + optional :filepath, type: String, desc: 'The filepath of the link' end post 'links' do authorize! :create_release, release @@ -73,6 +74,7 @@ module API params do optional :name, type: String, desc: 'The name of the link' optional :url, type: String, desc: 'The URL of the link' + optional :filepath, type: String, desc: 'The filepath of the link' at_least_one_of :name, :url end put do diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 6e7a99bf0bb..1be263ac80d 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -46,7 +46,7 @@ module API params do requires :tag_name, type: String, desc: 'The name of the tag', as: :tag optional :name, type: String, desc: 'The name of the release' - requires :description, type: String, desc: 'The release notes' + optional :description, type: String, desc: 'The release notes' optional :ref, type: String, desc: 'The commit sha or branch name' optional :assets, type: Hash do optional :links, type: Array do diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb index 95313966133..7e484eb8885 100644 --- a/lib/api/remote_mirrors.rb +++ b/lib/api/remote_mirrors.rb @@ -5,9 +5,6 @@ module API include PaginationParams before do - # TODO: Remove flag: https://gitlab.com/gitlab-org/gitlab/issues/38121 - not_found! unless Feature.enabled?(:remote_mirrors_api, user_project) - unauthorized! unless can?(current_user, :admin_remote_mirror, user_project) end @@ -26,6 +23,28 @@ module API with: Entities::RemoteMirror end + desc 'Create remote mirror for a project' do + success Entities::RemoteMirror + end + params do + requires :url, type: String, desc: 'The URL for a remote mirror' + optional :enabled, type: Boolean, desc: 'Determines if the mirror is enabled' + optional :only_protected_branches, type: Boolean, desc: 'Determines if only protected branches are mirrored' + optional :keep_divergent_refs, type: Boolean, desc: 'Determines if divergent refs are kept on the target' + end + post ':id/remote_mirrors' do + create_params = declared_params(include_missing: false) + create_params.delete(:keep_divergent_refs) unless ::Feature.enabled?(:keep_divergent_refs, user_project) + + new_mirror = user_project.remote_mirrors.create(create_params) + + if new_mirror.persisted? + present new_mirror, with: Entities::RemoteMirror + else + render_validation_error!(new_mirror) + end + end + desc 'Update the attributes of a single remote mirror' do success Entities::RemoteMirror end @@ -33,12 +52,15 @@ module API requires :mirror_id, type: String, desc: 'The ID of a remote mirror' optional :enabled, type: Boolean, desc: 'Determines if the mirror is enabled' optional :only_protected_branches, type: Boolean, desc: 'Determines if only protected branches are mirrored' + optional :keep_divergent_refs, type: Boolean, desc: 'Determines if divergent refs are kept on the target' end put ':id/remote_mirrors/:mirror_id' do mirror = user_project.remote_mirrors.find(params[:mirror_id]) mirror_params = declared_params(include_missing: false) mirror_params[:id] = mirror_params.delete(:mirror_id) + mirror_params.delete(:keep_divergent_refs) unless ::Feature.enabled?(:keep_divergent_refs, user_project) + update_params = { remote_mirrors_attributes: mirror_params } result = ::Projects::UpdateService diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 00473db1ff1..62f5b67af1e 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -13,6 +13,8 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do + include ::Gitlab::RateLimitHelpers + def handle_project_member_errors(errors) if errors[:project_access].any? error!(errors[:project_access], 422) @@ -89,6 +91,10 @@ module API optional :format, type: String, desc: 'The archive format' end get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do + if archive_rate_limit_reached?(current_user, user_project) + render_api_error!({ error: ::Gitlab::RateLimitHelpers::ARCHIVE_RATE_LIMIT_REACHED_MESSAGE }, 429) + end + send_git_archive user_project.repository, ref: params[:sha], format: params[:format], append_sha: true rescue not_found!('File') diff --git a/lib/api/runner.rb b/lib/api/runner.rb index e1c79aa8efe..0b6bad6708b 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -283,10 +283,12 @@ module API bad_request!('Missing artifacts file!') unless artifacts file_too_large! unless artifacts.size < max_artifacts_size(job) - if Ci::CreateJobArtifactsService.new.execute(job, artifacts, params, metadata_file: metadata) + result = Ci::CreateJobArtifactsService.new(job.project).execute(job, artifacts, params, metadata_file: metadata) + + if result[:status] == :success status :created else - render_validation_error!(job) + render_api_error!(result[:message], result[:http_status]) end end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index c2d371b6867..eba1b5499d0 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -115,7 +115,7 @@ module API params do requires :id, type: Integer, desc: 'The ID of the runner' optional :status, type: String, desc: 'Status of the job', values: Ci::Build::AVAILABLE_STATUSES - optional :order_by, type: String, desc: 'Order by `id` or not', values: RunnerJobsFinder::ALLOWED_INDEXED_COLUMNS + optional :order_by, type: String, desc: 'Order by `id` or not', values: Ci::RunnerJobsFinder::ALLOWED_INDEXED_COLUMNS optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Sort by asc (ascending) or desc (descending)' use :pagination end @@ -123,7 +123,7 @@ module API runner = get_runner(params[:id]) authenticate_list_runners_jobs!(runner) - jobs = RunnerJobsFinder.new(runner, params).execute + jobs = Ci::RunnerJobsFinder.new(runner, params).execute present paginate(jobs), with: Entities::JobBasicWithProject end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index e3f3aca27df..02b8bb55274 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -13,13 +13,6 @@ module API 'issues' => ->(iid) { find_project_issue(iid) } }.freeze - helpers do - # EE::API::Todos would override this method - def find_todos - TodosFinder.new(current_user, params).execute - end - end - params do requires :id, type: String, desc: 'The ID of a project' end @@ -48,6 +41,10 @@ module API resource :todos do helpers do + def find_todos + TodosFinder.new(current_user, params).execute + end + def issuable_and_awardable?(type) obj_type = Object.const_get(type, false) diff --git a/lib/api/users.rb b/lib/api/users.rb index c6dc7c08b11..1ca222b4ed5 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -52,8 +52,8 @@ module API optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 optional :avatar, type: File, desc: 'Avatar image for user' # rubocop:disable Scalability/FileUploads - optional :theme_id, type: Integer, default: 1, desc: 'The GitLab theme for the user' - optional :color_scheme_id, type: Integer, default: 1, desc: 'The color scheme for the file viewer' + optional :theme_id, type: Integer, desc: 'The GitLab theme for the user' + optional :color_scheme_id, type: Integer, desc: 'The color scheme for the file viewer' optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' all_or_none_of :extern_uid, :provider @@ -308,7 +308,7 @@ module API desc 'Add a GPG key to a specified user. Available only for admins.' do detail 'This feature was added in GitLab 10.0' - success Entities::GPGKey + success Entities::GpgKey end params do requires :id, type: Integer, desc: 'The ID of the user' @@ -324,7 +324,7 @@ module API key = user.gpg_keys.new(declared_params(include_missing: false)) if key.save - present key, with: Entities::GPGKey + present key, with: Entities::GpgKey else render_validation_error!(key) end @@ -333,7 +333,7 @@ module API desc 'Get the GPG keys of a specified user. Available only for admins.' do detail 'This feature was added in GitLab 10.0' - success Entities::GPGKey + success Entities::GpgKey end params do requires :id, type: Integer, desc: 'The ID of the user' @@ -346,7 +346,7 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user - present paginate(user.gpg_keys), with: Entities::GPGKey + present paginate(user.gpg_keys), with: Entities::GpgKey end # rubocop: enable CodeReuse/ActiveRecord @@ -528,11 +528,18 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user - if !user.ldap_blocked? - user.block - else + if user.ldap_blocked? forbidden!('LDAP blocked users cannot be modified by the API') end + + break if user.blocked? + + result = ::Users::BlockService.new(current_user).execute(user) + if result[:status] == :success + true + else + render_api_error!(result[:message], result[:http_status]) + end end # rubocop: enable CodeReuse/ActiveRecord @@ -739,18 +746,18 @@ module API desc "Get the currently authenticated user's GPG keys" do detail 'This feature was added in GitLab 10.0' - success Entities::GPGKey + success Entities::GpgKey end params do use :pagination end get 'gpg_keys' do - present paginate(current_user.gpg_keys), with: Entities::GPGKey + present paginate(current_user.gpg_keys), with: Entities::GpgKey end desc 'Get a single GPG key owned by currently authenticated user' do detail 'This feature was added in GitLab 10.0' - success Entities::GPGKey + success Entities::GpgKey end params do requires :key_id, type: Integer, desc: 'The ID of the GPG key' @@ -760,13 +767,13 @@ module API key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key - present key, with: Entities::GPGKey + present key, with: Entities::GpgKey end # rubocop: enable CodeReuse/ActiveRecord desc 'Add a new GPG key to the currently authenticated user' do detail 'This feature was added in GitLab 10.0' - success Entities::GPGKey + success Entities::GpgKey end params do requires :key, type: String, desc: 'The new GPG key' @@ -775,7 +782,7 @@ module API key = current_user.gpg_keys.new(declared_params) if key.save - present key, with: Entities::GPGKey + present key, with: Entities::GpgKey else render_validation_error!(key) end diff --git a/lib/api/version.rb b/lib/api/version.rb index f79bb3428f2..2d8c90260fa 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -3,6 +3,9 @@ module API class Version < Grape::API helpers ::API::Helpers::GraphqlHelpers + include APIGuard + + allow_access_with_scope :read_user, if: -> (request) { request.get? } before { authenticate! } diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 2b6b10cf044..915567f8106 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -12,7 +12,7 @@ module Backup @progress = progress end - def pack + def write_info # Make sure there is a connection ActiveRecord::Base.connection.reconnect! @@ -20,7 +20,11 @@ module Backup File.open("#{backup_path}/backup_information.yml", "w+") do |file| file << backup_information.to_yaml.gsub(/^---\n/, '') end + end + end + def pack + Dir.chdir(backup_path) do # create archive progress.print "Creating backup archive: #{tar_file} ... " # Set file permissions on open to prevent chmod races. @@ -31,8 +35,6 @@ module Backup puts "creating archive #{tar_file} failed".color(:red) raise Backup::Error, 'Backup failed' end - - upload end end @@ -105,8 +107,30 @@ module Backup end end - # rubocop: disable Metrics/AbcSize + def verify_backup_version + Dir.chdir(backup_path) do + # restoring mismatching backups can lead to unexpected problems + if settings[:gitlab_version] != Gitlab::VERSION + progress.puts(<<~HEREDOC.color(:red)) + GitLab version mismatch: + Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup! + Please switch to the following version and try again: + version: #{settings[:gitlab_version]} + HEREDOC + progress.puts + progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" + exit 1 + end + end + end + def unpack + if ENV['BACKUP'].blank? && non_tarred_backup? + progress.puts "Non tarred backup found in #{backup_path}, using that" + + return false + end + Dir.chdir(backup_path) do # check for existing backups in the backup dir if backup_file_list.empty? @@ -141,21 +165,6 @@ module Backup progress.puts 'unpacking backup failed'.color(:red) exit 1 end - - ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 - - # restoring mismatching backups can lead to unexpected problems - if settings[:gitlab_version] != Gitlab::VERSION - progress.puts(<<~HEREDOC.color(:red)) - GitLab version mismatch: - Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup! - Please switch to the following version and try again: - version: #{settings[:gitlab_version]} - HEREDOC - progress.puts - progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" - exit 1 - end end end @@ -170,6 +179,10 @@ module Backup private + def non_tarred_backup? + File.exist?(File.join(backup_path, 'backup_information.yml')) + end + def backup_path Gitlab.config.backup.path end @@ -252,7 +265,7 @@ module Backup def create_attributes attrs = { key: remote_target, - body: File.open(tar_file), + body: File.open(File.join(backup_path, tar_file)), multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, encryption: Gitlab.config.backup.upload.encryption, encryption_key: Gitlab.config.backup.upload.encryption_key, diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 123a695be13..1c5108b12ab 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -4,7 +4,6 @@ require 'yaml' module Backup class Repository - include Gitlab::ShellAdapter attr_reader :progress def initialize(progress) @@ -71,23 +70,14 @@ module Backup def restore Project.find_each(batch_size: 1000) do |project| progress.print " * #{project.full_path} ... " - path_to_project_bundle = path_to_bundle(project) - project.repository.remove rescue nil - restore_repo_success = nil - - if File.exist?(path_to_project_bundle) + restore_repo_success = begin - project.repository.create_from_bundle(path_to_project_bundle) - restore_custom_hooks(project) - restore_repo_success = true - rescue => e - restore_repo_success = false - progress.puts "Error: #{e}".color(:red) + try_restore_repository(project) + rescue => err + progress.puts "Error: #{err}".color(:red) + false end - else - restore_repo_success = gitlab_shell.create_project_repository(project) - end if restore_repo_success progress.puts "[DONE]".color(:green) @@ -118,6 +108,20 @@ module Backup protected + def try_restore_repository(project) + path_to_project_bundle = path_to_bundle(project) + project.repository.remove rescue nil + + if File.exist?(path_to_project_bundle) + project.repository.create_from_bundle(path_to_project_bundle) + restore_custom_hooks(project) + else + project.repository.create_repository + end + + true + end + def path_to_bundle(project) File.join(backup_repos_path, project.disk_path + '.bundle') end diff --git a/lib/banzai/filter/broadcast_message_placeholders_filter.rb b/lib/banzai/filter/broadcast_message_placeholders_filter.rb new file mode 100644 index 00000000000..5b5e2f643c5 --- /dev/null +++ b/lib/banzai/filter/broadcast_message_placeholders_filter.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # Replaces placeholders for broadcast messages with data from the current + # user or the instance. + class BroadcastMessagePlaceholdersFilter < HTML::Pipeline::Filter + def call + return doc unless context[:broadcast_message_placeholders] + + doc.traverse { |node| replace_placeholders(node) } + end + + private + + def replace_placeholders(node) + if node.text? && !node.content.empty? + node.content = replace_content(node.content) + elsif href = link_href(node) + href.value = replace_content(href.value, url_safe_encoding: true) + end + + node + end + + def link_href(node) + node.element? && + node.name == 'a' && + node.attribute_nodes.find { |a| a.name == "href" } + end + + def replace_content(content, url_safe_encoding: false) + placeholders.each do |placeholder, method| + regex = Regexp.new("{{#{placeholder}}}|#{CGI.escape("{{#{placeholder}}}")}") + value = url_safe_encoding ? CGI.escape(method.call.to_s) : method.call.to_s + content.gsub!(regex, value) + end + + content + end + + def placeholders + { + "email" => -> { current_user.try(:email) }, + "name" => -> { current_user.try(:name) }, + "user_id" => -> { current_user.try(:id) }, + "username" => -> { current_user.try(:username) }, + "instance_id" => -> { Gitlab::CurrentSettings.try(:uuid) } + } + end + + def current_user + context[:current_user] + end + end + end +end diff --git a/lib/banzai/filter/inline_embeds_filter.rb b/lib/banzai/filter/inline_embeds_filter.rb index 9f1ef0796f0..d7d78cf1866 100644 --- a/lib/banzai/filter/inline_embeds_filter.rb +++ b/lib/banzai/filter/inline_embeds_filter.rb @@ -21,11 +21,18 @@ module Banzai doc end - # Implement in child class. + # Child class must provide the metrics_dashboard_url. # # Return a Nokogiri::XML::Element to embed in the - # markdown. + # markdown which provides a url to the metric_dashboard endpoint where + # data can be requested through a prometheus proxy. InlineMetricsRedactorFilter + # is responsible for premissions to see this div (and relies on the class 'js-render-metrics' ). def create_element(params) + doc.document.create_element( + 'div', + class: 'js-render-metrics', + 'data-dashboard-url': metrics_dashboard_url(params) + ) end # Implement in child class unless overriding #embed_params @@ -60,6 +67,21 @@ module Banzai link_pattern.match(url) { |m| m.named_captures } end + + # Parses query params out from full url string into hash. + # + # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' + # --> { title: 'Title', group: 'Group' } + def query_params(url) + Gitlab::Metrics::Dashboard::Url.parse_query(url) + end + + # Implement in child class. + # + # Provides a full url to request the relevant panels of metric data. + def metrics_dashboard_url + raise NotImplementedError + end end end end diff --git a/lib/banzai/filter/inline_grafana_metrics_filter.rb b/lib/banzai/filter/inline_grafana_metrics_filter.rb index 321580b532f..07bde9858e8 100644 --- a/lib/banzai/filter/inline_grafana_metrics_filter.rb +++ b/lib/banzai/filter/inline_grafana_metrics_filter.rb @@ -10,20 +10,17 @@ module Banzai def create_element(params) begin_loading_dashboard(params[:url]) - doc.document.create_element( - 'div', - class: 'js-render-metrics', - 'data-dashboard-url': metrics_dashboard_url(params) - ) + super end + # @return [Hash<Symbol, String>] with keys :grafana_url, :start, and :end def embed_params(node) query_params = Gitlab::Metrics::Dashboard::Url.parse_query(node['href']) - return unless [:panelId, :from, :to].all? do |param| - query_params.include?(param) - end - { url: node['href'], start: query_params[:from], end: query_params[:to] } + time_window = Grafana::TimeWindow.new(query_params[:from], query_params[:to]) + url = url_with_window(node['href'], query_params, time_window.in_milliseconds) + + { grafana_url: url }.merge(time_window.formatted) end # Selects any links with an href contains the configured @@ -48,18 +45,24 @@ module Banzai Gitlab::Routing.url_helpers.project_grafana_api_metrics_dashboard_url( project, embedded: true, - grafana_url: params[:url], - start: format_time(params[:start]), - end: format_time(params[:end]) + **params ) end - # Formats a timestamp from Grafana for compatibility with - # parsing in JS via `new Date(timestamp)` + # If the provided url is missing time window parameters, + # this inserts the default window into the url, allowing + # the embed service to correctly format prometheus + # queries during embed processing. # - # @param time [String] Represents miliseconds since epoch - def format_time(time) - Time.at(time.to_i / 1000).utc.strftime('%FT%TZ') + # @param url [String] + # @param query_params [Hash<Symbol, String>] + # @param time_window_params [Hash<Symbol, Integer>] + # @return [String] + def url_with_window(url, query_params, time_window_params) + uri = URI(url) + uri.query = time_window_params.merge(query_params).to_query + + uri.to_s end # Fetches a dashboard and caches the result for the diff --git a/lib/banzai/filter/inline_metrics_filter.rb b/lib/banzai/filter/inline_metrics_filter.rb index c1f4bf1f97f..409e8db87f4 100644 --- a/lib/banzai/filter/inline_metrics_filter.rb +++ b/lib/banzai/filter/inline_metrics_filter.rb @@ -5,21 +5,12 @@ module Banzai # HTML filter that inserts a placeholder element for each # reference to a metrics dashboard. class InlineMetricsFilter < Banzai::Filter::InlineEmbedsFilter - # Placeholder element for the frontend to use as an - # injection point for charts. - def create_element(params) - doc.document.create_element( - 'div', - class: 'js-render-metrics', - 'data-dashboard-url': metrics_dashboard_url(params) - ) - end - # Search params for selecting metrics links. A few # simple checks is enough to boost performance without # the cost of doing a full regex match. def xpath_search "descendant-or-self::a[contains(@href,'metrics') and \ + contains(@href,'environments') and \ starts-with(@href, '#{Gitlab.config.gitlab.url}')]" end @@ -41,14 +32,6 @@ module Banzai **query_params(params['url']) ) end - - # Parses query params out from full url string into hash. - # - # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' - # --> { title: 'Title', group: 'Group' } - def query_params(url) - Gitlab::Metrics::Dashboard::Url.parse_query(url) - end end end end diff --git a/lib/banzai/filter/inline_metrics_redactor_filter.rb b/lib/banzai/filter/inline_metrics_redactor_filter.rb index ae830831a27..75bd3325bd4 100644 --- a/lib/banzai/filter/inline_metrics_redactor_filter.rb +++ b/lib/banzai/filter/inline_metrics_redactor_filter.rb @@ -9,8 +9,8 @@ module Banzai METRICS_CSS_CLASS = '.js-render-metrics' EMBED_LIMIT = 100 - URL = Gitlab::Metrics::Dashboard::Url + Route = Struct.new(:regex, :permission) Embed = Struct.new(:project_path, :permission) # Finds all embeds based on the css class the FE @@ -59,14 +59,28 @@ module Banzai embed = Embed.new url = node.attribute('data-dashboard-url').to_s - set_path_and_permission(embed, url, URL.metrics_regex, :read_environment) - set_path_and_permission(embed, url, URL.grafana_regex, :read_project) unless embed.permission + permissions_by_route.each do |route| + set_path_and_permission(embed, url, route.regex, route.permission) unless embed.permission + end embeds[node] = embed if embed.permission end end end + def permissions_by_route + [ + Route.new( + ::Gitlab::Metrics::Dashboard::Url.metrics_regex, + :read_environment + ), + Route.new( + ::Gitlab::Metrics::Dashboard::Url.grafana_regex, + :read_project + ) + ] + end + # Attempts to determine the path and permission attributes # of a url based on expected dashboard url formats and # sets the attributes on an Embed object @@ -129,3 +143,5 @@ module Banzai end end end + +Banzai::Filter::InlineMetricsRedactorFilter.prepend_if_ee('EE::Banzai::Filter::InlineMetricsRedactorFilter') diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index f9d8bf8a1fa..a88629ac105 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -18,7 +18,7 @@ module Banzai issuables = extractor.extract([doc]) issuables.each do |node, issuable| - next if !can_read_cross_project? && cross_reference?(issuable) + next if !can_read_cross_project? && cross_referenced?(issuable) if VISIBLE_STATES.include?(issuable.state) && issuable_reference?(node.inner_html, issuable) state = moved_issue?(issuable) ? s_("IssuableStatus|moved") : issuable.state @@ -39,7 +39,7 @@ module Banzai CGI.unescapeHTML(text) == issuable.reference_link_text(project || group) end - def cross_reference?(issuable) + def cross_referenced?(issuable) return true if issuable.project != project return true if issuable.respond_to?(:group) && issuable.group != group diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 609ea8fb5ca..60ffb178393 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -93,23 +93,26 @@ module Banzai end presenter = object.present(issuable_subject: parent) - LabelsHelper.render_colored_label(presenter, label_suffix: label_suffix, title: tooltip_title(presenter)) + LabelsHelper.render_colored_label(presenter, suffix: label_suffix) end - def tooltip_title(label) - nil + def wrap_link(link, label) + presenter = label.present(issuable_subject: project || group) + LabelsHelper.wrap_label_html(link, small: true, label: presenter) end def full_path_ref?(matches) matches[:namespace] && matches[:project] end + def reference_class(type, tooltip: true) + super + ' gl-link gl-label-link' + end + def object_link_title(object, matches) - # use title of wrapped element instead - nil + presenter = object.present(issuable_subject: project || group) + LabelsHelper.label_tooltip_title(presenter) end end end end - -Banzai::Filter::LabelReferenceFilter.prepend_if_ee('EE::Banzai::Filter::LabelReferenceFilter') diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index b3ce9200b49..38bbed3cf72 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -37,7 +37,8 @@ module Banzai attributes[:reference_type] ||= self.class.reference_type attributes[:container] ||= 'body' - attributes[:placement] ||= 'bottom' + attributes[:placement] ||= 'top' + attributes[:html] ||= 'true' attributes.delete(:original) if context[:no_original_data] attributes.map do |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb index d448238c6e4..24900217560 100644 --- a/lib/banzai/filter/repository_link_filter.rb +++ b/lib/banzai/filter/repository_link_filter.rb @@ -80,6 +80,13 @@ module Banzai end Gitlab::GitalyClient::BlobService.new(repository).get_blob_types(revision_paths, 1) + rescue GRPC::Unavailable, GRPC::DeadlineExceeded => e + # Handle Gitaly connection issues gracefully + Gitlab::ErrorTracking.track_exception(e, project_id: project.id) + # Return all links as blob types + paths.collect do |path| + [path, :blob] + end end def get_uri(html_attr) @@ -124,7 +131,7 @@ module Banzai path = cleaned_file_path(uri) nested_path = relative_file_path(uri) - file_exists?(nested_path) ? nested_path : path + path_exists?(nested_path) ? nested_path : path end def cleaned_file_path(uri) @@ -183,12 +190,12 @@ module Banzai parts.push(path).join('/') end - def file_exists?(path) - path.present? && uri_type(path).present? + def path_exists?(path) + path.present? && @uri_types[path] != :unknown end def uri_type(path) - @uri_types[path] == :unknown ? "" : @uri_types[path] + @uri_types[path] == :unknown ? :blob : @uri_types[path] end def current_commit diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index dad0d95685e..b6238dfe7f0 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -29,8 +29,7 @@ module Banzai Filter::AudioLinkFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, - Filter::InlineMetricsFilter, - Filter::InlineGrafanaMetricsFilter, + *metrics_filters, Filter::TableOfContentsFilter, Filter::TableOfContentsTagFilter, Filter::AutolinkFilter, @@ -48,6 +47,13 @@ module Banzai ] end + def self.metrics_filters + [ + Filter::InlineMetricsFilter, + Filter::InlineGrafanaMetricsFilter + ] + end + def self.reference_filters [ Filter::UserReferenceFilter, diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 5e02d972614..8236b702147 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -8,7 +8,8 @@ module Banzai def self.filters @filters ||= FilterArray[ *internal_link_filters, - Filter::AbsoluteLinkFilter + Filter::AbsoluteLinkFilter, + Filter::BroadcastMessagePlaceholdersFilter ] end diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index 9e9df88373a..e51f30af581 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -13,6 +13,8 @@ require_dependency 'declarative_policy/step' require_dependency 'declarative_policy/base' module DeclarativePolicy + extend PreferredScope + CLASS_CACHE_MUTEX = Mutex.new CLASS_CACHE_IVAR = :@__DeclarativePolicy_CLASS_CACHE diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb index 9b7d1548056..d653a0ec1e1 100644 --- a/lib/declarative_policy/preferred_scope.rb +++ b/lib/declarative_policy/preferred_scope.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true module DeclarativePolicy - PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope" + module PreferredScope + PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope" - class << self def with_preferred_scope(scope) Thread.current[PREFERRED_SCOPE_KEY], old_scope = scope, Thread.current[PREFERRED_SCOPE_KEY] yield diff --git a/lib/feature.rb b/lib/feature.rb index aadc2c64957..60a5c03a839 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -38,7 +38,7 @@ class Feature begin # We saw on GitLab.com, this database request was called 2300 # times/s. Let's cache it for a minute to avoid that load. - Gitlab::ThreadMemoryCache.cache_backend.fetch('flipper:persisted_names', expires_in: 1.minute) do + Gitlab::ProcessMemoryCache.cache_backend.fetch('flipper:persisted_names', expires_in: 1.minute) do FlipperFeature.feature_names end end @@ -134,7 +134,11 @@ class Feature end def l1_cache_backend - Gitlab::ThreadMemoryCache.cache_backend + if Gitlab::Utils.to_boolean(ENV['USE_THREAD_MEMORY_CACHE']) + Gitlab::ThreadMemoryCache.cache_backend + else + Gitlab::ProcessMemoryCache.cache_backend + end end def l2_cache_backend diff --git a/lib/gitlab/access/branch_protection.rb b/lib/gitlab/access/branch_protection.rb index f039e5c011f..339a99eb068 100644 --- a/lib/gitlab/access/branch_protection.rb +++ b/lib/gitlab/access/branch_protection.rb @@ -37,6 +37,10 @@ module Gitlab def developer_can_merge? level == PROTECTION_DEV_CAN_MERGE end + + def fully_protected? + level == PROTECTION_FULL + end end end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 49e1f1edfb9..211c59fe841 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -21,8 +21,9 @@ module Gitlab { project_export: { threshold: 1, interval: 5.minutes }, project_download_export: { threshold: 10, interval: 10.minutes }, + project_repositories_archive: { threshold: 5, interval: 1.minute }, project_generate_new_export: { threshold: 1, interval: 5.minutes }, - project_import: { threshold: 30, interval: 10.minutes }, + project_import: { threshold: 30, interval: 5.minutes }, play_pipeline_schedule: { threshold: 1, interval: 1.minute }, show_raw_controller: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute } }.freeze diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 1329357d0b8..c16c2ce96de 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -88,7 +88,7 @@ module Gitlab else # If no user is provided, try LDAP. # LDAP users are only authenticated via LDAP - authenticators << Gitlab::Auth::LDAP::Authentication + authenticators << Gitlab::Auth::Ldap::Authentication end authenticators.compact! @@ -134,7 +134,7 @@ module Gitlab end def authenticate_using_internal_or_ldap_password? - Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::Auth::LDAP::Config.enabled? + Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::Auth::Ldap::Config.enabled? end def service_request_check(login, password, project) diff --git a/lib/gitlab/auth/current_user_mode.rb b/lib/gitlab/auth/current_user_mode.rb index 1ef95c03cfc..06ae4d81870 100644 --- a/lib/gitlab/auth/current_user_mode.rb +++ b/lib/gitlab/auth/current_user_mode.rb @@ -23,15 +23,26 @@ module Gitlab class << self # Admin mode activation requires storing a flag in the user session. Using this - # method when scheduling jobs in Sidekiq will bypass the session check for a - # user that was already in admin mode + # method when scheduling jobs in sessionless environments (e.g. Sidekiq, API) + # will bypass the session check for a user that was already in admin mode + # + # If passed a block, it will surround the block execution and reset the session + # bypass at the end; otherwise use manually '.reset_bypass_session!' def bypass_session!(admin_id) Gitlab::SafeRequestStore[CURRENT_REQUEST_BYPASS_SESSION_ADMIN_ID_RS_KEY] = admin_id Gitlab::AppLogger.debug("Bypassing session in admin mode for: #{admin_id}") - yield - ensure + if block_given? + begin + yield + ensure + reset_bypass_session! + end + end + end + + def reset_bypass_session! Gitlab::SafeRequestStore.delete(CURRENT_REQUEST_BYPASS_SESSION_ADMIN_ID_RS_KEY) end @@ -90,10 +101,6 @@ module Gitlab current_session_data[ADMIN_MODE_START_TIME_KEY] = Time.now end - def enable_sessionless_admin_mode! - request_admin_mode! && enable_admin_mode!(skip_password_validation: true) - end - def disable_admin_mode! return unless user&.admin? diff --git a/lib/gitlab/auth/key_status_checker.rb b/lib/gitlab/auth/key_status_checker.rb new file mode 100644 index 00000000000..af654c0caad --- /dev/null +++ b/lib/gitlab/auth/key_status_checker.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + class KeyStatusChecker + include Gitlab::Utils::StrongMemoize + + attr_reader :key + + def initialize(key) + @key = key + end + + def show_console_message? + console_message.present? + end + + def console_message + strong_memoize(:console_message) do + if key.expired? + _('INFO: Your SSH key has expired. Please generate a new key.') + elsif key.expires_soon? + _('INFO: Your SSH key is expiring soon. Please generate a new key.') + end + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 940b802be7e..98eec0e4a7b 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -6,14 +6,14 @@ # module Gitlab module Auth - module LDAP + module Ldap class Access - prepend_if_ee('::EE::Gitlab::Auth::LDAP::Access') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_if_ee('::EE::Gitlab::Auth::Ldap::Access') # rubocop: disable Cop/InjectEnterpriseEditionModule attr_reader :provider, :user, :ldap_identity def self.open(user, &block) - Gitlab::Auth::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| + Gitlab::Auth::Ldap::Adapter.open(user.ldap_identity.provider) do |adapter| block.call(self.new(user, adapter)) end end @@ -50,7 +50,7 @@ module Gitlab end # Block user in GitLab if they were blocked in AD - if Gitlab::Auth::LDAP::Person.disabled_via_active_directory?(ldap_identity.extern_uid, adapter) + if Gitlab::Auth::Ldap::Person.disabled_via_active_directory?(ldap_identity.extern_uid, adapter) block_user(user, 'is disabled in Active Directory') false else @@ -62,7 +62,7 @@ module Gitlab block_user(user, 'does not exist anymore') false end - rescue LDAPConnectionError + rescue LdapConnectionError false end @@ -73,11 +73,11 @@ module Gitlab private def adapter - @adapter ||= Gitlab::Auth::LDAP::Adapter.new(provider) + @adapter ||= Gitlab::Auth::Ldap::Adapter.new(provider) end def ldap_config - Gitlab::Auth::LDAP::Config.new(provider) + Gitlab::Auth::Ldap::Config.new(provider) end def ldap_user @@ -87,7 +87,7 @@ module Gitlab end def find_ldap_user - Gitlab::Auth::LDAP::Person.find_by_dn(ldap_identity.extern_uid, adapter) + Gitlab::Auth::Ldap::Person.find_by_dn(ldap_identity.extern_uid, adapter) end def block_user(user, reason) diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 356579ef402..c5ec4e1981b 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -2,9 +2,9 @@ module Gitlab module Auth - module LDAP + module Ldap class Adapter - prepend_if_ee('::EE::Gitlab::Auth::LDAP::Adapter') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_if_ee('::EE::Gitlab::Auth::Ldap::Adapter') # rubocop: disable Cop/InjectEnterpriseEditionModule SEARCH_RETRY_FACTOR = [1, 1, 2, 3].freeze MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size.freeze @@ -18,7 +18,7 @@ module Gitlab end def self.config(provider) - Gitlab::Auth::LDAP::Config.new(provider) + Gitlab::Auth::Ldap::Config.new(provider) end def initialize(provider, ldap = nil) @@ -27,7 +27,7 @@ module Gitlab end def config - Gitlab::Auth::LDAP::Config.new(provider) + Gitlab::Auth::Ldap::Config.new(provider) end def users(fields, value, limit = nil) @@ -75,7 +75,7 @@ module Gitlab renew_connection_adapter retry else - raise LDAPConnectionError, error_message + raise LdapConnectionError, error_message end end @@ -91,13 +91,13 @@ module Gitlab end entries.map do |entry| - Gitlab::Auth::LDAP::Person.new(entry, provider) + Gitlab::Auth::Ldap::Person.new(entry, provider) end end def user_options(fields, value, limit) options = { - attributes: Gitlab::Auth::LDAP::Person.ldap_attributes(config), + attributes: Gitlab::Auth::Ldap::Person.ldap_attributes(config), base: config.base } diff --git a/lib/gitlab/auth/ldap/auth_hash.rb b/lib/gitlab/auth/ldap/auth_hash.rb index 83fdc8a8c76..5435355f136 100644 --- a/lib/gitlab/auth/ldap/auth_hash.rb +++ b/lib/gitlab/auth/ldap/auth_hash.rb @@ -4,10 +4,10 @@ # module Gitlab module Auth - module LDAP + module Ldap class AuthHash < Gitlab::Auth::OAuth::AuthHash def uid - @uid ||= Gitlab::Auth::LDAP::Person.normalize_dn(super) + @uid ||= Gitlab::Auth::Ldap::Person.normalize_dn(super) end def username @@ -42,7 +42,7 @@ module Gitlab end def ldap_config - @ldap_config ||= Gitlab::Auth::LDAP::Config.new(self.provider) + @ldap_config ||= Gitlab::Auth::Ldap::Config.new(self.provider) end end end diff --git a/lib/gitlab/auth/ldap/authentication.rb b/lib/gitlab/auth/ldap/authentication.rb index 174e81dd603..d9964f237b1 100644 --- a/lib/gitlab/auth/ldap/authentication.rb +++ b/lib/gitlab/auth/ldap/authentication.rb @@ -8,10 +8,10 @@ module Gitlab module Auth - module LDAP + module Ldap class Authentication < Gitlab::Auth::OAuth::Authentication def self.login(login, password) - return unless Gitlab::Auth::LDAP::Config.enabled? + return unless Gitlab::Auth::Ldap::Config.enabled? return unless login.present? && password.present? # return found user that was authenticated by first provider for given login credentials @@ -22,7 +22,7 @@ module Gitlab end def self.providers - Gitlab::Auth::LDAP::Config.providers + Gitlab::Auth::Ldap::Config.providers end def login(login, password) @@ -33,7 +33,7 @@ module Gitlab ) return unless result - @user = Gitlab::Auth::LDAP::User.find_by_uid_and_provider(result.dn, provider) + @user = Gitlab::Auth::Ldap::User.find_by_uid_and_provider(result.dn, provider) end def adapter @@ -41,7 +41,7 @@ module Gitlab end def config - Gitlab::Auth::LDAP::Config.new(provider) + Gitlab::Auth::Ldap::Config.new(provider) end def user_filter(login) diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 4bc0ceedae7..709cd0d787a 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -3,9 +3,9 @@ # Load a specific server configuration module Gitlab module Auth - module LDAP + module Ldap class Config - prepend_if_ee('::EE::Gitlab::Auth::LDAP::Config') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_if_ee('::EE::Gitlab::Auth::Ldap::Config') # rubocop: disable Cop/InjectEnterpriseEditionModule NET_LDAP_ENCRYPTION_METHOD = { simple_tls: :simple_tls, diff --git a/lib/gitlab/auth/ldap/dn.rb b/lib/gitlab/auth/ldap/dn.rb index 0b496da784d..ea88dedadf5 100644 --- a/lib/gitlab/auth/ldap/dn.rb +++ b/lib/gitlab/auth/ldap/dn.rb @@ -21,7 +21,7 @@ # class also helps take care of that. module Gitlab module Auth - module LDAP + module Ldap class DN FormatError = Class.new(StandardError) MalformedError = Class.new(FormatError) diff --git a/lib/gitlab/auth/ldap/ldap_connection_error.rb b/lib/gitlab/auth/ldap/ldap_connection_error.rb index d0e5f24d203..13b0d29e104 100644 --- a/lib/gitlab/auth/ldap/ldap_connection_error.rb +++ b/lib/gitlab/auth/ldap/ldap_connection_error.rb @@ -2,8 +2,8 @@ module Gitlab module Auth - module LDAP - LDAPConnectionError = Class.new(StandardError) + module Ldap + LdapConnectionError = Class.new(StandardError) end end end diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index 88ec22aa75c..430f94a9a28 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -2,9 +2,9 @@ module Gitlab module Auth - module LDAP + module Ldap class Person - prepend_if_ee('::EE::Gitlab::Auth::LDAP::Person') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_if_ee('::EE::Gitlab::Auth::Ldap::Person') # rubocop: disable Cop/InjectEnterpriseEditionModule # Active Directory-specific LDAP filter that checks if bit 2 of the # userAccountControl attribute is set. @@ -45,8 +45,8 @@ module Gitlab end def self.normalize_dn(dn) - ::Gitlab::Auth::LDAP::DN.new(dn).to_normalized_s - rescue ::Gitlab::Auth::LDAP::DN::FormatError => e + ::Gitlab::Auth::Ldap::DN.new(dn).to_normalized_s + rescue ::Gitlab::Auth::Ldap::DN::FormatError => e Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") # rubocop:disable Gitlab/RailsLogger dn @@ -57,8 +57,8 @@ module Gitlab # 1. Excess spaces are stripped # 2. The string is downcased (for case-insensitivity) def self.normalize_uid(uid) - ::Gitlab::Auth::LDAP::DN.normalize_value(uid) - rescue ::Gitlab::Auth::LDAP::DN::FormatError => e + ::Gitlab::Auth::Ldap::DN.normalize_value(uid) + rescue ::Gitlab::Auth::Ldap::DN::FormatError => e Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") # rubocop:disable Gitlab/RailsLogger uid @@ -103,7 +103,7 @@ module Gitlab attr_reader :entry def config - @config ||= Gitlab::Auth::LDAP::Config.new(provider) + @config ||= Gitlab::Auth::Ldap::Config.new(provider) end # Using the LDAP attributes configuration, find and return the first diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index 3b68230e193..df14e5fc3dc 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -8,10 +8,10 @@ # module Gitlab module Auth - module LDAP + module Ldap class User < Gitlab::Auth::OAuth::User extend ::Gitlab::Utils::Override - prepend_if_ee('::EE::Gitlab::Auth::LDAP::User') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_if_ee('::EE::Gitlab::Auth::Ldap::User') # rubocop: disable Cop/InjectEnterpriseEditionModule class << self # rubocop: disable CodeReuse/ActiveRecord @@ -46,7 +46,7 @@ module Gitlab end def allowed? - Gitlab::Auth::LDAP::Access.allowed?(gl_user) + Gitlab::Auth::Ldap::Access.allowed?(gl_user) end def valid_sign_in? @@ -54,11 +54,11 @@ module Gitlab end def ldap_config - Gitlab::Auth::LDAP::Config.new(auth_hash.provider) + Gitlab::Auth::Ldap::Config.new(auth_hash.provider) end def auth_hash=(auth_hash) - @auth_hash = Gitlab::Auth::LDAP::AuthHash.new(auth_hash) + @auth_hash = Gitlab::Auth::Ldap::AuthHash.new(auth_hash) end end end diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 3d44c83736a..f0811098b15 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -7,7 +7,8 @@ module Gitlab LABELS = { "github" => "GitHub", "gitlab" => "GitLab.com", - "google_oauth2" => "Google" + "google_oauth2" => "Google", + "azure_oauth2" => "Azure AD" }.freeze def self.authentication(user, provider) @@ -17,7 +18,7 @@ module Gitlab authenticator = case provider when /^ldap/ - Gitlab::Auth::LDAP::Authentication + Gitlab::Auth::Ldap::Authentication when 'database' Gitlab::Auth::Database::Authentication end @@ -59,8 +60,8 @@ module Gitlab def self.config_for(name) name = name.to_s if ldap_provider?(name) - if Gitlab::Auth::LDAP::Config.valid_provider?(name) - Gitlab::Auth::LDAP::Config.new(name).options + if Gitlab::Auth::Ldap::Config.valid_provider?(name) + Gitlab::Auth::Ldap::Config.new(name).options else nil end @@ -74,6 +75,12 @@ module Gitlab config = config_for(name) (config && config['label']) || LABELS[name] || name.titleize end + + def self.icon_for(name) + name = name.to_s + config = config_for(name) + config && config['icon'] + end end end end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 300181025a4..df595da1536 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -111,7 +111,7 @@ module Gitlab def find_or_build_ldap_user return unless ldap_person - user = Gitlab::Auth::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) + user = Gitlab::Auth::Ldap::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) if user log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." return user @@ -141,8 +141,8 @@ module Gitlab return @ldap_person if defined?(@ldap_person) # Look for a corresponding person with same uid in any of the configured LDAP providers - Gitlab::Auth::LDAP::Config.providers.each do |provider| - adapter = Gitlab::Auth::LDAP::Adapter.new(provider) + Gitlab::Auth::Ldap::Config.providers.each do |provider| + adapter = Gitlab::Auth::Ldap::Adapter.new(provider) @ldap_person = find_ldap_person(auth_hash, adapter) break if @ldap_person end @@ -150,15 +150,15 @@ module Gitlab end def find_ldap_person(auth_hash, adapter) - Gitlab::Auth::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || - Gitlab::Auth::LDAP::Person.find_by_email(auth_hash.uid, adapter) || - Gitlab::Auth::LDAP::Person.find_by_dn(auth_hash.uid, adapter) - rescue Gitlab::Auth::LDAP::LDAPConnectionError + Gitlab::Auth::Ldap::Person.find_by_uid(auth_hash.uid, adapter) || + Gitlab::Auth::Ldap::Person.find_by_email(auth_hash.uid, adapter) || + Gitlab::Auth::Ldap::Person.find_by_dn(auth_hash.uid, adapter) + rescue Gitlab::Auth::Ldap::LdapConnectionError nil end def ldap_config - Gitlab::Auth::LDAP::Config.new(ldap_person.provider) if ldap_person + Gitlab::Auth::Ldap::Config.new(ldap_person.provider) if ldap_person end def needs_blocking? diff --git a/lib/gitlab/authorized_keys.rb b/lib/gitlab/authorized_keys.rb index 820a78b653c..50cd15b7a10 100644 --- a/lib/gitlab/authorized_keys.rb +++ b/lib/gitlab/authorized_keys.rb @@ -70,7 +70,7 @@ module Gitlab # # @param [String] id identifier of the key to be removed prefixed by `key-` # @return [Boolean] - def rm_key(id) + def remove_key(id) lock do logger.info("Removing key (#{id})") open_authorized_keys_file('r+') do |f| diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb new file mode 100644 index 00000000000..fa6453abefb --- /dev/null +++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will fill the project_repositories table for projects that + # are on hashed storage and an entry is missing in this table. + class BackfillSnippetRepositories + MAX_RETRIES = 2 + + def perform(start_id, stop_id) + Snippet.includes(:author, snippet_repository: :shard).where(id: start_id..stop_id).find_each do |snippet| + # We need to expire the exists? value for the cached method in case it was cached + snippet.repository.expire_exists_cache + + next if repository_present?(snippet) + + retry_index = 0 + + begin + create_repository_and_files(snippet) + + logger.info(message: 'Snippet Migration: repository created and migrated', snippet: snippet.id) + rescue => e + retry_index += 1 + + retry if retry_index < MAX_RETRIES + + logger.error(message: "Snippet Migration: error migrating snippet. Reason: #{e.message}", snippet: snippet.id) + + destroy_snippet_repository(snippet) + delete_repository(snippet) + end + end + end + + private + + def repository_present?(snippet) + snippet.snippet_repository && !snippet.empty_repo? + end + + def create_repository_and_files(snippet) + snippet.create_repository + create_commit(snippet) + end + + def destroy_snippet_repository(snippet) + # Removing the db record + snippet.snippet_repository&.destroy + rescue => e + logger.error(message: "Snippet Migration: error destroying snippet repository. Reason: #{e.message}", snippet: snippet.id) + end + + def delete_repository(snippet) + # Removing the repository in disk + snippet.repository.remove if snippet.repository_exists? + rescue => e + logger.error(message: "Snippet Migration: error deleting repository. Reason: #{e.message}", snippet: snippet.id) + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + + def snippet_action(snippet) + # We don't need the previous_path param + # Because we're not updating any existing file + [{ file_path: filename(snippet), + content: snippet.content }] + end + + def filename(snippet) + snippet.file_name.presence || empty_file_name + end + + def empty_file_name + @empty_file_name ||= "#{SnippetRepository::DEFAULT_EMPTY_FILE_NAME}1.txt" + end + + def commit_attrs + @commit_attrs ||= { branch_name: 'master', message: 'Initial commit' } + end + + def create_commit(snippet) + snippet.snippet_repository.multi_files_action(snippet.author, snippet_action(snippet), commit_attrs) + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_optimistic_locking_nulls.rb b/lib/gitlab/background_migration/cleanup_optimistic_locking_nulls.rb new file mode 100644 index 00000000000..bf69ef352cc --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_optimistic_locking_nulls.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class CleanupOptimisticLockingNulls + QUERY_ITEM_SIZE = 1_000 + + # table - The name of the table the migration is performed for. + # start_id - The ID of the object to start at + # stop_id - The ID of the object to end at + def perform(start_id, stop_id, table) + model = define_model_for(table) + + # After analysis done, a batch size of 1,000 items per query was found to be + # the most optimal. Discussion in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18418#note_282285336 + (start_id..stop_id).each_slice(QUERY_ITEM_SIZE).each do |range| + model + .where(lock_version: nil) + .where("ID BETWEEN ? AND ?", range.first, range.last) + .update_all(lock_version: 0) + end + end + + def define_model_for(table) + Class.new(ActiveRecord::Base) do + self.table_name = table + end + end + end + end +end diff --git a/lib/gitlab/background_migration/link_lfs_objects_projects.rb b/lib/gitlab/background_migration/link_lfs_objects_projects.rb new file mode 100644 index 00000000000..983470c5121 --- /dev/null +++ b/lib/gitlab/background_migration/link_lfs_objects_projects.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Create missing LfsObjectsProject records for forks + class LinkLfsObjectsProjects + # Model specifically used for migration. + class LfsObjectsProject < ActiveRecord::Base + include EachBatch + + self.table_name = 'lfs_objects_projects' + + def self.linkable + where( + <<~SQL + lfs_objects_projects.project_id IN ( + SELECT fork_network_members.forked_from_project_id + FROM fork_network_members + WHERE fork_network_members.forked_from_project_id IS NOT NULL + ) + SQL + ) + end + end + + # Model specifically used for migration. + class ForkNetworkMember < ActiveRecord::Base + include EachBatch + + self.table_name = 'fork_network_members' + + def self.without_lfs_object(lfs_object_id) + where( + <<~SQL + fork_network_members.project_id NOT IN ( + SELECT lop.project_id + FROM lfs_objects_projects lop + WHERE lop.lfs_object_id = #{lfs_object_id} + ) + SQL + ) + end + end + + BATCH_SIZE = 1000 + + def perform(start_id, end_id) + lfs_objects_projects = + Gitlab::BackgroundMigration::LinkLfsObjectsProjects::LfsObjectsProject + .linkable + .where(id: start_id..end_id) + + return if lfs_objects_projects.empty? + + lfs_objects_projects.find_each do |lop| + ForkNetworkMember + .select("#{lop.lfs_object_id}, fork_network_members.project_id, NOW(), NOW()") + .without_lfs_object(lop.lfs_object_id) + .where(forked_from_project_id: lop.project_id) + .each_batch(of: BATCH_SIZE) do |batch, index| + execute <<~SQL + INSERT INTO lfs_objects_projects (lfs_object_id, project_id, created_at, updated_at) + #{batch.to_sql} + SQL + + logger.info(message: "LinkLfsObjectsProjects: created missing LfsObjectsProject records for LfsObject #{lop.lfs_object_id}") + end + end + end + + private + + def execute(sql) + ::ActiveRecord::Base.connection.execute(sql) + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_security_scans.rb b/lib/gitlab/background_migration/migrate_security_scans.rb new file mode 100644 index 00000000000..189a150cb87 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_security_scans.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class MigrateSecurityScans + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::MigrateSecurityScans.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateSecurityScans') diff --git a/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb b/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb new file mode 100644 index 00000000000..f137e41c728 --- /dev/null +++ b/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RemoveUndefinedOccurrenceSeverityLevel + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel') diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb new file mode 100644 index 00000000000..95540cd5f49 --- /dev/null +++ b/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RemoveUndefinedVulnerabilitySeverityLevel + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel') diff --git a/lib/gitlab/background_migration/update_authorized_keys_file_since.rb b/lib/gitlab/background_migration/update_authorized_keys_file_since.rb deleted file mode 100644 index dd80d4bab1a..00000000000 --- a/lib/gitlab/background_migration/update_authorized_keys_file_since.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class UpdateAuthorizedKeysFileSince - def perform(cutoff_datetime) - end - end - end -end - -Gitlab::BackgroundMigration::UpdateAuthorizedKeysFileSince.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateAuthorizedKeysFileSince') diff --git a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb index 40f45301727..cf0f582a2d4 100644 --- a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb @@ -8,7 +8,7 @@ module Gitlab # Resources that have mentions to be migrated: # issue, merge_request, epic, commit, snippet, design - BULK_INSERT_SIZE = 5000 + BULK_INSERT_SIZE = 1_000 ISOLATION_MODULE = 'Gitlab::BackgroundMigration::UserMentions::Models' def perform(resource_model, join, conditions, with_notes, start_id, end_id) @@ -21,7 +21,8 @@ module Gitlab records.in_groups_of(BULK_INSERT_SIZE, false).each do |records| mentions = [] records.each do |record| - mentions << record.build_mention_values(resource_user_mention_model.resource_foreign_key) + mention_record = record.build_mention_values(resource_user_mention_model.resource_foreign_key) + mentions << mention_record unless mention_record.blank? end Gitlab::Database.bulk_insert( diff --git a/lib/gitlab/background_migration/user_mentions/models/commit.rb b/lib/gitlab/background_migration/user_mentions/models/commit.rb new file mode 100644 index 00000000000..279e93dbf0d --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/commit.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + class Commit + include Concerns::IsolatedMentionable + include Concerns::MentionableMigrationMethods + + def self.user_mention_model + Gitlab::BackgroundMigration::UserMentions::Models::CommitUserMention + end + + def user_mention_model + self.class.user_mention_model + end + + def user_mention_resource_id + id + end + + def user_mention_note_id + 'NULL' + end + + def self.no_quote_columns + [:note_id] + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb new file mode 100644 index 00000000000..bdb4d6c7d48 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + class CommitUserMention < ActiveRecord::Base + self.table_name = 'commit_user_mentions' + + def self.resource_foreign_key + :commit_id + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb index b7fa92a6686..69ba3f9132b 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb @@ -4,89 +4,97 @@ module Gitlab module BackgroundMigration module UserMentions module Models - # == IsolatedMentionable concern - # - # Shortcutted for isolation version of Mentionable to be used in mentions migrations - # - module IsolatedMentionable - extend ::ActiveSupport::Concern - - class_methods do - # Indicate which attributes of the Mentionable to search for GFM references. - def attr_mentionable(attr, options = {}) - attr = attr.to_s - mentionable_attrs << [attr, options] + module Concerns + # == IsolatedMentionable concern + # + # Shortcutted for isolation version of Mentionable to be used in mentions migrations + # + module IsolatedMentionable + extend ::ActiveSupport::Concern + + class_methods do + # Indicate which attributes of the Mentionable to search for GFM references. + def attr_mentionable(attr, options = {}) + attr = attr.to_s + mentionable_attrs << [attr, options] + end end - end - included do - # Accessor for attributes marked mentionable. - cattr_accessor :mentionable_attrs, instance_accessor: false do - [] - end + included do + # Accessor for attributes marked mentionable. + cattr_accessor :mentionable_attrs, instance_accessor: false do + [] + end - if self < Participable - participant -> (user, ext) { all_references(user, extractor: ext) } + if self < Participable + participant -> (user, ext) { all_references(user, extractor: ext) } + end end - end - def all_references(current_user = nil, extractor: nil) - # Use custom extractor if it's passed in the function parameters. - if extractor - extractors[current_user] = extractor - else - extractor = extractors[current_user] ||= ::Gitlab::ReferenceExtractor.new(project, current_user) + def all_references(current_user = nil, extractor: nil) + # Use custom extractor if it's passed in the function parameters. + if extractor + extractors[current_user] = extractor + else + extractor = extractors[current_user] ||= ::Gitlab::ReferenceExtractor.new(project, current_user) - extractor.reset_memoized_values - end + extractor.reset_memoized_values + end - self.class.mentionable_attrs.each do |attr, options| - text = __send__(attr) # rubocop:disable GitlabSecurity/PublicSend - options = options.merge( - cache_key: [self, attr], - author: author, - skip_project_check: skip_project_check? - ).merge(mentionable_params) + self.class.mentionable_attrs.each do |attr, options| + text = __send__(attr) # rubocop:disable GitlabSecurity/PublicSend + options = options.merge( + cache_key: [self, attr], + author: author, + skip_project_check: skip_project_check? + ).merge(mentionable_params) - cached_html = self.try(:updated_cached_html_for, attr.to_sym) - options[:rendered] = cached_html if cached_html + cached_html = self.try(:updated_cached_html_for, attr.to_sym) + options[:rendered] = cached_html if cached_html - extractor.analyze(text, options) + extractor.analyze(text, options) + end + + extractor end - extractor - end + def extractors + @extractors ||= {} + end - def extractors - @extractors ||= {} - end + def skip_project_check? + false + end - def skip_project_check? - false - end + def build_mention_values(resource_foreign_key) + refs = all_references(author) - def build_mention_values(resource_foreign_key) - refs = all_references(author) + mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id)) + mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id)) + mentioned_groups_ids = array_to_sql(refs.mentioned_groups.pluck(:id)) - { - "#{resource_foreign_key}": user_mention_resource_id, - note_id: user_mention_note_id, - mentioned_users_ids: array_to_sql(refs.mentioned_users.pluck(:id)), - mentioned_projects_ids: array_to_sql(refs.mentioned_projects.pluck(:id)), - mentioned_groups_ids: array_to_sql(refs.mentioned_groups.pluck(:id)) - } - end + return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank? + + { + "#{resource_foreign_key}": user_mention_resource_id, + note_id: user_mention_note_id, + mentioned_users_ids: mentioned_users_ids, + mentioned_projects_ids: mentioned_projects_ids, + mentioned_groups_ids: mentioned_groups_ids + } + end - def array_to_sql(ids_array) - return unless ids_array.present? + def array_to_sql(ids_array) + return unless ids_array.present? - '{' + ids_array.join(", ") + '}' - end + '{' + ids_array.join(", ") + '}' + end - private + private - def mentionable_params - {} + def mentionable_params + {} + end end end end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/mentionable_migration_methods.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/mentionable_migration_methods.rb index fa479cb0ed3..efb08d44100 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/mentionable_migration_methods.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/mentionable_migration_methods.rb @@ -4,17 +4,19 @@ module Gitlab module BackgroundMigration module UserMentions module Models - # Extract common no_quote_columns method used in determining the columns that do not need - # to be quoted for corresponding models - module MentionableMigrationMethods - extend ::ActiveSupport::Concern + module Concerns + # Extract common no_quote_columns method used in determining the columns that do not need + # to be quoted for corresponding models + module MentionableMigrationMethods + extend ::ActiveSupport::Concern - class_methods do - def no_quote_columns - [ - :note_id, - user_mention_model.resource_foreign_key - ] + class_methods do + def no_quote_columns + [ + :note_id, + user_mention_model.resource_foreign_key + ] + end end end end diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb new file mode 100644 index 00000000000..0cdfc6447c7 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + module DesignManagement + class Design < ActiveRecord::Base + include Concerns::MentionableMigrationMethods + + def self.user_mention_model + Gitlab::BackgroundMigration::UserMentions::Models::DesignUserMention + end + + def user_mention_model + self.class.user_mention_model + end + + def user_mention_resource_id + id + end + + def user_mention_note_id + 'NULL' + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb new file mode 100644 index 00000000000..68205ecd3c2 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + class DesignUserMention < ActiveRecord::Base + self.table_name = 'design_user_mentions' + + def self.resource_foreign_key + :design_id + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb index 9797c86478e..dc2b7819800 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb @@ -6,9 +6,9 @@ module Gitlab module UserMentions module Models class Epic < ActiveRecord::Base - include IsolatedMentionable + include Concerns::IsolatedMentionable + include Concerns::MentionableMigrationMethods include CacheMarkdownField - include MentionableMigrationMethods attr_mentionable :title, pipeline: :single_line attr_mentionable :description diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb new file mode 100644 index 00000000000..655c1db71ae --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + class MergeRequest < ActiveRecord::Base + include Concerns::IsolatedMentionable + include CacheMarkdownField + include Concerns::MentionableMigrationMethods + + attr_mentionable :title, pipeline: :single_line + attr_mentionable :description + cache_markdown_field :title, pipeline: :single_line + cache_markdown_field :description, issuable_state_filter_enabled: true + + self.table_name = 'merge_requests' + + belongs_to :author, class_name: "User" + belongs_to :target_project, class_name: "Project" + belongs_to :source_project, class_name: "Project" + + alias_attribute :project, :target_project + + def self.user_mention_model + Gitlab::BackgroundMigration::UserMentions::Models::MergeRequestUserMention + end + + def user_mention_model + self.class.user_mention_model + end + + def user_mention_resource_id + id + end + + def user_mention_note_id + 'NULL' + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb new file mode 100644 index 00000000000..e9b85e9cb8c --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + class MergeRequestUserMention < ActiveRecord::Base + self.table_name = 'merge_request_user_mentions' + + def self.resource_foreign_key + :merge_request_id + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb index dc364d7af5a..7a1a0223bc7 100644 --- a/lib/gitlab/background_migration/user_mentions/models/note.rb +++ b/lib/gitlab/background_migration/user_mentions/models/note.rb @@ -6,7 +6,7 @@ module Gitlab module UserMentions module Models class Note < ActiveRecord::Base - include IsolatedMentionable + include Concerns::IsolatedMentionable include CacheMarkdownField self.table_name = 'notes' @@ -20,7 +20,7 @@ module Gitlab belongs_to :project def for_personal_snippet? - noteable.class.name == 'PersonalSnippet' + noteable && noteable.class.name == 'PersonalSnippet' end def for_project_noteable? @@ -32,7 +32,7 @@ module Gitlab end def for_epic? - noteable.class.name == 'Epic' + noteable && noteable_type == 'Epic' end def user_mention_resource_id @@ -43,6 +43,14 @@ module Gitlab id end + def noteable + super unless for_commit? + end + + def for_commit? + noteable_type == "Commit" + end + private def mentionable_params @@ -52,6 +60,8 @@ module Gitlab end def banzai_context_params + return {} unless noteable + { group: noteable.group, label_url_method: :group_epics_url } end end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 3a087a3ef83..5af839d8a32 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -3,8 +3,6 @@ module Gitlab module BitbucketImport class Importer - include Gitlab::ShellAdapter - LABELS = [{ title: 'bug', color: '#FF0000' }, { title: 'enhancement', color: '#428BCA' }, { title: 'proposal', color: '#69D100' }, @@ -80,7 +78,7 @@ module Gitlab wiki = WikiFormatter.new(project) - gitlab_shell.import_wiki_repository(project, wiki) + project.wiki.repository.import_repository(wiki.import_url) rescue StandardError => e errors << { type: :wiki, errors: e.message } end diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb new file mode 100644 index 00000000000..ead94761ae7 --- /dev/null +++ b/lib/gitlab/cache/import/caching.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + module Import + module Caching + # The default timeout of the cache keys. + TIMEOUT = 24.hours.to_i + + WRITE_IF_GREATER_SCRIPT = <<-EOF.strip_heredoc.freeze + local key, value, ttl = KEYS[1], tonumber(ARGV[1]), ARGV[2] + local existing = tonumber(redis.call("get", key)) + + if existing == nil or value > existing then + redis.call("set", key, value) + redis.call("expire", key, ttl) + return true + else + return false + end + EOF + + # Reads a cache key. + # + # If the key exists and has a non-empty value its TTL is refreshed + # automatically. + # + # raw_key - The cache key to read. + # timeout - The new timeout of the key if the key is to be refreshed. + def self.read(raw_key, timeout: TIMEOUT) + key = cache_key_for(raw_key) + value = Redis::Cache.with { |redis| redis.get(key) } + + if value.present? + # We refresh the expiration time so frequently used keys stick + # around, removing the need for querying the database as much as + # possible. + # + # A key may be empty when we looked up a GitHub user (for example) but + # did not find a matching GitLab user. In that case we _don't_ want to + # refresh the TTL so we automatically pick up the right data when said + # user were to register themselves on the GitLab instance. + Redis::Cache.with { |redis| redis.expire(key, timeout) } + end + + value + end + + # Reads an integer from the cache, or returns nil if no value was found. + # + # See Caching.read for more information. + def self.read_integer(raw_key, timeout: TIMEOUT) + value = read(raw_key, timeout: timeout) + + value.to_i if value.present? + end + + # Sets a cache key to the given value. + # + # key - The cache key to write. + # value - The value to set. + # timeout - The time after which the cache key should expire. + def self.write(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.set(key, value, ex: timeout) + end + + value + end + + # Adds a value to a set. + # + # raw_key - The key of the set to add the value to. + # value - The value to add to the set. + # timeout - The new timeout of the key. + def self.set_add(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.multi do |m| + m.sadd(key, value) + m.expire(key, timeout) + end + end + end + + # Returns true if the given value is present in the set. + # + # raw_key - The key of the set to check. + # value - The value to check for. + def self.set_includes?(raw_key, value) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.sismember(key, value) + end + end + + # Sets multiple keys to a given value. + # + # mapping - A Hash mapping the cache keys to their values. + # timeout - The time after which the cache key should expire. + def self.write_multiple(mapping, timeout: TIMEOUT) + Redis::Cache.with do |redis| + redis.multi do |multi| + mapping.each do |raw_key, value| + multi.set(cache_key_for(raw_key), value, ex: timeout) + end + end + end + end + + # Sets the expiration time of a key. + # + # raw_key - The key for which to change the timeout. + # timeout - The new timeout. + def self.expire(raw_key, timeout) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.expire(key, timeout) + end + end + + # Sets a key to the given integer but only if the existing value is + # smaller than the given value. + # + # This method uses a Lua script to ensure the read and write are atomic. + # + # raw_key - The key to set. + # value - The new value for the key. + # timeout - The key timeout in seconds. + # + # Returns true when the key was overwritten, false otherwise. + def self.write_if_greater(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + val = Redis::Cache.with do |redis| + redis + .eval(WRITE_IF_GREATER_SCRIPT, keys: [key], argv: [value, timeout]) + end + + val ? true : false + end + + def self.cache_key_for(raw_key) + "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}" + end + end + end + end +end diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index 4ddc1c718c7..7be0ef05a49 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -28,7 +28,7 @@ module Gitlab logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do if deletion? && branch_name == project.default_branch - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_default_branch] end end @@ -42,7 +42,7 @@ module Gitlab return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks if forced_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:force_push_protected_branch] end end @@ -62,15 +62,15 @@ module Gitlab break if user_access.can_push_to_branch?(branch_name) unless user_access.can_merge_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_branch] end unless safe_commit_for_new_protected_branch? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:invalid_commit_create_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:invalid_commit_create_protected_branch] end unless updated_from_web? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_create_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_web_create_protected_branch] end end end @@ -78,11 +78,11 @@ module Gitlab def protected_branch_deletion_checks logger.log_timed(LOG_MESSAGES[:protected_branch_deletion_checks]) do unless user_access.can_delete_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_master_delete_protected_branch] end unless updated_from_web? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_web_delete_protected_branch] end end end @@ -91,11 +91,11 @@ module Gitlab logger.log_timed(LOG_MESSAGES[:protected_branch_push_checks]) do if matching_merge_request? unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:merge_protected_branch] end else unless user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message + raise GitAccess::ForbiddenError, push_to_protected_branch_rejected_message end end end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index 5de71addd5f..0eb2b4c79ef 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -46,7 +46,7 @@ module Gitlab def validate_diff(diff) validations_for_diff.each do |validation| if error = validation.call(diff) - raise ::Gitlab::GitAccess::UnauthorizedError, error + raise ::Gitlab::GitAccess::ForbiddenError, error end end end @@ -77,7 +77,7 @@ module Gitlab logger.log_timed(LOG_MESSAGES[__method__]) do path_validations.each do |validation| if error = validation.call(file_paths) - raise ::Gitlab::GitAccess::UnauthorizedError, error + raise ::Gitlab::GitAccess::ForbiddenError, error end end end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index 7b013567a03..f81c215d847 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -15,7 +15,7 @@ module Gitlab lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) if lfs_check.objects_missing? - raise GitAccess::UnauthorizedError, ERROR_MESSAGE + raise GitAccess::ForbiddenError, ERROR_MESSAGE end end end diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index 1652d5a30a4..e18cf6ff8f2 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -9,7 +9,6 @@ module Gitlab @time_left = time_left end - # rubocop: disable CodeReuse/ActiveRecord def objects_missing? return false unless @newrev && @project.lfs_enabled? @@ -19,12 +18,11 @@ module Gitlab return false unless new_lfs_pointers.present? existing_count = @project.all_lfs_objects - .where(oid: new_lfs_pointers.map(&:lfs_oid)) + .for_oids(new_lfs_pointers.map(&:lfs_oid)) .count existing_count != new_lfs_pointers.count end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/checks/post_push_message.rb b/lib/gitlab/checks/post_push_message.rb index 492dbb5a596..b3c981d252b 100644 --- a/lib/gitlab/checks/post_push_message.rb +++ b/lib/gitlab/checks/post_push_message.rb @@ -3,8 +3,8 @@ module Gitlab module Checks class PostPushMessage - def initialize(project, user, protocol) - @project = project + def initialize(repository, user, protocol) + @repository = repository @user = user @protocol = protocol end @@ -34,14 +34,21 @@ module Gitlab protected - attr_reader :project, :user, :protocol + attr_reader :repository, :user, :protocol + + delegate :project, to: :repository, allow_nil: true + delegate :container, to: :repository, allow_nil: false def self.message_key(user_id, project_id) raise NotImplementedError end def url_to_repo - protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + protocol == 'ssh' ? message_subject.ssh_url_to_repo : message_subject.http_url_to_repo + end + + def message_subject + repository.repo_type.wiki? ? project.wiki : container end end end diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index 6f04fddc6c4..4cc35de9c2d 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -5,10 +5,10 @@ module Gitlab class ProjectMoved < PostPushMessage REDIRECT_NAMESPACE = "redirect_namespace" - def initialize(project, user, protocol, redirected_path) + def initialize(repository, user, protocol, redirected_path) @redirected_path = redirected_path - super(project, user, protocol) + super(repository, user, protocol) end def message diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb index 91f8d0bdbc8..7cc5bc56cbb 100644 --- a/lib/gitlab/checks/push_check.rb +++ b/lib/gitlab/checks/push_check.rb @@ -6,7 +6,7 @@ module Gitlab def validate! logger.log_timed("Checking if you are allowed to push...") do unless can_push? - raise GitAccess::UnauthorizedError, GitAccess::ERROR_MESSAGES[:push_code] + raise GitAccess::ForbiddenError, GitAccess::ERROR_MESSAGES[:push_code] end end end diff --git a/lib/gitlab/checks/push_file_count_check.rb b/lib/gitlab/checks/push_file_count_check.rb new file mode 100644 index 00000000000..288a7e0d41a --- /dev/null +++ b/lib/gitlab/checks/push_file_count_check.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class PushFileCountCheck < BaseChecker + attr_reader :repository, :newrev, :limit, :logger + + LOG_MESSAGES = { + diff_content_check: "Validating diff contents being single file..." + }.freeze + + ERROR_MESSAGES = { + upper_limit: "The repository can contain at most %{limit} file(s).", + lower_limit: "The repository must contain at least 1 file." + }.freeze + + def initialize(change, repository:, limit:, logger:) + @repository = repository + @newrev = change[:newrev] + @limit = limit + @logger = logger + end + + def validate! + file_count = repository.ls_files(newrev).size + + if file_count > limit + raise ::Gitlab::GitAccess::ForbiddenError, ERROR_MESSAGES[:upper_limit] % { limit: limit } + end + + if file_count == 0 + raise ::Gitlab::GitAccess::ForbiddenError, ERROR_MESSAGES[:lower_limit] + end + end + end + end +end diff --git a/lib/gitlab/checks/snippet_check.rb b/lib/gitlab/checks/snippet_check.rb new file mode 100644 index 00000000000..bcecd0fc251 --- /dev/null +++ b/lib/gitlab/checks/snippet_check.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class SnippetCheck < BaseChecker + DEFAULT_BRANCH = 'master'.freeze + ERROR_MESSAGES = { + create_delete_branch: 'You can not create or delete branches.' + }.freeze + + ATTRIBUTES = %i[oldrev newrev ref branch_name tag_name logger].freeze + attr_reader(*ATTRIBUTES) + + def initialize(change, logger:) + @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) + @branch_name = Gitlab::Git.branch_name(@ref) + @tag_name = Gitlab::Git.tag_name(@ref) + + @logger = logger + @logger.append_message("Running checks for ref: #{@branch_name || @tag_name}") + end + + def validate! + if creation? || deletion? + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_delete_branch] + end + + true + end + + private + + def creation? + @branch_name != DEFAULT_BRANCH && super + end + end + end +end diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb index ced0612a7a3..a47e55cb160 100644 --- a/lib/gitlab/checks/tag_check.rb +++ b/lib/gitlab/checks/tag_check.rb @@ -20,7 +20,7 @@ module Gitlab logger.log_timed(LOG_MESSAGES[:tag_checks]) do if tag_exists? && user_access.cannot_do_action?(:admin_tag) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:change_existing_tags] end end @@ -33,11 +33,11 @@ module Gitlab logger.log_timed(LOG_MESSAGES[__method__]) do return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:update_protected_tag]) if update? + raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? unless user_access.can_create_tag?(tag_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_tag] end end end diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb new file mode 100644 index 00000000000..c2d17cc176e --- /dev/null +++ b/lib/gitlab/ci/artifact_file_reader.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +# This class takes in input a Ci::Build object and an artifact path to read. +# It downloads and extracts the artifacts archive, then returns the content +# of the artifact, if found. +module Gitlab + module Ci + class ArtifactFileReader + Error = Class.new(StandardError) + + MAX_ARCHIVE_SIZE = 5.megabytes + + def initialize(job) + @job = job + + raise ArgumentError, 'Job does not have artifacts' unless @job.artifacts? + + validate! + end + + def read(path) + return unless job.artifacts_metadata + + metadata_entry = job.artifacts_metadata_entry(path) + + if metadata_entry.total_size > MAX_ARCHIVE_SIZE + raise Error, "Artifacts archive for job `#{job.name}` is too large: max #{max_archive_size_in_mb}" + end + + read_zip_file!(path) + end + + private + + attr_reader :job + + def validate! + if job.job_artifacts_archive.size > MAX_ARCHIVE_SIZE + raise Error, "Artifacts archive for job `#{job.name}` is too large: max #{max_archive_size_in_mb}" + end + + unless job.artifacts_metadata? + raise Error, "Job `#{job.name}` has missing artifacts metadata and cannot be extracted!" + end + end + + def read_zip_file!(file_path) + job.artifacts_file.use_file do |archive_path| + Zip::File.open(archive_path) do |zip_file| + entry = zip_file.find_entry(file_path) + unless entry + raise Error, "Path `#{file_path}` does not exist inside the `#{job.name}` artifacts archive!" + end + + if entry.name_is_directory? + raise Error, "Path `#{file_path}` was expected to be a file but it was a directory!" + end + + zip_file.get_input_stream(entry) do |is| + is.read + end + end + end + end + + def max_archive_size_in_mb + ActiveSupport::NumberHelper.number_to_human_size(MAX_ARCHIVE_SIZE) + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 38ab3475d01..10e0f4b8e4d 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -18,12 +18,9 @@ module Gitlab attr_reader :root - def initialize(config, project: nil, sha: nil, user: nil) - @context = build_context(project: project, sha: sha, user: user) - - if Feature.enabled?(:ci_limit_yaml_expansion, project, default_enabled: true) - @context.set_deadline(TIMEOUT_SECONDS) - end + def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil) + @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline) + @context.set_deadline(TIMEOUT_SECONDS) @config = expand_config(config) @@ -79,19 +76,17 @@ module Gitlab initial_config = Gitlab::Config::Loader::Yaml.new(config).load! initial_config = Config::External::Processor.new(initial_config, @context).perform initial_config = Config::Extendable.new(initial_config).to_hash - - if Feature.enabled?(:ci_pre_post_pipeline_stages, @context.project, default_enabled: true) - initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash - end + initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash initial_config end - def build_context(project:, sha:, user:) + def build_context(project:, sha:, user:, parent_pipeline:) Config::External::Context.new( project: project, sha: sha || project&.repository&.root_ref_sha, - user: user) + user: user, + parent_pipeline: parent_pipeline) end def track_and_raise_for_dev_exception(error) diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index aebc1675bec..241c73db3bb 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -44,8 +44,6 @@ module Gitlab end end - helpers :reports - def value @config[:reports] = reports_value if @config.key?(:reports) @config diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index c0247dca73d..f4362d3b0ce 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -9,34 +9,21 @@ module Gitlab # defining a downstream project trigger. # class Bridge < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Configurable - include ::Gitlab::Config::Entry::Attributable - include ::Gitlab::Config::Entry::Inheritable + include ::Gitlab::Ci::Config::Entry::Processable - ALLOWED_KEYS = %i[trigger stage allow_failure only except - when extends variables needs rules].freeze + ALLOWED_KEYS = %i[trigger allow_failure when needs].freeze validations do - validates :config, allowed_keys: ALLOWED_KEYS - validates :config, presence: true - validates :name, presence: true - validates :name, type: Symbol - validates :config, disallowed_keys: { - in: %i[only except when start_in], - message: 'key may not be used with `rules`' - }, - if: :has_rules? + validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS with_options allow_nil: true do validates :when, inclusion: { in: %w[on_success on_failure always], message: 'should be on_success, on_failure or always' } - validates :extends, type: String - validates :rules, array_of_hashes: true end validate on: :composed do - unless trigger.present? || bridge_needs.present? + unless trigger_defined? || bridge_needs.present? errors.add(:config, 'should contain either a trigger or a needs:pipeline') end end @@ -58,32 +45,7 @@ module Gitlab inherit: false, metadata: { allowed_needs: %i[job bridge] } - entry :stage, ::Gitlab::Ci::Config::Entry::Stage, - description: 'Pipeline stage this job will be executed into.', - inherit: false - - entry :only, ::Gitlab::Ci::Config::Entry::Policy, - description: 'Refs policy this job will be executed for.', - default: ::Gitlab::Ci::Config::Entry::Policy::DEFAULT_ONLY, - inherit: false - - entry :except, ::Gitlab::Ci::Config::Entry::Policy, - description: 'Refs policy this job will be executed for.', - inherit: false - - entry :rules, ::Gitlab::Ci::Config::Entry::Rules, - description: 'List of evaluable Rules to determine job inclusion.', - inherit: false, - metadata: { - allowed_when: %w[on_success on_failure always never manual delayed].freeze - } - - entry :variables, ::Gitlab::Ci::Config::Entry::Variables, - description: 'Environment variables available for this job.', - inherit: false - - helpers(*ALLOWED_KEYS) - attributes(*ALLOWED_KEYS) + attributes :when, :allow_failure def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -95,56 +57,19 @@ module Gitlab true end - def compose!(deps = nil) - super do - has_workflow_rules = deps&.workflow&.has_rules? - - # If workflow:rules: or rules: are used - # they are considered not compatible - # with `only/except` defaults - # - # Context: https://gitlab.com/gitlab-org/gitlab/merge_requests/21742 - if has_rules? || has_workflow_rules - # Remove only/except defaults - # defaults are not considered as defined - @entries.delete(:only) unless only_defined? - @entries.delete(:except) unless except_defined? - end - end - end - - def has_rules? - @config&.key?(:rules) - end - - def name - @metadata[:name] - end - def value - { name: name, + super.merge( trigger: (trigger_value if trigger_defined?), needs: (needs_value if needs_defined?), ignore: !!allow_failure, - stage: stage_value, - when: when_value, - extends: extends_value, - variables: (variables_value if variables_defined?), - rules: (rules_value if has_rules?), - only: only_value, - except: except_value, - scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage }.compact + when: self.when, + scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage + ).compact end def bridge_needs needs_value[:bridge] if needs_value end - - private - - def overwrite_entry(deps, key, current_entry) - deps.default[key] unless current_entry.specified? - end end end end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index ef07c319ce4..a304d9b724f 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -28,8 +28,6 @@ module Gitlab entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' - helpers :key - attributes :policy def value diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index 88db17a75da..ab493ff7d78 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -61,8 +61,6 @@ module Gitlab description: 'Default artifacts.', inherit: false - helpers :before_script, :image, :services, :after_script, :cache - private def overwrite_entry(deps, key, current_entry) diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb index f2f3dd84eda..cd09d83b728 100644 --- a/lib/gitlab/ci/config/entry/include.rb +++ b/lib/gitlab/ci/config/entry/include.rb @@ -10,7 +10,7 @@ module Gitlab class Include < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable - ALLOWED_KEYS = %i[local file remote template].freeze + ALLOWED_KEYS = %i[local file remote template artifact job].freeze validations do validates :config, hash_or_string: true diff --git a/lib/gitlab/ci/config/entry/inherit.rb b/lib/gitlab/ci/config/entry/inherit.rb new file mode 100644 index 00000000000..b806d77b155 --- /dev/null +++ b/lib/gitlab/ci/config/entry/inherit.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # This class represents a inherit entry + # + class Inherit < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[default variables].freeze + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + end + + entry :default, ::Gitlab::Ci::Config::Entry::Inherit::Default, + description: 'Indicates whether to inherit `default:`.', + default: true + + entry :variables, ::Gitlab::Ci::Config::Entry::Inherit::Variables, + description: 'Indicates whether to inherit `variables:`.', + default: true + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/inherit/default.rb b/lib/gitlab/ci/config/entry/inherit/default.rb new file mode 100644 index 00000000000..74386baf62f --- /dev/null +++ b/lib/gitlab/ci/config/entry/inherit/default.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # This class represents a default inherit entry + # + class Inherit + class Default < ::Gitlab::Config::Entry::Simplifiable + strategy :BooleanStrategy, if: -> (config) { [true, false].include?(config) } + strategy :ArrayStrategy, if: -> (config) { config.is_a?(Array) } + + class BooleanStrategy < ::Gitlab::Config::Entry::Boolean + def inherit?(_key) + value + end + end + + class ArrayStrategy < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + ALLOWED_VALUES = ::Gitlab::Ci::Config::Entry::Default::ALLOWED_KEYS.map(&:to_s).freeze + + validations do + validates :config, type: Array + validates :config, array_of_strings: true + validates :config, allowed_array_values: { in: ALLOWED_VALUES } + end + + def inherit?(key) + value.include?(key.to_s) + end + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be a bool or array of strings"] + end + + def inherit?(key) + false + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/inherit/variables.rb b/lib/gitlab/ci/config/entry/inherit/variables.rb new file mode 100644 index 00000000000..aa68833bdb8 --- /dev/null +++ b/lib/gitlab/ci/config/entry/inherit/variables.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # This class represents a variables inherit entry + # + class Inherit + class Variables < ::Gitlab::Config::Entry::Simplifiable + strategy :BooleanStrategy, if: -> (config) { [true, false].include?(config) } + strategy :ArrayStrategy, if: -> (config) { config.is_a?(Array) } + + class BooleanStrategy < ::Gitlab::Config::Entry::Boolean + def inherit?(_key) + value + end + end + + class ArrayStrategy < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: Array + validates :config, array_of_strings: true + end + + def inherit?(key) + value.include?(key.to_s) + end + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be a bool or array of strings"] + end + + def inherit?(key) + false + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 666c6e23eb4..1ea59491378 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -8,33 +8,21 @@ module Gitlab # Entry that represents a concrete CI/CD job. # class Job < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Configurable - include ::Gitlab::Config::Entry::Attributable - include ::Gitlab::Config::Entry::Inheritable + include ::Gitlab::Ci::Config::Entry::Processable ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze - ALLOWED_KEYS = %i[tags script only except rules type image services - allow_failure type stage when start_in artifacts cache - dependencies before_script needs after_script variables - environment coverage retry parallel extends interruptible timeout + ALLOWED_KEYS = %i[tags script type image services + allow_failure type when start_in artifacts cache + dependencies before_script needs after_script + environment coverage retry parallel interruptible timeout resource_group release].freeze REQUIRED_BY_NEEDS = %i[stage].freeze validations do - validates :config, type: Hash - validates :config, allowed_keys: ALLOWED_KEYS + validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs? - validates :config, presence: true validates :script, presence: true - validates :name, presence: true - validates :name, type: Symbol - validates :config, - disallowed_keys: { - in: %i[only except when start_in], - message: 'key may not be used with `rules`' - }, - if: :has_rules? validates :config, disallowed_keys: { in: %i[release], @@ -53,8 +41,6 @@ module Gitlab } validates :dependencies, array_of_strings: true - validates :extends, array_of_strings_or_string: true - validates :rules, array_of_hashes: true validates :resource_group, type: String end @@ -81,10 +67,6 @@ module Gitlab description: 'Commands that will be executed in this job.', inherit: false - entry :stage, Entry::Stage, - description: 'Pipeline stage this job will be executed into.', - inherit: false - entry :type, Entry::Stage, description: 'Deprecated: stage this job will be executed into.', inherit: false @@ -125,31 +107,11 @@ module Gitlab description: 'Artifacts configuration for this job.', inherit: true - entry :only, Entry::Policy, - description: 'Refs policy this job will be executed for.', - default: ::Gitlab::Ci::Config::Entry::Policy::DEFAULT_ONLY, - inherit: false - - entry :except, Entry::Policy, - description: 'Refs policy this job will be executed for.', - inherit: false - - entry :rules, Entry::Rules, - description: 'List of evaluable Rules to determine job inclusion.', - inherit: false, - metadata: { - allowed_when: %w[on_success on_failure always never manual delayed].freeze - } - entry :needs, Entry::Needs, description: 'Needs configuration for this job.', metadata: { allowed_needs: %i[job cross_dependency] }, inherit: false - entry :variables, Entry::Variables, - description: 'Environment variables available for this job.', - inherit: false - entry :environment, Entry::Environment, description: 'Environment configuration for this job.', inherit: false @@ -162,13 +124,8 @@ module Gitlab description: 'This job will produce a release.', inherit: false - helpers :before_script, :script, :stage, :type, :after_script, - :cache, :image, :services, :only, :except, :variables, - :artifacts, :environment, :coverage, :retry, :rules, - :parallel, :needs, :interruptible, :release, :tags - attributes :script, :tags, :allow_failure, :when, :dependencies, - :needs, :retry, :parallel, :extends, :start_in, :rules, + :needs, :retry, :parallel, :start_in, :interruptible, :timeout, :resource_group, :release def self.matching?(name, config) @@ -187,31 +144,9 @@ module Gitlab end @entries.delete(:type) - - has_workflow_rules = deps&.workflow&.has_rules? - - # If workflow:rules: or rules: are used - # they are considered not compatible - # with `only/except` defaults - # - # Context: https://gitlab.com/gitlab-org/gitlab/merge_requests/21742 - if has_rules? || has_workflow_rules - # Remove only/except defaults - # defaults are not considered as defined - @entries.delete(:only) unless only_defined? - @entries.delete(:except) unless except_defined? - end end end - def name - @metadata[:name] - end - - def value - @config.merge(to_hash.compact) - end - def manual_action? self.when == 'manual' end @@ -220,38 +155,26 @@ module Gitlab self.when == 'delayed' end - def has_rules? - @config.try(:key?, :rules) - end - def ignored? allow_failure.nil? ? manual_action? : allow_failure end - private - - def overwrite_entry(deps, key, current_entry) - deps.default[key] unless current_entry.specified? - end - - def to_hash - { name: name, + def value + super.merge( before_script: before_script_value, script: script_value, image: image_value, services: services_value, - stage: stage_value, cache: cache_value, tags: tags_value, - only: only_value, - except: except_value, - rules: has_rules? ? rules_value : nil, - variables: variables_defined? ? variables_value : {}, + when: self.when, + start_in: self.start_in, + dependencies: dependencies, environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, retry: retry_defined? ? retry_value : nil, - parallel: parallel_defined? ? parallel_value.to_i : nil, + parallel: has_parallel? ? parallel.to_i : nil, interruptible: interruptible_defined? ? interruptible_value : nil, timeout: has_timeout? ? ChronicDuration.parse(timeout.to_s) : nil, artifacts: artifacts_value, @@ -260,7 +183,8 @@ module Gitlab ignore: ignored?, needs: needs_defined? ? needs_value : nil, resource_group: resource_group, - scheduling_type: needs_defined? ? :dag : :stage } + scheduling_type: needs_defined? ? :dag : :stage + ).compact end end end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb new file mode 100644 index 00000000000..81211acbec7 --- /dev/null +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a CI/CD Processable (a job) + # + module Processable + extend ActiveSupport::Concern + + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Inheritable + + PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables inherit].freeze + + included do + validations do + validates :config, presence: true + validates :name, presence: true + validates :name, type: Symbol + + validates :config, disallowed_keys: { + in: %i[only except when start_in], + message: 'key may not be used with `rules`' + }, + if: :has_rules? + + with_options allow_nil: true do + validates :extends, array_of_strings_or_string: true + validates :rules, array_of_hashes: true + end + end + + entry :stage, Entry::Stage, + description: 'Pipeline stage this job will be executed into.', + inherit: false + + entry :only, ::Gitlab::Ci::Config::Entry::Policy, + description: 'Refs policy this job will be executed for.', + default: ::Gitlab::Ci::Config::Entry::Policy::DEFAULT_ONLY, + inherit: false + + entry :except, ::Gitlab::Ci::Config::Entry::Policy, + description: 'Refs policy this job will be executed for.', + inherit: false + + entry :rules, ::Gitlab::Ci::Config::Entry::Rules, + description: 'List of evaluable Rules to determine job inclusion.', + inherit: false, + metadata: { + allowed_when: %w[on_success on_failure always never manual delayed].freeze + } + + entry :variables, ::Gitlab::Ci::Config::Entry::Variables, + description: 'Environment variables available for this job.', + inherit: false + + entry :inherit, ::Gitlab::Ci::Config::Entry::Inherit, + description: 'Indicates whether to inherit defaults or not.', + inherit: false, + default: {} + + attributes :extends, :rules + end + + def compose!(deps = nil) + super do + has_workflow_rules = deps&.workflow_entry&.has_rules? + + # If workflow:rules: or rules: are used + # they are considered not compatible + # with `only/except` defaults + # + # Context: https://gitlab.com/gitlab-org/gitlab/merge_requests/21742 + if has_rules? || has_workflow_rules + # Remove only/except defaults + # defaults are not considered as defined + @entries.delete(:only) unless only_defined? # rubocop:disable Gitlab/ModuleWithInstanceVariables + @entries.delete(:except) unless except_defined? # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + # inherit root variables + @root_variables_value = deps&.variables_value # rubocop:disable Gitlab/ModuleWithInstanceVariables + + yield if block_given? + end + end + + def name + metadata[:name] + end + + def overwrite_entry(deps, key, current_entry) + return unless inherit_entry&.default_entry&.inherit?(key) + return unless deps.default_entry + + deps.default_entry[key] unless current_entry.specified? + end + + def value + { name: name, + stage: stage_value, + extends: extends, + rules: rules_value, + variables: root_and_job_variables_value, + only: only_value, + except: except_value }.compact + end + + def root_and_job_variables_value + root_variables = @root_variables_value.to_h # rubocop:disable Gitlab/ModuleWithInstanceVariables + root_variables = root_variables.select do |key, _| + inherit_entry&.variables_entry&.inherit?(key) + end + + root_variables.merge(variables_value.to_h) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/release.rb b/lib/gitlab/ci/config/entry/release.rb index 3eceaa0ccd9..b4e4c149730 100644 --- a/lib/gitlab/ci/config/entry/release.rb +++ b/lib/gitlab/ci/config/entry/release.rb @@ -33,8 +33,6 @@ module Gitlab validates :description, type: String, presence: true end - helpers :assets - def value @config[:assets] = assets_value if @config.key?(:assets) @config diff --git a/lib/gitlab/ci/config/entry/release/assets.rb b/lib/gitlab/ci/config/entry/release/assets.rb index 82ed39f51e0..1f7057d1bf6 100644 --- a/lib/gitlab/ci/config/entry/release/assets.rb +++ b/lib/gitlab/ci/config/entry/release/assets.rb @@ -23,8 +23,6 @@ module Gitlab validates :links, array_of_hashes: true, presence: true end - helpers :links - def value @config[:links] = links_value if @config.key?(:links) @config diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 571e056e096..40d37f3601a 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -11,7 +11,10 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management license_scanning metrics lsif].freeze + ALLOWED_KEYS = + %i[junit codequality sast dependency_scanning container_scanning + dast performance license_management license_scanning metrics lsif + dotenv cobertura].freeze attributes ALLOWED_KEYS @@ -31,6 +34,8 @@ module Gitlab validates :license_scanning, array_of_strings_or_string: true validates :metrics, array_of_strings_or_string: true validates :lsif, array_of_strings_or_string: true + validates :dotenv, array_of_strings_or_string: true + validates :cobertura, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 12dd942fc1c..19d6a470941 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -65,15 +65,16 @@ module Gitlab reserved: true entry :workflow, Entry::Workflow, - description: 'List of evaluable rules to determine Pipeline status' + description: 'List of evaluable rules to determine Pipeline status', + default: {} - helpers :default, :jobs, :stages, :types, :variables, :workflow + dynamic_helpers :jobs delegate :before_script_value, :image_value, :services_value, :after_script_value, - :cache_value, to: :default + :cache_value, to: :default_entry attr_reader :jobs_config @@ -102,14 +103,6 @@ module Gitlab end end - def default - self[:default] - end - - def workflow - self[:workflow] if workflow_defined? - end - private # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 8d16371e857..247bf930d3b 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -7,8 +7,13 @@ module Gitlab ## # Entry that represents a configuration of Docker service. # - class Service < Image + # TODO: remove duplication with Image superclass by defining a common + # Imageable concern. + # https://gitlab.com/gitlab-org/gitlab/issues/208774 + class Service < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable ALLOWED_KEYS = %i[name entrypoint command alias ports].freeze @@ -16,9 +21,9 @@ module Gitlab validates :config, hash_or_string: true validates :config, allowed_keys: ALLOWED_KEYS validates :config, disallowed_keys: %i[ports], unless: :with_image_ports? - validates :name, type: String, presence: true validates :entrypoint, array_of_strings: true, allow_nil: true + validates :command, array_of_strings: true, allow_nil: true validates :alias, type: String, allow_nil: true validates :alias, type: String, presence: true, unless: ->(record) { record.ports.blank? } @@ -27,6 +32,8 @@ module Gitlab entry :ports, Entry::Ports, description: 'Ports used to expose the service' + attributes :ports + def alias value[:alias] end @@ -34,6 +41,29 @@ module Gitlab def command value[:command] end + + def name + value[:name] + end + + def entrypoint + value[:entrypoint] + end + + def value + return { name: @config } if string? + return @config if hash? + + {} + end + + def with_image_ports? + opt(:with_image_ports) + end + + def skip_config_hash_validation? + true + end end end end diff --git a/lib/gitlab/ci/config/entry/workflow.rb b/lib/gitlab/ci/config/entry/workflow.rb index 1d9007c9b9b..5bc992a38a0 100644 --- a/lib/gitlab/ci/config/entry/workflow.rb +++ b/lib/gitlab/ci/config/entry/workflow.rb @@ -12,7 +12,6 @@ module Gitlab validations do validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS - validates :config, presence: true end entry :rules, Entry::Rules, diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index bb4439cd069..814dcc66362 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -7,13 +7,14 @@ module Gitlab class Context TimeoutError = Class.new(StandardError) - attr_reader :project, :sha, :user + attr_reader :project, :sha, :user, :parent_pipeline attr_reader :expandset, :execution_deadline - def initialize(project: nil, sha: nil, user: nil) + def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil) @project = project @sha = sha @user = user + @parent_pipeline = parent_pipeline @expandset = Set.new @execution_deadline = 0 diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb new file mode 100644 index 00000000000..a8f78b62d8d --- /dev/null +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module File + class Artifact < Base + extend ::Gitlab::Utils::Override + include Gitlab::Utils::StrongMemoize + + attr_reader :job_name + + def initialize(params, context) + @location = params[:artifact] + @job_name = params[:job] + + super + end + + def content + strong_memoize(:content) do + next unless artifact_job + + Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location) + rescue Gitlab::Ci::ArtifactFileReader::Error => error + errors.push(error.message) + end + end + + def matching? + super && + Feature.enabled?(:ci_dynamic_child_pipeline, project, default_enabled: true) + end + + private + + def project + context&.parent_pipeline&.project + end + + def validate_content! + return unless ensure_preconditions_satisfied! + + errors.push("File `#{location}` is empty!") unless content.present? + end + + def ensure_preconditions_satisfied! + unless creating_child_pipeline? + errors.push('Including configs from artifacts is only allowed when triggering child pipelines') + return false + end + + unless job_name.present? + errors.push("Job must be provided when including configs from artifacts") + return false + end + + unless artifact_job.present? + errors.push("Job `#{job_name}` not found in parent pipeline or does not have artifacts!") + return false + end + + true + end + + def artifact_job + strong_memoize(:artifact_job) do + next unless creating_child_pipeline? + + context.parent_pipeline.find_job_with_archive_artifacts(job_name) + end + end + + def creating_child_pipeline? + context.parent_pipeline.present? + end + + override :expand_context_attrs + def expand_context_attrs + { + project: context.project, + sha: context.sha, + user: context.user, + parent_pipeline: context.parent_pipeline + } + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 8cb1575a3e1..e74f5b33de7 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -40,7 +40,8 @@ module Gitlab { project: context.project, sha: context.sha, - user: context.user + user: context.user, + parent_pipeline: context.parent_pipeline } end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index c7b49b495fa..be479741784 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -71,7 +71,8 @@ module Gitlab { project: project, sha: sha, - user: context.user + user: context.user, + parent_pipeline: context.parent_pipeline } end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 0143d7784fa..97ae6c4ceba 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -13,7 +13,8 @@ module Gitlab External::File::Remote, External::File::Template, External::File::Local, - External::File::Project + External::File::Project, + External::File::Artifact ].freeze Error = Class.new(StandardError) diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index c76cd5ff285..a44105d53c2 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -9,7 +9,8 @@ module Gitlab def self.parsers { - junit: ::Gitlab::Ci::Parsers::Test::Junit + junit: ::Gitlab::Ci::Parsers::Test::Junit, + cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura } end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb new file mode 100644 index 00000000000..006d5097148 --- /dev/null +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Coverage + class Cobertura + CoberturaParserError = Class.new(Gitlab::Ci::Parsers::ParserError) + + def parse!(xml_data, coverage_report) + root = Hash.from_xml(xml_data) + + parse_all(root, coverage_report) + rescue Nokogiri::XML::SyntaxError + raise CoberturaParserError, "XML parsing failed" + rescue + raise CoberturaParserError, "Cobertura parsing failed" + end + + private + + def parse_all(root, coverage_report) + return unless root.present? + + root.each do |key, value| + parse_node(key, value, coverage_report) + end + end + + def parse_node(key, value, coverage_report) + if key == 'class' + Array.wrap(value).each do |item| + parse_class(item, coverage_report) + end + elsif value.is_a?(Hash) + parse_all(value, coverage_report) + elsif value.is_a?(Array) + value.each do |item| + parse_all(item, coverage_report) + end + end + end + + def parse_class(file, coverage_report) + return unless file["filename"].present? && file["lines"].present? + + parsed_lines = parse_lines(file["lines"]) + + coverage_report.add_file(file["filename"], Hash[parsed_lines]) + end + + def parse_lines(lines) + line_array = Array.wrap(lines["line"]) + + line_array.map do |line| + # Using `Integer()` here to raise exception on invalid values + [Integer(line["number"]), Integer(line["hits"])] + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index 133eb16a83e..0ce901fa5aa 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -6,6 +6,7 @@ module Gitlab module Test class Junit JunitParserError = Class.new(Gitlab::Ci::Parsers::ParserError) + ATTACHMENT_TAG_REGEX = /\[\[ATTACHMENT\|(?<path>.+?)\]\]/.freeze def parse!(xml_data, test_suite) root = Hash.from_xml(xml_data) @@ -49,6 +50,7 @@ module Gitlab if data['failure'] status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED system_output = data['failure'] + attachment = attachment_path(data['system_out']) elsif data['error'] status = ::Gitlab::Ci::Reports::TestCase::STATUS_ERROR system_output = data['error'] @@ -63,9 +65,17 @@ module Gitlab file: data['file'], execution_time: data['time'], status: status, - system_output: system_output + system_output: system_output, + attachment: attachment ) end + + def attachment_path(data) + return unless data + + matches = data.match(ATTACHMENT_TAG_REGEX) + matches[:path] if matches + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index aabdf7ce47d..9b494f3a7ec 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -7,7 +7,7 @@ module Gitlab class Base attr_reader :pipeline, :command, :config - delegate :project, :current_user, to: :command + delegate :project, :current_user, :parent_pipeline, to: :command def initialize(pipeline, command) @pipeline = pipeline diff --git a/lib/gitlab/ci/pipeline/chain/build/associations.rb b/lib/gitlab/ci/pipeline/chain/build/associations.rb new file mode 100644 index 00000000000..eb49c56bcd7 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/build/associations.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class Build + class Associations < Chain::Base + def perform! + return unless @command.bridge + + @pipeline.build_source_pipeline( + source_pipeline: @command.bridge.pipeline, + source_project: @command.bridge.project, + source_bridge: @command.bridge, + project: @command.project + ) + end + + def break? + false + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 6a16e6df23d..fa46114615c 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -72,6 +72,10 @@ module Gitlab project.repository.ambiguous_ref?(origin_ref) end end + + def parent_pipeline + bridge&.parent_pipeline + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 09d1b0edc93..1e47be21b93 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -15,7 +15,8 @@ module Gitlab @command.config_content, { project: project, sha: @pipeline.sha, - user: current_user + user: current_user, + parent_pipeline: parent_pipeline } ) rescue Gitlab::Ci::YamlProcessor::ValidationError => ex diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 98b4b4593e0..114a46ca9f6 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -7,6 +7,8 @@ module Gitlab class Build < Seed::Base include Gitlab::Utils::StrongMemoize + EnvironmentCreationFailure = Class.new(StandardError) + delegate :dig, to: :@seed_attributes # When the `ci_dag_limit_needs` is enabled it uses the lower limit @@ -77,14 +79,39 @@ module Gitlab if bridge? ::Ci::Bridge.new(attributes) else - ::Ci::Build.new(attributes).tap do |job| - job.deployment = Seed::Deployment.new(job).to_resource - job.resource_group = Seed::Build::ResourceGroup.new(job, @resource_group_key).to_resource + ::Ci::Build.new(attributes).tap do |build| + build.assign_attributes(self.class.environment_attributes_for(build)) + build.resource_group = Seed::Build::ResourceGroup.new(build, @resource_group_key).to_resource end end end end + def self.environment_attributes_for(build) + return {} unless build.has_environment? + + environment = Seed::Environment.new(build).to_resource + + # If there is a validation error on environment creation, such as + # the name contains invalid character, the build falls back to a + # non-environment job. + unless environment.persisted? + Gitlab::ErrorTracking.track_exception( + EnvironmentCreationFailure.new, + project_id: build.project_id, + reason: environment.errors.full_messages.to_sentence) + + return { environment: nil } + end + + { + deployment: Seed::Deployment.new(build, environment).to_resource, + metadata_attributes: { + expanded_environment_name: environment.name + } + } + end + private def all_of_only? diff --git a/lib/gitlab/ci/pipeline/seed/deployment.rb b/lib/gitlab/ci/pipeline/seed/deployment.rb index cc63fb4c609..69dfd6be8d5 100644 --- a/lib/gitlab/ci/pipeline/seed/deployment.rb +++ b/lib/gitlab/ci/pipeline/seed/deployment.rb @@ -7,9 +7,9 @@ module Gitlab class Deployment < Seed::Base attr_reader :job, :environment - def initialize(job) + def initialize(job, environment) @job = job - @environment = Seed::Environment.new(@job) + @environment = environment end def to_resource @@ -17,19 +17,18 @@ module Gitlab return unless job.starts_environment? deployment = ::Deployment.new(attributes) - deployment.environment = environment.to_resource # If there is a validation error on environment creation, such as # the name contains invalid character, the job will fall back to a # non-environment job. return unless deployment.valid? && deployment.environment.persisted? - if cluster_id = deployment.environment.deployment_platform&.cluster_id + if cluster = deployment.environment.deployment_platform&.cluster # double write cluster_id until 12.9: https://gitlab.com/gitlab-org/gitlab/issues/202628 - deployment.cluster_id = cluster_id + deployment.cluster_id = cluster.id deployment.deployment_cluster = ::DeploymentCluster.new( - cluster_id: cluster_id, - kubernetes_namespace: deployment.environment.deployment_namespace + cluster_id: cluster.id, + kubernetes_namespace: cluster.kubernetes_namespace_for(deployment.environment, deployable: job) ) end @@ -45,6 +44,7 @@ module Gitlab def attributes { project: job.project, + environment: environment, user: job.user, ref: job.ref, tag: job.tag, diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb index 2d3a1e702f9..42e8c365824 100644 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ b/lib/gitlab/ci/pipeline/seed/environment.rb @@ -12,25 +12,15 @@ module Gitlab end def to_resource - find_environment || ::Environment.create(attributes) + job.project.environments + .safe_find_or_create_by(name: expanded_environment_name) end private - def find_environment - job.project.environments.find_by_name(expanded_environment_name) - end - def expanded_environment_name job.expanded_environment_name end - - def attributes - { - project: job.project, - name: expanded_environment_name - } - end end end end diff --git a/lib/gitlab/ci/reports/coverage_reports.rb b/lib/gitlab/ci/reports/coverage_reports.rb new file mode 100644 index 00000000000..31afb636d2f --- /dev/null +++ b/lib/gitlab/ci/reports/coverage_reports.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CoverageReports + attr_reader :files + + def initialize + @files = {} + end + + def pick(keys) + coverage_files = files.select do |key| + keys.include?(key) + end + + { files: coverage_files } + end + + def add_file(name, line_coverage) + if files[name].present? + line_coverage.each { |line, hits| combine_lines(name, line, hits) } + + else + files[name] = line_coverage + end + end + + private + + def combine_lines(name, line, hits) + if files[name][line].present? + files[name][line] += hits + + else + files[name][line] = hits + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb index fdeaad698b9..55856f64533 100644 --- a/lib/gitlab/ci/reports/test_case.rb +++ b/lib/gitlab/ci/reports/test_case.rb @@ -10,9 +10,9 @@ module Gitlab STATUS_ERROR = 'error' STATUS_TYPES = [STATUS_SUCCESS, STATUS_FAILED, STATUS_SKIPPED, STATUS_ERROR].freeze - attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key + attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment - def initialize(name:, classname:, execution_time:, status:, file: nil, system_output: nil, stack_trace: nil) + def initialize(name:, classname:, execution_time:, status:, file: nil, system_output: nil, stack_trace: nil, attachment: nil) @name = name @classname = classname @file = file @@ -21,6 +21,11 @@ module Gitlab @system_output = system_output @stack_trace = stack_trace @key = sanitize_key_name("#{classname}_#{name}") + @attachment = attachment + end + + def has_attachment? + attachment.present? end private diff --git a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml new file mode 100644 index 00000000000..ecca1731579 --- /dev/null +++ b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml @@ -0,0 +1,36 @@ +stages: + - build + - test + - review + - deploy + - production + +include: + - template: Jobs/Build.gitlab-ci.yml + +.deploy_to_ecs: + image: registry.gitlab.com/gitlab-org/cloud-deploy:latest + script: + - ecs update-task-definition + +review: + extends: .deploy_to_ecs + stage: review + environment: + name: review/$CI_COMMIT_REF_NAME + only: + refs: + - branches + - tags + except: + refs: + - master + +production: + extends: .deploy_to_ecs + stage: production + environment: + name: production + only: + refs: + - master diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index c3ca44eea9e..20063cf6a69 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -1,12 +1,10 @@ performance: stage: performance - # pin to a version matching the dind service, just to be safe image: docker:19.03.5 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" services: - # pin to a known working version until https://gitlab.com/gitlab-org/gitlab-runner/issues/6697 is fixed - docker:19.03.5-dind script: - | diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 488945ffa3e..bb0de9df8bf 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -4,7 +4,6 @@ build: variables: DOCKER_TLS_CERTDIR: "" services: - # pin to a known working version until https://gitlab.com/gitlab-org/gitlab-runner/issues/6697 is fixed - docker:19.03.5-dind script: - | diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index dd5144e28a7..a6338ff6925 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -1,15 +1,13 @@ code_quality: stage: test - # pin to a version matching the dind service, just to be safe image: docker:19.03.5 allow_failure: true services: - # pin to a known working version until https://gitlab.com/gitlab-org/gitlab-runner/issues/6697 is fixed - docker:19.03.5-dind variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.6" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.9" script: - | if ! docker info &>/dev/null; then diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 78ee9b28605..3cf4910fe86 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.9.1" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.10.0" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 47cc6caa192..c6c8256b4bb 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.9.3" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.12.1" review: extends: .auto-deploy @@ -40,6 +40,7 @@ stop_review: environment: name: review/$CI_COMMIT_REF_NAME action: stop + dependencies: [] when: manual allow_failure: true only: diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 73ae63c3092..4ef6a4d3bef 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.8.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.12.0" environment: name: production variables: @@ -11,9 +11,12 @@ apply: SENTRY_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/sentry/values.yaml GITLAB_RUNNER_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/gitlab-runner/values.yaml CILIUM_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/cilium/values.yaml + CILIUM_HUBBLE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/cilium/hubble-values.yaml JUPYTERHUB_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/jupyterhub/values.yaml PROMETHEUS_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/prometheus/values.yaml ELASTIC_STACK_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/elastic-stack/values.yaml + VAULT_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/vault/values.yaml + CROSSPLANE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/crossplane/values.yaml script: - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml only: diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index e7dacd3a1fc..0c8859dc779 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -1,12 +1,13 @@ # Template project: https://gitlab.com/pages/jekyll # Docs: https://docs.gitlab.com/ce/pages/ -image: ruby:2.3 +image: ruby:2.6 variables: JEKYLL_ENV: production LC_ALL: C.UTF-8 before_script: + - gem install bundler - bundle install test: diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml index 57ac323dfdf..462b4737c4e 100644 --- a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -1,5 +1,5 @@ # Full project: https://gitlab.com/pages/middleman -image: ruby:2.3 +image: ruby:2.6 cache: paths: diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml index 7f037b5f5cf..b512f8d77e9 100644 --- a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -1,5 +1,5 @@ # Full project: https://gitlab.com/pages/nanoc -image: ruby:2.3 +image: ruby:2.6 pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml index 6d912a89bc1..4318aadcaa6 100644 --- a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -1,5 +1,5 @@ # Full project: https://gitlab.com/pages/octopress -image: ruby:2.3 +image: ruby:2.6 pages: script: diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index f708e95c2cf..6efb6b4e273 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -5,9 +5,7 @@ variables: container_scanning: stage: test - image: - name: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CS_MAJOR_VERSION - entrypoint: [] + image: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CS_MAJOR_VERSION variables: # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes @@ -22,10 +20,7 @@ container_scanning: - name: $CLAIR_DB_IMAGE alias: clair-vulnerabilities-db script: - # the kubernetes executor currently ignores the Docker image entrypoint value, so the start.sh script must - # be explicitly executed here in order for this to work with both the kubernetes and docker executors - # see this issue for more details https://gitlab.com/gitlab-org/gitlab-runner/issues/4125 - - /container-scanner/start.sh + - /analyzer run artifacts: reports: container_scanning: gl-container-scanning-report.json diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 94b9d94fd39..020d1f323ee 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -18,6 +18,7 @@ dast: image: name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" variables: + GIT_STRATEGY: none # URL to scan: # DAST_WEBSITE: https://example.com/ # diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 5ff6413898f..3200220a332 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -5,7 +5,8 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - DS_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURITY_SCANNER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products" + DS_ANALYZER_IMAGE_PREFIX: "$SECURITY_SCANNER_IMAGE_PREFIX/analyzers" DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_MAJOR_VERSION: 2 DS_DISABLE_DIND: "false" @@ -59,10 +60,12 @@ dependency_scanning: BUNDLER_AUDIT_UPDATE_DISABLED \ BUNDLER_AUDIT_ADVISORY_DB_URL \ BUNDLER_AUDIT_ADVISORY_DB_REF_NAME \ + RETIREJS_JS_ADVISORY_DB \ + RETIREJS_NODE_ADVISORY_DB \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_VERSION" /code + "$SECURITY_SCANNER_IMAGE_PREFIX/dependency-scanning:$DS_VERSION" /code artifacts: reports: dependency_scanning: gl-dependency-scanning-report.json diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 51a1f4e549b..9f9975f9e1c 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -36,9 +36,9 @@ sast: export DOCKER_HOST='tcp://localhost:2375' fi fi + - ENVS=`printenv | grep -vE '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | sed -n '/^[^\t]/s/=.*//p' | sed '/^$/d' | sed 's/^/-e /g' | tr '\n' ' '` - | - ENVS=`printenv | grep -vE '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | sed -n '/^[^\t]/s/=.*//p' | sed '/^$/d' | sed 's/^/-e /g' | tr '\n' ' '` - docker run "$ENVS" \ + docker run $ENVS \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml new file mode 100644 index 00000000000..5d9d3c74def --- /dev/null +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -0,0 +1,19 @@ +# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/accessibility_testing.html + +stages: + - build + - test + - deploy + - accessibility + +a11y: + stage: accessibility + image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.2 + script: /gitlab-accessibility.sh $a11y_urls + allow_failure: true + artifacts: + when: always + expose_as: 'Accessibility Reports' + paths: ['reports/'] + rules: + - if: $a11y_urls diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index ae3ff4a51e2..764047dae6d 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -57,7 +57,7 @@ module Gitlab when: job[:when] || 'on_success', environment: job[:environment_name], coverage_regex: job[:coverage], - yaml_variables: transform_to_yaml_variables(job_variables(name)), + yaml_variables: transform_to_yaml_variables(job[:variables]), needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], only: job[:only], @@ -146,13 +146,6 @@ module Gitlab end end - def job_variables(name) - job_variables = @jobs.dig(name.to_sym, :variables) - - @variables.to_h - .merge(job_variables.to_h) - end - def transform_to_yaml_variables(variables) variables.to_h.map do |key, value| { key: key.to_s, value: value, public: true } diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb index 4deb233d10e..d266d5218de 100644 --- a/lib/gitlab/config/entry/attributable.rb +++ b/lib/gitlab/config/entry/attributable.rb @@ -10,7 +10,7 @@ module Gitlab def attributes(*attributes) attributes.flatten.each do |attribute| if method_defined?(attribute) - raise ArgumentError, "Method already defined: #{attribute}" + raise ArgumentError, "Method '#{attribute}' already defined in '#{name}'" end define_method(attribute) do diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb index e7d441bb21c..571e7a5127e 100644 --- a/lib/gitlab/config/entry/configurable.rb +++ b/lib/gitlab/config/entry/configurable.rb @@ -75,6 +75,9 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def entry(key, entry, description: nil, default: nil, inherit: nil, reserved: nil, metadata: {}) + entry_name = key.to_sym + raise ArgumentError, "Entry '#{key}' already defined in '#{name}'" if @nodes.to_h[entry_name] + factory = ::Gitlab::Config::Entry::Factory.new(entry) .with(description: description) .with(default: default) @@ -82,20 +85,38 @@ module Gitlab .with(reserved: reserved) .metadata(metadata) - (@nodes ||= {}).merge!(key.to_sym => factory) + @nodes ||= {} + @nodes[entry_name] = factory + + helpers(entry_name) end # rubocop: enable CodeReuse/ActiveRecord - def helpers(*nodes) + def dynamic_helpers(*nodes) + helpers(*nodes, dynamic: true) + end + + def helpers(*nodes, dynamic: false) nodes.each do |symbol| + if method_defined?("#{symbol}_defined?") || method_defined?("#{symbol}_entry") || method_defined?("#{symbol}_value") + raise ArgumentError, "Method '#{symbol}_defined?', '#{symbol}_entry' or '#{symbol}_value' already defined in '#{name}'" + end + + unless @nodes.to_h[symbol] + raise ArgumentError, "Entry for #{symbol} is undefined" unless dynamic + end + define_method("#{symbol}_defined?") do entries[symbol]&.specified? end - define_method("#{symbol}_value") do - return unless entries[symbol] && entries[symbol].valid? + define_method("#{symbol}_entry") do + entries[symbol] + end - entries[symbol].value + define_method("#{symbol}_value") do + entry = entries[symbol] + entry.value if entry&.valid? end end end diff --git a/lib/gitlab/config_checker/puma_rugged_checker.rb b/lib/gitlab/config_checker/puma_rugged_checker.rb new file mode 100644 index 00000000000..82c59f3328b --- /dev/null +++ b/lib/gitlab/config_checker/puma_rugged_checker.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module ConfigChecker + module PumaRuggedChecker + extend self + extend Gitlab::Git::RuggedImpl::UseRugged + + def check + notices = [] + + if running_puma_with_multiple_threads? && rugged_enabled_through_feature_flag? + link_start = '<a href="https://docs.gitlab.com/ee/administration/operations/puma.html#performance-caveat-when-using-puma-with-rugged">' + link_end = '</a>' + notices << { + type: 'warning', + message: _('Puma is running with a thread count above 1 and the Rugged '\ + 'service is enabled. This may decrease performance in some environments. '\ + 'See our %{link_start}documentation%{link_end} '\ + 'for details of this issue.') % { link_start: link_start, link_end: link_end } + } + end + + notices + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/usage_data.rb b/lib/gitlab/cycle_analytics/usage_data.rb index acfb641aeec..e58def57e69 100644 --- a/lib/gitlab/cycle_analytics/usage_data.rb +++ b/lib/gitlab/cycle_analytics/usage_data.rb @@ -3,15 +3,32 @@ module Gitlab module CycleAnalytics class UsageData + include Gitlab::Utils::StrongMemoize PROJECTS_LIMIT = 10 - attr_reader :projects, :options + attr_reader :options def initialize - @projects = Project.sorted_by_activity.limit(PROJECTS_LIMIT) @options = { from: 7.days.ago } end + def projects + strong_memoize(:projects) do + projects = Project.where.not(last_activity_at: nil).order(last_activity_at: :desc).limit(10) + + Project.where.not(last_repository_updated_at: nil).order(last_repository_updated_at: :desc).limit(10) + + projects = projects.uniq.sort_by do |project| + [project.last_activity_at, project.last_repository_updated_at].min + end + + if projects.size < 10 + projects.concat(Project.where(last_activity_at: nil, last_repository_updated_at: nil).limit(10)) + end + + projects.uniq.first(10) + end + end + def to_json(*) total = 0 diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb index c0748a4b8e6..8f51ef05f69 100644 --- a/lib/gitlab/danger/commit_linter.rb +++ b/lib/gitlab/danger/commit_linter.rb @@ -14,6 +14,7 @@ module Gitlab MAX_CHANGED_LINES_IN_COMMIT = 30 SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(#|!|&|%)\d+\b}.freeze DEFAULT_SUBJECT_DESCRIPTION = 'commit subject' + WIP_PREFIX = 'WIP: ' PROBLEMS = { subject_too_short: "The %s must contain at least #{MIN_SUBJECT_WORDS_COUNT} words", subject_too_long: "The %s may not be longer than #{MAX_LINE_LENGTH} characters", @@ -164,7 +165,7 @@ module Gitlab end def subject - message_parts[0] + message_parts[0].delete_prefix(WIP_PREFIX) end def separator @@ -199,7 +200,9 @@ module Gitlab end def subject_starts_with_lowercase? - first_char = subject[0] + first_char = subject.sub(/\A\[.+\]\s/, '')[0] + first_char_downcased = first_char.downcase + return true unless ('a'..'z').cover?(first_char_downcased) first_char.downcase == first_char end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 5363533ace5..c5174da4b7c 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -118,19 +118,22 @@ module Gitlab \.haml-lint_todo.yml | babel\.config\.js | jest\.config\.js | - karma\.config\.js | - webpack\.config\.js | package\.json | yarn\.lock | + config/.+\.js | \.gitlab/ci/frontend\.gitlab-ci\.yml )\z}x => :frontend, %r{\A(ee/)?db/(?!fixtures)[^/]+} => :database, %r{\A(ee/)?lib/gitlab/(database|background_migration|sql|github_import)(/|\.rb)} => :database, %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => :database, + %r{\A(ee/)?app/finders/} => :database, %r{\Arubocop/cop/migration(/|\.rb)} => :database, %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity, + %r{\A\.overcommit\.yml\.example\z} => :engineering_productivity, + %r{\Atooling/overcommit/} => :engineering_productivity, + %r{\A.editorconfig\z} => :engineering_productivity, %r{Dangerfile\z} => :engineering_productivity, %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity, %r{\A(ee/)?scripts/} => :engineering_productivity, diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 41ceeb329b3..af363705bed 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -35,7 +35,8 @@ module Gitlab commits: [ { id: "c5feabde2d8cd023215af4d2ceeb7a64839fc428", - message: "Add simple search to projects in public area", + message: "Add simple search to projects in public area\n\ncommit message body", + title: "Add simple search to projects in public area", timestamp: "2013-05-13T18:18:08+00:00", url: "https://test.example.com/gitlab/gitlab/-/commit/c5feabde2d8cd023215af4d2ceeb7a64839fc428", author: { diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index a9d4665bc5f..728e0d423af 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -28,7 +28,7 @@ module Gitlab class BatchCounter FALLBACK = -1 - MIN_REQUIRED_BATCH_SIZE = 2_000 + MIN_REQUIRED_BATCH_SIZE = 1_250 MAX_ALLOWED_LOOPS = 10_000 SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep # Each query should take <<500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 diff --git a/lib/gitlab/database/connection_timer.rb b/lib/gitlab/database/connection_timer.rb new file mode 100644 index 00000000000..ef8d52ba71c --- /dev/null +++ b/lib/gitlab/database/connection_timer.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class ConnectionTimer + DEFAULT_INTERVAL = 3600 + RANDOMIZATION_INTERVAL = 600 + + class << self + def configure + yield self + end + + def starting_now + # add a small amount of randomization to the interval, so reconnects don't all occur at once + new(interval_with_randomization, current_clock_value) + end + + attr_writer :interval + + def interval + @interval ||= DEFAULT_INTERVAL + end + + def interval_with_randomization + interval + rand(RANDOMIZATION_INTERVAL) if interval.positive? + end + + def current_clock_value + Concurrent.monotonic_time + end + end + + attr_reader :interval, :starting_clock_value + + def initialize(interval, starting_clock_value) + @interval = interval + @starting_clock_value = starting_clock_value + end + + def expired? + interval&.positive? && self.class.current_clock_value > (starting_clock_value + interval) + end + + def reset! + @starting_clock_value = self.class.current_clock_value + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 3b6684b861c..82a84508959 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -215,7 +215,7 @@ module Gitlab fk_name = name || concurrent_foreign_key_name(source, column) unless foreign_key_exists?(source, name: fk_name) - raise "cannot find #{fk_name} on #{source} table" + raise missing_schema_object_message(source, "foreign key", fk_name) end disable_statement_timeout do @@ -235,11 +235,17 @@ module Gitlab # PostgreSQL constraint names have a limit of 63 bytes. The logic used # here is based on Rails' foreign_key_name() method, which unfortunately # is private so we can't rely on it directly. - def concurrent_foreign_key_name(table, column) + # + # prefix: + # - The default prefix is `fk_` for backward compatibility with the existing + # concurrent foreign key helpers. + # - For standard rails foreign keys the prefix is `fk_rails_` + # + def concurrent_foreign_key_name(table, column, prefix: 'fk_') identifier = "#{table}_#{column}_fk" hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) - "fk_#{hashed_identifier}" + "#{prefix}#{hashed_identifier}" end # Long-running migrations may take more than the timeout allowed by @@ -688,7 +694,7 @@ module Gitlab start_id, end_id = batch.pluck('MIN(id), MAX(id)').first max_index = index - BackgroundMigrationWorker.perform_in( + migrate_in( index * interval, 'CopyColumn', [table, column, temp_column, start_id, end_id] @@ -697,7 +703,7 @@ module Gitlab # Schedule the renaming of the column to happen (initially) 1 hour after # the last batch finished. - BackgroundMigrationWorker.perform_in( + migrate_in( (max_index * interval) + 1.hour, 'CleanupConcurrentTypeChange', [table, column, temp_column] @@ -779,7 +785,7 @@ module Gitlab start_id, end_id = batch.pluck('MIN(id), MAX(id)').first max_index = index - BackgroundMigrationWorker.perform_in( + migrate_in( index * interval, 'CopyColumn', [table, old_column, new_column, start_id, end_id] @@ -788,7 +794,7 @@ module Gitlab # Schedule the renaming of the column to happen (initially) 1 hour after # the last batch finished. - BackgroundMigrationWorker.perform_in( + migrate_in( (max_index * interval) + 1.hour, 'CleanupConcurrentRename', [table, old_column, new_column] @@ -925,7 +931,10 @@ module Gitlab def column_for(table, name) name = name.to_s - columns(table).find { |column| column.name == name } + column = columns(table).find { |column| column.name == name } + raise(missing_schema_object_message(table, "column", name)) if column.nil? + + column end # This will replace the first occurrence of a string in a column with @@ -1024,14 +1033,14 @@ into similar problems in the future (e.g. when new tables are created). # We push multiple jobs at a time to reduce the time spent in # Sidekiq/Redis operations. We're using this buffer based approach so we # don't need to run additional queries for every range. - BackgroundMigrationWorker.bulk_perform_async(jobs) + bulk_migrate_async(jobs) jobs.clear end jobs << [job_class_name, [start_id, end_id]] end - BackgroundMigrationWorker.bulk_perform_async(jobs) unless jobs.empty? + bulk_migrate_async(jobs) unless jobs.empty? end # Queues background migration jobs for an entire table, batched by ID range. @@ -1042,6 +1051,7 @@ into similar problems in the future (e.g. when new tables are created). # job_class_name - The background migration job class as a string # delay_interval - The duration between each job's scheduled time (must respond to `to_f`) # batch_size - The maximum number of rows per job + # other_arguments - Other arguments to send to the job # # Example: # @@ -1059,7 +1069,7 @@ into similar problems in the future (e.g. when new tables are created). # # do something # end # end - def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE) + def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_arguments: []) raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') # To not overload the worker too much we enforce a minimum interval both @@ -1074,7 +1084,7 @@ into similar problems in the future (e.g. when new tables are created). # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for # the same time, which is not helpful in most cases where we wish to # spread the work over time. - BackgroundMigrationWorker.perform_in(delay_interval * index, job_class_name, [start_id, end_id]) + migrate_in(delay_interval * index, job_class_name, [start_id, end_id] + other_arguments) end end @@ -1133,8 +1143,44 @@ into similar problems in the future (e.g. when new tables are created). execute(sql) end + def migrate_async(*args) + with_migration_context do + BackgroundMigrationWorker.perform_async(*args) + end + end + + def migrate_in(*args) + with_migration_context do + BackgroundMigrationWorker.perform_in(*args) + end + end + + def bulk_migrate_in(*args) + with_migration_context do + BackgroundMigrationWorker.bulk_perform_in(*args) + end + end + + def bulk_migrate_async(*args) + with_migration_context do + BackgroundMigrationWorker.bulk_perform_async(*args) + end + end + private + def missing_schema_object_message(table, type, name) + <<~MESSAGE + Could not find #{type} "#{name}" on table "#{table}" which was referenced during the migration. + This issue could be caused by the database schema straying from the expected state. + + To resolve this issue, please verify: + 1. all previous migrations have completed + 2. the database objects used in this migration match the Rails definition in schema.rb or structure.sql + + MESSAGE + end + def tables_match?(target_table, foreign_key_table) target_table.blank? || foreign_key_table == target_table end @@ -1191,6 +1237,10 @@ into similar problems in the future (e.g. when new tables are created). your migration class ERROR end + + def with_migration_context(&block) + Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block) + end end end end diff --git a/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb new file mode 100644 index 00000000000..9f664fa2137 --- /dev/null +++ b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PostgresqlAdapter + module ForceDisconnectableMixin + extend ActiveSupport::Concern + + prepended do + set_callback :checkin, :after, :force_disconnect_if_old! + end + + def force_disconnect_if_old! + if force_disconnect_timer.expired? + disconnect! + reset_force_disconnect_timer! + end + end + + def reset_force_disconnect_timer! + force_disconnect_timer.reset! + end + + def force_disconnect_timer + @force_disconnect_timer ||= ConnectionTimer.starting_now + end + end + end + end +end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 0a8fbb9a673..e79127108b4 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -17,6 +17,14 @@ module Gitlab buckets [100, 1000, 10000, 100000, 1000000, 10000000] end + define_counter :gitlab_redis_diff_caching_hit do + docstring 'Redis diff caching hits' + end + + define_counter :gitlab_redis_diff_caching_miss do + docstring 'Redis diff caching misses' + end + def initialize(diff_collection) @diff_collection = diff_collection end @@ -93,6 +101,8 @@ module Gitlab # redis.expire(key, EXPIRATION) end + + record_memory_usage(fetch_memory_usage(redis, key)) end # Subsequent read_file calls would need the latest cache. @@ -101,6 +111,23 @@ module Gitlab clear_memoization(:cacheable_files) end + def record_memory_usage(memory_usage) + if memory_usage + self.class.gitlab_redis_diff_caching_memory_usage_bytes.observe({}, memory_usage) + end + end + + def fetch_memory_usage(redis, key) + # Redis versions prior to 4.0.0 do not support memory usage reporting + # for a specific key. As of 11-March-2020 we support Redis 3.x, so + # need to account for this. We can remove this check once we + # officially cease supporting versions <4.0.0. + # + return if Gem::Version.new(redis.info["redis_version"]) < Gem::Version.new("4") + + redis.memory("USAGE", key) + end + def file_paths strong_memoize(:file_paths) do diff_files.collect(&:file_path) diff --git a/lib/gitlab/elasticsearch/logs.rb b/lib/gitlab/elasticsearch/logs.rb new file mode 100644 index 00000000000..f976f6ce305 --- /dev/null +++ b/lib/gitlab/elasticsearch/logs.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +module Gitlab + module Elasticsearch + class Logs + InvalidCursor = Class.new(RuntimeError) + + # How many log lines to fetch in a query + LOGS_LIMIT = 500 + + def initialize(client) + @client = client + end + + def pod_logs(namespace, pod_name, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil) + query = { bool: { must: [] } }.tap do |q| + filter_pod_name(q, pod_name) + filter_namespace(q, namespace) + filter_container_name(q, container_name) + filter_search(q, search) + filter_times(q, start_time, end_time) + end + + body = build_body(query, cursor) + response = @client.search body: body + + format_response(response) + end + + private + + def build_body(query, cursor = nil) + body = { + query: query, + # reverse order so we can query N-most recent records + sort: [ + { "@timestamp": { order: :desc } }, + { "offset": { order: :desc } } + ], + # only return these fields in the response + _source: ["@timestamp", "message"], + # fixed limit for now, we should support paginated queries + size: ::Gitlab::Elasticsearch::Logs::LOGS_LIMIT + } + + unless cursor.nil? + body[:search_after] = decode_cursor(cursor) + end + + body + end + + def filter_pod_name(query, pod_name) + query[:bool][:must] << { + match_phrase: { + "kubernetes.pod.name" => { + query: pod_name + } + } + } + end + + def filter_namespace(query, namespace) + query[:bool][:must] << { + match_phrase: { + "kubernetes.namespace" => { + query: namespace + } + } + } + end + + def filter_container_name(query, container_name) + # A pod can contain multiple containers. + # By default we return logs from every container + return if container_name.nil? + + query[:bool][:must] << { + match_phrase: { + "kubernetes.container.name" => { + query: container_name + } + } + } + end + + def filter_search(query, search) + return if search.nil? + + query[:bool][:must] << { + simple_query_string: { + query: search, + fields: [:message], + default_operator: :and + } + } + end + + def filter_times(query, start_time, end_time) + return unless start_time || end_time + + time_range = { range: { :@timestamp => {} } }.tap do |tr| + tr[:range][:@timestamp][:gte] = start_time if start_time + tr[:range][:@timestamp][:lt] = end_time if end_time + end + + query[:bool][:filter] = [time_range] + end + + def format_response(response) + results = response.fetch("hits", {}).fetch("hits", []) + last_result = results.last + results = results.map do |hit| + { + timestamp: hit["_source"]["@timestamp"], + message: hit["_source"]["message"] + } + end + + # we queried for the N-most recent records but we want them ordered oldest to newest + { + logs: results.reverse, + cursor: last_result.nil? ? nil : encode_cursor(last_result["sort"]) + } + end + + # we want to hide the implementation details of the search_after parameter from the frontend + # behind a single easily transmitted value + def encode_cursor(obj) + obj.join(',') + end + + def decode_cursor(obj) + cursor = obj.split(',').map(&:to_i) + + unless valid_cursor(cursor) + raise InvalidCursor, "invalid cursor format" + end + + cursor + end + + def valid_cursor(cursor) + cursor.instance_of?(Array) && + cursor.length == 2 && + cursor.map {|i| i.instance_of?(Integer)}.reduce(:&) + end + end + end +end diff --git a/lib/gitlab/email.rb b/lib/gitlab/email.rb new file mode 100644 index 00000000000..5f935880764 --- /dev/null +++ b/lib/gitlab/email.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Email + ProcessingError = Class.new(StandardError) + EmailUnparsableError = Class.new(ProcessingError) + SentNotificationNotFoundError = Class.new(ProcessingError) + ProjectNotFound = Class.new(ProcessingError) + EmptyEmailError = Class.new(ProcessingError) + AutoGeneratedEmailError = Class.new(ProcessingError) + UserNotFoundError = Class.new(ProcessingError) + UserBlockedError = Class.new(ProcessingError) + UserNotAuthorizedError = Class.new(ProcessingError) + NoteableNotFoundError = Class.new(ProcessingError) + InvalidRecordError = Class.new(ProcessingError) + InvalidNoteError = Class.new(InvalidRecordError) + InvalidIssueError = Class.new(InvalidRecordError) + InvalidMergeRequestError = Class.new(InvalidRecordError) + UnknownIncomingEmail = Class.new(ProcessingError) + InvalidAttachment = Class.new(ProcessingError) + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index f028102da9b..bf6c28b9f90 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -5,23 +5,6 @@ require_dependency 'gitlab/email/handler' # Inspired in great part by Discourse's Email::Receiver module Gitlab module Email - ProcessingError = Class.new(StandardError) - EmailUnparsableError = Class.new(ProcessingError) - SentNotificationNotFoundError = Class.new(ProcessingError) - ProjectNotFound = Class.new(ProcessingError) - EmptyEmailError = Class.new(ProcessingError) - AutoGeneratedEmailError = Class.new(ProcessingError) - UserNotFoundError = Class.new(ProcessingError) - UserBlockedError = Class.new(ProcessingError) - UserNotAuthorizedError = Class.new(ProcessingError) - NoteableNotFoundError = Class.new(ProcessingError) - InvalidRecordError = Class.new(ProcessingError) - InvalidNoteError = Class.new(InvalidRecordError) - InvalidIssueError = Class.new(InvalidRecordError) - InvalidMergeRequestError = Class.new(InvalidRecordError) - UnknownIncomingEmail = Class.new(ProcessingError) - InvalidAttachment = Class.new(ProcessingError) - class Receiver def initialize(raw) @raw = raw @@ -34,8 +17,7 @@ module Gitlab ignore_auto_reply!(mail) - mail_key = extract_mail_key(mail) - handler = Handler.for(mail, mail_key) + handler = find_handler(mail) raise UnknownIncomingEmail unless handler @@ -46,6 +28,11 @@ module Gitlab private + def find_handler(mail) + mail_key = extract_mail_key(mail) + Handler.for(mail, mail_key) + end + def build_mail Mail::Message.new(@raw) rescue Encoding::UndefinedConversionError, diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 88729babb2b..67f8d691a77 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -50,7 +50,7 @@ module Gitlab detect && detect[:type] == :binary end - def encode_utf8(message) + def encode_utf8(message, replace: "") message = force_encode_utf8(message) return message if message.valid_encoding? @@ -64,7 +64,7 @@ module Gitlab '' end else - clean(message) + clean(message, replace: replace) end rescue ArgumentError nil @@ -94,8 +94,13 @@ module Gitlab message.force_encoding("UTF-8") end - def clean(message) - message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "".encode("UTF-16BE")) + def clean(message, replace: "") + message.encode( + "UTF-16BE", + undef: :replace, + invalid: :replace, + replace: replace.encode("UTF-16BE") + ) .encode("UTF-8") .gsub("\0".encode("UTF-8"), "") end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 7c59267c0b6..30c8eaf605a 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -40,7 +40,7 @@ module Gitlab extend ActiveSupport::Concern included do - before_action :set_experimentation_subject_id_cookie + before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? helper_method :experiment_enabled? end @@ -56,7 +56,12 @@ module Gitlab end def experiment_enabled?(experiment_key) - Experimentation.enabled_for_user?(experiment_key, experimentation_subject_index) || forced_enabled?(experiment_key) + return false if dnt_enabled? + + return true if Experimentation.enabled_for_user?(experiment_key, experimentation_subject_index) + return true if forced_enabled?(experiment_key) + + false end def track_experiment_event(experiment_key, action, value = nil) @@ -73,6 +78,10 @@ module Gitlab private + def dnt_enabled? + Gitlab::Utils.to_boolean(request.headers['DNT']) + end + def experimentation_subject_id cookies.signed[:experimentation_subject_id] end diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb index e052792675a..475d50e37bf 100644 --- a/lib/gitlab/file_type_detection.rb +++ b/lib/gitlab/file_type_detection.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# The method `filename` must be defined in classes that use this module. +# The method `filename` must be defined in classes that mix in this module. # # This module is intended to be used as a helper and not a security gate # to validate that a file is safe, as it identifies files only by the @@ -35,6 +35,13 @@ module Gitlab DANGEROUS_VIDEO_EXT = [].freeze # None, yet DANGEROUS_AUDIO_EXT = [].freeze # None, yet + def self.extension_match?(filename, extensions) + return false unless filename.present? + + extension = File.extname(filename).delete('.') + extensions.include?(extension.downcase) + end + def image? extension_match?(SAFE_IMAGE_EXT) end @@ -74,10 +81,7 @@ module Gitlab private def extension_match?(extensions) - return false unless filename - - extension = File.extname(filename).delete('.') - extensions.include?(extension.downcase) + ::Gitlab::FileTypeDetection.extension_match?(filename, extensions) end end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index f2a6211f270..5579449bf57 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -35,6 +35,11 @@ module Gitlab docstring 'blob.truncated? == false' end + define_histogram :gitlab_blob_size do + docstring 'Gitlab::Git::Blob size' + buckets [1_000, 5_000, 10_000, 50_000, 100_000, 500_000, 1_000_000] + end + class << self def find(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) tree_entry(repository, sha, path, limit) @@ -122,6 +127,9 @@ module Gitlab # Retain the actual size before it is encoded @loaded_size = @data.bytesize if @data @loaded_all_data = @loaded_size == size + + record_metric_blob_size + record_metric_truncated(truncated?) end def binary_in_repo? @@ -157,7 +165,9 @@ module Gitlab end def truncated? - size && (size > loaded_size) + return false unless size && loaded_size + + size > loaded_size end # Valid LFS object pointer is a text file consisting of @@ -197,6 +207,20 @@ module Gitlab private + def record_metric_blob_size + return unless size + + self.class.gitlab_blob_size.observe({}, size) + end + + def record_metric_truncated(bool) + if bool + self.class.gitlab_blob_truncated_true.increment + else + self.class.gitlab_blob_truncated_false.increment + end + end + def has_lfs_version_key? !empty? && text_in_repo? && data.start_with?("version https://git-lfs.github.com/spec") end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 6bfe744a5cd..9adabd4e8fe 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -152,6 +152,12 @@ module Gitlab end end + def replicate(source_repository) + wrapped_gitaly_errors do + gitaly_repository_client.replicate(source_repository) + end + end + def expire_has_local_branches_cache clear_memoization(:has_local_branches) end @@ -322,6 +328,7 @@ module Gitlab limit: 10, offset: 0, path: nil, + author: nil, follow: false, skip_merges: false, after: nil, @@ -766,12 +773,6 @@ module Gitlab !has_visible_content? end - def fetch_repository_as_mirror(repository) - wrapped_gitaly_errors do - gitaly_remote_client.fetch_internal_remote(repository) - end - end - # Fetch remote for repository # # remote - remote name @@ -792,6 +793,14 @@ module Gitlab end end + def import_repository(url) + raise ArgumentError, "don't use disk paths with import_repository: #{url.inspect}" if url.start_with?('.', '/') + + wrapped_gitaly_errors do + gitaly_repository_client.import_repository(url) + end + end + def blob_at(sha, path) Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) end @@ -841,10 +850,9 @@ module Gitlab end end - def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) + def squash(user, squash_id, start_sha:, end_sha:, author:, message:) wrapped_gitaly_errors do - gitaly_operation_client.user_squash(user, squash_id, branch, - start_sha, end_sha, author, message) + gitaly_operation_client.user_squash(user, squash_id, start_sha, end_sha, author, message) end end diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb index f63e35030c1..f9573bedba7 100644 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -15,12 +15,6 @@ module Gitlab Gitlab::GitalyClient.can_use_disk?(repo.storage) end - def running_puma_with_multiple_threads? - return false unless Gitlab::Runtime.puma? - - ::Puma.respond_to?(:cli_config) && ::Puma.cli_config.options[:max_threads] > 1 - end - def execute_rugged_call(method_name, *args) Gitlab::GitalyClient::StorageSettings.allow_disk_access do start = Gitlab::Metrics::System.monotonic_time @@ -43,6 +37,22 @@ module Gitlab result end end + + def running_puma_with_multiple_threads? + return false unless Gitlab::Runtime.puma? + + ::Puma.respond_to?(:cli_config) && ::Puma.cli_config.options[:max_threads] > 1 + end + + def rugged_feature_keys + Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS + end + + def rugged_enabled_through_feature_flag? + rugged_feature_keys.any? do |feature_key| + Feature.enabled?(feature_key) + end + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 906350e57c5..c400e1cd4fd 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -6,7 +6,7 @@ module Gitlab class GitAccess include Gitlab::Utils::StrongMemoize - UnauthorizedError = Class.new(StandardError) + ForbiddenError = Class.new(StandardError) NotFoundError = Class.new(StandardError) ProjectCreationError = Class.new(StandardError) TimeoutError = Class.new(StandardError) @@ -43,15 +43,15 @@ module Gitlab PUSH_COMMANDS = %w{git-receive-pack}.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes, :logger + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :repository_path, :redirected_path, :auth_result_type, :changes, :logger - def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil) + def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, repository_path: nil, redirected_path: nil, auth_result_type: nil) @actor = actor @project = project @protocol = protocol - @authentication_abilities = authentication_abilities + @authentication_abilities = Array(authentication_abilities) @namespace_path = namespace_path || project&.namespace&.full_path - @project_path = project_path || project&.path + @repository_path = repository_path || project&.path @redirected_path = redirected_path @auth_result_type = auth_result_type end @@ -60,7 +60,6 @@ module Gitlab @logger = Checks::TimedLogger.new(timeout: INTERNAL_TIMEOUT, header: LOG_HEADER) @changes = changes - check_namespace! check_protocol! check_valid_actor! check_active_user! @@ -72,11 +71,7 @@ module Gitlab return custom_action if custom_action check_db_accessibility!(cmd) - - ensure_project_on_push!(cmd, changes) - - check_project_accessibility! - add_project_moved_message! + check_project!(changes, cmd) check_repository_existence! case cmd @@ -86,7 +81,7 @@ module Gitlab check_push_access! end - success_result(cmd) + success_result end def guest_can_download_code? @@ -113,19 +108,38 @@ module Gitlab private + def check_project!(changes, cmd) + check_namespace! + ensure_project_on_push!(cmd, changes) + check_project_accessibility! + add_project_moved_message! + end + def check_custom_action(cmd) nil end - def check_for_console_messages(cmd) + def check_for_console_messages + return console_messages unless key? + + key_status = Gitlab::Auth::KeyStatusChecker.new(actor) + + if key_status.show_console_message? + console_messages.push(key_status.console_message) + else + console_messages + end + end + + def console_messages [] end def check_valid_actor! - return unless actor.is_a?(Key) + return unless key? unless actor.valid? - raise UnauthorizedError, "Your SSH key #{actor.errors[:key].first}." + raise ForbiddenError, "Your SSH key #{actor.errors[:key].first}." end end @@ -133,7 +147,7 @@ module Gitlab return if request_from_ci_build? unless protocol_allowed? - raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" + raise ForbiddenError, "Git access over #{protocol.upcase} is not allowed" end end @@ -148,7 +162,7 @@ module Gitlab unless user_access.allowed? message = Gitlab::Auth::UserAccessDeniedReason.new(user).rejection_message - raise UnauthorizedError, message + raise ForbiddenError, message end end @@ -156,11 +170,11 @@ module Gitlab case cmd when *DOWNLOAD_COMMANDS unless authentication_abilities.include?(:download_code) || authentication_abilities.include?(:build_download_code) - raise UnauthorizedError, ERROR_MESSAGES[:auth_download] + raise ForbiddenError, ERROR_MESSAGES[:auth_download] end when *PUSH_COMMANDS unless authentication_abilities.include?(:push_code) - raise UnauthorizedError, ERROR_MESSAGES[:auth_upload] + raise ForbiddenError, ERROR_MESSAGES[:auth_upload] end end end @@ -174,7 +188,7 @@ module Gitlab def add_project_moved_message! return if redirected_path.nil? - project_moved = Checks::ProjectMoved.new(project, user, protocol, redirected_path) + project_moved = Checks::ProjectMoved.new(repository, user, protocol, redirected_path) project_moved.add_message end @@ -189,19 +203,19 @@ module Gitlab def check_upload_pack_disabled! if http? && upload_pack_disabled_over_http? - raise UnauthorizedError, ERROR_MESSAGES[:upload_pack_disabled_over_http] + raise ForbiddenError, ERROR_MESSAGES[:upload_pack_disabled_over_http] end end def check_receive_pack_disabled! if http? && receive_pack_disabled_over_http? - raise UnauthorizedError, ERROR_MESSAGES[:receive_pack_disabled_over_http] + raise ForbiddenError, ERROR_MESSAGES[:receive_pack_disabled_over_http] end end def check_command_existence!(cmd) unless ALL_COMMANDS.include?(cmd) - raise UnauthorizedError, ERROR_MESSAGES[:command_not_allowed] + raise ForbiddenError, ERROR_MESSAGES[:command_not_allowed] end end @@ -209,7 +223,7 @@ module Gitlab return unless receive_pack?(cmd) if Gitlab::Database.read_only? - raise UnauthorizedError, push_to_read_only_message + raise ForbiddenError, push_to_read_only_message end end @@ -222,7 +236,7 @@ module Gitlab return unless user&.can?(:create_projects, namespace) project_params = { - path: project_path, + path: repository_path, namespace_id: namespace.id, visibility_level: Gitlab::VisibilityLevel::PRIVATE } @@ -236,7 +250,7 @@ module Gitlab @project = project user_access.project = @project - Checks::ProjectCreated.new(project, user, protocol).add_message + Checks::ProjectCreated.new(repository, user, protocol).add_message end def check_repository_existence! @@ -253,23 +267,23 @@ module Gitlab guest_can_download_code? unless passed - raise UnauthorizedError, ERROR_MESSAGES[:download] + raise ForbiddenError, ERROR_MESSAGES[:download] end end def check_push_access! if project.repository_read_only? - raise UnauthorizedError, ERROR_MESSAGES[:read_only] + raise ForbiddenError, ERROR_MESSAGES[:read_only] end if deploy_key? unless deploy_key.can_push_to?(project) - raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] + raise ForbiddenError, ERROR_MESSAGES[:deploy_key_upload] end elsif user # User access is verified in check_change_access! else - raise UnauthorizedError, ERROR_MESSAGES[:upload] + raise ForbiddenError, ERROR_MESSAGES[:upload] end check_change_access! @@ -284,7 +298,7 @@ module Gitlab project.any_branch_allows_collaboration?(user_access.user) unless can_push - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] + raise ForbiddenError, ERROR_MESSAGES[:push_code] end else # If there are worktrees with a HEAD pointing to a non-existent object, @@ -338,6 +352,10 @@ module Gitlab actor == :ci end + def key? + actor.is_a?(Key) + end + def can_read_project? if deploy_key? deploy_key.has_access_to?(project) @@ -372,8 +390,8 @@ module Gitlab protected - def success_result(cmd) - ::Gitlab::GitAccessResult::Success.new(console_messages: check_for_console_messages(cmd)) + def success_result + ::Gitlab::GitAccessResult::Success.new(console_messages: check_for_console_messages) end def changes_list diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index d99b9c3fe89..e11c1ea527c 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -2,7 +2,12 @@ module Gitlab class GitAccessSnippet < GitAccess + extend ::Gitlab::Utils::Override + ERROR_MESSAGES = { + authentication_mechanism: 'The authentication mechanism is not supported.', + read_snippet: 'You are not allowed to read this snippet.', + update_snippet: 'You are not allowed to update this snippet.', snippet_not_found: 'The snippet you were looking for could not be found.', repository_not_found: 'The snippet repository you were looking for could not be found.' }.freeze @@ -12,25 +17,43 @@ module Gitlab def initialize(actor, snippet, protocol, **kwargs) @snippet = snippet - super(actor, project, protocol, **kwargs) + super(actor, snippet&.project, protocol, **kwargs) + + @auth_result_type = nil + @authentication_abilities &= [:download_code, :push_code] end - def check(cmd, _changes) - unless Feature.enabled?(:version_snippets, user) - raise NotFoundError, ERROR_MESSAGES[:snippet_not_found] + def check(cmd, changes) + # TODO: Investigate if expanding actor/authentication types are needed. + # https://gitlab.com/gitlab-org/gitlab/issues/202190 + if actor && !actor.is_a?(User) && !actor.instance_of?(Key) + raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism] end check_snippet_accessibility! - success_result(cmd) + super end - def project - snippet&.project + private + + override :check_project! + def check_project!(cmd, changes) + return unless snippet.is_a?(ProjectSnippet) + + check_namespace! + check_project_accessibility! + add_project_moved_message! end - private + override :check_push_access! + def check_push_access! + raise ForbiddenError, ERROR_MESSAGES[:update_snippet] unless user + check_change_access! + end + + override :repository def repository snippet&.repository end @@ -39,10 +62,63 @@ module Gitlab if snippet.blank? raise NotFoundError, ERROR_MESSAGES[:snippet_not_found] end + end - unless repository&.exists? + override :check_download_access! + def check_download_access! + passed = guest_can_download_code? || user_can_download_code? + + unless passed + raise ForbiddenError, ERROR_MESSAGES[:read_snippet] + end + end + + override :guest_can_download_code? + def guest_can_download_code? + Guest.can?(:read_snippet, snippet) + end + + override :user_can_download_code? + def user_can_download_code? + authentication_abilities.include?(:download_code) && user_access.can_do_action?(:read_snippet) + end + + override :check_change_access! + def check_change_access! + unless user_access.can_do_action?(:update_snippet) + raise ForbiddenError, ERROR_MESSAGES[:update_snippet] + end + + changes_list.each do |change| + # If user does not have access to make at least one change, cancel all + # push by allowing the exception to bubble up + check_single_change_access(change) + end + end + + def check_single_change_access(change) + Checks::SnippetCheck.new(change, logger: logger).validate! + Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet::MAX_FILE_COUNT, logger: logger).validate! + rescue Checks::TimedLogger::TimeoutError + raise TimeoutError, logger.full_message + end + + override :check_repository_existence! + def check_repository_existence! + unless repository.exists? raise NotFoundError, ERROR_MESSAGES[:repository_not_found] end end + + override :user_access + def user_access + @user_access ||= UserAccessSnippet.new(user, snippet: snippet) + end + + # TODO: Implement EE/Geo https://gitlab.com/gitlab-org/gitlab/issues/205629 + override :check_custom_action + def check_custom_action(cmd) + nil + end end end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 3d0db753f6e..aad46937c32 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -19,11 +19,11 @@ module Gitlab def check_change_access! unless user_access.can_do_action?(:create_wiki) - raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] + raise ForbiddenError, ERROR_MESSAGES[:write_to_wiki] end if Gitlab::Database.read_only? - raise UnauthorizedError, push_to_read_only_message + raise ForbiddenError, push_to_read_only_message end true diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index 5264bae47a1..13d991cdfbd 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -3,10 +3,10 @@ module Gitlab class GitPostReceive include Gitlab::Identifier - attr_reader :project, :identifier, :changes, :push_options + attr_reader :container, :identifier, :changes, :push_options - def initialize(project, identifier, changes, push_options = {}) - @project = project + def initialize(container, identifier, changes, push_options = {}) + @container = container @identifier = identifier @changes = parse_changes(changes) @push_options = push_options @@ -27,10 +27,10 @@ module Gitlab def includes_default_branch? # If the branch doesn't have a default branch yet, we presume the # first branch pushed will be the default. - return true unless project.default_branch.present? + return true unless container.default_branch.present? changes.branch_changes.any? do |change| - Gitlab::Git.branch_name(change[:ref]) == project.default_branch + Gitlab::Git.branch_name(change[:ref]) == container.default_branch end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 4eb1ccf32ba..3b9402da0dd 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -42,7 +42,7 @@ module Gitlab klass = stub_class(name) addr = stub_address(storage) creds = stub_creds(storage) - klass.new(addr, creds, interceptors: interceptors) + klass.new(addr, creds, interceptors: interceptors, channel_args: channel_args) end end end @@ -54,6 +54,16 @@ module Gitlab end private_class_method :interceptors + def self.channel_args + # These values match the go Gitaly client + # https://gitlab.com/gitlab-org/gitaly/-/blob/bf9f52bc/client/dial.go#L78 + { + 'grpc.keepalive_time_ms': 20000, + 'grpc.keepalive_permit_without_calls': 1 + } + end + private_class_method :channel_args + def self.stub_cert_paths cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"] cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE @@ -141,6 +151,20 @@ module Gitlab # kwargs.merge(deadline: Time.now + 10) # end # + # The optional remote_storage keyword argument is used to enable + # inter-gitaly calls. Say you have an RPC that needs to pull data from + # one repository to another. For example, to fetch a branch from a + # (non-deduplicated) fork into the fork parent. In that case you would + # send an RPC call to the Gitaly server hosting the fork parent, and in + # the request, you would tell that Gitaly server to pull Git data from + # the fork. How does that Gitaly server connect to the Gitaly server the + # forked repo lives on? This is the problem `remote_storage:` solves: it + # adds address and authentication information to the call, as gRPC + # metadata (under the `gitaly-servers` header). The request would say + # "pull from repo X on gitaly-2". In the Ruby code you pass + # `remote_storage: 'gitaly-2'`. And then the metadata would say + # "gitaly-2 is at network address tcp://10.0.1.2:8075". + # def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block) self.measure_timings(service, rpc, request) do self.execute(storage, service, rpc, request, remote_storage: remote_storage, timeout: timeout, &block) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index ac22f5bf419..1f914dc95d1 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -324,7 +324,8 @@ module Gitlab request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] request.revision = encode_binary(options[:ref]) if options[:ref] - request.order = options[:order].upcase.sub('DEFAULT', 'NONE') if options[:order].present? + request.author = encode_binary(options[:author]) if options[:author] + request.order = options[:order].upcase.sub('DEFAULT', 'NONE') if options[:order].present? request.paths = encode_repeated(Array(options[:path])) if options[:path].present? diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 67fb0ab9608..9ed4b2da09a 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -248,12 +248,11 @@ module Gitlab request_enum.close end - def user_squash(user, squash_id, branch, start_sha, end_sha, author, message) + def user_squash(user, squash_id, start_sha, end_sha, author, message) request = Gitaly::UserSquashRequest.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, squash_id: squash_id.to_s, - branch: encode_binary(branch), start_sha: start_sha, end_sha: end_sha, author: Gitlab::Git::User.from_gitlab(author).to_gitaly, diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 0e95b0ef469..2405f3be197 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -41,20 +41,6 @@ module Gitlab GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.long_timeout).result end - def fetch_internal_remote(repository) - request = Gitaly::FetchInternalRemoteRequest.new( - repository: @gitaly_repo, - remote_repository: repository.gitaly_repository - ) - - response = GitalyClient.call(@storage, :remote_service, - :fetch_internal_remote, request, - timeout: GitalyClient.long_timeout, - remote_storage: repository.storage) - - response.result - end - def find_remote_root_ref(remote_name) request = Gitaly::FindRemoteRootRefRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 597ae4651ea..f74c9ea4192 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -359,6 +359,22 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :remove_repository, request, timeout: GitalyClient.long_timeout) end + def replicate(source_repository) + request = Gitaly::ReplicateRepositoryRequest.new( + repository: @gitaly_repo, + source: source_repository.gitaly_repository + ) + + GitalyClient.call( + @storage, + :repository_service, + :replicate_repository, + request, + remote_storage: source_repository.storage, + timeout: GitalyClient.long_timeout + ) + end + private def search_results_from_response(gitaly_response, options = {}) diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 14a6d6443ec..9a7c406d981 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -16,7 +16,7 @@ module Gitlab def self.ghost_user_id key = 'github-import/ghost-user-id' - Caching.read_integer(key) || Caching.write(key, User.select(:id).ghost.id) + Gitlab::Cache::Import::Caching.read_integer(key) || Gitlab::Cache::Import::Caching.write(key, User.select(:id).ghost.id) end end end diff --git a/lib/gitlab/github_import/caching.rb b/lib/gitlab/github_import/caching.rb deleted file mode 100644 index b08f133794f..00000000000 --- a/lib/gitlab/github_import/caching.rb +++ /dev/null @@ -1,151 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - module Caching - # The default timeout of the cache keys. - TIMEOUT = 24.hours.to_i - - WRITE_IF_GREATER_SCRIPT = <<-EOF.strip_heredoc.freeze - local key, value, ttl = KEYS[1], tonumber(ARGV[1]), ARGV[2] - local existing = tonumber(redis.call("get", key)) - - if existing == nil or value > existing then - redis.call("set", key, value) - redis.call("expire", key, ttl) - return true - else - return false - end - EOF - - # Reads a cache key. - # - # If the key exists and has a non-empty value its TTL is refreshed - # automatically. - # - # raw_key - The cache key to read. - # timeout - The new timeout of the key if the key is to be refreshed. - def self.read(raw_key, timeout: TIMEOUT) - key = cache_key_for(raw_key) - value = Redis::Cache.with { |redis| redis.get(key) } - - if value.present? - # We refresh the expiration time so frequently used keys stick - # around, removing the need for querying the database as much as - # possible. - # - # A key may be empty when we looked up a GitHub user (for example) but - # did not find a matching GitLab user. In that case we _don't_ want to - # refresh the TTL so we automatically pick up the right data when said - # user were to register themselves on the GitLab instance. - Redis::Cache.with { |redis| redis.expire(key, timeout) } - end - - value - end - - # Reads an integer from the cache, or returns nil if no value was found. - # - # See Caching.read for more information. - def self.read_integer(raw_key, timeout: TIMEOUT) - value = read(raw_key, timeout: timeout) - - value.to_i if value.present? - end - - # Sets a cache key to the given value. - # - # key - The cache key to write. - # value - The value to set. - # timeout - The time after which the cache key should expire. - def self.write(raw_key, value, timeout: TIMEOUT) - key = cache_key_for(raw_key) - - Redis::Cache.with do |redis| - redis.set(key, value, ex: timeout) - end - - value - end - - # Adds a value to a set. - # - # raw_key - The key of the set to add the value to. - # value - The value to add to the set. - # timeout - The new timeout of the key. - def self.set_add(raw_key, value, timeout: TIMEOUT) - key = cache_key_for(raw_key) - - Redis::Cache.with do |redis| - redis.multi do |m| - m.sadd(key, value) - m.expire(key, timeout) - end - end - end - - # Returns true if the given value is present in the set. - # - # raw_key - The key of the set to check. - # value - The value to check for. - def self.set_includes?(raw_key, value) - key = cache_key_for(raw_key) - - Redis::Cache.with do |redis| - redis.sismember(key, value) - end - end - - # Sets multiple keys to a given value. - # - # mapping - A Hash mapping the cache keys to their values. - # timeout - The time after which the cache key should expire. - def self.write_multiple(mapping, timeout: TIMEOUT) - Redis::Cache.with do |redis| - redis.multi do |multi| - mapping.each do |raw_key, value| - multi.set(cache_key_for(raw_key), value, ex: timeout) - end - end - end - end - - # Sets the expiration time of a key. - # - # raw_key - The key for which to change the timeout. - # timeout - The new timeout. - def self.expire(raw_key, timeout) - key = cache_key_for(raw_key) - - Redis::Cache.with do |redis| - redis.expire(key, timeout) - end - end - - # Sets a key to the given integer but only if the existing value is - # smaller than the given value. - # - # This method uses a Lua script to ensure the read and write are atomic. - # - # raw_key - The key to set. - # value - The new value for the key. - # timeout - The key timeout in seconds. - # - # Returns true when the key was overwritten, false otherwise. - def self.write_if_greater(raw_key, value, timeout: TIMEOUT) - key = cache_key_for(raw_key) - val = Redis::Cache.with do |redis| - redis - .eval(WRITE_IF_GREATER_SCRIPT, keys: [key], argv: [value, timeout]) - end - - val ? true : false - end - - def self.cache_key_for(raw_key) - "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}" - end - end - end -end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 6aad7955415..7ae91912b8a 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -4,7 +4,6 @@ module Gitlab module GithubImport module Importer class RepositoryImporter - include Gitlab::ShellAdapter include Gitlab::Utils::StrongMemoize attr_reader :project, :client, :wiki_formatter @@ -65,10 +64,10 @@ module Gitlab end def import_wiki_repository - gitlab_shell.import_wiki_repository(project, wiki_formatter) + project.wiki.repository.import_repository(wiki_formatter.import_url) true - rescue Gitlab::Shell::Error => e + rescue ::Gitlab::Git::CommandError => e if e.message !~ /repository not exported/ project.create_wiki fail_import("Failed to import the wiki: #{e.message}") diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb index c81603a1aa9..136531505ea 100644 --- a/lib/gitlab/github_import/issuable_finder.rb +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -23,7 +23,7 @@ module Gitlab # # This method will return `nil` if no ID could be found. def database_id - val = Caching.read(cache_key) + val = Gitlab::Cache::Import::Caching.read(cache_key) val.to_i if val.present? end @@ -32,7 +32,7 @@ module Gitlab # # database_id - The ID of the corresponding database row. def cache_database_id(database_id) - Caching.write(cache_key, database_id) + Gitlab::Cache::Import::Caching.write(cache_key, database_id) end private diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb index cad39e48e43..39e669dbba4 100644 --- a/lib/gitlab/github_import/label_finder.rb +++ b/lib/gitlab/github_import/label_finder.rb @@ -15,7 +15,7 @@ module Gitlab # Returns the label ID for the given name. def id_for(name) - Caching.read_integer(cache_key_for(name)) + Gitlab::Cache::Import::Caching.read_integer(cache_key_for(name)) end # rubocop: disable CodeReuse/ActiveRecord @@ -27,7 +27,7 @@ module Gitlab hash[cache_key_for(name)] = id end - Caching.write_multiple(mapping) + Gitlab::Cache::Import::Caching.write_multiple(mapping) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb index a157a1e1ff5..d9290e36ea1 100644 --- a/lib/gitlab/github_import/milestone_finder.rb +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -18,7 +18,7 @@ module Gitlab def id_for(issuable) return unless issuable.milestone_number - Caching.read_integer(cache_key_for(issuable.milestone_number)) + Gitlab::Cache::Import::Caching.read_integer(cache_key_for(issuable.milestone_number)) end # rubocop: disable CodeReuse/ActiveRecord @@ -30,7 +30,7 @@ module Gitlab hash[cache_key_for(iid)] = id end - Caching.write_multiple(mapping) + Gitlab::Cache::Import::Caching.write_multiple(mapping) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/github_import/page_counter.rb b/lib/gitlab/github_import/page_counter.rb index a3e7b3c1afc..3b4fd42ba2a 100644 --- a/lib/gitlab/github_import/page_counter.rb +++ b/lib/gitlab/github_import/page_counter.rb @@ -19,12 +19,12 @@ module Gitlab # # Returns true if the page number was overwritten, false otherwise. def set(page) - Caching.write_if_greater(cache_key, page) + Gitlab::Cache::Import::Caching.write_if_greater(cache_key, page) end # Returns the current value from the cache. def current - Caching.read_integer(cache_key) || 1 + Gitlab::Cache::Import::Caching.read_integer(cache_key) || 1 end end end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 849a66d47ed..cabc615ea11 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -42,7 +42,7 @@ module Gitlab # still scheduling duplicates while. Since all work has already been # completed those jobs will just cycle through any remaining pages while # not scheduling anything. - Caching.expire(already_imported_cache_key, 15.minutes.to_i) + Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, 15.minutes.to_i) retval end @@ -112,14 +112,14 @@ module Gitlab def already_imported?(object) id = id_for_already_imported_cache(object) - Caching.set_includes?(already_imported_cache_key, id) + Gitlab::Cache::Import::Caching.set_includes?(already_imported_cache_key, id) end # Marks the given object as "already imported". def mark_as_imported(object) id = id_for_already_imported_cache(object) - Caching.set_add(already_imported_cache_key, id) + Gitlab::Cache::Import::Caching.set_add(already_imported_cache_key, id) end # Returns the ID to use for the cache used for checking if an object has diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 51a532437bd..9da986ae921 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -102,11 +102,11 @@ module Gitlab def email_for_github_username(username) cache_key = EMAIL_FOR_USERNAME_CACHE_KEY % username - email = Caching.read(cache_key) + email = Gitlab::Cache::Import::Caching.read(cache_key) unless email user = client.user(username) - email = Caching.write(cache_key, user.email) if user + email = Gitlab::Cache::Import::Caching.write(cache_key, user.email) if user end email @@ -125,7 +125,7 @@ module Gitlab def id_for_github_id(id) gitlab_id = query_id_for_github_id(id) || nil - Caching.write(ID_CACHE_KEY % id, gitlab_id) + Gitlab::Cache::Import::Caching.write(ID_CACHE_KEY % id, gitlab_id) end # Queries and caches the GitLab user ID for a GitHub email, if one was @@ -133,7 +133,7 @@ module Gitlab def id_for_github_email(email) gitlab_id = query_id_for_github_email(email) || nil - Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id) + Gitlab::Cache::Import::Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id) end # rubocop: disable CodeReuse/ActiveRecord @@ -155,7 +155,7 @@ module Gitlab # 1. A boolean indicating if the key was present or not. # 2. The ID as an Integer, or nil in case no ID could be found. def read_id_from_cache(key) - value = Caching.read(key) + value = Gitlab::Cache::Import::Caching.read(key) exists = !value.nil? number = value.to_i diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index fcebcb463cd..26440e6f82d 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -7,19 +7,21 @@ module Gitlab PROJECT = RepoType.new( name: :project, access_checker_class: Gitlab::GitAccess, - repository_resolver: -> (project) { project.repository } + repository_resolver: -> (project) { project&.repository } ).freeze WIKI = RepoType.new( name: :wiki, access_checker_class: Gitlab::GitAccessWiki, - repository_resolver: -> (project) { project.wiki.repository }, + repository_resolver: -> (project) { project&.wiki&.repository }, suffix: :wiki ).freeze SNIPPET = RepoType.new( name: :snippet, access_checker_class: Gitlab::GitAccessSnippet, - repository_resolver: -> (snippet) { snippet.repository }, - container_resolver: -> (id) { Snippet.find_by_id(id) } + repository_resolver: -> (snippet) { snippet&.repository }, + container_resolver: -> (id) { Snippet.find_by_id(id) }, + project_resolver: -> (snippet) { snippet&.project }, + guest_read_ability: :read_snippet ).freeze TYPES = { @@ -42,7 +44,7 @@ module Gitlab container = type.fetch_container!(gl_repository) - [container, type] + [container, type.project_for(container), type] end def self.default_type diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 9663fd7de8f..052ce578881 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -7,6 +7,8 @@ module Gitlab :access_checker_class, :repository_resolver, :container_resolver, + :project_resolver, + :guest_read_ability, :suffix def initialize( @@ -14,11 +16,15 @@ module Gitlab access_checker_class:, repository_resolver:, container_resolver: default_container_resolver, + project_resolver: nil, + guest_read_ability: :download_code, suffix: nil) @name = name @access_checker_class = access_checker_class @repository_resolver = repository_resolver @container_resolver = container_resolver + @project_resolver = project_resolver + @guest_read_ability = guest_read_ability @suffix = suffix end @@ -59,8 +65,18 @@ module Gitlab repository_resolver.call(container) end + def project_for(container) + return container unless project_resolver + + project_resolver.call(container) + end + def valid?(repository_path) - repository_path.end_with?(path_suffix) + repository_path.end_with?(path_suffix) && + ( + !snippet? || + repository_path.match?(Gitlab::PathRegex.full_snippets_repository_path_regex) + ) end private diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 3db6c3b51c0..e4e69241bd9 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -46,6 +46,7 @@ module Gitlab push_frontend_feature_flag(:monaco_snippets, default_enabled: false) push_frontend_feature_flag(:monaco_blobs, default_enabled: false) push_frontend_feature_flag(:monaco_ci, default_enabled: false) + push_frontend_feature_flag(:snippets_edit_vue, default_enabled: false) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb index 837473d47cd..045a341f2ed 100644 --- a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb +++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb @@ -6,6 +6,8 @@ module Gitlab class LogrageWithTimestamp include Gitlab::EncodingHelper + EMPTY_ARRAY = [].freeze + def call(severity, datetime, _, data) time = data.delete :time data[:params] = process_params(data) @@ -16,29 +18,27 @@ module Gitlab duration: time[:total], db: time[:db], view: time[:view] - }.merge(data) - ::Lograge.formatter.call(attributes) + "\n" + }.merge!(data) + + ::Lograge.formatter.call(attributes) << "\n" end private def process_params(data) - return [] unless data.has_key?(:params) + return EMPTY_ARRAY unless data.has_key?(:params) - params_array = - data[:params] - .each_pair - .map { |k, v| { key: k, value: utf8_encode_values(v) } } + params_array = data[:params].map { |k, v| { key: k, value: utf8_encode_values(v) } } - Gitlab::Utils::LogLimitedArray.log_limited_array(params_array) + Gitlab::Utils::LogLimitedArray.log_limited_array(params_array, sentinel: Gitlab::Lograge::CustomOptions::LIMITED_ARRAY_SENTINEL) end def utf8_encode_values(data) case data when Hash - data.merge(data) { |k, v| utf8_encode_values(v) } + data.merge!(data) { |k, v| utf8_encode_values(v) } when Array - data.map { |v| utf8_encode_values(v) } + data.map! { |v| utf8_encode_values(v) } when String encode_utf8(data) end diff --git a/lib/gitlab/graphql/connections.rb b/lib/gitlab/graphql/connections.rb index 08d5cd0b72e..0c0bfe5a458 100644 --- a/lib/gitlab/graphql/connections.rb +++ b/lib/gitlab/graphql/connections.rb @@ -16,6 +16,10 @@ module Gitlab Gitlab::Graphql::ExternallyPaginatedArray, Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection ) + GraphQL::Relay::BaseConnection.register_connection_implementation( + Gitlab::Graphql::Pagination::Relations::OffsetActiveRecordRelation, + Gitlab::Graphql::Pagination::OffsetActiveRecordRelationConnection + ) end end end diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb index 56524120ffd..0dd28b32511 100644 --- a/lib/gitlab/graphql/docs/helper.rb +++ b/lib/gitlab/graphql/docs/helper.rb @@ -25,6 +25,28 @@ module Gitlab fields.sort_by { |field| field[:name] } end + def render_field(field) + '| %s | %s | %s |' % [ + render_field_name(field), + render_field_type(field[:type][:info]), + render_field_description(field) + ] + end + + def render_field_name(field) + rendered_name = "`#{field[:name]}`" + rendered_name += ' **{warning-solid}**' if field[:is_deprecated] + rendered_name + end + + # Returns the field description. If the field has been deprecated, + # the deprecation reason will be returned in place of the description. + def render_field_description(field) + return field[:description] unless field[:is_deprecated] + + "**Deprecated:** #{field[:deprecation_reason]}" + end + # Some fields types are arrays of other types and are displayed # on docs wrapped in square brackets, for example: [String!]. # This makes GitLab docs renderer thinks they are links so here diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index b126a22c301..8c033526557 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -11,6 +11,9 @@ Each table below documents a GraphQL type. Types match loosely to models, but not all fields and methods on a model are available via GraphQL. + + CAUTION: **Caution:** + Fields that are deprecated are marked with **{warning-solid}**. \ - objects.each do |type| - unless type[:fields].empty? @@ -22,5 +25,5 @@ ~ "| Name | Type | Description |" ~ "| --- | ---- | ---------- |" - sorted_fields(type[:fields]).each do |field| - = "| `#{field[:name]}` | #{render_field_type(field[:type][:info])} | #{field[:description]} |" + = render_field(field) \ diff --git a/lib/gitlab/graphql/pagination/offset_active_record_relation_connection.rb b/lib/gitlab/graphql/pagination/offset_active_record_relation_connection.rb new file mode 100644 index 00000000000..c852fbf0ab8 --- /dev/null +++ b/lib/gitlab/graphql/pagination/offset_active_record_relation_connection.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# We use the Keyset / Stable cursor connection by default for ActiveRecord::Relation. +# However, there are times when that may not be powerful enough (yet), and we +# want to use standard offset pagination. +module Gitlab + module Graphql + module Pagination + class OffsetActiveRecordRelationConnection < GraphQL::Relay::RelationConnection + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/relations/offset_active_record_relation.rb b/lib/gitlab/graphql/pagination/relations/offset_active_record_relation.rb new file mode 100644 index 00000000000..2e5a0d66d4e --- /dev/null +++ b/lib/gitlab/graphql/pagination/relations/offset_active_record_relation.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Pagination + module Relations + class OffsetActiveRecordRelation < ::ActiveRecord::Relation + end + end + end + end +end diff --git a/lib/gitlab/graphql/timeout.rb b/lib/gitlab/graphql/timeout.rb new file mode 100644 index 00000000000..4282c46a19e --- /dev/null +++ b/lib/gitlab/graphql/timeout.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + class Timeout < GraphQL::Schema::Timeout + def handle_timeout(error, query) + Gitlab::GraphqlLogger.error(message: error.message, query: query.query_string, query_variables: query.provided_variables) + end + end + end +end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 8ce6549c0c7..52102b6f508 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -42,8 +42,20 @@ module Gitlab "project.wiki.bundle" end + def snippet_repo_bundle_dir + 'snippets' + end + + def snippets_repo_bundle_path(absolute_path) + File.join(absolute_path, ::Gitlab::ImportExport.snippet_repo_bundle_dir) + end + + def snippet_repo_bundle_filename_for(snippet) + "#{snippet.hexdigest}.bundle" + end + def config_file - Rails.root.join('lib/gitlab/import_export/import_export.yml') + Rails.root.join('lib/gitlab/import_export/project/import_export.yml') end def version_filename @@ -77,7 +89,7 @@ module Gitlab end def group_config_file - Rails.root.join('lib/gitlab/import_export/group_import_export.yml') + Rails.root.join('lib/gitlab/import_export/group/import_export.yml') end end end diff --git a/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb b/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb new file mode 100644 index 00000000000..2e3136936f8 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module AfterExportStrategies + class MoveFileStrategy < BaseAfterExportStrategy + def initialize(archive_path:) + @archive_path = archive_path + end + + private + + def strategy_execute + FileUtils.mv(project.export_file.path, @archive_path) + end + end + end + end +end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index d1c20dff799..3bfc059dcd3 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -4,8 +4,8 @@ module Gitlab module ImportExport class AttributeCleaner ALLOWED_REFERENCES = [ - *ProjectRelationFactory::PROJECT_REFERENCES, - *ProjectRelationFactory::USER_REFERENCES, + *Gitlab::ImportExport::Project::RelationFactory::PROJECT_REFERENCES, + *Gitlab::ImportExport::Project::RelationFactory::USER_REFERENCES, 'group_id', 'commit_id', 'discussion_id', diff --git a/lib/gitlab/import_export/base/object_builder.rb b/lib/gitlab/import_export/base/object_builder.rb new file mode 100644 index 00000000000..109d2e233a5 --- /dev/null +++ b/lib/gitlab/import_export/base/object_builder.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Base + # Base class for Group & Project Object Builders. + # This class is not intended to be used on its own but + # rather inherited from. + # + # Cache keeps 1000 entries at most, 1000 is chosen based on: + # - one cache entry uses around 0.5K memory, 1000 items uses around 500K. + # (leave some buffer it should be less than 1M). It is afforable cost for project import. + # - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough. + # For example, gitlab has ~970 labels and 26 milestones. + LRU_CACHE_SIZE = 1000 + + class ObjectBuilder + def self.build(*args) + new(*args).find + end + + def initialize(klass, attributes) + @klass = klass.ancestors.include?(Label) ? Label : klass + @attributes = attributes + + if Gitlab::SafeRequestStore.active? + @lru_cache = cache_from_request_store + @cache_key = [klass, attributes] + end + end + + def find + find_with_cache do + find_object || klass.create(prepare_attributes) + end + end + + protected + + def where_clauses + raise NotImplementedError + end + + # attributes wrapped in a method to be + # adjusted in sub-class if needed + def prepare_attributes + attributes + end + + private + + attr_reader :klass, :attributes, :lru_cache, :cache_key + + def find_with_cache + return yield unless lru_cache && cache_key + + lru_cache[cache_key] ||= yield + end + + def cache_from_request_store + Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE) + end + + def find_object + klass.where(where_clause).first + end + + def where_clause + where_clauses.reduce(:and) + end + + def table + @table ||= klass.arel_table + end + + # Returns Arel clause: + # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"` + # from the given Hash of attributes. + def attrs_to_arel(attrs) + attrs.map do |key, value| + table[key].eq(value) + end.reduce(:and) + end + + # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'` + # if attributes has 'title key, otherwise `nil`. + def where_clause_for_title + attrs_to_arel(attributes.slice('title')) + end + + # Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'` + # if attributes has 'description key, otherwise `nil`. + def where_clause_for_description + attrs_to_arel(attributes.slice('description')) + end + + # Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'` + # if attributes has 'created_at key, otherwise `nil`. + def where_clause_for_created_at + attrs_to_arel(attributes.slice('created_at')) + end + end + end + end +end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb new file mode 100644 index 00000000000..05b69362976 --- /dev/null +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -0,0 +1,312 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Base + class RelationFactory + include Gitlab::Utils::StrongMemoize + + IMPORTED_OBJECT_MAX_RETRIES = 5.freeze + + OVERRIDES = {}.freeze + EXISTING_OBJECT_RELATIONS = %i[].freeze + + # This represents all relations that have unique key on `project_id` or `group_id` + UNIQUE_RELATIONS = %i[].freeze + + USER_REFERENCES = %w[ + author_id + assignee_id + updated_by_id + merged_by_id + latest_closed_by_id + user_id + created_by_id + last_edited_by_id + merge_user_id + resolved_by_id + closed_by_id + owner_id + ].freeze + + TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze + + def self.create(*args) + new(*args).create + end + + def self.relation_class(relation_name) + # There are scenarios where the model is pluralized (e.g. + # MergeRequest::Metrics), and we don't want to force it to singular + # with #classify. + relation_name.to_s.classify.constantize + rescue NameError + relation_name.to_s.constantize + end + + def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) + @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym + @relation_hash = relation_hash.except('noteable_id') + @members_mapper = members_mapper + @object_builder = object_builder + @user = user + @importable = importable + @imported_object_retries = 0 + @relation_hash[importable_column_name] = @importable.id + + # Remove excluded keys from relation_hash + # We don't do this in the parsed_relation_hash because of the 'transformed attributes' + # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, + # in the create method that attribute is renamed to diff. And because diff is an excluded key, + # if we clean the excluded keys in the parsed_relation_hash, it will be removed + # from the object attributes and the export will fail. + @relation_hash.except!(*excluded_keys) + end + + # Creates an object from an actual model with name "relation_sym" with params from + # the relation_hash, updating references with new object IDs, mapping users using + # the "members_mapper" object, also updating notes if required. + def create + return if invalid_relation? || predefined_relation? + + setup_base_models + setup_models + + generate_imported_object + end + + def self.overrides + self::OVERRIDES + end + + def self.existing_object_relations + self::EXISTING_OBJECT_RELATIONS + end + + private + + def invalid_relation? + false + end + + def predefined_relation? + relation_class.try(:predefined_id?, @relation_hash['id']) + end + + def setup_models + raise NotImplementedError + end + + def unique_relations + # define in sub-class if any + self.class::UNIQUE_RELATIONS + end + + def setup_base_models + update_user_references + remove_duplicate_assignees + reset_tokens! + remove_encrypted_attributes! + end + + def update_user_references + self.class::USER_REFERENCES.each do |reference| + if @relation_hash[reference] + @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] + end + end + end + + def remove_duplicate_assignees + return unless @relation_hash['issue_assignees'] + + # When an assignee did not exist in the members mapper, the importer is + # assigned. We only need to assign each user once. + @relation_hash['issue_assignees'].uniq!(&:user_id) + end + + def generate_imported_object + imported_object + end + + def reset_tokens! + return unless Gitlab::ImportExport.reset_tokens? && self.class::TOKEN_RESET_MODELS.include?(@relation_name) + + # If we import/export to the same instance, tokens will have to be reset. + # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. + relation_class.attribute_names.select { |name| name.include?('token') }.each do |token| + @relation_hash[token] = nil + end + end + + def remove_encrypted_attributes! + return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? + + relation_class.encrypted_attributes.each_key do |key| + @relation_hash[key.to_s] = nil + end + end + + def relation_class + @relation_class ||= self.class.relation_class(@relation_name) + end + + def importable_column_name + importable_class_name.concat('_id') + end + + def importable_class_name + @importable.class.to_s.downcase + end + + def imported_object + if existing_or_new_object.respond_to?(:importing) + existing_or_new_object.importing = true + end + + existing_or_new_object + rescue ActiveRecord::RecordNotUnique + # as the operation is not atomic, retry in the unlikely scenario an INSERT is + # performed on the same object between the SELECT and the INSERT + @imported_object_retries += 1 + retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES + end + + def parsed_relation_hash + @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, + relation_class: relation_class) + end + + def existing_or_new_object + # Only find existing records to avoid mapping tables such as milestones + # Otherwise always create the record, skipping the extra SELECT clause. + @existing_or_new_object ||= begin + if existing_object? + attribute_hash = attribute_hash_for(['events']) + + existing_object.assign_attributes(attribute_hash) if attribute_hash.any? + + existing_object + else + # Because of single-type inheritance, we need to be careful to use the `type` field + # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 + inheritance_column = relation_class.try(:inheritance_column) + inheritance_attributes = parsed_relation_hash.slice(inheritance_column) + object = relation_class.new(inheritance_attributes) + object.assign_attributes(parsed_relation_hash) + object + end + end + end + + def attribute_hash_for(attributes) + attributes.each_with_object({}) do |hash, value| + hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value] + hash + end + end + + def existing_object + @existing_object ||= find_or_create_object! + end + + def unique_relation_object + unique_relation_object = relation_class.find_or_create_by(importable_column_name => @importable.id) + unique_relation_object.assign_attributes(parsed_relation_hash) + unique_relation_object + end + + def find_or_create_object! + return unique_relation_object if unique_relation? + + # Can't use IDs as validation exists calling `group` or `project` attributes + finder_hash = parsed_relation_hash.tap do |hash| + if relation_class.attribute_method?('group_id') && @importable.is_a?(::Project) + hash['group'] = @importable.group + end + + hash[importable_class_name] = @importable if relation_class.reflect_on_association(importable_class_name.to_sym) + hash.delete(importable_column_name) + end + + @object_builder.build(relation_class, finder_hash) + end + + def setup_note + set_note_author + # attachment is deprecated and note uploads are handled by Markdown uploader + @relation_hash['attachment'] = nil + end + + # Sets the author for a note. If the user importing the project + # has admin access, an actual mapping with new project members + # will be used. Otherwise, a note stating the original author name + # is left. + def set_note_author + old_author_id = @relation_hash['author_id'] + author = @relation_hash.delete('author') + + update_note_for_missing_author(author['name']) unless has_author?(old_author_id) + end + + def has_author?(old_author_id) + admin_user? && @members_mapper.include?(old_author_id) + end + + def missing_author_note(updated_at, author_name) + timestamp = updated_at.split('.').first + "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" + end + + def update_note_for_missing_author(author_name) + @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? + @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" + end + + def admin_user? + @user.admin? + end + + def existing_object? + strong_memoize(:_existing_object) do + self.class.existing_object_relations.include?(@relation_name) || unique_relation? + end + end + + def unique_relation? + strong_memoize(:unique_relation) do + importable_foreign_key.present? && + (has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?) + end + end + + def has_unique_index_on_importable_fk? + cache = cached_has_unique_index_on_importable_fk + table_name = relation_class.table_name + return cache[table_name] if cache.has_key?(table_name) + + index_exists = + ActiveRecord::Base.connection.index_exists?( + relation_class.table_name, + importable_foreign_key, + unique: true) + + cache[table_name] = index_exists + end + + # Avoid unnecessary DB requests + def cached_has_unique_index_on_importable_fk + Thread.current[:cached_has_unique_index_on_importable_fk] ||= {} + end + + def uses_importable_fk_as_primary_key? + relation_class.primary_key == importable_foreign_key + end + + def importable_foreign_key + relation_class.reflect_on_association(importable_class_name.to_sym)&.foreign_key + end + end + end + end +end diff --git a/lib/gitlab/import_export/base_object_builder.rb b/lib/gitlab/import_export/base_object_builder.rb deleted file mode 100644 index ec66b7a7a4f..00000000000 --- a/lib/gitlab/import_export/base_object_builder.rb +++ /dev/null @@ -1,103 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - # Base class for Group & Project Object Builders. - # This class is not intended to be used on its own but - # rather inherited from. - # - # Cache keeps 1000 entries at most, 1000 is chosen based on: - # - one cache entry uses around 0.5K memory, 1000 items uses around 500K. - # (leave some buffer it should be less than 1M). It is afforable cost for project import. - # - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough. - # For example, gitlab has ~970 labels and 26 milestones. - LRU_CACHE_SIZE = 1000 - - class BaseObjectBuilder - def self.build(*args) - new(*args).find - end - - def initialize(klass, attributes) - @klass = klass.ancestors.include?(Label) ? Label : klass - @attributes = attributes - - if Gitlab::SafeRequestStore.active? - @lru_cache = cache_from_request_store - @cache_key = [klass, attributes] - end - end - - def find - find_with_cache do - find_object || klass.create(prepare_attributes) - end - end - - protected - - def where_clauses - raise NotImplementedError - end - - # attributes wrapped in a method to be - # adjusted in sub-class if needed - def prepare_attributes - attributes - end - - private - - attr_reader :klass, :attributes, :lru_cache, :cache_key - - def find_with_cache - return yield unless lru_cache && cache_key - - lru_cache[cache_key] ||= yield - end - - def cache_from_request_store - Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE) - end - - def find_object - klass.where(where_clause).first - end - - def where_clause - where_clauses.reduce(:and) - end - - def table - @table ||= klass.arel_table - end - - # Returns Arel clause: - # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"` - # from the given Hash of attributes. - def attrs_to_arel(attrs) - attrs.map do |key, value| - table[key].eq(value) - end.reduce(:and) - end - - # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'` - # if attributes has 'title key, otherwise `nil`. - def where_clause_for_title - attrs_to_arel(attributes.slice('title')) - end - - # Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'` - # if attributes has 'description key, otherwise `nil`. - def where_clause_for_description - attrs_to_arel(attributes.slice('description')) - end - - # Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'` - # if attributes has 'created_at key, otherwise `nil`. - def where_clause_for_created_at - attrs_to_arel(attributes.slice('created_at')) - end - end - end -end diff --git a/lib/gitlab/import_export/base_relation_factory.rb b/lib/gitlab/import_export/base_relation_factory.rb deleted file mode 100644 index d3c8802bcce..00000000000 --- a/lib/gitlab/import_export/base_relation_factory.rb +++ /dev/null @@ -1,307 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class BaseRelationFactory - include Gitlab::Utils::StrongMemoize - - IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - - OVERRIDES = {}.freeze - EXISTING_OBJECT_RELATIONS = %i[].freeze - - # This represents all relations that have unique key on `project_id` or `group_id` - UNIQUE_RELATIONS = %i[].freeze - - USER_REFERENCES = %w[ - author_id - assignee_id - updated_by_id - merged_by_id - latest_closed_by_id - user_id - created_by_id - last_edited_by_id - merge_user_id - resolved_by_id - closed_by_id - owner_id - ].freeze - - TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze - - def self.create(*args) - new(*args).create - end - - def self.relation_class(relation_name) - # There are scenarios where the model is pluralized (e.g. - # MergeRequest::Metrics), and we don't want to force it to singular - # with #classify. - relation_name.to_s.classify.constantize - rescue NameError - relation_name.to_s.constantize - end - - def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, merge_requests_mapping: nil, user:, importable:, excluded_keys: []) - @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym - @relation_hash = relation_hash.except('noteable_id') - @members_mapper = members_mapper - @object_builder = object_builder - @merge_requests_mapping = merge_requests_mapping - @user = user - @importable = importable - @imported_object_retries = 0 - @relation_hash[importable_column_name] = @importable.id - - # Remove excluded keys from relation_hash - # We don't do this in the parsed_relation_hash because of the 'transformed attributes' - # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, - # in the create method that attribute is renamed to diff. And because diff is an excluded key, - # if we clean the excluded keys in the parsed_relation_hash, it will be removed - # from the object attributes and the export will fail. - @relation_hash.except!(*excluded_keys) - end - - # Creates an object from an actual model with name "relation_sym" with params from - # the relation_hash, updating references with new object IDs, mapping users using - # the "members_mapper" object, also updating notes if required. - def create - return if invalid_relation? - - setup_base_models - setup_models - - generate_imported_object - end - - def self.overrides - self::OVERRIDES - end - - def self.existing_object_relations - self::EXISTING_OBJECT_RELATIONS - end - - private - - def invalid_relation? - false - end - - def setup_models - raise NotImplementedError - end - - def unique_relations - # define in sub-class if any - self.class::UNIQUE_RELATIONS - end - - def setup_base_models - update_user_references - remove_duplicate_assignees - reset_tokens! - remove_encrypted_attributes! - end - - def update_user_references - self.class::USER_REFERENCES.each do |reference| - if @relation_hash[reference] - @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] - end - end - end - - def remove_duplicate_assignees - return unless @relation_hash['issue_assignees'] - - # When an assignee did not exist in the members mapper, the importer is - # assigned. We only need to assign each user once. - @relation_hash['issue_assignees'].uniq!(&:user_id) - end - - def generate_imported_object - imported_object - end - - def reset_tokens! - return unless Gitlab::ImportExport.reset_tokens? && self.class::TOKEN_RESET_MODELS.include?(@relation_name) - - # If we import/export to the same instance, tokens will have to be reset. - # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. - relation_class.attribute_names.select { |name| name.include?('token') }.each do |token| - @relation_hash[token] = nil - end - end - - def remove_encrypted_attributes! - return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? - - relation_class.encrypted_attributes.each_key do |key| - @relation_hash[key.to_s] = nil - end - end - - def relation_class - @relation_class ||= self.class.relation_class(@relation_name) - end - - def importable_column_name - importable_class_name.concat('_id') - end - - def importable_class_name - @importable.class.to_s.downcase - end - - def imported_object - if existing_or_new_object.respond_to?(:importing) - existing_or_new_object.importing = true - end - - existing_or_new_object - rescue ActiveRecord::RecordNotUnique - # as the operation is not atomic, retry in the unlikely scenario an INSERT is - # performed on the same object between the SELECT and the INSERT - @imported_object_retries += 1 - retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES - end - - def parsed_relation_hash - @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, - relation_class: relation_class) - end - - def existing_or_new_object - # Only find existing records to avoid mapping tables such as milestones - # Otherwise always create the record, skipping the extra SELECT clause. - @existing_or_new_object ||= begin - if existing_object? - attribute_hash = attribute_hash_for(['events']) - - existing_object.assign_attributes(attribute_hash) if attribute_hash.any? - - existing_object - else - # Because of single-type inheritance, we need to be careful to use the `type` field - # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 - inheritance_column = relation_class.try(:inheritance_column) - inheritance_attributes = parsed_relation_hash.slice(inheritance_column) - object = relation_class.new(inheritance_attributes) - object.assign_attributes(parsed_relation_hash) - object - end - end - end - - def attribute_hash_for(attributes) - attributes.each_with_object({}) do |hash, value| - hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value] - hash - end - end - - def existing_object - @existing_object ||= find_or_create_object! - end - - def unique_relation_object - unique_relation_object = relation_class.find_or_create_by(importable_column_name => @importable.id) - unique_relation_object.assign_attributes(parsed_relation_hash) - unique_relation_object - end - - def find_or_create_object! - return unique_relation_object if unique_relation? - - # Can't use IDs as validation exists calling `group` or `project` attributes - finder_hash = parsed_relation_hash.tap do |hash| - if relation_class.attribute_method?('group_id') && @importable.is_a?(Project) - hash['group'] = @importable.group - end - - hash[importable_class_name] = @importable if relation_class.reflect_on_association(importable_class_name.to_sym) - hash.delete(importable_column_name) - end - - @object_builder.build(relation_class, finder_hash) - end - - def setup_note - set_note_author - # attachment is deprecated and note uploads are handled by Markdown uploader - @relation_hash['attachment'] = nil - end - - # Sets the author for a note. If the user importing the project - # has admin access, an actual mapping with new project members - # will be used. Otherwise, a note stating the original author name - # is left. - def set_note_author - old_author_id = @relation_hash['author_id'] - author = @relation_hash.delete('author') - - update_note_for_missing_author(author['name']) unless has_author?(old_author_id) - end - - def has_author?(old_author_id) - admin_user? && @members_mapper.include?(old_author_id) - end - - def missing_author_note(updated_at, author_name) - timestamp = updated_at.split('.').first - "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" - end - - def update_note_for_missing_author(author_name) - @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? - @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" - end - - def admin_user? - @user.admin? - end - - def existing_object? - strong_memoize(:_existing_object) do - self.class.existing_object_relations.include?(@relation_name) || unique_relation? - end - end - - def unique_relation? - strong_memoize(:unique_relation) do - importable_foreign_key.present? && - (has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?) - end - end - - def has_unique_index_on_importable_fk? - cache = cached_has_unique_index_on_importable_fk - table_name = relation_class.table_name - return cache[table_name] if cache.has_key?(table_name) - - index_exists = - ActiveRecord::Base.connection.index_exists?( - relation_class.table_name, - importable_foreign_key, - unique: true) - - cache[table_name] = index_exists - end - - # Avoid unnecessary DB requests - def cached_has_unique_index_on_importable_fk - Thread.current[:cached_has_unique_index_on_importable_fk] ||= {} - end - - def uses_importable_fk_as_primary_key? - relation_class.primary_key == importable_foreign_key - end - - def importable_foreign_key - relation_class.reflect_on_association(importable_class_name.to_sym)&.foreign_key - end - end - end -end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index 454dc778b6b..f11b7a0a298 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -2,6 +2,13 @@ module Gitlab module ImportExport - Error = Class.new(StandardError) + class Error < StandardError + def self.permission_error(user, importable) + self.new( + "User with ID: %s does not have required permissions for %s: %s with ID: %s" % + [user.id, importable.class.name, importable.name, importable.id] + ) + end + end end end diff --git a/lib/gitlab/import_export/fast_hash_serializer.rb b/lib/gitlab/import_export/fast_hash_serializer.rb index 5a067b5c9f3..c6ecf13ded8 100644 --- a/lib/gitlab/import_export/fast_hash_serializer.rb +++ b/lib/gitlab/import_export/fast_hash_serializer.rb @@ -136,6 +136,12 @@ module Gitlab data = [] record.in_batches(of: @batch_size) do |batch| # rubocop:disable Cop/InBatches + # order each batch by it's primary key to ensure + # consistent and predictable ordering of each exported relation + # as additional `WHERE` clauses can impact the order in which data is being + # returned by database when no `ORDER` is specified + batch = batch.reorder(batch.klass.primary_key) + if Feature.enabled?(:export_fast_serialize_with_raw_json, default_enabled: true) data.append(JSONBatchRelation.new(batch, options, preloads[key]).tap(&:raw_json)) else diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group/import_export.yml index d4e0ff12373..2721198860c 100644 --- a/lib/gitlab/import_export/group_import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -70,6 +70,7 @@ ee: - :push_event_payload - boards: - :board_assignee + - :milestone - labels: - :priorities - lists: diff --git a/lib/gitlab/import_export/group/object_builder.rb b/lib/gitlab/import_export/group/object_builder.rb new file mode 100644 index 00000000000..e171a31348e --- /dev/null +++ b/lib/gitlab/import_export/group/object_builder.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + # Given a class, it finds or creates a new object at group level. + # + # Example: + # `Group::ObjectBuilder.build(Label, label_attributes)` + # finds or initializes a label with the given attributes. + class ObjectBuilder < Base::ObjectBuilder + def self.build(*args) + ::Group.transaction do + super + end + end + + def initialize(klass, attributes) + super + + @group = @attributes['group'] + + update_description + end + + private + + attr_reader :group + + # Convert description empty string to nil + # due to existing object being saved with description: nil + # Which makes object lookup to fail since nil != '' + def update_description + attributes['description'] = nil if attributes['description'] == '' + end + + def where_clauses + [ + where_clause_base, + where_clause_for_title, + where_clause_for_description, + where_clause_for_created_at + ].compact + end + + # Returns Arel clause `"{table_name}"."group_id" = {group.id}` + def where_clause_base + table[:group_id].in(group_and_ancestor_ids) + end + + def group_and_ancestor_ids + group.ancestors.map(&:id) << group.id + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb new file mode 100644 index 00000000000..91637161377 --- /dev/null +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class RelationFactory < Base::RelationFactory + OVERRIDES = { + labels: :group_labels, + priorities: :label_priorities, + label: :group_label, + parent: :epic + }.freeze + + EXISTING_OBJECT_RELATIONS = %i[ + epic + epics + milestone + milestones + label + labels + group_label + group_labels + ].freeze + + private + + def setup_models + setup_note if @relation_name == :notes + + update_group_references + end + + def update_group_references + return unless self.class.existing_object_relations.include?(@relation_name) + return unless @relation_hash['group_id'] + + @relation_hash['group_id'] = @importable.id + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb new file mode 100644 index 00000000000..247e39a68b9 --- /dev/null +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class TreeRestorer + attr_reader :user + attr_reader :shared + attr_reader :group + + def initialize(user:, shared:, group:, group_hash:) + @path = File.join(shared.export_path, 'group.json') + @user = user + @shared = shared + @group = group + @group_hash = group_hash + end + + def restore + @relation_reader ||= + if @group_hash.present? + ImportExport::JSON::LegacyReader::User.new(@group_hash, reader.group_relation_names) + else + ImportExport::JSON::LegacyReader::File.new(@path, reader.group_relation_names) + end + + @group_members = @relation_reader.consume_relation('members') + @children = @relation_reader.consume_attribute('children') + @relation_reader.consume_attribute('name') + @relation_reader.consume_attribute('path') + + if members_mapper.map && restorer.restore + @children&.each do |group_hash| + group = create_group(group_hash: group_hash, parent_group: @group) + shared = Gitlab::ImportExport::Shared.new(group) + + self.class.new( + user: @user, + shared: shared, + group: group, + group_hash: group_hash + ).restore + end + end + + return false if @shared.errors.any? + + true + rescue => e + @shared.error(e) + false + end + + private + + def restorer + @relation_tree_restorer ||= RelationTreeRestorer.new( + user: @user, + shared: @shared, + importable: @group, + relation_reader: @relation_reader, + members_mapper: members_mapper, + object_builder: object_builder, + relation_factory: relation_factory, + reader: reader + ) + end + + def create_group(group_hash:, parent_group:) + group_params = { + name: group_hash['name'], + path: group_hash['path'], + parent_id: parent_group&.id, + visibility_level: sub_group_visibility_level(group_hash, parent_group) + } + + ::Groups::CreateService.new(@user, group_params).execute + end + + def sub_group_visibility_level(group_hash, parent_group) + original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE + + if parent_group && parent_group.visibility_level < original_visibility_level + Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) + else + original_visibility_level + end + end + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group) + end + + def relation_factory + Gitlab::ImportExport::Group::RelationFactory + end + + def object_builder + Gitlab::ImportExport::Group::ObjectBuilder + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new( + shared: @shared, + config: Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + ) + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb new file mode 100644 index 00000000000..fd1eb329ad2 --- /dev/null +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class TreeSaver + attr_reader :full_path, :shared + + def initialize(group:, current_user:, shared:, params: {}) + @params = params + @current_user = current_user + @shared = shared + @group = group + @full_path = File.join(@shared.export_path, ImportExport.group_filename) + end + + def save + group_tree = serialize(@group, reader.group_tree) + tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + def serialize(group, relations_tree) + group_tree = tree_saver.serialize(group, relations_tree) + + group.children.each do |child| + group_tree['children'] ||= [] + group_tree['children'] << serialize(child, relations_tree) + end + + group_tree + rescue => e + @shared.error(e) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new( + shared: @shared, + config: Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + ) + end + + def tree_saver + @tree_saver ||= LegacyRelationTreeSaver.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/group_object_builder.rb b/lib/gitlab/import_export/group_object_builder.rb deleted file mode 100644 index 9796bfa07d4..00000000000 --- a/lib/gitlab/import_export/group_object_builder.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - # Given a class, it finds or creates a new object at group level. - # - # Example: - # `GroupObjectBuilder.build(Label, label_attributes)` - # finds or initializes a label with the given attributes. - class GroupObjectBuilder < BaseObjectBuilder - def self.build(*args) - Group.transaction do - super - end - end - - def initialize(klass, attributes) - super - - @group = @attributes['group'] - - update_description - end - - private - - attr_reader :group - - # Convert description empty string to nil - # due to existing object being saved with description: nil - # Which makes object lookup to fail since nil != '' - def update_description - attributes['description'] = nil if attributes['description'] == '' - end - - def where_clauses - [ - where_clause_base, - where_clause_for_title, - where_clause_for_description, - where_clause_for_created_at - ].compact - end - - # Returns Arel clause `"{table_name}"."group_id" = {group.id}` - def where_clause_base - table[:group_id].in(group_and_ancestor_ids) - end - - def group_and_ancestor_ids - group.ancestors.map(&:id) << group.id - end - end - end -end diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb deleted file mode 100644 index 9e8f9d11393..00000000000 --- a/lib/gitlab/import_export/group_project_object_builder.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - # Given a class, it finds or creates a new object - # (initializes in the case of Label) at group or project level. - # If it does not exist in the group, it creates it at project level. - # - # Example: - # `GroupProjectObjectBuilder.build(Label, label_attributes)` - # finds or initializes a label with the given attributes. - # - # It also adds some logic around Group Labels/Milestones for edge cases. - class GroupProjectObjectBuilder < BaseObjectBuilder - def self.build(*args) - Project.transaction do - super - end - end - - def initialize(klass, attributes) - super - - @group = @attributes['group'] - @project = @attributes['project'] - end - - def find - return if epic? && group.nil? - - super - end - - private - - attr_reader :group, :project - - def where_clauses - [ - where_clause_base, - where_clause_for_title, - where_clause_for_klass - ].compact - end - - # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present - # For example: merge_request has :target_project_id, and we are searching by :iid - # or, if group is present: - # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}` - def where_clause_base - [].tap do |clauses| - clauses << table[:project_id].eq(project.id) if project - clauses << table[:group_id].in(group.self_and_ancestors_ids) if group - end.reduce(:or) - end - - # Returns Arel clause for a particular model or `nil`. - def where_clause_for_klass - attrs_to_arel(attributes.slice('iid')) if merge_request? - end - - def prepare_attributes - attributes.dup.tap do |atts| - atts.delete('group') unless epic? - - if label? - atts['type'] = 'ProjectLabel' # Always create project labels - elsif milestone? - if atts['group_id'] # Transform new group milestones into project ones - atts['iid'] = nil - atts.delete('group_id') - else - claim_iid - end - end - - atts['importing'] = true if klass.ancestors.include?(Importable) - end - end - - def label? - klass == Label - end - - def milestone? - klass == Milestone - end - - def merge_request? - klass == MergeRequest - end - - def epic? - klass == Epic - end - - # If an existing group milestone used the IID - # claim the IID back and set the group milestone to use one available - # This is necessary to fix situations like the following: - # - Importing into a user namespace project with exported group milestones - # where the IID of the Group milestone could conflict with a project one. - def claim_iid - # The milestone has to be a group milestone, as it's the only case where - # we set the IID as the maximum. The rest of them are fixed. - milestone = project.milestones.find_by(iid: attributes['iid']) - - return unless milestone - - milestone.iid = nil - milestone.ensure_project_iid! - milestone.save! - end - end - end -end - -Gitlab::ImportExport::GroupProjectObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::GroupProjectObjectBuilder') diff --git a/lib/gitlab/import_export/group_relation_factory.rb b/lib/gitlab/import_export/group_relation_factory.rb deleted file mode 100644 index e3597af44d2..00000000000 --- a/lib/gitlab/import_export/group_relation_factory.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class GroupRelationFactory < BaseRelationFactory - OVERRIDES = { - labels: :group_labels, - priorities: :label_priorities, - label: :group_label, - parent: :epic - }.freeze - - EXISTING_OBJECT_RELATIONS = %i[ - epic - epics - milestone - milestones - label - labels - group_label - group_labels - ].freeze - - private - - def setup_models - setup_note if @relation_name == :notes - - update_group_references - end - - def update_group_references - return unless self.class.existing_object_relations.include?(@relation_name) - return unless @relation_hash['group_id'] - - @relation_hash['group_id'] = @importable.id - end - end - end -end diff --git a/lib/gitlab/import_export/group_tree_restorer.rb b/lib/gitlab/import_export/group_tree_restorer.rb deleted file mode 100644 index 2f42843ed6c..00000000000 --- a/lib/gitlab/import_export/group_tree_restorer.rb +++ /dev/null @@ -1,116 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class GroupTreeRestorer - attr_reader :user - attr_reader :shared - attr_reader :group - - def initialize(user:, shared:, group:, group_hash:) - @path = File.join(shared.export_path, 'group.json') - @user = user - @shared = shared - @group = group - @group_hash = group_hash - end - - def restore - @tree_hash = @group_hash || read_tree_hash - @group_members = @tree_hash.delete('members') - @children = @tree_hash.delete('children') - - if members_mapper.map && restorer.restore - @children&.each do |group_hash| - group = create_group(group_hash: group_hash, parent_group: @group) - shared = Gitlab::ImportExport::Shared.new(group) - - self.class.new( - user: @user, - shared: shared, - group: group, - group_hash: group_hash - ).restore - end - end - - return false if @shared.errors.any? - - true - rescue => e - @shared.error(e) - false - end - - private - - def read_tree_hash - json = IO.read(@path) - ActiveSupport::JSON.decode(json) - rescue => e - @shared.logger.error( - group_id: @group.id, - group_name: @group.name, - message: "Import/Export error: #{e.message}" - ) - - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') - end - - def restorer - @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - importable: @group, - tree_hash: @tree_hash.except('name', 'path'), - members_mapper: members_mapper, - object_builder: object_builder, - relation_factory: relation_factory, - reader: reader - ) - end - - def create_group(group_hash:, parent_group:) - group_params = { - name: group_hash['name'], - path: group_hash['path'], - parent_id: parent_group&.id, - visibility_level: sub_group_visibility_level(group_hash, parent_group) - } - - ::Groups::CreateService.new(@user, group_params).execute - end - - def sub_group_visibility_level(group_hash, parent_group) - original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE - - if parent_group && parent_group.visibility_level < original_visibility_level - Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) - else - original_visibility_level - end - end - - def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group) - end - - def relation_factory - Gitlab::ImportExport::GroupRelationFactory - end - - def object_builder - Gitlab::ImportExport::GroupObjectBuilder - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new( - shared: @shared, - config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.group_config_file - ).to_h - ) - end - end - end -end diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb deleted file mode 100644 index 2effcd01e30..00000000000 --- a/lib/gitlab/import_export/group_tree_saver.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class GroupTreeSaver - attr_reader :full_path, :shared - - def initialize(group:, current_user:, shared:, params: {}) - @params = params - @current_user = current_user - @shared = shared - @group = group - @full_path = File.join(@shared.export_path, ImportExport.group_filename) - end - - def save - group_tree = serialize(@group, reader.group_tree) - tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) - - true - rescue => e - @shared.error(e) - false - end - - private - - def serialize(group, relations_tree) - group_tree = tree_saver.serialize(group, relations_tree) - - group.children.each do |child| - group_tree['children'] ||= [] - group_tree['children'] << serialize(child, relations_tree) - end - - group_tree - rescue => e - @shared.error(e) - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new( - shared: @shared, - config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.group_config_file - ).to_h - ) - end - - def tree_saver - @tree_saver ||= RelationTreeSaver.new - end - end - end -end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index a6463ed678c..4b761eb86ae 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -35,7 +35,7 @@ module Gitlab def restorers [repo_restorer, wiki_restorer, project_tree, avatar_restorer, - uploads_restorer, lfs_restorer, statistics_restorer] + uploads_restorer, lfs_restorer, statistics_restorer, snippets_repo_restorer] end def import_file @@ -49,7 +49,7 @@ module Gitlab end def project_tree - @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: current_user, + @project_tree ||= Gitlab::ImportExport::Project::TreeRestorer.new(user: current_user, shared: shared, project: project) end @@ -79,6 +79,12 @@ module Gitlab Gitlab::ImportExport::LfsRestorer.new(project: project, shared: shared) end + def snippets_repo_restorer + Gitlab::ImportExport::SnippetsRepoRestorer.new(project: project, + shared: shared, + user: current_user) + end + def statistics_restorer Gitlab::ImportExport::StatisticsRestorer.new(project: project, shared: shared) end @@ -125,7 +131,7 @@ module Gitlab def project_to_overwrite strong_memoize(:project_to_overwrite) do - Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}") + ::Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}") end end end diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb new file mode 100644 index 00000000000..477e41ae3eb --- /dev/null +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module JSON + class LegacyReader + class File < LegacyReader + def initialize(path, relation_names) + @path = path + super(relation_names) + end + + def valid? + ::File.exist?(@path) + end + + private + + def tree_hash + @tree_hash ||= read_hash + end + + def read_hash + ActiveSupport::JSON.decode(IO.read(@path)) + rescue => e + Gitlab::ErrorTracking.log_exception(e) + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + end + + class User < LegacyReader + def initialize(tree_hash, relation_names) + @tree_hash = tree_hash + super(relation_names) + end + + def valid? + @tree_hash.present? + end + + protected + + attr_reader :tree_hash + end + + def initialize(relation_names) + @relation_names = relation_names.map(&:to_s) + end + + def valid? + raise NotImplementedError + end + + def legacy? + true + end + + def root_attributes(excluded_attributes = []) + attributes.except(*excluded_attributes.map(&:to_s)) + end + + def consume_relation(key) + value = relations.delete(key) + + return value unless block_given? + + return if value.nil? + + if value.is_a?(Array) + value.each.with_index do |item, idx| + yield(item, idx) + end + else + yield(value, 0) + end + end + + def consume_attribute(key) + attributes.delete(key) + end + + def sort_ci_pipelines_by_id + relations['ci_pipelines']&.sort_by! { |hash| hash['id'] } + end + + private + + attr_reader :relation_names + + def tree_hash + raise NotImplementedError + end + + def attributes + @attributes ||= tree_hash.slice!(*relation_names) + end + + def relations + @relations ||= tree_hash.extract!(*relation_names) + end + end + end + end +end diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb new file mode 100644 index 00000000000..c935e360a65 --- /dev/null +++ b/lib/gitlab/import_export/json/legacy_writer.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module JSON + class LegacyWriter + include Gitlab::ImportExport::CommandLineUtil + + attr_reader :path + + def initialize(path) + @path = path + @last_array = nil + @keys = Set.new + + mkdir_p(File.dirname(@path)) + file.write('{}') + end + + def close + @file&.close + @file = nil + end + + def set(hash) + hash.each do |key, value| + write(key, value) + end + end + + def write(key, value) + raise ArgumentError, "key '#{key}' already written" if @keys.include?(key) + + # rewind by one byte, to overwrite '}' + file.pos = file.size - 1 + + file.write(',') if @keys.any? + file.write(key.to_json) + file.write(':') + file.write(value.to_json) + file.write('}') + + @keys.add(key) + @last_array = nil + @last_array_count = nil + end + + def append(key, value) + unless @last_array == key + write(key, []) + + @last_array = key + @last_array_count = 0 + end + + # rewind by two bytes, to overwrite ']}' + file.pos = file.size - 2 + + file.write(',') if @last_array_count > 0 + file.write(value.to_json) + file.write(']}') + @last_array_count += 1 + end + + private + + def file + @file ||= File.open(@path, "wb") + end + end + end + end +end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb new file mode 100644 index 00000000000..d053bf16166 --- /dev/null +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module JSON + class StreamingSerializer + include Gitlab::ImportExport::CommandLineUtil + + BATCH_SIZE = 100 + + class Raw < String + def to_json(*_args) + to_s + end + end + + def initialize(exportable, relations_schema, json_writer) + @exportable = exportable + @relations_schema = relations_schema + @json_writer = json_writer + end + + def execute + serialize_root + + includes.each do |relation_definition| + serialize_relation(relation_definition) + end + end + + private + + attr_reader :json_writer, :relations_schema, :exportable + + def serialize_root + attributes = exportable.as_json( + relations_schema.merge(include: nil, preloads: nil)) + json_writer.set(attributes) + end + + def serialize_relation(definition) + raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash) + raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one? + + key, options = definition.first + + record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend + if record.is_a?(ActiveRecord::Relation) + serialize_many_relations(key, record, options) + else + serialize_single_relation(key, record, options) + end + end + + def serialize_many_relations(key, records, options) + key_preloads = preloads&.dig(key) + records = records.preload(key_preloads) if key_preloads + + records.find_each(batch_size: BATCH_SIZE) do |record| + json = Raw.new(record.to_json(options)) + + json_writer.append(key, json) + end + end + + def serialize_single_relation(key, record, options) + json = Raw.new(record.to_json(options)) + + json_writer.write(key, json) + end + + def includes + relations_schema[:include] + end + + def preloads + relations_schema[:preload] + end + end + end + end +end diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb index a0452071ccf..fe3e64358e5 100644 --- a/lib/gitlab/import_export/relation_tree_saver.rb +++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - class RelationTreeSaver + class LegacyRelationTreeSaver include Gitlab::ImportExport::CommandLineUtil def serialize(exportable, relations_tree) @@ -18,7 +18,7 @@ module Gitlab def save(tree, dir_path, filename) mkdir_p(dir_path) - tree_json = JSON.generate(tree) + tree_json = ::JSON.generate(tree) File.write(File.join(dir_path, filename), tree_json) end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 2a70344374b..fd76252eb36 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -51,7 +51,7 @@ module Gitlab @importable.members.destroy_all # rubocop: disable DestroyAll - relation_class.create!(user: @user, access_level: relation_class::MAINTAINER, source_id: @importable.id, importing: true) + relation_class.create!(user: @user, access_level: highest_access_level, source_id: @importable.id, importing: true) rescue => e raise e, "Error adding importer user to #{@importable.class} members. #{e.message}" end @@ -59,7 +59,7 @@ module Gitlab def user_already_member? member = @importable.members&.first - member&.user == @user && member.access_level >= relation_class::MAINTAINER + member&.user == @user && member.access_level >= highest_access_level end def add_team_member(member, existing_user = nil) @@ -72,7 +72,7 @@ module Gitlab parsed_hash(member).merge( 'source_id' => @importable.id, 'importing' => true, - 'access_level' => [member['access_level'], relation_class::MAINTAINER].min + 'access_level' => [member['access_level'], highest_access_level].min ).except('user_id') end @@ -91,12 +91,18 @@ module Gitlab def relation_class case @importable - when Project + when ::Project ProjectMember - when Group + when ::Group GroupMember end end + + def highest_access_level + return relation_class::OWNER if relation_class == GroupMember + + relation_class::MAINTAINER + end end end end diff --git a/lib/gitlab/import_export/project/base_task.rb b/lib/gitlab/import_export/project/base_task.rb new file mode 100644 index 00000000000..6a7b24421c9 --- /dev/null +++ b/lib/gitlab/import_export/project/base_task.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class BaseTask + include Gitlab::WithRequestStore + + def initialize(opts, logger: Logger.new($stdout)) + @project_path = opts.fetch(:project_path) + @file_path = opts.fetch(:file_path) + @namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path)) + @current_user = User.find_by_username(opts.fetch(:username)) + @measurement_enabled = opts.fetch(:measurement_enabled) + @measurement = Gitlab::Utils::Measuring.new(logger: logger) if @measurement_enabled + @logger = logger + end + + private + + attr_reader :measurement, :project, :namespace, :current_user, :file_path, :project_path, :logger + + def measurement_enabled? + @measurement_enabled + end + + def success(message) + logger.info(message) + + true + end + + def error(message) + logger.error(message) + + false + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/export_task.rb b/lib/gitlab/import_export/project/export_task.rb new file mode 100644 index 00000000000..ec287380c48 --- /dev/null +++ b/lib/gitlab/import_export/project/export_task.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class ExportTask < BaseTask + def initialize(*) + super + + @project = namespace.projects.find_by_path(@project_path) + end + + def export + return error("Project with path: #{project_path} was not found. Please provide correct project path") unless project + return error("Invalid file path: #{file_path}. Please provide correct file path") unless file_path_exists? + + with_export do + ::Projects::ImportExport::ExportService.new(project, current_user) + .execute(Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy.new(archive_path: file_path)) + end + + success('Done!') + end + + private + + def file_path_exists? + directory = File.dirname(file_path) + + Dir.exist?(directory) + end + + def with_export + with_request_store do + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + measurement_enabled? ? measurement.with_measuring { yield } : yield + end + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index e55ad898263..aa6085de4f9 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -65,6 +65,7 @@ tree: - resource_label_events: - label: - :priorities + - :external_pull_requests - ci_pipelines: - notes: - :author @@ -74,7 +75,6 @@ tree: - :statuses - :external_pull_request - :merge_request - - :external_pull_requests - :auto_devops - :triggers - :pipeline_schedules @@ -173,7 +173,6 @@ excluded_attributes: - :secret - :encrypted_secret_token - :encrypted_secret_token_iv - - :repository_storage merge_request_diff: - :external_diff - :stored_externally @@ -189,6 +188,7 @@ excluded_attributes: issues: - :milestone_id - :moved_to_id + - :sent_notifications - :state_id - :duplicated_to_id - :promoted_to_epic_id @@ -248,6 +248,7 @@ excluded_attributes: - :token_encrypted services: - :template + - :instance error_tracking_setting: - :encrypted_token - :encrypted_token_iv @@ -319,6 +320,9 @@ excluded_attributes: - :state_id - :start_date_sourcing_epic_id - :due_date_sourcing_epic_id + epic_issue: + - :epic_id + - :issue_id methods: notes: - :type @@ -371,9 +375,13 @@ ee: - design_versions: - actions: - :design # Duplicate export of issues.designs in order to link the record to both Issue and Action - - :epic + - epic_issue: + - :epic - protected_branches: - :unprotect_access_levels - protected_environments: - :deploy_access_levels - :service_desk_setting + excluded_attributes: + actions: + - image_v432x230 diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb new file mode 100644 index 00000000000..ae654ddbeaf --- /dev/null +++ b/lib/gitlab/import_export/project/import_task.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class ImportTask < BaseTask + def import + show_import_start_message + + run_isolated_sidekiq_job + + show_import_failures_count + + return error(project.import_state.last_error) if project.import_state&.last_error + return error(project.errors.full_messages.to_sentence) if project.errors.any? + + success('Done!') + end + + private + + # We want to ensure that all Sidekiq jobs are executed + # synchronously as part of that process. + # This ensures that all expensive operations do not escape + # to general Sidekiq clusters/nodes. + def with_isolated_sidekiq_job + Sidekiq::Testing.fake! do + with_request_store do + # If you are attempting to import a large project into a development environment, + # you may see Gitaly throw an error about too many calls or invocations. + # This is due to a n+1 calls limit being set for development setups (not enforced in production) + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635 + # For development setups, this code-path will be excluded from n+1 detection. + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + measurement_enabled? ? measurement.with_measuring { yield } : yield + end + end + + true + end + end + + def run_isolated_sidekiq_job + with_isolated_sidekiq_job do + @project = create_project + + execute_sidekiq_job + end + end + + def create_project + # We are disabling ObjectStorage for `import` + # as it is too slow to handle big archives: + # 1. DB transaction timeouts on upload + # 2. Download of archive before unpacking + disable_upload_object_storage do + service = Projects::GitlabProjectsImportService.new( + current_user, + { + namespace_id: namespace.id, + path: project_path, + file: File.open(file_path) + } + ) + + service.execute + end + end + + def execute_sidekiq_job + Sidekiq::Worker.drain_all + end + + def disable_upload_object_storage + overwrite_uploads_setting('background_upload', false) do + overwrite_uploads_setting('direct_upload', false) do + yield + end + end + end + + def overwrite_uploads_setting(key, value) + old_value = Settings.uploads.object_store[key] + Settings.uploads.object_store[key] = value + + yield + + ensure + Settings.uploads.object_store[key] = old_value + end + + def full_path + "#{namespace.full_path}/#{project_path}" + end + + def show_import_start_message + logger.info "Importing GitLab export: #{file_path} into GitLab" \ + " #{full_path}" \ + " as #{current_user.name}" + end + + def show_import_failures_count + return unless project.import_failures.exists? + + logger.info "Total number of not imported relations: #{project.import_failures.count}" + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/legacy_tree_saver.rb b/lib/gitlab/import_export/project/legacy_tree_saver.rb new file mode 100644 index 00000000000..2ed98f52c58 --- /dev/null +++ b/lib/gitlab/import_export/project/legacy_tree_saver.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class LegacyTreeSaver + attr_reader :full_path + + def initialize(project:, current_user:, shared:, params: {}) + @params = params + @project = project + @current_user = current_user + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) + end + + def save + project_tree = tree_saver.serialize(@project, reader.project_tree) + fix_project_tree(project_tree) + tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + # Aware that the resulting hash needs to be pure-hash and + # does not include any AR objects anymore, only objects that run `.to_json` + def fix_project_tree(project_tree) + if @params[:description].present? + project_tree['description'] = @params[:description] + end + + project_tree['project_members'] += group_members_array + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def group_members_array + group_members.as_json(reader.group_members_tree).each do |group_member| + group_member['source_type'] = 'Project' # Make group members project members of the future import + end + end + + def group_members + return [] unless @current_user.can?(:admin_group, @project.group) + + # We need `.where.not(user_id: nil)` here otherwise when a group has an + # invitee, it would make the following query return 0 rows since a NULL + # user_id would be present in the subquery + # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values + non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id) + + GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) + end + + def tree_saver + @tree_saver ||= Gitlab::ImportExport::LegacyRelationTreeSaver.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb new file mode 100644 index 00000000000..c3637b1c115 --- /dev/null +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + # Given a class, it finds or creates a new object + # (initializes in the case of Label) at group or project level. + # If it does not exist in the group, it creates it at project level. + # + # Example: + # `ObjectBuilder.build(Label, label_attributes)` + # finds or initializes a label with the given attributes. + # + # It also adds some logic around Group Labels/Milestones for edge cases. + class ObjectBuilder < Base::ObjectBuilder + def self.build(*args) + ::Project.transaction do + super + end + end + + def initialize(klass, attributes) + super + + @group = @attributes['group'] + @project = @attributes['project'] + end + + def find + return if epic? && group.nil? + + super + end + + private + + attr_reader :group, :project + + def where_clauses + [ + where_clause_base, + where_clause_for_title, + where_clause_for_klass + ].compact + end + + # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present + # For example: merge_request has :target_project_id, and we are searching by :iid + # or, if group is present: + # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}` + def where_clause_base + [].tap do |clauses| + clauses << table[:project_id].eq(project.id) if project + clauses << table[:group_id].in(group.self_and_ancestors_ids) if group + end.reduce(:or) + end + + # Returns Arel clause for a particular model or `nil`. + def where_clause_for_klass + attrs_to_arel(attributes.slice('iid')) if merge_request? + end + + def prepare_attributes + attributes.dup.tap do |atts| + atts.delete('group') unless epic? + + if label? + atts['type'] = 'ProjectLabel' # Always create project labels + elsif milestone? + if atts['group_id'] # Transform new group milestones into project ones + atts['iid'] = nil + atts.delete('group_id') + else + claim_iid + end + end + + atts['importing'] = true if klass.ancestors.include?(Importable) + end + end + + def label? + klass == Label + end + + def milestone? + klass == Milestone + end + + def merge_request? + klass == MergeRequest + end + + def epic? + klass == Epic + end + + # If an existing group milestone used the IID + # claim the IID back and set the group milestone to use one available + # This is necessary to fix situations like the following: + # - Importing into a user namespace project with exported group milestones + # where the IID of the Group milestone could conflict with a project one. + def claim_iid + # The milestone has to be a group milestone, as it's the only case where + # we set the IID as the maximum. The rest of them are fixed. + milestone = project.milestones.find_by(iid: attributes['iid']) + + return unless milestone + + milestone.iid = nil + milestone.ensure_project_iid! + milestone.save! + end + end + end + end +end + +Gitlab::ImportExport::Project::ObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::Project::ObjectBuilder') diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb new file mode 100644 index 00000000000..2405176c518 --- /dev/null +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class RelationFactory < Base::RelationFactory + prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule + + OVERRIDES = { snippets: :project_snippets, + ci_pipelines: 'Ci::Pipeline', + pipelines: 'Ci::Pipeline', + stages: 'Ci::Stage', + statuses: 'commit_status', + triggers: 'Ci::Trigger', + pipeline_schedules: 'Ci::PipelineSchedule', + builds: 'Ci::Build', + runners: 'Ci::Runner', + hooks: 'ProjectHook', + merge_access_levels: 'ProtectedBranch::MergeAccessLevel', + push_access_levels: 'ProtectedBranch::PushAccessLevel', + create_access_levels: 'ProtectedTag::CreateAccessLevel', + labels: :project_labels, + priorities: :label_priorities, + auto_devops: :project_auto_devops, + label: :project_label, + custom_attributes: 'ProjectCustomAttribute', + project_badges: 'Badge', + metrics: 'MergeRequest::Metrics', + ci_cd_settings: 'ProjectCiCdSetting', + error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', + links: 'Releases::Link', + metrics_setting: 'ProjectMetricsSetting' }.freeze + + BUILD_MODELS = %i[Ci::Build commit_status].freeze + + GROUP_REFERENCES = %w[group_id].freeze + + PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze + + EXISTING_OBJECT_RELATIONS = %i[ + milestone + milestones + label + labels + project_label + project_labels + group_label + group_labels + project_feature + merge_request + epic + ProjectCiCdSetting + container_expiration_policy + external_pull_request + external_pull_requests + ].freeze + + def create + @object = super + + # We preload the project, user, and group to re-use objects + @object = preload_keys(@object, PROJECT_REFERENCES, @importable) + @object = preload_keys(@object, GROUP_REFERENCES, @importable.group) + @object = preload_keys(@object, USER_REFERENCES, @user) + end + + private + + def invalid_relation? + # Do not create relation if it is: + # - An unknown service + # - A legacy trigger + unknown_service? || + (!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?) + end + + def setup_models + case @relation_name + when :merge_request_diff_files then setup_diff + when :notes then setup_note + when :'Ci::Pipeline' then setup_pipeline + when *BUILD_MODELS then setup_build + end + + update_project_references + update_group_references + end + + def generate_imported_object + if @relation_name == :merge_requests + MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse! + else + super + end + end + + def update_project_references + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID + end + + @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] + end + + def same_source_and_target? + @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] + end + + def update_group_references + return unless existing_object? + return unless @relation_hash['group_id'] + + @relation_hash['group_id'] = @importable.namespace_id + end + + def setup_build + @relation_hash.delete('trace') # old export files have trace + @relation_hash.delete('token') + @relation_hash.delete('commands') + @relation_hash.delete('artifacts_file_store') + @relation_hash.delete('artifacts_metadata_store') + @relation_hash.delete('artifacts_size') + end + + def setup_diff + @relation_hash['diff'] = @relation_hash.delete('utf8_diff') + end + + def setup_pipeline + @relation_hash.fetch('stages', []).each do |stage| + stage.statuses.each do |status| + status.pipeline = imported_object + end + end + end + + def unknown_service? + @relation_name == :services && parsed_relation_hash['type'] && + !Object.const_defined?(parsed_relation_hash['type']) + end + + def legacy_trigger? + @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? + end + + def preload_keys(object, references, value) + return object unless value + + references.each do |key| + attribute = "#{key.delete_suffix('_id')}=".to_sym + next unless object.respond_to?(key) && object.respond_to?(attribute) + + if object.read_attribute(key) == value&.id + object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend + end + end + + object + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb new file mode 100644 index 00000000000..f8d25e14c02 --- /dev/null +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class TreeRestorer + attr_reader :user + attr_reader :shared + attr_reader :project + + def initialize(user:, shared:, project:) + @user = user + @shared = shared + @project = project + end + + def restore + @relation_reader = ImportExport::JSON::LegacyReader::File.new(File.join(shared.export_path, 'project.json'), reader.project_relation_names) + + @project_members = @relation_reader.consume_relation('project_members') + + if relation_tree_restorer.restore + import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do + @project.merge_requests.set_latest_merge_request_diff_ids! + end + + true + else + false + end + rescue => e + @shared.error(e) + false + end + + private + + def relation_tree_restorer + @relation_tree_restorer ||= RelationTreeRestorer.new( + user: @user, + shared: @shared, + importable: @project, + relation_reader: @relation_reader, + object_builder: object_builder, + members_mapper: members_mapper, + relation_factory: relation_factory, + reader: reader + ) + end + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + importable: @project) + end + + def object_builder + Project::ObjectBuilder + end + + def relation_factory + Project::RelationFactory + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def import_failure_service + @import_failure_service ||= ImportFailureService.new(@project) + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb new file mode 100644 index 00000000000..01000c9d6d9 --- /dev/null +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class TreeSaver + attr_reader :full_path + + def initialize(project:, current_user:, shared:, params: {}) + @params = params + @project = project + @current_user = current_user + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) + end + + def save + json_writer = ImportExport::JSON::LegacyWriter.new(@full_path) + + serializer = ImportExport::JSON::StreamingSerializer.new(exportable, reader.project_tree, json_writer) + serializer.execute + + true + rescue => e + @shared.error(e) + false + ensure + json_writer&.close + end + + private + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def exportable + @project.present(exportable_params) + end + + def exportable_params + params = { + presenter_class: presenter_class, + current_user: @current_user + } + params[:override_description] = @params[:description] if @params[:description].present? + params + end + + def presenter_class + Projects::ImportExport::ProjectExportPresenter + end + end + end + end +end diff --git a/lib/gitlab/import_export/project_relation_factory.rb b/lib/gitlab/import_export/project_relation_factory.rb deleted file mode 100644 index e27bb9d3af1..00000000000 --- a/lib/gitlab/import_export/project_relation_factory.rb +++ /dev/null @@ -1,184 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectRelationFactory < BaseRelationFactory - prepend_if_ee('::EE::Gitlab::ImportExport::ProjectRelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule - - OVERRIDES = { snippets: :project_snippets, - ci_pipelines: 'Ci::Pipeline', - pipelines: 'Ci::Pipeline', - stages: 'Ci::Stage', - statuses: 'commit_status', - triggers: 'Ci::Trigger', - pipeline_schedules: 'Ci::PipelineSchedule', - builds: 'Ci::Build', - runners: 'Ci::Runner', - hooks: 'ProjectHook', - merge_access_levels: 'ProtectedBranch::MergeAccessLevel', - push_access_levels: 'ProtectedBranch::PushAccessLevel', - create_access_levels: 'ProtectedTag::CreateAccessLevel', - labels: :project_labels, - priorities: :label_priorities, - auto_devops: :project_auto_devops, - label: :project_label, - custom_attributes: 'ProjectCustomAttribute', - project_badges: 'Badge', - metrics: 'MergeRequest::Metrics', - ci_cd_settings: 'ProjectCiCdSetting', - error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', - links: 'Releases::Link', - metrics_setting: 'ProjectMetricsSetting' }.freeze - - BUILD_MODELS = %i[Ci::Build commit_status].freeze - - GROUP_REFERENCES = %w[group_id].freeze - - PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze - - EXISTING_OBJECT_RELATIONS = %i[ - milestone - milestones - label - labels - project_label - project_labels - group_label - group_labels - project_feature - merge_request - epic - ProjectCiCdSetting - container_expiration_policy - ].freeze - - def create - @object = super - - # We preload the project, user, and group to re-use objects - @object = preload_keys(@object, PROJECT_REFERENCES, @importable) - @object = preload_keys(@object, GROUP_REFERENCES, @importable.group) - @object = preload_keys(@object, USER_REFERENCES, @user) - end - - private - - def invalid_relation? - # Do not create relation if it is: - # - An unknown service - # - A legacy trigger - unknown_service? || - (!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?) - end - - def setup_models - case @relation_name - when :merge_request_diff_files then setup_diff - when :notes then setup_note - when :'Ci::Pipeline' then setup_pipeline - when *BUILD_MODELS then setup_build - end - - update_project_references - update_group_references - end - - def generate_imported_object - if @relation_name == :merge_requests - MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse! - else - super - end - end - - def update_project_references - # If source and target are the same, populate them with the new project ID. - if @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID - end - - @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] - end - - def same_source_and_target? - @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] - end - - def update_group_references - return unless existing_object? - return unless @relation_hash['group_id'] - - @relation_hash['group_id'] = @importable.namespace_id - end - - # This code is a workaround for broken project exports that don't - # export merge requests with CI pipelines (i.e. exports that were - # generated from - # https://gitlab.com/gitlab-org/gitlab/merge_requests/17844). - # This method can be removed in GitLab 12.6. - def update_merge_request_references - # If a merge request was properly created, we don't need to fix - # up this export. - return if @relation_hash['merge_request'] - - merge_request_id = @relation_hash['merge_request_id'] - - return unless merge_request_id - - new_merge_request_id = @merge_requests_mapping[merge_request_id] - - return unless new_merge_request_id - - @relation_hash['merge_request_id'] = new_merge_request_id - parsed_relation_hash['merge_request_id'] = new_merge_request_id - end - - def setup_build - @relation_hash.delete('trace') # old export files have trace - @relation_hash.delete('token') - @relation_hash.delete('commands') - @relation_hash.delete('artifacts_file_store') - @relation_hash.delete('artifacts_metadata_store') - @relation_hash.delete('artifacts_size') - end - - def setup_diff - @relation_hash['diff'] = @relation_hash.delete('utf8_diff') - end - - def setup_pipeline - update_merge_request_references - - @relation_hash.fetch('stages', []).each do |stage| - stage.statuses.each do |status| - status.pipeline = imported_object - end - end - end - - def unknown_service? - @relation_name == :services && parsed_relation_hash['type'] && - !Object.const_defined?(parsed_relation_hash['type']) - end - - def legacy_trigger? - @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? - end - - def preload_keys(object, references, value) - return object unless value - - references.each do |key| - attribute = "#{key.delete_suffix('_id')}=".to_sym - next unless object.respond_to?(key) && object.respond_to?(attribute) - - if object.read_attribute(key) == value&.id - object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend - end - end - - object - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_loader.rb b/lib/gitlab/import_export/project_tree_loader.rb deleted file mode 100644 index fc21858043d..00000000000 --- a/lib/gitlab/import_export/project_tree_loader.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectTreeLoader - def load(path, dedup_entries: false) - tree_hash = ActiveSupport::JSON.decode(IO.read(path)) - - if dedup_entries - dedup_tree(tree_hash) - else - tree_hash - end - end - - private - - # This function removes duplicate entries from the given tree recursively - # by caching nodes it encounters repeatedly. We only consider nodes for - # which there can actually be multiple equivalent instances (e.g. strings, - # hashes and arrays, but not `nil`s, numbers or booleans.) - # - # The algorithm uses a recursive depth-first descent with 3 cases, starting - # with a root node (the tree/hash itself): - # - a node has already been cached; in this case we return it from the cache - # - a node has not been cached yet but should be; descend into its children - # - a node is neither cached nor qualifies for caching; this is a no-op - def dedup_tree(node, nodes_seen = {}) - if nodes_seen.key?(node) && distinguishable?(node) - yield nodes_seen[node] - elsif should_dedup?(node) - nodes_seen[node] = node - - case node - when Array - node.each_index do |idx| - dedup_tree(node[idx], nodes_seen) do |cached_node| - node[idx] = cached_node - end - end - when Hash - node.each do |k, v| - dedup_tree(v, nodes_seen) do |cached_node| - node[k] = cached_node - end - end - end - else - node - end - end - - # We do not need to consider nodes for which there cannot be multiple instances - def should_dedup?(node) - node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass)) - end - - # We can only safely de-dup values that are distinguishable. True value objects - # are always distinguishable by nature. Hashes however can represent entities, - # which are identified by ID, not value. We therefore disallow de-duping hashes - # that do not have an `id` field, since we might risk dropping entities that - # have equal attributes yet different identities. - def distinguishable?(node) - if node.is_a?(Hash) - node.key?('id') - else - true - end - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb deleted file mode 100644 index aae07657ea0..00000000000 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectTreeRestorer - LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte - - attr_reader :user - attr_reader :shared - attr_reader :project - - def initialize(user:, shared:, project:) - @user = user - @shared = shared - @project = project - @tree_loader = ProjectTreeLoader.new - end - - def restore - @tree_hash = read_tree_hash - @project_members = @tree_hash.delete('project_members') - - RelationRenameService.rename(@tree_hash) - - if relation_tree_restorer.restore - import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do - @project.merge_requests.set_latest_merge_request_diff_ids! - end - - true - else - false - end - rescue => e - @shared.error(e) - false - end - - private - - def large_project?(path) - File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES - end - - def read_tree_hash - path = File.join(@shared.export_path, 'project.json') - dedup_entries = large_project?(path) && - Feature.enabled?(:dedup_project_import_metadata, project.group) - - @tree_loader.load(path, dedup_entries: dedup_entries) - rescue => e - Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') - end - - def relation_tree_restorer - @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - importable: @project, - tree_hash: @tree_hash, - object_builder: object_builder, - members_mapper: members_mapper, - relation_factory: relation_factory, - reader: reader - ) - end - - def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, - user: @user, - importable: @project) - end - - def object_builder - Gitlab::ImportExport::GroupProjectObjectBuilder - end - - def relation_factory - Gitlab::ImportExport::ProjectRelationFactory - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) - end - - def import_failure_service - @import_failure_service ||= ImportFailureService.new(@project) - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb deleted file mode 100644 index 386a4cfdfc6..00000000000 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ /dev/null @@ -1,68 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class ProjectTreeSaver - attr_reader :full_path - - def initialize(project:, current_user:, shared:, params: {}) - @params = params - @project = project - @current_user = current_user - @shared = shared - @full_path = File.join(@shared.export_path, ImportExport.project_filename) - end - - def save - project_tree = tree_saver.serialize(@project, reader.project_tree) - fix_project_tree(project_tree) - tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) - - true - rescue => e - @shared.error(e) - false - end - - private - - # Aware that the resulting hash needs to be pure-hash and - # does not include any AR objects anymore, only objects that run `.to_json` - def fix_project_tree(project_tree) - if @params[:description].present? - project_tree['description'] = @params[:description] - end - - project_tree['project_members'] += group_members_array - - RelationRenameService.add_new_associations(project_tree) - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) - end - - def group_members_array - group_members.as_json(reader.group_members_tree).each do |group_member| - group_member['source_type'] = 'Project' # Make group members project members of the future import - end - end - - def group_members - return [] unless @current_user.can?(:admin_group, @project.group) - - # We need `.where.not(user_id: nil)` here otherwise when a group has an - # invitee, it would make the following query return 0 rows since a NULL - # user_id would be present in the subquery - # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values - non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id) - - GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) - end - - def tree_saver - @tree_saver ||= RelationTreeSaver.new - end - end - end -end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 1390770acef..8d36d05ca6f 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -17,10 +17,18 @@ module Gitlab tree_by_key(:project) end + def project_relation_names + attributes_finder.find_relations_tree(:project).keys + end + def group_tree tree_by_key(:group) end + def group_relation_names + attributes_finder.find_relations_tree(:group).keys + end + def group_members_tree tree_by_key(:group_members) end diff --git a/lib/gitlab/import_export/relation_rename_service.rb b/lib/gitlab/import_export/relation_rename_service.rb deleted file mode 100644 index 03aaa6aefc3..00000000000 --- a/lib/gitlab/import_export/relation_rename_service.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -# This class is intended to help with relation renames within Gitlab versions -# and allow compatibility between versions. -# If you have to change one relationship name that is imported/exported, -# you should add it to the RENAMES constant indicating the old name and the -# new one. -# The behavior of these renamed relationships should be transient and it should -# only last one release until you completely remove the renaming from the list. -# -# When importing, this class will check the hash and: -# - if only the old relationship name is found, it will rename it with the new one -# - if only the new relationship name is found, it will do nothing -# - if it finds both, it will use the new relationship data -# -# When exporting, this class will duplicate the keys in the resulting file. -# This way, if we open the file in an old version of the exporter it will work -# and also it will with the newer versions. -module Gitlab - module ImportExport - class RelationRenameService - RENAMES = { - 'pipelines' => 'ci_pipelines' # Added in 11.6, remove in 11.7 - }.freeze - - def self.rename(tree_hash) - return unless tree_hash&.present? - - RENAMES.each do |old_name, new_name| - old_entry = tree_hash.delete(old_name) - - next if tree_hash[new_name] - next unless old_entry - - tree_hash[new_name] = old_entry - end - end - - def self.add_new_associations(tree_hash) - RENAMES.each do |old_name, new_name| - next if tree_hash.key?(old_name) - - tree_hash[old_name] = tree_hash[new_name] - end - end - end - end -end diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index cc01d70db16..466cb03862e 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -9,13 +9,13 @@ module Gitlab attr_reader :user attr_reader :shared attr_reader :importable - attr_reader :tree_hash + attr_reader :relation_reader - def initialize(user:, shared:, importable:, tree_hash:, members_mapper:, object_builder:, relation_factory:, reader:) + def initialize(user:, shared:, importable:, relation_reader:, members_mapper:, object_builder:, relation_factory:, reader:) @user = user @shared = shared @importable = importable - @tree_hash = tree_hash + @relation_reader = relation_reader @members_mapper = members_mapper @object_builder = object_builder @relation_factory = relation_factory @@ -26,7 +26,13 @@ module Gitlab ActiveRecord::Base.uncached do ActiveRecord::Base.no_touching do update_params! - create_relations! + + bulk_inserts_enabled = @importable.class == ::Project && + Feature.enabled?(:import_bulk_inserts, @importable.group) + BulkInsertableAssociations.with_bulk_insert(enabled: bulk_inserts_enabled) do + fix_ci_pipelines_not_sorted_on_legacy_project_json! + create_relations! + end end end @@ -51,33 +57,21 @@ module Gitlab end def process_relation!(relation_key, relation_definition) - data_hashes = @tree_hash.delete(relation_key) - return unless data_hashes - - # we do not care if we process array or hash - data_hashes = [data_hashes] unless data_hashes.is_a?(Array) - - relation_index = 0 - - # consume and remove objects from memory - while data_hash = data_hashes.shift + @relation_reader.consume_relation(relation_key) do |data_hash, relation_index| process_relation_item!(relation_key, relation_definition, relation_index, data_hash) - relation_index += 1 end end def process_relation_item!(relation_key, relation_definition, relation_index, data_hash) relation_object = build_relation(relation_key, relation_definition, data_hash) return unless relation_object - return if importable_class == Project && group_model?(relation_object) + return if importable_class == ::Project && group_model?(relation_object) relation_object.assign_attributes(importable_class_sym => @importable) import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do relation_object.save! end - - save_id_mapping(relation_key, data_hash, relation_object) rescue => e import_failure_service.log_import_failure( source: 'process_relation_item!', @@ -90,17 +84,6 @@ module Gitlab @import_failure_service ||= ImportFailureService.new(@importable) end - # Older, serialized CI pipeline exports may only have a - # merge_request_id and not the full hash of the merge request. To - # import these pipelines, we need to preserve the mapping between - # the old and new the merge request ID. - def save_id_mapping(relation_key, data_hash, relation_object) - return unless importable_class == Project - return unless relation_key == 'merge_requests' - - merge_requests_mapping[data_hash['id']] = relation_object.id - end - def relations @relations ||= @reader @@ -110,10 +93,7 @@ module Gitlab end def update_params! - params = @tree_hash.reject do |key, _| - relations.include?(key) - end - + params = @relation_reader.root_attributes(relations.keys) params = params.merge(present_override_params) # Cleaning all imported and overridden params @@ -123,7 +103,7 @@ module Gitlab excluded_keys: excluded_keys_for_relation(importable_class_sym)) @importable.assign_attributes(params) - @importable.drop_visibility_level! if importable_class == Project + @importable.drop_visibility_level! if importable_class == ::Project Gitlab::Timeless.timeless(@importable) do @importable.save! @@ -182,7 +162,7 @@ module Gitlab # if object is a hash we can create simple object # as it means that this is 1-to-1 vs 1-to-many - sub_data_hash = + current_item = if sub_data_hash.is_a?(Array) build_relations( sub_relation_key, @@ -195,9 +175,8 @@ module Gitlab sub_data_hash) end - # persist object(s) or delete from relation - if sub_data_hash - data_hash[sub_relation_key] = sub_data_hash + if current_item + data_hash[sub_relation_key] = current_item else data_hash.delete(sub_relation_key) end @@ -219,13 +198,8 @@ module Gitlab importable_class.to_s.downcase.to_sym end - # A Hash of the imported merge request ID -> imported ID. - def merge_requests_mapping - @merge_requests_mapping ||= {} - end - def relation_factory_params(relation_key, data_hash) - base_params = { + { relation_sym: relation_key.to_sym, relation_hash: data_hash, importable: @importable, @@ -234,9 +208,15 @@ module Gitlab user: @user, excluded_keys: excluded_keys_for_relation(relation_key) } + end + + # Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json + # This should be removed once legacy JSON format is deprecated. + # Ndjson export file will fix the order during project export. + def fix_ci_pipelines_not_sorted_on_legacy_project_json! + return unless relation_reader.legacy? - base_params[:merge_requests_mapping] = merge_requests_mapping if importable_class == Project - base_params + relation_reader.sort_ci_pipelines_by_id end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 8d81b2af065..09ed4eb568d 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -94,14 +94,6 @@ module Gitlab end end - def log_error(details) - @logger.error(log_base_data.merge(details)) - end - - def log_debug(details) - @logger.debug(log_base_data.merge(details)) - end - def log_base_data log = { importer: 'Import/Export', diff --git a/lib/gitlab/import_export/snippet_repo_restorer.rb b/lib/gitlab/import_export/snippet_repo_restorer.rb new file mode 100644 index 00000000000..079681dfac5 --- /dev/null +++ b/lib/gitlab/import_export/snippet_repo_restorer.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class SnippetRepoRestorer < RepoRestorer + attr_reader :snippet + + def initialize(snippet:, user:, shared:, path_to_bundle:) + @snippet = snippet + @user = user + @repository = snippet.repository + @path_to_bundle = path_to_bundle.to_s + @shared = shared + end + + def restore + if File.exist?(path_to_bundle) + create_repository_from_bundle + else + create_repository_from_db + end + + true + rescue => e + shared.error(e) + false + end + + private + + def create_repository_from_bundle + repository.create_from_bundle(path_to_bundle) + snippet.track_snippet_repository + end + + def create_repository_from_db + snippet.create_repository + + commit_attrs = { + branch_name: 'master', + message: 'Initial commit' + } + + repository.create_file(@user, snippet.file_name, snippet.content, commit_attrs) + end + end + end +end diff --git a/lib/gitlab/import_export/snippet_repo_saver.rb b/lib/gitlab/import_export/snippet_repo_saver.rb new file mode 100644 index 00000000000..cab96c78232 --- /dev/null +++ b/lib/gitlab/import_export/snippet_repo_saver.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class SnippetRepoSaver < RepoSaver + def initialize(project:, shared:, repository:) + @project = project + @shared = shared + @repository = repository + end + + private + + def bundle_full_path + File.join(shared.export_path, + ::Gitlab::ImportExport.snippet_repo_bundle_dir, + ::Gitlab::ImportExport.snippet_repo_bundle_filename_for(repository.container)) + end + end + end +end diff --git a/lib/gitlab/import_export/snippets_repo_restorer.rb b/lib/gitlab/import_export/snippets_repo_restorer.rb new file mode 100644 index 00000000000..8fe83225812 --- /dev/null +++ b/lib/gitlab/import_export/snippets_repo_restorer.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class SnippetsRepoRestorer + def initialize(project:, shared:, user:) + @project = project + @shared = shared + @user = user + end + + def restore + return true unless Feature.enabled?(:version_snippets, @user) + return true unless Dir.exist?(snippets_repo_bundle_path) + + @project.snippets.find_each.all? do |snippet| + Gitlab::ImportExport::SnippetRepoRestorer.new(snippet: snippet, + user: @user, + shared: @shared, + path_to_bundle: snippet_repo_bundle_path(snippet)) + .restore + end + end + + private + + def snippet_repo_bundle_path(snippet) + File.join(snippets_repo_bundle_path, ::Gitlab::ImportExport.snippet_repo_bundle_filename_for(snippet)) + end + + def snippets_repo_bundle_path + @snippets_repo_bundle_path ||= ::Gitlab::ImportExport.snippets_repo_bundle_path(@shared.export_path) + end + end + end +end diff --git a/lib/gitlab/import_export/snippets_repo_saver.rb b/lib/gitlab/import_export/snippets_repo_saver.rb new file mode 100644 index 00000000000..85e094c0d15 --- /dev/null +++ b/lib/gitlab/import_export/snippets_repo_saver.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class SnippetsRepoSaver + include Gitlab::ImportExport::CommandLineUtil + + def initialize(current_user:, project:, shared:) + @project = project + @shared = shared + @current_user = current_user + end + + def save + return true unless Feature.enabled?(:version_snippets, @current_user) + + create_snippets_repo_directory + + @project.snippets.find_each.all? do |snippet| + Gitlab::ImportExport::SnippetRepoSaver.new(project: @project, + shared: @shared, + repository: snippet.repository) + .save + end + end + + private + + def create_snippets_repo_directory + mkdir_p(::Gitlab::ImportExport.snippets_repo_bundle_path(@shared.export_path)) + end + end + end +end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 4547a9b0a01..2889dbc68cc 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -28,8 +28,9 @@ module Gitlab config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") end - def key_from_address(address) - regex = address_regex + def key_from_address(address, wildcard_address: nil) + wildcard_address ||= config.address + regex = address_regex(wildcard_address) return unless regex match = address.match(regex) @@ -55,8 +56,7 @@ module Gitlab private - def address_regex - wildcard_address = config.address + def address_regex(wildcard_address) return unless wildcard_address regex = Regexp.escape(wildcard_address) diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index c09d8170d17..b973244a531 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -12,7 +12,12 @@ module Gitlab def request(*args) result = make_request(*args) - raise JIRA::HTTPError.new(result.response) unless result.response.is_a?(Net::HTTPSuccess) + unless result.response.is_a?(Net::HTTPSuccess) + Gitlab::ErrorTracking.track_and_raise_exception( + JIRA::HTTPError.new(result.response), + response: result.body + ) + end result end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 90dbe4d005d..e7a8cc6305a 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -19,6 +19,9 @@ module Gitlab class JobWaiter KEY_PREFIX = "gitlab:job_waiter" + STARTED_METRIC = :gitlab_job_waiter_started_total + TIMEOUTS_METRIC = :gitlab_job_waiter_timeouts_total + def self.notify(key, jid) Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } end @@ -27,15 +30,16 @@ module Gitlab key.is_a?(String) && key =~ /\A#{KEY_PREFIX}:\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/ end - attr_reader :key, :finished + attr_reader :key, :finished, :worker_label attr_accessor :jobs_remaining # jobs_remaining - the number of jobs left to wait for # key - The key of this waiter. - def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}") + def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}", worker_label: nil) @key = key @jobs_remaining = jobs_remaining @finished = [] + @worker_label = worker_label end # Waits for all the jobs to be completed. @@ -45,6 +49,7 @@ module Gitlab # long to process, or is never processed. def wait(timeout = 10) deadline = Time.now.utc + timeout + increment_counter(STARTED_METRIC) Gitlab::Redis::SharedState.with do |redis| # Fallback key expiry: allow a long grace period to reduce the chance of @@ -60,7 +65,12 @@ module Gitlab break if seconds_left <= 0 list, jid = redis.blpop(key, timeout: seconds_left) - break unless list && jid # timed out + + # timed out + unless list && jid + increment_counter(TIMEOUTS_METRIC) + break + end @finished << jid @jobs_remaining -= 1 @@ -72,5 +82,20 @@ module Gitlab finished end + + private + + def increment_counter(metric) + return unless worker_label + + metrics[metric].increment(worker: worker_label) + end + + def metrics + @metrics ||= { + STARTED_METRIC => Gitlab::Metrics.counter(STARTED_METRIC, 'JobWaiter attempts started'), + TIMEOUTS_METRIC => Gitlab::Metrics.counter(TIMEOUTS_METRIC, 'JobWaiter attempts timed out') + } + end end end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index c7c348ce9eb..3e201d68297 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -3,13 +3,19 @@ module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.16.1' + HELM_VERSION = '2.16.3' KUBECTL_VERSION = '1.13.12' NAMESPACE = 'gitlab-managed-apps' NAMESPACE_LABELS = { 'app.gitlab.com/managed_by' => :gitlab }.freeze SERVICE_ACCOUNT = 'tiller' CLUSTER_ROLE_BINDING = 'tiller-admin' CLUSTER_ROLE = 'cluster-admin' + + MANAGED_APPS_LOCAL_TILLER_FEATURE_FLAG = :managed_apps_local_tiller + + def self.local_tiller_enabled? + Feature.enabled?(MANAGED_APPS_LOCAL_TILLER_FEATURE_FLAG) + end end end end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index 3ed07818302..3b843799d66 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -3,7 +3,7 @@ module Gitlab module Kubernetes module Helm - class Api + class API def initialize(kubeclient) @kubeclient = kubeclient @namespace = Gitlab::Kubernetes::Namespace.new( diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb index b953ce24c4a..e7ade7e4d39 100644 --- a/lib/gitlab/kubernetes/helm/client_command.rb +++ b/lib/gitlab/kubernetes/helm/client_command.rb @@ -59,7 +59,7 @@ module Gitlab end def local_tiller_enabled? - Feature.enabled?(:managed_apps_local_tiller) + ::Gitlab::Kubernetes::Helm.local_tiller_enabled? end end end diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb index 9862861118b..68e4aeb4bae 100644 --- a/lib/gitlab/kubernetes/namespace.rb +++ b/lib/gitlab/kubernetes/namespace.rb @@ -35,12 +35,14 @@ module Gitlab def log_create_failed(error) logger.error({ - exception: error.class.name, + exception: { + class: error.class.name, + message: error.message + }, status_code: error.error_code, namespace: name, class_name: self.class.name, - event: :failed_to_create_namespace, - message: error.message + event: :failed_to_create_namespace }) end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 751726d4810..3f9fd1b1a19 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -3,8 +3,6 @@ module Gitlab module LegacyGithubImport class Importer - include Gitlab::ShellAdapter - def self.refmap Gitlab::GithubImport.refmap end @@ -264,11 +262,11 @@ module Gitlab end def import_wiki - unless project.wiki.repository_exists? - wiki = WikiFormatter.new(project) - gitlab_shell.import_wiki_repository(project, wiki) - end - rescue Gitlab::Shell::Error => e + return if project.wiki.repository_exists? + + wiki = WikiFormatter.new(project) + project.wiki.repository.import_repository(wiki.import_url) + rescue ::Gitlab::Git::CommandError => e # GitHub error message when the wiki repo has not been created, # this means that repo has wiki enabled, but have no pages. So, # we can skip the import. diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb new file mode 100644 index 00000000000..5dbff7d9102 --- /dev/null +++ b/lib/gitlab/lograge/custom_options.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Lograge + module CustomOptions + LIMITED_ARRAY_SENTINEL = { key: 'truncated', value: '...' }.freeze + IGNORE_PARAMS = Set.new(%w(controller action format)).freeze + + def self.call(event) + params = event + .payload[:params] + .each_with_object([]) { |(k, v), array| array << { key: k, value: v } unless IGNORE_PARAMS.include?(k) } + + payload = { + time: Time.now.utc.iso8601(3), + params: Gitlab::Utils::LogLimitedArray.log_limited_array(params, sentinel: LIMITED_ARRAY_SENTINEL), + remote_ip: event.payload[:remote_ip], + user_id: event.payload[:user_id], + username: event.payload[:username], + ua: event.payload[:ua], + queue_duration: event.payload[:queue_duration] + } + + ::Gitlab::InstrumentationHelper.add_instrumentation_data(payload) + + payload[:response] = event.payload[:response] if event.payload[:response] + payload[:etag_route] = event.payload[:etag_route] if event.payload[:etag_route] + payload[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id + + if cpu_s = Gitlab::Metrics::System.thread_cpu_duration(::Gitlab::RequestContext.instance.start_thread_cpu_time) + payload[:cpu_s] = cpu_s + end + + # https://github.com/roidrage/lograge#logging-errors--exceptions + exception = event.payload[:exception_object] + + ::Gitlab::ExceptionLogFormatter.format!(exception, payload) + + payload + end + end + end +end diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb index 3dfaec48311..d7a0a9b6518 100644 --- a/lib/gitlab/markdown_cache.rb +++ b/lib/gitlab/markdown_cache.rb @@ -3,7 +3,7 @@ module Gitlab module MarkdownCache # Increment this number every time the renderer changes its output - CACHE_COMMONMARK_VERSION = 18 + CACHE_COMMONMARK_VERSION = 20 CACHE_COMMONMARK_VERSION_START = 10 BaseError = Class.new(StandardError) diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index 3dd86c8685d..990fd57bf41 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -29,9 +29,11 @@ module Gitlab # Used by embedded dashboards. # @param options - y_label [String] Y-Axis label of # a panel. Used by embedded dashboards. - # @param options - cluster [Cluster] + # @param options - cluster [Cluster]. Used by + # embedded and un-embedded dashboards. # @param options - cluster_type [Symbol] The level of - # cluster, one of [:admin, :project, :group] + # cluster, one of [:admin, :project, :group]. Used by + # embedded and un-embedded dashboards. # @param options - grafana_url [String] URL pointing # to a grafana dashboard panel # @param options - prometheus_alert_id [Integer] ID of diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb index 24ea85a5a95..993e508cbc6 100644 --- a/lib/gitlab/metrics/dashboard/service_selector.rb +++ b/lib/gitlab/metrics/dashboard/service_selector.rb @@ -3,7 +3,8 @@ # Responsible for determining which dashboard service should # be used to fetch or generate a dashboard hash. # The services can be considered in two categories - embeds -# and dashboards. Embeds are all portions of dashboards. +# and dashboards. Embed hashes are identical to dashboard hashes except +# that they contain a subset of panels. module Gitlab module Metrics module Dashboard diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb index ce75c54d014..c90c1e3f0bc 100644 --- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb +++ b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb @@ -13,12 +13,7 @@ module Gitlab # Reformats the specified panel in the Gitlab # dashboard-yml format def transform! - InputFormatValidator.new( - grafana_dashboard, - datasource, - panel, - query_params - ).validate! + validate_input! new_dashboard = formatted_dashboard @@ -28,6 +23,17 @@ module Gitlab private + def validate_input! + ::Grafana::Validator.new( + grafana_dashboard, + datasource, + panel, + query_params + ).validate! + rescue ::Grafana::Validator::Error => e + raise ::Gitlab::Metrics::Dashboard::Errors::DashboardProcessingError, e.message + end + def formatted_dashboard { panel_groups: [{ panels: [formatted_panel] }] } end @@ -56,11 +62,25 @@ module Gitlab def panel strong_memoize(:panel) do grafana_dashboard[:dashboard][:panels].find do |panel| - panel[:id].to_s == query_params[:panelId] + query_params[:panelId] ? matching_panel?(panel) : valid_panel?(panel) end end end + # Determines whether a given panel is the one + # specified by the linked grafana url + def matching_panel?(panel) + panel[:id].to_s == query_params[:panelId] + end + + # Determines whether any given panel has the potenial + # to return valid results from grafana/prometheus + def valid_panel?(panel) + ::Grafana::Validator + .new(grafana_dashboard, datasource, panel, query_params) + .valid? + end + # Grafana url query parameters. Includes information # on which panel to select and time range. def query_params @@ -141,83 +161,6 @@ module Gitlab params[:grafana_url] end end - - class InputFormatValidator - include ::Gitlab::Metrics::Dashboard::Errors - - attr_reader :grafana_dashboard, :datasource, :panel, :query_params - - UNSUPPORTED_GRAFANA_GLOBAL_VARS = %w( - $__interval_ms - $__timeFilter - $__name - $timeFilter - $interval - ).freeze - - def initialize(grafana_dashboard, datasource, panel, query_params) - @grafana_dashboard = grafana_dashboard - @datasource = datasource - @panel = panel - @query_params = query_params - end - - def validate! - validate_query_params! - validate_datasource! - validate_panel_type! - validate_variable_definitions! - validate_global_variables! - end - - private - - def validate_datasource! - return if datasource[:access] == 'proxy' && datasource[:type] == 'prometheus' - - raise_error 'Only Prometheus datasources with proxy access in Grafana are supported.' - end - - def validate_query_params! - return if [:panelId, :from, :to].all? { |param| query_params.include?(param) } - - raise_error 'Grafana query parameters must include panelId, from, and to.' - end - - def validate_panel_type! - return if panel[:type] == 'graph' && panel[:lines] - - raise_error 'Panel type must be a line graph.' - end - - def validate_variable_definitions! - return unless grafana_dashboard[:dashboard][:templating] - - return if grafana_dashboard[:dashboard][:templating][:list].all? do |variable| - query_params[:"var-#{variable[:name]}"].present? - end - - raise_error 'All Grafana variables must be defined in the query parameters.' - end - - def validate_global_variables! - return unless panel_contains_unsupported_vars? - - raise_error 'Prometheus must not include' - end - - def panel_contains_unsupported_vars? - panel[:targets].any? do |target| - UNSUPPORTED_GRAFANA_GLOBAL_VARS.any? do |variable| - target[:expr].include?(variable) - end - end - end - - def raise_error(message) - raise DashboardProcessingError.new(message) - end - end end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 53508938c49..abdbccd3aa8 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -53,8 +53,9 @@ module Gitlab repository_url = if Gitlab::CurrentSettings.enabled_git_access_protocol == 'ssh' shell = config.gitlab_shell + user = "#{shell.ssh_user}@" unless shell.ssh_user.empty? port = ":#{shell.ssh_port}" unless shell.ssh_port == 22 - "ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git" + "ssh://#{user}#{shell.ssh_host}#{port}/#{path}.git" else "#{project_url}.git" end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index ca8f4e34802..cdab86540f8 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -90,12 +90,14 @@ module Gitlab # Overridden in EE module def whitelisted_routes - grack_route? || internal_route? || lfs_route? || compare_git_revisions_route? || sidekiq_route? || session_route? || graphql_query? + workhorse_passthrough_route? || internal_route? || lfs_route? || compare_git_revisions_route? || sidekiq_route? || session_route? || graphql_query? end - def grack_route? + # URL for requests passed through gitlab-workhorse to rails-web + # https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/12 + def workhorse_passthrough_route? # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless + return false unless request.post? && request.path.end_with?('.git/git-upload-pack', '.git/git-receive-pack') WHITELISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 74057bbc493..41d80fe9aa6 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -51,7 +51,7 @@ module Gitlab # and all their ancestors (recursively). # # Passing an `upto` will stop the recursion once the specified parent_id is - # reached. So all ancestors *lower* than the specified acestor will be + # reached. So all ancestors *lower* than the specified ancestor will be # included. # # Passing a `hierarchy_order` with either `:asc` or `:desc` will cause the diff --git a/lib/gitlab/omniauth_logging/json_formatter.rb b/lib/gitlab/omniauth_logging/json_formatter.rb new file mode 100644 index 00000000000..cdd4da31803 --- /dev/null +++ b/lib/gitlab/omniauth_logging/json_formatter.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'json' + +module Gitlab + module OmniauthLogging + class JSONFormatter + def call(severity, datetime, progname, msg) + { severity: severity, timestamp: datetime.utc.iso8601(3), pid: $$, progname: progname, message: msg }.to_json << "\n" + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 9606e3e134c..5fa0fbf874c 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -42,7 +42,6 @@ module Gitlab invites jwt login - notification_settings oauth profile projects @@ -237,8 +236,32 @@ module Gitlab }x end + def full_snippets_repository_path_regex + %r{\A(#{personal_snippet_repository_path_regex}|#{project_snippet_repository_path_regex})\z} + end + + def personal_and_project_snippets_path_regex + %r{#{personal_snippet_path_regex}|#{project_snippet_path_regex}} + end + private + def personal_snippet_path_regex + /snippets/ + end + + def personal_snippet_repository_path_regex + %r{#{personal_snippet_path_regex}/\d+} + end + + def project_snippet_path_regex + %r{#{full_namespace_route_regex}/#{project_route_regex}/snippets} + end + + def project_snippet_repository_path_regex + %r{#{project_snippet_path_regex}/\d+} + end + def single_line_regexp(regex) # Turns a multiline extended regexp into a single line one, # because `rake routes` breaks on multiline regexes. diff --git a/lib/gitlab/process_memory_cache.rb b/lib/gitlab/process_memory_cache.rb new file mode 100644 index 00000000000..5e8578711b2 --- /dev/null +++ b/lib/gitlab/process_memory_cache.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + class ProcessMemoryCache + # ActiveSupport::Cache::MemoryStore is thread-safe: + # https://github.com/rails/rails/blob/2f1fefe456932a6d7d2b155d27b5315c33f3daa1/activesupport/lib/active_support/cache/memory_store.rb#L19 + @cache = ActiveSupport::Cache::MemoryStore.new + + def self.cache_backend + @cache + end + end +end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index f47ccb8fed9..e10cdf0d8fb 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -2,6 +2,8 @@ module Gitlab module Profiler + extend WithRequestStore + FILTERED_STRING = '[FILTERED]' IGNORE_BACKTRACES = %w[ @@ -58,28 +60,26 @@ module Gitlab logger = create_custom_logger(logger, private_token: private_token) - RequestStore.begin! - - # Make an initial call for an asset path in development mode to avoid - # sprockets dominating the profiler output. - ActionController::Base.helpers.asset_path('katex.css') if Rails.env.development? + result = with_request_store do + # Make an initial call for an asset path in development mode to avoid + # sprockets dominating the profiler output. + ActionController::Base.helpers.asset_path('katex.css') if Rails.env.development? - # Rails loads internationalization files lazily the first time a - # translation is needed. Running this prevents this overhead from showing - # up in profiles. - ::I18n.t('.')[:test_string] + # Rails loads internationalization files lazily the first time a + # translation is needed. Running this prevents this overhead from showing + # up in profiles. + ::I18n.t('.')[:test_string] - # Remove API route mounting from the profile. - app.get('/api/v4/users') + # Remove API route mounting from the profile. + app.get('/api/v4/users') - result = with_custom_logger(logger) do - with_user(user) do - RubyProf.profile { app.public_send(verb, url, params: post_data, headers: headers) } # rubocop:disable GitlabSecurity/PublicSend + with_custom_logger(logger) do + with_user(user) do + RubyProf.profile { app.public_send(verb, url, params: post_data, headers: headers) } # rubocop:disable GitlabSecurity/PublicSend + end end end - RequestStore.end! - log_load_times_by_model(logger) result diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index b4ee8818925..9ed6a23632c 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -40,10 +40,11 @@ module Gitlab ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'), ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'), ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'), - ProjectTemplate.new('iosswift', 'iOS (Swift)', _('A ready-to-go template for use with iOS Swift apps.'), 'https://gitlab.com/gitlab-org/project-templates/iosswift'), + ProjectTemplate.new('iosswift', 'iOS (Swift)', _('A ready-to-go template for use with iOS Swift apps.'), 'https://gitlab.com/gitlab-org/project-templates/iosswift', 'illustrations/logos/swift.svg'), ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/logos/dotnet.svg'), ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps.'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'), ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro'), + ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby.'), 'https://gitlab.com/pages/gatsby'), ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo'), ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll'), ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'), diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb index ba2d33ee1c1..4d48c4a3af7 100644 --- a/lib/gitlab/prometheus/query_variables.rb +++ b/lib/gitlab/prometheus/query_variables.rb @@ -7,7 +7,11 @@ module Gitlab { ci_environment_slug: environment.slug, kube_namespace: environment.deployment_namespace || '', - environment_filter: %{container_name!="POD",environment="#{environment.slug}"} + environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, + ci_project_name: environment.project.name, + ci_project_namespace: environment.project.namespace.name, + ci_project_path: environment.project.full_path, + ci_environment_name: environment.name } end end diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 6f87968e286..cd07122ffd9 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -13,6 +13,7 @@ module Gitlab def initialize(command_definitions) @command_definitions = command_definitions + @commands_regex = {} end # Extracts commands from content and return an array of commands. @@ -58,7 +59,8 @@ module Gitlab content = content.dup content.delete!("\r") - content.gsub!(commands_regex(only: only)) do + names = command_names(limit_to_commands: only).map(&:to_s) + content.gsub!(commands_regex(names: names)) do command, output = process_commands($~, redact) commands << command output @@ -91,10 +93,8 @@ module Gitlab # It looks something like: # # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ - def commands_regex(only:) - names = command_names(limit_to_commands: only).map(&:to_s) - - @commands_regex ||= %r{ + def commands_regex(names:) + @commands_regex[names] ||= %r{ (?<code> # Code blocks: # ``` @@ -106,6 +106,17 @@ module Gitlab \n```$ ) | + (?<inline_code> + # Inline code on separate rows: + # ` + # Anything, including `/cmd arg` which are ignored by this filter + # ` + + ^.*`\n* + .+? + \n*`$ + ) + | (?<html> # HTML block: # <tag> @@ -151,14 +162,18 @@ module Gitlab end substitution_definitions.each do |substitution| - match_data = substitution.match(content.downcase) - if match_data - command = [substitution.name.to_s] - command << match_data[1] unless match_data[1].empty? - commands << command + regex = commands_regex(names: substitution.all_names) + content = content.gsub(regex) do |text| + if $~[:cmd] + command = [substitution.name.to_s] + command << $~[:arg] if $~[:arg].present? + commands << command + + substitution.perform_substitution(self, text) + else + text + end end - - content = substitution.perform_substitution(self, content) end [content, commands] diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb index b7231aa3a8b..cd4d202e8d0 100644 --- a/lib/gitlab/quick_actions/substitution_definition.rb +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -17,7 +17,7 @@ module Gitlab return unless content all_names.each do |a_name| - content = content.gsub(%r{/#{a_name}(?![\S]) ?(.*)$}i, execute_block(action_block, context, '\1')) + content = content.sub(%r{/#{a_name}(?![\S]) ?(.*)$}i, execute_block(action_block, context, '\1')) end content diff --git a/lib/gitlab/rate_limit_helpers.rb b/lib/gitlab/rate_limit_helpers.rb new file mode 100644 index 00000000000..2dcc888892b --- /dev/null +++ b/lib/gitlab/rate_limit_helpers.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module RateLimitHelpers + ARCHIVE_RATE_LIMIT_REACHED_MESSAGE = 'This archive has been requested too many times. Try again later.' + ARCHIVE_RATE_ANONYMOUS_THRESHOLD = 100 # Allow 100 requests/min for anonymous users + ARCHIVE_RATE_THROTTLE_KEY = :project_repositories_archive + + def archive_rate_limit_reached?(user, project) + return false unless Feature.enabled?(:archive_rate_limit, default_enabled: true) + + key = ARCHIVE_RATE_THROTTLE_KEY + + if rate_limiter.throttled?(key, scope: [project, user], threshold: archive_rate_threshold_by_user(user)) + rate_limiter.log_request(request, "#{key}_request_limit".to_sym, user) + + return true + end + + false + end + + def archive_rate_threshold_by_user(user) + if user + nil # Use the defaults + else + ARCHIVE_RATE_ANONYMOUS_THRESHOLD + end + end + + def rate_limiter + ::Gitlab::ApplicationRateLimiter + end + end +end diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb new file mode 100644 index 00000000000..609087d8137 --- /dev/null +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +# Interface to the Redis-backed cache store to keep track of complete cache keys +# for a ReactiveCache resource. +module Gitlab + class ReactiveCacheSetCache < Gitlab::SetCache + attr_reader :expires_in + + def initialize(expires_in: 10.minutes) + @expires_in = expires_in + end + + def cache_key(key) + "#{cache_type}:#{key}:set" + end + + def clear_cache!(key) + with do |redis| + keys = read(key).map { |value| "#{cache_type}:#{value}" } + keys << cache_key(key) + + redis.pipelined do + keys.each_slice(1000) { |subset| redis.del(*subset) } + end + end + end + + private + + def cache_type + Gitlab::Redis::Cache::CACHE_NAMESPACE + end + end +end diff --git a/lib/gitlab/redacted_search_results_logger.rb b/lib/gitlab/redacted_search_results_logger.rb new file mode 100644 index 00000000000..07dbf6fe97d --- /dev/null +++ b/lib/gitlab/redacted_search_results_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class RedactedSearchResultsLogger < ::Gitlab::JsonLogger + def self.file_name_noext + 'redacted_search_results' + end + end +end diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb index 1c43de35816..5fdfa5e75ed 100644 --- a/lib/gitlab/reference_counter.rb +++ b/lib/gitlab/reference_counter.rb @@ -1,20 +1,42 @@ # frozen_string_literal: true module Gitlab + # Reference Counter + # + # A reference counter is used as a mechanism to identify when + # a repository is being accessed by a writable operation. + # + # Maintenance operations would use this as a clue to when it should + # execute significant changes in order to avoid disrupting running traffic class ReferenceCounter REFERENCE_EXPIRE_TIME = 600 attr_reader :gl_repository, :key + # Reference Counter instance + # + # @example + # Gitlab::ReferenceCounter.new('project-1') + # + # @see Gitlab::GlRepository::RepoType.identifier_for_repositorable + # @param [String] gl_repository repository identifier def initialize(gl_repository) @gl_repository = gl_repository @key = "git-receive-pack-reference-counter:#{gl_repository}" end + # Return the actual counter value + # + # @return [Integer] value def value - Gitlab::Redis::SharedState.with { |redis| (redis.get(key) || 0).to_i } + Gitlab::Redis::SharedState.with do |redis| + (redis.get(key) || 0).to_i + end end + # Increase the counter + # + # @return [Boolean] whether operation was a success def increase redis_cmd do |redis| redis.incr(key) @@ -22,26 +44,51 @@ module Gitlab end end - # rubocop:disable Gitlab/RailsLogger + # Decrease the counter + # + # @return [Boolean] whether operation was a success def decrease redis_cmd do |redis| current_value = redis.decr(key) if current_value < 0 + # rubocop:disable Gitlab/RailsLogger Rails.logger.warn("Reference counter for #{gl_repository} decreased" \ - " when its value was less than 1. Reseting the counter.") + " when its value was less than 1. Resetting the counter.") + # rubocop:enable Gitlab/RailsLogger redis.del(key) end end end - # rubocop:enable Gitlab/RailsLogger + + # Reset the reference counter + # + # @private Used internally by SRE and debugging purpose + # @return [Boolean] whether reset was a success + def reset! + redis_cmd do |redis| + redis.del(key) + end + end + + # When the reference counter would expire + # + # @api private Used internally by SRE and debugging purpose + # @return [Integer] Number in seconds until expiration or false if never + def expires_in + Gitlab::Redis::SharedState.with do |redis| + redis.ttl(key) + end + end private def redis_cmd Gitlab::Redis::SharedState.with { |redis| yield(redis) } + true rescue => e Rails.logger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") # rubocop:disable Gitlab/RailsLogger + false end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 519eb49658a..d07d6440c6b 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -44,7 +44,7 @@ module Gitlab end def issues - if project && project.jira_tracker? + if project&.external_references_supported? if project.issues_enabled? @references[:all_issues] ||= references(:external_issue) + references(:issue) else diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index fd6e24a96d8..38281fb1c91 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -112,7 +112,7 @@ module Gitlab # Based on Jira's project key format # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html def jira_issue_key_regex - @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+/ + @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+\b/ end def jira_transition_id_regex @@ -144,6 +144,10 @@ module Gitlab def utc_date_regex @utc_date_regex ||= /\A[0-9]{4}-[0-9]{2}-[0-9]{2}\z/.freeze end + + def issue + @issue ||= /(?<issue>\d+\b)/ + end end end diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index e8c749cac14..67e23624045 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -19,30 +19,62 @@ module Gitlab # Removing the suffix (.wiki, .design, ...) from the project path full_path = repo_path.chomp(type.path_suffix) + container, project, redirected_path = find_container(type, full_path) - project, was_redirected = find_project(full_path) - redirected_path = repo_path if was_redirected - - # If we found a matching project, then the type was matched, no need to - # continue looking. - return [project, type, redirected_path] if project + return [container, project, type, redirected_path] if container end # When a project did not exist, the parsed repo_type would be empty. # In that case, we want to continue with a regular project repository. As we # could create the project if the user pushing is allowed to do so. - [nil, Gitlab::GlRepository.default_type, nil] + [nil, nil, Gitlab::GlRepository.default_type, nil] + end + + def self.find_container(type, full_path) + if type.snippet? + snippet, redirected_path = find_snippet(full_path) + + [snippet, snippet&.project, redirected_path] + else + project, redirected_path = find_project(full_path) + + [project, project, redirected_path] + end end def self.find_project(project_path) + return [nil, nil] if project_path.blank? + project = Project.find_by_full_path(project_path, follow_redirects: true) + redirected_path = redirected?(project, project_path) ? project_path : nil - [project, redirected?(project, project_path)] + [project, redirected_path] end def self.redirected?(project, project_path) project && project.full_path.casecmp(project_path) != 0 end + + # Snippet_path can be either: + # - snippets/1 + # - h5bp/html5-boilerplate/snippets/53 + def self.find_snippet(snippet_path) + return [nil, nil] if snippet_path.blank? + + snippet_id, project_path = extract_snippet_info(snippet_path) + project, redirected_path = find_project(project_path) + + [Snippet.find_by_id_and_project(id: snippet_id, project: project), redirected_path] + end + + def self.extract_snippet_info(snippet_path) + path_segments = snippet_path.split('/') + snippet_id = path_segments.pop + path_segments.pop # Remove snippets from path + project_path = File.join(path_segments) + + [snippet_id, project_path] + end end end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 304f53b58c4..688a4a39dba 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -237,7 +237,7 @@ module Gitlab end def expire_redis_set_method_caches(methods) - methods.each { |name| redis_set_cache.expire(name) } + redis_set_cache.expire(*methods) end def expire_redis_hash_method_caches(methods) diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index 4797ec0b116..1e2d86b7ad2 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -2,7 +2,7 @@ # Interface to the Redis-backed cache store for keys that use a Redis set module Gitlab - class RepositorySetCache + class RepositorySetCache < Gitlab::SetCache attr_reader :repository, :namespace, :expires_in def initialize(repository, extra_namespace: nil, expires_in: 2.weeks) @@ -17,18 +17,6 @@ module Gitlab "#{type}:#{namespace}:set" end - def expire(key) - with { |redis| redis.del(cache_key(key)) } - end - - def exist?(key) - with { |redis| redis.exists(cache_key(key)) } - end - - def read(key) - with { |redis| redis.smembers(cache_key(key)) } - end - def write(key, value) full_key = cache_key(key) @@ -54,15 +42,5 @@ module Gitlab write(key, yield) end end - - def include?(key, value) - with { |redis| redis.sismember(cache_key(key), value) } - end - - private - - def with(&blk) - Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord - end end end diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb index 99958d7a211..7050aee3847 100644 --- a/lib/gitlab/request_profiler/middleware.rb +++ b/lib/gitlab/request_profiler/middleware.rb @@ -51,7 +51,7 @@ module Gitlab def call_with_call_stack_profiling(env) ret = nil report = RubyProf::Profile.profile do - ret = catch(:warden) do + ret = catch(:warden) do # rubocop:disable Cop/BanCatchThrow @app.call(env) end end @@ -67,7 +67,7 @@ module Gitlab def call_with_memory_profiling(env) ret = nil report = MemoryProfiler.report do - ret = catch(:warden) do + ret = catch(:warden) do # rubocop:disable Cop/BanCatchThrow @app.call(env) end end @@ -99,7 +99,7 @@ module Gitlab if ret.is_a?(Array) ret else - throw(:warden, ret) + throw(:warden, ret) # rubocop:disable Cop/BanCatchThrow end end end diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb index f472c70446c..fc1abc064c7 100644 --- a/lib/gitlab/search/found_blob.rb +++ b/lib/gitlab/search/found_blob.rb @@ -155,7 +155,7 @@ module Gitlab end def repository - @repository ||= project.repository + @repository ||= project&.repository end end end diff --git a/lib/gitlab/serverless/domain.rb b/lib/gitlab/serverless/domain.rb deleted file mode 100644 index ec7c68764d1..00000000000 --- a/lib/gitlab/serverless/domain.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Serverless - class Domain - UUID_LENGTH = 14 - - def self.generate_uuid - SecureRandom.hex(UUID_LENGTH / 2) - end - end - end -end diff --git a/lib/gitlab/serverless/function_uri.rb b/lib/gitlab/serverless/function_uri.rb deleted file mode 100644 index c0e0cf00f35..00000000000 --- a/lib/gitlab/serverless/function_uri.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Serverless - class FunctionURI < URI::HTTPS - SERVERLESS_DOMAIN_REGEXP = %r{^(?<scheme>https?://)?(?<function>[^.]+)-(?<cluster_left>\h{2})a1(?<cluster_middle>\h{10})f2(?<cluster_right>\h{2})(?<environment_id>\h+)-(?<environment_slug>[^.]+)\.(?<domain>.+)}.freeze - - attr_reader :function, :cluster, :environment - - def initialize(function: nil, cluster: nil, environment: nil) - initialize_required_argument(:function, function) - initialize_required_argument(:cluster, cluster) - initialize_required_argument(:environment, environment) - - @host = "#{function}-#{cluster.uuid[0..1]}a1#{cluster.uuid[2..-3]}f2#{cluster.uuid[-2..-1]}#{"%x" % environment.id}-#{environment.slug}.#{cluster.domain}" - - super('https', nil, host, nil, nil, nil, nil, nil, nil) - end - - def self.parse(uri) - match = SERVERLESS_DOMAIN_REGEXP.match(uri) - return unless match - - cluster = ::Serverless::DomainCluster.find(match[:cluster_left] + match[:cluster_middle] + match[:cluster_right]) - return unless cluster - - environment = ::Environment.find(match[:environment_id].to_i(16)) - return unless environment&.slug == match[:environment_slug] - - new( - function: match[:function], - cluster: cluster, - environment: environment - ) - end - - private - - def initialize_required_argument(name, value) - raise ArgumentError.new("missing argument: #{name}") unless value - - instance_variable_set("@#{name}".to_sym, value) - end - end - end -end diff --git a/lib/gitlab/serverless/service.rb b/lib/gitlab/serverless/service.rb index 643e076c587..c3ab2e9ddeb 100644 --- a/lib/gitlab/serverless/service.rb +++ b/lib/gitlab/serverless/service.rb @@ -60,7 +60,11 @@ class Gitlab::Serverless::Service def proxy_url if cluster&.serverless_domain - Gitlab::Serverless::FunctionURI.new(function: name, cluster: cluster.serverless_domain, environment: environment) + ::Serverless::Domain.new( + function_name: name, + serverless_domain_cluster: cluster.serverless_domain, + environment: environment + ).uri.to_s end end diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb new file mode 100644 index 00000000000..d1151a431bb --- /dev/null +++ b/lib/gitlab/set_cache.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +# Interface to the Redis-backed cache store to keep track of complete cache keys +# for a ReactiveCache resource. +module Gitlab + class SetCache + attr_reader :expires_in + + def initialize(expires_in: 2.weeks) + @expires_in = expires_in + end + + def cache_key(key) + "#{key}:set" + end + + # Returns the number of keys deleted by Redis + def expire(*keys) + return 0 if keys.empty? + + with do |redis| + keys = keys.map { |key| cache_key(key) } + unlink_or_delete(redis, keys) + end + end + + def exist?(key) + with { |redis| redis.exists(cache_key(key)) } + end + + def write(key, value) + with do |redis| + redis.pipelined do + redis.sadd(cache_key(key), value) + + redis.expire(cache_key(key), expires_in) + end + end + + value + end + + def read(key) + with { |redis| redis.smembers(cache_key(key)) } + end + + def include?(key, value) + with { |redis| redis.sismember(cache_key(key), value) } + end + + def ttl(key) + with { |redis| redis.ttl(cache_key(key)) } + end + + private + + def with(&blk) + Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end + + def unlink_or_delete(redis, keys) + if Feature.enabled?(:repository_set_cache_unlink, default_enabled: true) + redis.unlink(*keys) + else + redis.del(*keys) + end + rescue ::Redis::CommandError + redis.del(*keys) + end + end +end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index c449c6879bc..99a7e617884 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -33,8 +33,6 @@ module Gitlab if Rails.env.test? storage_path = Rails.root.join('tmp', 'tests', 'second_storage').to_s - - FileUtils.mkdir(storage_path) unless File.exist?(storage_path) storages << { name: 'test_second_storage', path: storage_path } end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 726ecd81824..1f8a45e5481 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,13 +1,17 @@ # frozen_string_literal: true -# Gitaly note: SSH key operations are not part of Gitaly so will never be migrated. - require 'securerandom' module Gitlab + # This class is an artifact of a time when common repository operations were + # performed by calling out to scripts in the gitlab-shell project. Now, these + # operations are all performed by Gitaly, and are mostly accessible through + # the Repository class. Prefer using a Repository to functionality here. + # + # Legacy code relating to namespaces still relies on Gitlab::Shell; it can be + # converted to a module once https://gitlab.com/groups/gitlab-org/-/epics/2320 + # is completed. https://gitlab.com/gitlab-org/gitlab/-/issues/25095 tracks it. class Shell - GITLAB_SHELL_ENV_VARS = %w(GIT_TERMINAL_PROMPT).freeze - Error = Class.new(StandardError) class << self @@ -36,8 +40,31 @@ module Gitlab .join('GITLAB_SHELL_VERSION')).strip end + # Return GitLab shell version + # + # @return [String] version + def version + @version ||= File.read(gitlab_shell_version_file).chomp if File.readable?(gitlab_shell_version_file) + end + + # Return a SSH url for a given project path + # + # @param [String] full_path project path (URL) + # @return [String] SSH URL + def url_to_repo(full_path) + Gitlab.config.gitlab_shell.ssh_path_prefix + "#{full_path}.git" + end + private + def gitlab_shell_path + File.expand_path(Gitlab.config.gitlab_shell.path) + end + + def gitlab_shell_version_file + File.join(gitlab_shell_path, 'VERSION') + end + # Create (if necessary) and link the secret token file def generate_and_link_secret_token secret_file = Gitlab.config.gitlab_shell.secret_file @@ -56,88 +83,6 @@ module Gitlab end end - # Initialize a new project repository using a Project model - # - # @param [Project] project - # @return [Boolean] whether repository could be created - def create_project_repository(project) - create_repository(project.repository_storage, project.disk_path, project.full_path) - end - - # Initialize a new wiki repository using a Project model - # - # @param [Project] project - # @return [Boolean] whether repository could be created - def create_wiki_repository(project) - create_repository(project.repository_storage, project.wiki.disk_path, project.wiki.full_path) - end - - # Init new repository - # - # @example Create a repository - # create_repository("default", "path/to/gitlab-ci", "gitlab/gitlab-ci") - # - # @param [String] storage the shard key - # @param [String] disk_path project path on disk - # @param [String] gl_project_path project name - # @return [Boolean] whether repository could be created - def create_repository(storage, disk_path, gl_project_path) - relative_path = disk_path.dup - relative_path << '.git' unless relative_path.end_with?('.git') - - # During creation of a repository, gl_repository may not be known - # because that depends on a yet-to-be assigned project ID in the - # database (e.g. project-1234), so for now it is blank. - repository = Gitlab::Git::Repository.new(storage, relative_path, '', gl_project_path) - wrapped_gitaly_errors { repository.gitaly_repository_client.create_repository } - - true - rescue => err # Once the Rugged codes gets removes this can be improved - Rails.logger.error("Failed to add repository #{storage}/#{disk_path}: #{err}") # rubocop:disable Gitlab/RailsLogger - false - end - - # Import wiki repository from external service - # - # @param [Project] project - # @param [Gitlab::LegacyGithubImport::WikiFormatter, Gitlab::BitbucketImport::WikiFormatter] wiki_formatter - # @return [Boolean] whether repository could be imported - def import_wiki_repository(project, wiki_formatter) - import_repository(project.repository_storage, wiki_formatter.disk_path, wiki_formatter.import_url, project.wiki.full_path) - end - - # Import project repository from external service - # - # @param [Project] project - # @return [Boolean] whether repository could be imported - def import_project_repository(project) - import_repository(project.repository_storage, project.disk_path, project.import_url, project.full_path) - end - - # Import repository - # - # @example Import a repository - # import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git", "gitlab/gitlab-ci") - # - # @param [String] storage project's storage name - # @param [String] disk_path project path on disk - # @param [String] url from external resource to import from - # @param [String] gl_project_path project name - # @return [Boolean] whether repository could be imported - def import_repository(storage, disk_path, url, gl_project_path) - if url.start_with?('.', '/') - raise Error.new("don't use disk paths with import_repository: #{url.inspect}") - end - - relative_path = "#{disk_path}.git" - cmd = GitalyGitlabProjects.new(storage, relative_path, gl_project_path) - - success = cmd.import_project(url, git_timeout) - raise Error, cmd.output unless success - - success - end - # Move or rename a repository # # @example Move/rename a repository @@ -147,6 +92,8 @@ module Gitlab # @param [String] disk_path current project path on disk # @param [String] new_disk_path new project path on disk # @return [Boolean] whether repository could be moved/renamed on disk + # + # @deprecated def mv_repository(storage, disk_path, new_disk_path) return false if disk_path.empty? || new_disk_path.empty? @@ -159,17 +106,6 @@ module Gitlab false end - # Fork repository to new path - # - # @param [Project] source_project forked-from Project - # @param [Project] target_project forked-to Project - def fork_repository(source_project, target_project) - forked_from_relative_path = "#{source_project.disk_path}.git" - fork_args = [target_project.repository_storage, "#{target_project.disk_path}.git", target_project.full_path] - - GitalyGitlabProjects.new(source_project.repository_storage, forked_from_relative_path, source_project.full_path).fork_repository(*fork_args) - end - # Removes a repository from file system, using rm_diretory which is an alias # for rm_namespace. Given the underlying implementation removes the name # passed as second argument on the passed storage. @@ -179,6 +115,8 @@ module Gitlab # # @param [String] storage project's storage path # @param [String] disk_path current project path on disk + # + # @deprecated def remove_repository(storage, disk_path) return false if disk_path.empty? @@ -192,84 +130,6 @@ module Gitlab false end - # Add new key to authorized_keys - # - # @example Add new key - # add_key("key-42", "sha-rsa ...") - # - # @param [String] key_id identifier of the key - # @param [String] key_content key content (public certificate) - # @return [Boolean] whether key could be added - def add_key(key_id, key_content) - return unless self.authorized_keys_enabled? - - gitlab_authorized_keys.add_key(key_id, key_content) - end - - # Batch-add keys to authorized_keys - # - # @example - # batch_add_keys(Key.all) - # - # @param [Array<Key>] keys - # @return [Boolean] whether keys could be added - def batch_add_keys(keys) - return unless self.authorized_keys_enabled? - - gitlab_authorized_keys.batch_add_keys(keys) - end - - # Remove SSH key from authorized_keys - # - # @example Remove a key - # remove_key("key-342") - # - # @param [String] key_id - # @return [Boolean] whether key could be removed or not - def remove_key(key_id, _ = nil) - return unless self.authorized_keys_enabled? - - gitlab_authorized_keys.rm_key(key_id) - end - - # Remove all SSH keys from gitlab shell - # - # @example Remove all keys - # remove_all_keys - # - # @return [Boolean] whether keys could be removed or not - def remove_all_keys - return unless self.authorized_keys_enabled? - - gitlab_authorized_keys.clear - end - - # Remove SSH keys from gitlab shell that are not in the DB - # - # @example Remove keys not on the database - # remove_keys_not_found_in_db - # - # rubocop: disable CodeReuse/ActiveRecord - def remove_keys_not_found_in_db - return unless self.authorized_keys_enabled? - - Rails.logger.info("Removing keys not found in DB") # rubocop:disable Gitlab/RailsLogger - - batch_read_key_ids do |ids_in_file| - ids_in_file.uniq! - keys_in_db = Key.where(id: ids_in_file) - - next unless ids_in_file.size > keys_in_db.count # optimization - - ids_to_remove = ids_in_file - keys_in_db.pluck(:id) - ids_to_remove.each do |id| - Rails.logger.info("Removing key-#{id} not found in DB") # rubocop:disable Gitlab/RailsLogger - remove_key("key-#{id}") - end - end - end - # rubocop: enable CodeReuse/ActiveRecord - # Add empty directory for storing repositories # # @example Add new namespace directory @@ -277,6 +137,8 @@ module Gitlab # # @param [String] storage project's storage path # @param [String] name namespace name + # + # @deprecated def add_namespace(storage, name) Gitlab::GitalyClient.allow_n_plus_1_calls do Gitlab::GitalyClient::NamespaceService.new(storage).add(name) @@ -293,6 +155,8 @@ module Gitlab # # @param [String] storage project's storage path # @param [String] name namespace name + # + # @deprecated def rm_namespace(storage, name) Gitlab::GitalyClient::NamespaceService.new(storage).remove(name) rescue GRPC::InvalidArgument => e @@ -308,6 +172,8 @@ module Gitlab # @param [String] storage project's storage path # @param [String] old_name current namespace name # @param [String] new_name new namespace name + # + # @deprecated def mv_namespace(storage, old_name, new_name) Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name) rescue GRPC::InvalidArgument => e @@ -316,25 +182,6 @@ module Gitlab false end - # Return a SSH url for a given project path - # - # @param [String] full_path project path (URL) - # @return [String] SSH URL - def url_to_repo(full_path) - Gitlab.config.gitlab_shell.ssh_path_prefix + "#{full_path}.git" - end - - # Return GitLab shell version - # - # @return [String] version - def version - gitlab_shell_version_file = "#{gitlab_shell_path}/VERSION" - - if File.readable?(gitlab_shell_version_file) - File.read(gitlab_shell_version_file).chomp - end - end - # Check if repository exists on disk # # @example Check if repository exists @@ -343,116 +190,12 @@ module Gitlab # @return [Boolean] whether repository exists or not # @param [String] storage project's storage path # @param [Object] dir_name repository dir name + # + # @deprecated def repository_exists?(storage, dir_name) Gitlab::Git::Repository.new(storage, dir_name, nil, nil).exists? rescue GRPC::Internal false end - - # Return hooks folder path used by projects - # - # @return [String] path - def hooks_path - File.join(gitlab_shell_path, 'hooks') - end - - protected - - def gitlab_shell_path - File.expand_path(Gitlab.config.gitlab_shell.path) - end - - def gitlab_shell_user_home - File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") - end - - def full_path(storage, dir_name) - raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? - - File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, dir_name) - end - - def authorized_keys_enabled? - # Return true if nil to ensure the authorized_keys methods work while - # fixing the authorized_keys file during migration. - return true if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled.nil? - - Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled - end - - private - - def git_timeout - Gitlab.config.gitlab_shell.git_timeout - end - - def wrapped_gitaly_errors - yield - rescue GRPC::NotFound, GRPC::BadStatus => e - # Old Popen code returns [Error, output] to the caller, so we - # need to do the same here... - raise Error, e - end - - def gitlab_authorized_keys - @gitlab_authorized_keys ||= Gitlab::AuthorizedKeys.new - end - - def batch_read_key_ids(batch_size: 100, &block) - return unless self.authorized_keys_enabled? - - gitlab_authorized_keys.list_key_ids.lazy.each_slice(batch_size) do |key_ids| - yield(key_ids) - end - end - - def strip_key(key) - key.split(/[ ]+/)[0, 2].join(' ') - end - - def add_keys_to_io(keys, io) - keys.each do |k| - key = strip_key(k.key) - - raise Error.new("Invalid key: #{key.inspect}") if key.include?("\t") || key.include?("\n") - - io.puts("#{k.shell_id}\t#{key}") - end - end - - class GitalyGitlabProjects - attr_reader :shard_name, :repository_relative_path, :output, :gl_project_path - - def initialize(shard_name, repository_relative_path, gl_project_path) - @shard_name = shard_name - @repository_relative_path = repository_relative_path - @output = '' - @gl_project_path = gl_project_path - end - - def import_project(source, _timeout) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil, gl_project_path) - - Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) - true - rescue GRPC::BadStatus => e - @output = e.message - false - end - - def fork_repository(new_shard_name, new_repository_relative_path, new_project_name) - target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil, new_project_name) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil, gl_project_path) - - Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) - rescue GRPC::BadStatus => e - logger.error "fork-repository failed: #{e.message}" - false - end - - def logger - Rails.logger # rubocop:disable Gitlab/RailsLogger - end - end end end diff --git a/lib/gitlab/sidekiq_cluster.rb b/lib/gitlab/sidekiq_cluster.rb new file mode 100644 index 00000000000..c19bef1389a --- /dev/null +++ b/lib/gitlab/sidekiq_cluster.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqCluster + # The signals that should terminate both the master and workers. + TERMINATE_SIGNALS = %i(INT TERM).freeze + + # The signals that should simply be forwarded to the workers. + FORWARD_SIGNALS = %i(TTIN USR1 USR2 HUP).freeze + + # Traps the given signals and yields the block whenever these signals are + # received. + # + # The block is passed the name of the signal. + # + # Example: + # + # trap_signals(%i(HUP TERM)) do |signal| + # ... + # end + def self.trap_signals(signals) + signals.each do |signal| + trap(signal) do + yield signal + end + end + end + + def self.trap_terminate(&block) + trap_signals(TERMINATE_SIGNALS, &block) + end + + def self.trap_forward(&block) + trap_signals(FORWARD_SIGNALS, &block) + end + + def self.signal(pid, signal) + Process.kill(signal, pid) + true + rescue Errno::ESRCH + false + end + + def self.signal_processes(pids, signal) + pids.each { |pid| signal(pid, signal) } + end + + # Starts Sidekiq workers for the pairs of processes. + # + # Example: + # + # start([ ['foo'], ['bar', 'baz'] ], :production) + # + # This would start two Sidekiq processes: one processing "foo", and one + # processing "bar" and "baz". Each one is placed in its own process group. + # + # queues - An Array containing Arrays. Each sub Array should specify the + # queues to use for a single process. + # + # directory - The directory of the Rails application. + # + # Returns an Array containing the PIDs of the started processes. + def self.start(queues, env: :development, directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, dryrun: false) + queues.map.with_index do |pair, index| + start_sidekiq(pair, env: env, directory: directory, max_concurrency: max_concurrency, min_concurrency: min_concurrency, worker_id: index, dryrun: dryrun) + end + end + + # Starts a Sidekiq process that processes _only_ the given queues. + # + # Returns the PID of the started process. + def self.start_sidekiq(queues, env:, directory:, max_concurrency:, min_concurrency:, worker_id:, dryrun:) + counts = count_by_queue(queues) + + cmd = %w[bundle exec sidekiq] + cmd << "-c #{self.concurrency(queues, min_concurrency, max_concurrency)}" + cmd << "-e#{env}" + cmd << "-gqueues: #{proc_details(counts)}" + cmd << "-r#{directory}" + + counts.each do |queue, count| + cmd << "-q#{queue},#{count}" + end + + if dryrun + puts "Sidekiq command: #{cmd}" # rubocop:disable Rails/Output + return + end + + pid = Process.spawn( + { 'ENABLE_SIDEKIQ_CLUSTER' => '1', + 'SIDEKIQ_WORKER_ID' => worker_id.to_s }, + *cmd, + pgroup: true, + err: $stderr, + out: $stdout + ) + + wait_async(pid) + + pid + end + + def self.count_by_queue(queues) + queues.each_with_object(Hash.new(0)) { |element, hash| hash[element] += 1 } + end + + def self.proc_details(counts) + counts.map do |queue, count| + if count == 1 + queue + else + "#{queue} (#{count})" + end + end.join(', ') + end + + def self.concurrency(queues, min_concurrency, max_concurrency) + concurrency_from_queues = queues.length + 1 + max = max_concurrency.positive? ? max_concurrency : concurrency_from_queues + min = [min_concurrency, max].min + + concurrency_from_queues.clamp(min, max) + end + + # Waits for the given process to complete using a separate thread. + def self.wait_async(pid) + Thread.new do + Process.wait(pid) rescue Errno::ECHILD + end + end + + # Returns true if all the processes are alive. + def self.all_alive?(pids) + pids.each do |pid| + return false unless process_alive?(pid) + end + + true + end + + def self.any_alive?(pids) + pids_alive(pids).any? + end + + def self.pids_alive(pids) + pids.select { |pid| process_alive?(pid) } + end + + def self.process_alive?(pid) + # Signal 0 tests whether the process exists and we have access to send signals + # but is otherwise a noop (doesn't actually send a signal to the process) + signal(pid, 0) + end + + def self.write_pid(path) + File.open(path, 'w') do |handle| + handle.write(Process.pid.to_s) + end + end + end +end diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb new file mode 100644 index 00000000000..0a9624950c2 --- /dev/null +++ b/lib/gitlab/sidekiq_cluster/cli.rb @@ -0,0 +1,184 @@ +# frozen_string_literal: true + +require 'optparse' +require 'logger' +require 'time' + +module Gitlab + module SidekiqCluster + class CLI + CHECK_TERMINATE_INTERVAL_SECONDS = 1 + # How long to wait in total when asking for a clean termination + # Sidekiq default to self-terminate is 25s + TERMINATE_TIMEOUT_SECONDS = 30 + + CommandError = Class.new(StandardError) + + def initialize(log_output = STDERR) + require_relative '../../../lib/gitlab/sidekiq_logging/json_formatter' + + # As recommended by https://github.com/mperham/sidekiq/wiki/Advanced-Options#concurrency + @max_concurrency = 50 + @min_concurrency = 0 + @environment = ENV['RAILS_ENV'] || 'development' + @pid = nil + @interval = 5 + @alive = true + @processes = [] + @logger = Logger.new(log_output) + @logger.formatter = ::Gitlab::SidekiqLogging::JSONFormatter.new + @rails_path = Dir.pwd + @dryrun = false + end + + def run(argv = ARGV) + if argv.empty? + raise CommandError, + 'You must specify at least one queue to start a worker for' + end + + option_parser.parse!(argv) + + all_queues = SidekiqConfig::CliMethods.all_queues(@rails_path) + queue_names = SidekiqConfig::CliMethods.worker_queues(@rails_path) + + queue_groups = argv.map do |queues| + next queue_names if queues == '*' + + # When using the experimental queue query syntax, we treat + # each queue group as a worker attribute query, and resolve + # the queues for the queue group using this query. + if @experimental_queue_selector + SidekiqConfig::CliMethods.query_workers(queues, all_queues) + else + SidekiqConfig::CliMethods.expand_queues(queues.split(','), queue_names) + end + end + + if @negate_queues + queue_groups.map! { |queues| queue_names - queues } + end + + if queue_groups.all?(&:empty?) + raise CommandError, + 'No queues found, you must select at least one queue' + end + + @logger.info("Starting cluster with #{queue_groups.length} processes") + + @processes = SidekiqCluster.start( + queue_groups, + env: @environment, + directory: @rails_path, + max_concurrency: @max_concurrency, + min_concurrency: @min_concurrency, + dryrun: @dryrun + ) + + return if @dryrun + + write_pid + trap_signals + start_loop + end + + def write_pid + SidekiqCluster.write_pid(@pid) if @pid + end + + def monotonic_time + Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) + end + + def continue_waiting?(deadline) + SidekiqCluster.any_alive?(@processes) && monotonic_time < deadline + end + + def hard_stop_stuck_pids + SidekiqCluster.signal_processes(SidekiqCluster.pids_alive(@processes), :KILL) + end + + def wait_for_termination + deadline = monotonic_time + TERMINATE_TIMEOUT_SECONDS + sleep(CHECK_TERMINATE_INTERVAL_SECONDS) while continue_waiting?(deadline) + + hard_stop_stuck_pids + end + + def trap_signals + SidekiqCluster.trap_terminate do |signal| + @alive = false + SidekiqCluster.signal_processes(@processes, signal) + wait_for_termination + end + + SidekiqCluster.trap_forward do |signal| + SidekiqCluster.signal_processes(@processes, signal) + end + end + + def start_loop + while @alive + sleep(@interval) + + unless SidekiqCluster.all_alive?(@processes) + # If a child process died we'll just terminate the whole cluster. It's up to + # runit and such to then restart the cluster. + @logger.info('A worker terminated, shutting down the cluster') + + SidekiqCluster.signal_processes(@processes, :TERM) + break + end + end + end + + def option_parser + OptionParser.new do |opt| + opt.banner = "#{File.basename(__FILE__)} [QUEUE,QUEUE] [QUEUE] ... [OPTIONS]" + + opt.separator "\nOptions:\n" + + opt.on('-h', '--help', 'Shows this help message') do + abort opt.to_s + end + + opt.on('-m', '--max-concurrency INT', 'Maximum threads to use with Sidekiq (default: 50, 0 to disable)') do |int| + @max_concurrency = int.to_i + end + + opt.on('--min-concurrency INT', 'Minimum threads to use with Sidekiq (default: 0)') do |int| + @min_concurrency = int.to_i + end + + opt.on('-e', '--environment ENV', 'The application environment') do |env| + @environment = env + end + + opt.on('-P', '--pidfile PATH', 'Path to the PID file') do |pid| + @pid = pid + end + + opt.on('-r', '--require PATH', 'Location of the Rails application') do |path| + @rails_path = path + end + + opt.on('--experimental-queue-selector', 'EXPERIMENTAL: Run workers based on the provided selector') do |experimental_queue_selector| + @experimental_queue_selector = experimental_queue_selector + end + + opt.on('-n', '--negate', 'Run workers for all queues in sidekiq_queues.yml except the given ones') do + @negate_queues = true + end + + opt.on('-i', '--interval INT', 'The number of seconds to wait between worker checks') do |int| + @interval = int.to_i + end + + opt.on('-d', '--dryrun', 'Print commands that would be run without this flag, and quit') do |int| + @dryrun = true + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb index 8f19b557d24..c49432f0fc6 100644 --- a/lib/gitlab/sidekiq_config/cli_methods.rb +++ b/lib/gitlab/sidekiq_config/cli_methods.rb @@ -21,14 +21,14 @@ module Gitlab QUERY_OR_OPERATOR = '|' QUERY_AND_OPERATOR = '&' QUERY_CONCATENATE_OPERATOR = ',' - QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze + QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze QUERY_PREDICATES = { feature_category: :to_sym, has_external_dependencies: lambda { |value| value == 'true' }, - latency_sensitive: lambda { |value| value == 'true' }, name: :to_s, - resource_boundary: :to_sym + resource_boundary: :to_sym, + urgency: :to_sym }.freeze QueryError = Class.new(StandardError) diff --git a/lib/gitlab/sidekiq_config/dummy_worker.rb b/lib/gitlab/sidekiq_config/dummy_worker.rb index 858ff0db0c9..bd205c81931 100644 --- a/lib/gitlab/sidekiq_config/dummy_worker.rb +++ b/lib/gitlab/sidekiq_config/dummy_worker.rb @@ -9,8 +9,9 @@ module Gitlab ATTRIBUTE_METHODS = { feature_category: :get_feature_category, has_external_dependencies: :worker_has_external_dependencies?, - latency_sensitive: :latency_sensitive_worker?, + urgency: :get_urgency, resource_boundary: :get_worker_resource_boundary, + idempotent: :idempotent?, weight: :get_weight }.freeze diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb index 6cbe327e6b2..ec7a82f6459 100644 --- a/lib/gitlab/sidekiq_config/worker.rb +++ b/lib/gitlab/sidekiq_config/worker.rb @@ -7,8 +7,8 @@ module Gitlab attr_reader :klass delegate :feature_category_not_owned?, :get_feature_category, - :get_weight, :get_worker_resource_boundary, - :latency_sensitive_worker?, :queue, :queue_namespace, + :get_urgency, :get_weight, :get_worker_resource_boundary, + :idempotent?, :queue, :queue_namespace, :worker_has_external_dependencies?, to: :klass @@ -49,9 +49,10 @@ module Gitlab name: queue, feature_category: get_feature_category, has_external_dependencies: worker_has_external_dependencies?, - latency_sensitive: latency_sensitive_worker?, + urgency: get_urgency, resource_boundary: get_worker_resource_boundary, - weight: get_weight + weight: get_weight, + idempotent: idempotent? } end diff --git a/lib/gitlab/sidekiq_logging/client_logger.rb b/lib/gitlab/sidekiq_logging/client_logger.rb new file mode 100644 index 00000000000..8be755a55db --- /dev/null +++ b/lib/gitlab/sidekiq_logging/client_logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqLogging + class ClientLogger < Gitlab::Logger + def self.file_name_noext + 'sidekiq_client' + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/deduplication_logger.rb b/lib/gitlab/sidekiq_logging/deduplication_logger.rb new file mode 100644 index 00000000000..01810e474dc --- /dev/null +++ b/lib/gitlab/sidekiq_logging/deduplication_logger.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqLogging + class DeduplicationLogger + include Singleton + include LogsJobs + + def log(job, deduplication_type) + payload = parse_job(job) + payload['job_status'] = 'deduplicated' + payload['message'] = "#{base_message(payload)}: deduplicated: #{deduplication_type}" + payload['deduplication_type'] = deduplication_type + + Sidekiq.logger.info payload + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb index e0b0d684bea..c20e929ae36 100644 --- a/lib/gitlab/sidekiq_logging/json_formatter.rb +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# This is needed for sidekiq-cluster +require 'json' + module Gitlab module SidekiqLogging class JSONFormatter diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb new file mode 100644 index 00000000000..55d711c54ae --- /dev/null +++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqLogging + module LogsJobs + def base_message(payload) + "#{payload['class']} JID-#{payload['jid']}" + end + + def parse_job(job) + # Error information from the previous try is in the payload for + # displaying in the Sidekiq UI, but is very confusing in logs! + job = job.except('error_backtrace', 'error_class', 'error_message') + + # Add process id params + job['pid'] = ::Process.pid + + job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] + job['args'] = Gitlab::Utils::LogLimitedArray.log_limited_array(job['args'].map(&:to_s)) if job['args'] + + job + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index b45014d283f..af9072ea201 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -6,6 +6,8 @@ require 'active_record/log_subscriber' module Gitlab module SidekiqLogging class StructuredLogger + include LogsJobs + def call(job, queue) started_time = get_time base_payload = parse_job(job) @@ -24,10 +26,6 @@ module Gitlab private - def base_message(payload) - "#{payload['class']} JID-#{payload['jid']}" - end - def add_instrumentation_keys!(job, output_payload) output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS)) end @@ -76,20 +74,6 @@ module Gitlab payload['completed_at'] = Time.now.utc.to_f end - def parse_job(job) - # Error information from the previous try is in the payload for - # displaying in the Sidekiq UI, but is very confusing in logs! - job = job.except('error_backtrace', 'error_class', 'error_message') - - # Add process id params - job['pid'] = ::Process.pid - - job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] - job['args'] = Gitlab::Utils::LogLimitedArray.log_limited_array(job['args']) if job['args'] - - job - end - def elapsed(t0) t1 = get_time { diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 6c27213df49..37165d787c7 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -9,17 +9,18 @@ module Gitlab # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)` def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true, request_store: true) lambda do |chain| - chain.add Gitlab::SidekiqMiddleware::Monitor - chain.add Gitlab::SidekiqMiddleware::ServerMetrics if metrics - chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger - chain.add Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer - chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware if request_store - chain.add Gitlab::SidekiqMiddleware::BatchLoader - chain.add Labkit::Middleware::Sidekiq::Server - chain.add Gitlab::SidekiqMiddleware::InstrumentationLogger - chain.add Gitlab::SidekiqMiddleware::AdminMode::Server - chain.add Gitlab::SidekiqStatus::ServerMiddleware - chain.add Gitlab::SidekiqMiddleware::WorkerContext::Server + chain.add ::Gitlab::SidekiqMiddleware::Monitor + chain.add ::Gitlab::SidekiqMiddleware::ServerMetrics if metrics + chain.add ::Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger + chain.add ::Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer + chain.add ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware if request_store + chain.add ::Gitlab::SidekiqMiddleware::BatchLoader + chain.add ::Labkit::Middleware::Sidekiq::Server + chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger + chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Server + chain.add ::Gitlab::SidekiqStatus::ServerMiddleware + chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server + chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server end end @@ -28,11 +29,12 @@ module Gitlab # eg: `config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)` def self.client_configurator lambda do |chain| - chain.add Gitlab::SidekiqStatus::ClientMiddleware - chain.add Gitlab::SidekiqMiddleware::ClientMetrics - chain.add Gitlab::SidekiqMiddleware::WorkerContext::Client # needs to be before the Labkit middleware - chain.add Labkit::Middleware::Sidekiq::Client - chain.add Gitlab::SidekiqMiddleware::AdminMode::Client + chain.add ::Gitlab::SidekiqStatus::ClientMiddleware + chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics + chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Client # needs to be before the Labkit middleware + chain.add ::Labkit::Middleware::Sidekiq::Client + chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client + chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client end end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs.rb new file mode 100644 index 00000000000..23222430902 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'digest' + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + def self.drop_duplicates? + Feature.enabled?(:drop_duplicate_sidekiq_jobs) + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb new file mode 100644 index 00000000000..bb0c18735bb --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + class Client + def call(worker_class, job, queue, _redis_pool, &block) + DuplicateJob.new(job, queue).schedule(&block) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb new file mode 100644 index 00000000000..c6fb50b4610 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +require 'digest' + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + # This class defines an identifier of a job in a queue + # The identifier based on a job's class and arguments. + # + # As strategy decides when to keep track of the job in redis and when to + # remove it. + # + # Storing the deduplication key in redis can be done by calling `check!` + # check returns the `jid` of the job if it was scheduled, or the `jid` of + # the duplicate job if it was already scheduled + # + # When new jobs can be scheduled again, the strategy calls `#delete`. + class DuplicateJob + DUPLICATE_KEY_TTL = 6.hours + + attr_reader :existing_jid + + def initialize(job, queue_name, strategy: :until_executing) + @job = job + @queue_name = queue_name + @strategy = strategy + end + + # This will continue the middleware chain if the job should be scheduled + # It will return false if the job needs to be cancelled + def schedule(&block) + Strategies.for(strategy).new(self).schedule(job, &block) + end + + # This will continue the server middleware chain if the job should be + # executed. + # It will return false if the job should not be executed. + def perform(&block) + Strategies.for(strategy).new(self).perform(job, &block) + end + + # This method will return the jid that was set in redis + def check! + read_jid = nil + + Sidekiq.redis do |redis| + redis.multi do |multi| + redis.set(idempotency_key, jid, ex: DUPLICATE_KEY_TTL, nx: true) + read_jid = redis.get(idempotency_key) + end + end + + self.existing_jid = read_jid.value + end + + def delete! + Sidekiq.redis do |redis| + redis.del(idempotency_key) + end + end + + def duplicate? + raise "Call `#check!` first to check for existing duplicates" unless existing_jid + + jid != existing_jid + end + + def droppable? + idempotent? && duplicate? && DuplicateJobs.drop_duplicates? + end + + private + + attr_reader :queue_name, :strategy, :job + attr_writer :existing_jid + + def worker_class_name + job['class'] + end + + def arguments + job['args'] + end + + def jid + job['jid'] + end + + def idempotency_key + @idempotency_key ||= "#{namespace}:#{idempotency_hash}" + end + + def idempotency_hash + Digest::SHA256.hexdigest(idempotency_string) + end + + def namespace + "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:duplicate:#{queue_name}" + end + + def idempotency_string + "#{worker_class_name}:#{arguments.join('-')}" + end + + def idempotent? + worker_class = worker_class_name.to_s.safe_constantize + return false unless worker_class + return false unless worker_class.respond_to?(:idempotent?) + + worker_class.idempotent? + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb new file mode 100644 index 00000000000..a35edc5774e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + class Server + def call(worker, job, queue, &block) + DuplicateJob.new(job, queue).perform(&block) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb new file mode 100644 index 00000000000..a08310a58ff --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + module Strategies + UnknownStrategyError = Class.new(StandardError) + + STRATEGIES = { + until_executing: UntilExecuting + }.freeze + + def self.for(name) + STRATEGIES.fetch(name) + rescue KeyError + raise UnknownStrategyError, "Unknown deduplication strategy #{name}" + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb new file mode 100644 index 00000000000..674e436b714 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + module Strategies + # This strategy takes a lock before scheduling the job in a queue and + # removes the lock before the job starts allowing a new job to be queued + # while a job is still executing. + class UntilExecuting + def initialize(duplicate_job) + @duplicate_job = duplicate_job + end + + def schedule(job) + if duplicate_job.check! && duplicate_job.duplicate? + job['duplicate-of'] = duplicate_job.existing_jid + end + + if duplicate_job.droppable? + Gitlab::SidekiqLogging::DeduplicationLogger.instance.log(job, "dropped until executing") + return false + end + + yield + end + + def perform(_job) + duplicate_job.delete! + + yield + end + + private + + attr_reader :duplicate_job + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb index fbc34357323..693e35f2500 100644 --- a/lib/gitlab/sidekiq_middleware/metrics.rb +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -9,10 +9,10 @@ module Gitlab private def create_labels(worker_class, queue) - labels = { queue: queue.to_s, latency_sensitive: FALSE_LABEL, external_dependencies: FALSE_LABEL, feature_category: "", boundary: "" } + labels = { queue: queue.to_s, urgency: "", external_dependencies: FALSE_LABEL, feature_category: "", boundary: "" } return labels unless worker_class && worker_class.include?(WorkerAttributes) - labels[:latency_sensitive] = bool_as_label(worker_class.latency_sensitive_worker?) + labels[:urgency] = worker_class.get_urgency.to_s labels[:external_dependencies] = bool_as_label(worker_class.worker_has_external_dependencies?) feature_category = worker_class.get_feature_category diff --git a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb index 8824f81e8e3..f6142bd6ca5 100644 --- a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb +++ b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb @@ -3,12 +3,12 @@ module Gitlab module SidekiqMiddleware class RequestStoreMiddleware + include Gitlab::WithRequestStore + def call(worker, job, queue) - RequestStore.begin! - yield - ensure - RequestStore.end! - RequestStore.clear! + with_request_store do + yield + end end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index fa7f56b8d9c..60618787b24 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -45,6 +45,8 @@ module Gitlab labels[:job_status] = job_succeeded ? "done" : "fail" @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) + @metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000) + @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(job)) end end @@ -54,6 +56,8 @@ module Gitlab { sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), @@ -65,6 +69,10 @@ module Gitlab def get_thread_cputime defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 end + + def get_gitaly_time(job) + job.fetch(:gitaly_duration, 0) / 1000.0 + end end end end diff --git a/lib/gitlab/sidekiq_queue.rb b/lib/gitlab/sidekiq_queue.rb new file mode 100644 index 00000000000..807c27a71ff --- /dev/null +++ b/lib/gitlab/sidekiq_queue.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + class SidekiqQueue + include Gitlab::Utils::StrongMemoize + + NoMetadataError = Class.new(StandardError) + InvalidQueueError = Class.new(StandardError) + + attr_reader :queue_name + + def initialize(queue_name) + @queue_name = queue_name + end + + def drop_jobs!(search_metadata, timeout:) + start_time = Gitlab::Metrics::System.monotonic_time + completed = true + deleted_jobs = 0 + + job_search_metadata = + search_metadata + .stringify_keys + .slice(*Labkit::Context::KNOWN_KEYS) + .transform_keys { |key| "meta.#{key}" } + .compact + + raise NoMetadataError if job_search_metadata.empty? + raise InvalidQueueError unless queue + + queue.each do |job| + if timeout_exceeded?(start_time, timeout) + completed = false + break + end + + next unless job_matches?(job, job_search_metadata) + + job.delete + deleted_jobs += 1 + end + + { + completed: completed, + deleted_jobs: deleted_jobs, + queue_size: queue.size + } + end + + private + + def queue + strong_memoize(:queue) do + # Sidekiq::Queue.new always returns a queue, even if it doesn't + # exist. + Sidekiq::Queue.all.find { |queue| queue.name == queue_name } + end + end + + def job_matches?(job, job_search_metadata) + job_search_metadata.all? { |key, value| job[key] == value } + end + + def timeout_exceeded?(start_time, timeout) + (Gitlab::Metrics::System.monotonic_time - start_time) > timeout + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index 54d74ed3998..08de9df14f8 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -63,7 +63,7 @@ module Gitlab # Convert Markdown to slacks format def format(string) - Slack::Notifier::LinkFormatter.format(string) + Slack::Messenger::Util::LinkFormatter.format(string) end def resource_url diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 2dd4b7a4092..3669d652fd3 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -5,9 +5,11 @@ module Gitlab module Template module Finders class GlobalTemplateFinder < BaseTemplateFinder - def initialize(base_dir, extension, categories = {}) + def initialize(base_dir, extension, categories = {}, exclusions: []) @categories = categories @extension = extension + @exclusions = exclusions + super(base_dir) end @@ -16,6 +18,8 @@ module Gitlab end def find(key) + return if excluded?(key) + file_name = "#{key}#{@extension}" # The key is untrusted input, so ensure we can't be directed outside @@ -28,11 +32,20 @@ module Gitlab def list_files_for(dir) dir = "#{dir}/" unless dir.end_with?('/') - Dir.glob(File.join(dir, "*#{@extension}")).select { |f| f =~ self.class.filter_regex(@extension) } + + Dir.glob(File.join(dir, "*#{@extension}")).select do |f| + next if excluded?(f) + + f =~ self.class.filter_regex(@extension) + end end private + def excluded?(file_name) + @exclusions.include?(file_name) + end + def select_directory(file_name) @categories.keys.find do |category| File.exist?(File.join(category_directory(category), file_name)) diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index ee91f1200cd..26a9dc9fd38 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -17,16 +17,25 @@ module Gitlab { 'General' => '', 'Pages' => 'Pages', + 'Verify' => 'Verify', 'Auto deploy' => 'autodeploy' } end + def disabled_templates + %w[ + Verify/Browser-Performance + ] + end + def base_dir Rails.root.join('lib/gitlab/ci/templates') end def finder(project = nil) - Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) + Gitlab::Template::Finders::GlobalTemplateFinder.new( + self.base_dir, self.extension, self.categories, exclusions: self.disabled_templates + ) end end end diff --git a/lib/gitlab/testing/clear_thread_memory_cache_middleware.rb b/lib/gitlab/testing/clear_thread_memory_cache_middleware.rb new file mode 100644 index 00000000000..6f54038ae22 --- /dev/null +++ b/lib/gitlab/testing/clear_thread_memory_cache_middleware.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Testing + class ClearThreadMemoryCacheMiddleware + def initialize(app) + @app = app + end + + def call(env) + Gitlab::ThreadMemoryCache.cache_backend.clear + + @app.call(env) + end + end + end +end diff --git a/lib/gitlab/tracing.rb b/lib/gitlab/tracing.rb deleted file mode 100644 index 7732d7c9d9c..00000000000 --- a/lib/gitlab/tracing.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Tracing - # Only enable tracing when the `GITLAB_TRACING` env var is configured. Note that we avoid using ApplicationSettings since - # the same environment variable needs to be configured for Workhorse, Gitaly and any other components which - # emit tracing. Since other components may start before Rails, and may not have access to ApplicationSettings, - # an env var makes more sense. - def self.enabled? - connection_string.present? - end - - def self.connection_string - ENV['GITLAB_TRACING'] - end - - def self.tracing_url_template - ENV['GITLAB_TRACING_URL'] - end - - def self.tracing_url_enabled? - enabled? && tracing_url_template.present? - end - - # This will provide a link into the distributed tracing for the current trace, - # if it has been captured. - def self.tracing_url - return unless tracing_url_enabled? - - # Avoid using `format` since it can throw TypeErrors - # which we want to avoid on unsanitised env var input - tracing_url_template.to_s - .gsub(/\{\{\s*correlation_id\s*\}\}/, Labkit::Correlation::CorrelationId.current_id.to_s) - .gsub(/\{\{\s*service\s*\}\}/, Gitlab.process_name) - end - end -end diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb index 4ff064007f1..96ee6f0e8e6 100644 --- a/lib/gitlab/uploads/migration_helper.rb +++ b/lib/gitlab/uploads/migration_helper.rb @@ -21,6 +21,10 @@ module Gitlab prepare_variables(args, logger) end + def self.categories + CATEGORIES + end + def migrate_to_remote_storage @to_store = ObjectStorage::Store::REMOTE @@ -70,3 +74,5 @@ module Gitlab end end end + +Gitlab::Uploads::MigrationHelper.prepend_if_ee('EE::Gitlab::Uploads::MigrationHelper') diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 0adca34440c..88094839062 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -49,7 +49,7 @@ module Gitlab return [uri, nil] unless address_info ip_address = ip_address(address_info) - return [uri, nil] if domain_whitelisted?(uri) || ip_whitelisted?(ip_address) + return [uri, nil] if domain_whitelisted?(uri) || ip_whitelisted?(ip_address, port: get_port(uri)) protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, dns_rebind_protection) @@ -254,11 +254,11 @@ module Gitlab end def domain_whitelisted?(uri) - Gitlab::UrlBlockers::UrlWhitelist.domain_whitelisted?(uri.normalized_host) + Gitlab::UrlBlockers::UrlWhitelist.domain_whitelisted?(uri.normalized_host, port: get_port(uri)) end - def ip_whitelisted?(ip_address) - Gitlab::UrlBlockers::UrlWhitelist.ip_whitelisted?(ip_address) + def ip_whitelisted?(ip_address, port: nil) + Gitlab::UrlBlockers::UrlWhitelist.ip_whitelisted?(ip_address, port: port) end def config diff --git a/lib/gitlab/url_blockers/domain_whitelist_entry.rb b/lib/gitlab/url_blockers/domain_whitelist_entry.rb new file mode 100644 index 00000000000..b94e8ee3f69 --- /dev/null +++ b/lib/gitlab/url_blockers/domain_whitelist_entry.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module UrlBlockers + class DomainWhitelistEntry + attr_reader :domain, :port + + def initialize(domain, port: nil) + @domain = domain + @port = port + end + + def match?(requested_domain, requested_port = nil) + return false unless domain == requested_domain + return true if port.nil? + + port == requested_port + end + end + end +end diff --git a/lib/gitlab/url_blockers/ip_whitelist_entry.rb b/lib/gitlab/url_blockers/ip_whitelist_entry.rb new file mode 100644 index 00000000000..88c76574d3d --- /dev/null +++ b/lib/gitlab/url_blockers/ip_whitelist_entry.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module UrlBlockers + class IpWhitelistEntry + attr_reader :ip, :port + + # Argument ip should be an IPAddr object + def initialize(ip, port: nil) + @ip = ip + @port = port + end + + def match?(requested_ip, requested_port = nil) + return false unless ip.include?(requested_ip) + return true if port.nil? + + port == requested_port + end + end + end +end diff --git a/lib/gitlab/url_blockers/url_whitelist.rb b/lib/gitlab/url_blockers/url_whitelist.rb index 7622de4fdbe..59f74dde7fc 100644 --- a/lib/gitlab/url_blockers/url_whitelist.rb +++ b/lib/gitlab/url_blockers/url_whitelist.rb @@ -4,21 +4,25 @@ module Gitlab module UrlBlockers class UrlWhitelist class << self - def ip_whitelisted?(ip_string) + def ip_whitelisted?(ip_string, port: nil) return false if ip_string.blank? ip_whitelist, _ = outbound_local_requests_whitelist_arrays ip_obj = Gitlab::Utils.string_to_ip_object(ip_string) - ip_whitelist.any? { |ip| ip.include?(ip_obj) } + ip_whitelist.any? do |ip_whitelist_entry| + ip_whitelist_entry.match?(ip_obj, port) + end end - def domain_whitelisted?(domain_string) + def domain_whitelisted?(domain_string, port: nil) return false if domain_string.blank? _, domain_whitelist = outbound_local_requests_whitelist_arrays - domain_whitelist.include?(domain_string) + domain_whitelist.any? do |domain_whitelist_entry| + domain_whitelist_entry.match?(domain_string, port) + end end private diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 4bedf7a301e..cc53e3b7577 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -13,7 +13,8 @@ module Gitlab end def url - case object + # Objects are sometimes wrapped in a BatchLoader instance + case object.itself when Commit commit_url when Issue @@ -33,7 +34,7 @@ module Gitlab when User user_url(object) else - raise NotImplementedError.new("No URL builder defined for #{object.class}") + raise NotImplementedError.new("No URL builder defined for #{object.inspect}") end end diff --git a/lib/gitlab/usage_counters/common.rb b/lib/gitlab/usage_counters/common.rb new file mode 100644 index 00000000000..a5bdac430f4 --- /dev/null +++ b/lib/gitlab/usage_counters/common.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module UsageCounters + class Common + class << self + def increment(project_id) + Gitlab::Redis::SharedState.with { |redis| redis.hincrby(base_key, project_id, 1) } + end + + def usage_totals + Gitlab::Redis::SharedState.with do |redis| + total_sum = 0 + + totals = redis.hgetall(base_key).each_with_object({}) do |(project_id, count), result| + total_sum += result[project_id.to_i] = count.to_i + end + + totals[:total] = total_sum + totals + end + end + + def base_key + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/usage_counters/pod_logs.rb b/lib/gitlab/usage_counters/pod_logs.rb new file mode 100644 index 00000000000..94e29d2fad7 --- /dev/null +++ b/lib/gitlab/usage_counters/pod_logs.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module UsageCounters + class PodLogs < Common + def self.base_key + 'POD_LOGS_USAGE_COUNTS' + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6e29a3e4cc4..b9cd4d74914 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -2,7 +2,6 @@ module Gitlab class UsageData - APPROXIMATE_COUNT_MODELS = [Label, MergeRequest, Note, Todo].freeze BATCH_SIZE = 100 class << self @@ -67,8 +66,8 @@ module Gitlab clusters_disabled: count(::Clusters::Cluster.disabled), project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type), group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type), - clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled, batch: false), - clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled, batch: false), + clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled), + clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), clusters_applications_helm: count(::Clusters::Applications::Helm.available), clusters_applications_ingress: count(::Clusters::Applications::Ingress.available), @@ -85,7 +84,7 @@ module Gitlab issues: count(Issue), issues_created_from_gitlab_error_tracking_ui: count(SentryIssue), issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue), - issues_using_zoom_quick_actions: count(ZoomMeeting.select(:issue_id).distinct, batch: false), + issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), issues_with_embedded_grafana_charts_approx: ::Gitlab::GrafanaEmbedUsageData.issue_count, incident_issues: count(::Issue.authored(::User.alert_bot)), keys: count(Key), @@ -107,10 +106,12 @@ module Gitlab suggestions: count(Suggestion), todos: count(Todo), uploads: count(Upload), - web_hooks: count(WebHook) + web_hooks: count(WebHook), + labels: count(Label), + merge_requests: count(MergeRequest), + notes: count(Note) }.merge( services_usage, - approximate_counts, usage_counters, user_preferences_usage, ingress_modsecurity_usage @@ -122,6 +123,8 @@ module Gitlab def cycle_analytics_usage_data Gitlab::CycleAnalytics::UsageData.new.to_json + rescue ActiveRecord::StatementInvalid + { avg_cycle_analytics: {} } end def features_usage_data @@ -181,10 +184,8 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def services_usage - service_counts = count(Service.active.where(template: false).where.not(type: 'JiraService').group(:type), fallback: Hash.new(-1), batch: false) - - results = Service.available_services_names.each_with_object({}) do |service_name, response| - response["projects_#{service_name}_active".to_sym] = service_counts["#{service_name}_service".camelize] || 0 + results = Service.available_services_names.without('jira').each_with_object({}) do |service_name, response| + response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, type: "#{service_name}_service".camelize)) end # Keep old Slack keys for backward compatibility, https://gitlab.com/gitlab-data/analytics/issues/3241 @@ -232,7 +233,7 @@ module Gitlab end def count(relation, column = nil, fallback: -1, batch: true) - if batch && Feature.enabled?(:usage_ping_batch_counter) + if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true) Gitlab::Database::BatchCount.batch_count(relation, column) else relation.count @@ -242,7 +243,7 @@ module Gitlab end def distinct_count(relation, column = nil, fallback: -1, batch: true) - if batch && Feature.enabled?(:usage_ping_batch_counter) + if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true) Gitlab::Database::BatchCount.batch_distinct_count(relation, column) else relation.distinct_count_by(column) @@ -251,16 +252,6 @@ module Gitlab fallback end - def approximate_counts - approx_counts = Gitlab::Database::Count.approximate_counts(APPROXIMATE_COUNT_MODELS) - - APPROXIMATE_COUNT_MODELS.each_with_object({}) do |model, result| - key = model.name.underscore.pluralize.to_sym - - result[key] = approx_counts[model] || -1 - end - end - def installation_type if Rails.env.production? Gitlab::INSTALLATION_TYPE diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index a00e72f7aad..5e0a4faeba8 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -33,7 +33,7 @@ module Gitlab return false unless can_access_git? if user.requires_ldap_check? && user.try_obtain_ldap_lease - return false unless Gitlab::Auth::LDAP::Access.allowed?(user) + return false unless Gitlab::Auth::Ldap::Access.allowed?(user) end true @@ -104,7 +104,7 @@ module Gitlab @permission_cache ||= {} end - def can_access_git? + request_cache def can_access_git? user && user.can?(:access_git) end diff --git a/lib/gitlab/user_access_snippet.rb b/lib/gitlab/user_access_snippet.rb new file mode 100644 index 00000000000..bfed86c4df4 --- /dev/null +++ b/lib/gitlab/user_access_snippet.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + class UserAccessSnippet < UserAccess + extend ::Gitlab::Cache::RequestCache + # TODO: apply override check https://gitlab.com/gitlab-org/gitlab/issues/205677 + + request_cache_key do + [user&.id, snippet&.id] + end + + attr_reader :snippet + + def initialize(user, snippet: nil) + @user = user + @snippet = snippet + @project = snippet&.project + end + + def can_do_action?(action) + return false unless can_access_git? + + permission_cache[action] = + permission_cache.fetch(action) do + Ability.allowed?(user, action, snippet) + end + end + + def can_create_tag?(ref) + false + end + + def can_delete_branch?(ref) + false + end + + def can_push_to_branch?(ref) + super + return false unless snippet + return false unless can_do_action?(:update_snippet) + + true + end + + def can_merge_to_branch?(ref) + false + end + end +end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 3c567fad68d..ad6b213bb50 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -5,10 +5,20 @@ module Gitlab extend self # Ensure that the relative path will not traverse outside the base directory - def check_path_traversal!(path) - raise StandardError.new("Invalid path") if path.start_with?("..#{File::SEPARATOR}") || + # We url decode the path to avoid passing invalid paths forward in url encoded format. + # We are ok to pass some double encoded paths to File.open since they won't resolve. + # Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580 + # It also checks for ALT_SEPARATOR aka '\' (forward slash) + def check_path_traversal!(path, allowed_absolute: false) + path = CGI.unescape(path) + + if path.start_with?("..#{File::SEPARATOR}", "..#{File::ALT_SEPARATOR}") || path.include?("#{File::SEPARATOR}..#{File::SEPARATOR}") || - path.end_with?("#{File::SEPARATOR}..") + path.end_with?("#{File::SEPARATOR}..") || + (!allowed_absolute && Pathname.new(path).absolute?) + + raise StandardError.new("Invalid path") + end path end diff --git a/lib/gitlab/utils/json_size_estimator.rb b/lib/gitlab/utils/json_size_estimator.rb new file mode 100644 index 00000000000..9f8ea3e61f9 --- /dev/null +++ b/lib/gitlab/utils/json_size_estimator.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + # This class estimates the JSON blob byte size of a ruby object using as + # little allocations as possible. + # The estimation should be quite accurate when using simple objects. + # + # Example: + # + # Gitlab::Utils::JsonSizeEstimator.estimate(["a", { b: 12, c: nil }]) + class JsonSizeEstimator + ARRAY_BRACKETS_SIZE = 2 # [] + OBJECT_BRACKETS_SIZE = 2 # {} + DOUBLEQUOTE_SIZE = 2 # "" + COLON_SIZE = 1 # : character size from {"a": 1} + MINUS_SIGN_SIZE = 1 # - character size from -1 + NULL_SIZE = 4 # null + + class << self + # Returns: integer (number of bytes) + def estimate(object) + case object + when Hash + estimate_hash(object) + when Array + estimate_array(object) + when String + estimate_string(object) + when Integer + estimate_integer(object) + when Float + estimate_float(object) + when DateTime, Time + estimate_time(object) + when NilClass + NULL_SIZE + else + # might be incorrect, but #to_s is safe, #to_json might be disabled for some objects: User + estimate_string(object.to_s) + end + end + + private + + def estimate_hash(hash) + size = 0 + item_count = 0 + + hash.each do |key, value| + item_count += 1 + + size += estimate(key.to_s) + COLON_SIZE + estimate(value) + end + + size + OBJECT_BRACKETS_SIZE + comma_count(item_count) + end + + def estimate_array(array) + size = 0 + item_count = 0 + + array.each do |item| + item_count += 1 + + size += estimate(item) + end + + size + ARRAY_BRACKETS_SIZE + comma_count(item_count) + end + + def estimate_string(string) + string.bytesize + DOUBLEQUOTE_SIZE + end + + def estimate_float(float) + float.to_s.bytesize + end + + def estimate_integer(integer) + if integer > 0 + integer_string_size(integer) + elsif integer < 0 + integer_string_size(integer.abs) + MINUS_SIGN_SIZE + else # 0 + 1 + end + end + + def estimate_time(time) + time.to_json.size + end + + def integer_string_size(integer) + Math.log10(integer).floor + 1 + end + + def comma_count(item_count) + item_count == 0 ? 0 : item_count - 1 + end + end + end + end +end diff --git a/lib/gitlab/utils/log_limited_array.rb b/lib/gitlab/utils/log_limited_array.rb index fe8aadf9020..e0589c3df4c 100644 --- a/lib/gitlab/utils/log_limited_array.rb +++ b/lib/gitlab/utils/log_limited_array.rb @@ -6,19 +6,19 @@ module Gitlab MAXIMUM_ARRAY_LENGTH = 10.kilobytes # Prepare an array for logging by limiting its JSON representation - # to around 10 kilobytes. Once we hit the limit, add "..." as the - # last item in the returned array. - def self.log_limited_array(array) + # to around 10 kilobytes. Once we hit the limit, add the sentinel + # value as the last item in the returned array. + def self.log_limited_array(array, sentinel: '...') return [] unless array.is_a?(Array) total_length = 0 limited_array = array.take_while do |arg| - total_length += arg.to_json.length + total_length += JsonSizeEstimator.estimate(arg) total_length <= MAXIMUM_ARRAY_LENGTH end - limited_array.push('...') if total_length > MAXIMUM_ARRAY_LENGTH + limited_array.push(sentinel) if total_length > MAXIMUM_ARRAY_LENGTH limited_array end diff --git a/lib/gitlab/utils/measuring.rb b/lib/gitlab/utils/measuring.rb new file mode 100644 index 00000000000..c9e6cb9c039 --- /dev/null +++ b/lib/gitlab/utils/measuring.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require 'prometheus/pid_provider' + +module Gitlab + module Utils + class Measuring + def initialize(logger: Logger.new($stdout)) + @logger = logger + end + + def with_measuring + logger.info "Measuring enabled..." + with_gc_counter do + with_count_queries do + with_measure_time do + yield + end + end + end + + logger.info "Memory usage: #{Gitlab::Metrics::System.memory_usage.to_f / 1024 / 1024} MiB" + logger.info "Label: #{::Prometheus::PidProvider.worker_id}" + end + + private + + attr_reader :logger + + def with_count_queries(&block) + count = 0 + + counter_f = ->(_name, _started, _finished, _unique_id, payload) { + count += 1 unless payload[:name].in? %w[CACHE SCHEMA] + } + + ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block) + + logger.info "Number of sql calls: #{count}" + end + + def with_gc_counter + gc_counts_before = GC.stat.select { |k, _v| k =~ /count/ } + yield + gc_counts_after = GC.stat.select { |k, _v| k =~ /count/ } + stats = gc_counts_before.merge(gc_counts_after) { |_k, vb, va| va - vb } + + logger.info "Total GC count: #{stats[:count]}" + logger.info "Minor GC count: #{stats[:minor_gc_count]}" + logger.info "Major GC count: #{stats[:major_gc_count]}" + end + + def with_measure_time + timing = Benchmark.realtime do + yield + end + + logger.info "Time to finish: #{duration_in_numbers(timing)}" + end + + def duration_in_numbers(duration_in_seconds) + milliseconds = duration_in_seconds.in_milliseconds % 1.second.in_milliseconds + seconds = duration_in_seconds % 1.minute + minutes = (duration_in_seconds / 1.minute) % (1.hour / 1.minute) + hours = duration_in_seconds / 1.hour + + if hours == 0 + "%02d:%02d:%03d" % [minutes, seconds, milliseconds] + else + "%02d:%02d:%02d:%03d" % [hours, minutes, seconds, milliseconds] + end + end + end + end +end diff --git a/lib/gitlab/with_request_store.rb b/lib/gitlab/with_request_store.rb new file mode 100644 index 00000000000..d6c05e1e256 --- /dev/null +++ b/lib/gitlab/with_request_store.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module WithRequestStore + def with_request_store + RequestStore.begin! + yield + ensure + RequestStore.end! + RequestStore.clear! + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 8696e23cbc7..7da20b49d9d 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -24,7 +24,7 @@ module Gitlab attrs = { GL_ID: Gitlab::GlId.gl_id(user), - GL_REPOSITORY: repo_type.identifier_for_container(repository.project), + GL_REPOSITORY: repo_type.identifier_for_container(repository.container), GL_USERNAME: user&.username, ShowAllRefs: show_all_refs, Repository: repository.gitaly_repository.to_h, diff --git a/lib/gitlab/x509/commit.rb b/lib/gitlab/x509/commit.rb index b1d15047981..4b35c0ef7d2 100644 --- a/lib/gitlab/x509/commit.rb +++ b/lib/gitlab/x509/commit.rb @@ -184,11 +184,13 @@ module Gitlab commit_sha: @commit.sha, project: @commit.project, x509_certificate_id: certificate.id, - verification_status: verification_status + verification_status: verification_status(certificate) } end - def verification_status + def verification_status(certificate) + return :unverified if certificate.revoked? + if verified_signature && certificate_email == @commit.committer_email :verified else diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb index e776e2b7ea3..ee0951f18ca 100644 --- a/lib/gitlab_danger.rb +++ b/lib/gitlab_danger.rb @@ -3,14 +3,15 @@ class GitlabDanger LOCAL_RULES ||= %w[ changes_size - gemfile documentation frozen_string duplicate_yarn_dependencies prettier eslint + karma database commit_messages + telemetry ].freeze CI_ONLY_RULES ||= %w[ diff --git a/lib/grafana/time_window.rb b/lib/grafana/time_window.rb new file mode 100644 index 00000000000..111e3ab7de2 --- /dev/null +++ b/lib/grafana/time_window.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +module Grafana + # Allows for easy formatting and manipulations of timestamps + # coming from a Grafana url + class TimeWindow + include ::Gitlab::Utils::StrongMemoize + + def initialize(from, to) + @from = from + @to = to + end + + def formatted + { + start: window[:from].formatted, + end: window[:to].formatted + } + end + + def in_milliseconds + window.transform_values(&:to_ms) + end + + private + + def window + strong_memoize(:window) do + specified_window + rescue Timestamp::Error + default_window + end + end + + def specified_window + RangeWithDefaults.new( + from: Timestamp.from_ms_since_epoch(@from), + to: Timestamp.from_ms_since_epoch(@to) + ).to_hash + end + + def default_window + RangeWithDefaults.new.to_hash + end + end + + # For incomplete time ranges, adds default parameters to + # achieve a complete range. If both full range is provided, + # range will be returned. + class RangeWithDefaults + DEFAULT_RANGE = 8.hours + + # @param from [Grafana::Timestamp, nil] Start of the expected range + # @param to [Grafana::Timestamp, nil] End of the expected range + def initialize(from: nil, to: nil) + @from = from + @to = to + + apply_defaults! + end + + def to_hash + { from: @from, to: @to }.compact + end + + private + + def apply_defaults! + @to ||= @from ? relative_end : Timestamp.new(Time.now) + @from ||= relative_start + end + + def relative_start + Timestamp.new(DEFAULT_RANGE.before(@to.time)) + end + + def relative_end + Timestamp.new(DEFAULT_RANGE.since(@from.time)) + end + end + + # Offers a consistent API for timestamps originating from + # Grafana or other sources, allowing for formatting of timestamps + # as consumed by Grafana-related utilities + class Timestamp + Error = Class.new(StandardError) + + attr_accessor :time + + # @param timestamp [Time] + def initialize(time) + @time = time + end + + # Formats a timestamp from Grafana for compatibility with + # parsing in JS via `new Date(timestamp)` + def formatted + time.utc.strftime('%FT%TZ') + end + + # Converts to milliseconds since epoch + def to_ms + time.to_i * 1000 + end + + class << self + # @param time [String] Representing milliseconds since epoch. + # This is what JS "decided" unix is. + def from_ms_since_epoch(time) + return if time.nil? + + raise Error.new('Expected milliseconds since epoch') unless ms_since_epoch?(time) + + new(cast_ms_to_time(time)) + end + + private + + def cast_ms_to_time(time) + Time.at(time.to_i / 1000.0) + end + + def ms_since_epoch?(time) + ms = time.to_i + + ms.to_s == time && ms.bit_length < 64 + end + end + end +end diff --git a/lib/grafana/validator.rb b/lib/grafana/validator.rb new file mode 100644 index 00000000000..760263f7ec9 --- /dev/null +++ b/lib/grafana/validator.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +# Performs checks on whether resources from Grafana can be handled +# We have certain restrictions on which formats we accept. +# Some are technical requirements, others are simplifications. +module Grafana + class Validator + Error = Class.new(StandardError) + + attr_reader :grafana_dashboard, :datasource, :panel, :query_params + + UNSUPPORTED_GRAFANA_GLOBAL_VARS = %w( + $__interval_ms + $__timeFilter + $__name + $timeFilter + $interval + ).freeze + + def initialize(grafana_dashboard, datasource, panel, query_params) + @grafana_dashboard = grafana_dashboard + @datasource = datasource + @panel = panel + @query_params = query_params + end + + def validate! + validate_query_params! + validate_panel_type! + validate_variable_definitions! + validate_global_variables! + validate_datasource! if datasource + end + + def valid? + validate! + + true + rescue ::Grafana::Validator::Error + false + end + + private + + # See defaults in Banzai::Filter::InlineGrafanaMetricsFilter. + def validate_query_params! + return if [:from, :to].all? { |param| query_params.include?(param) } + + raise_error 'Grafana query parameters must include from and to.' + end + + # We may choose to support other panel types in future. + def validate_panel_type! + return if panel && panel[:type] == 'graph' && panel[:lines] + + raise_error 'Panel type must be a line graph.' + end + + # We must require variable definitions to create valid prometheus queries. + def validate_variable_definitions! + return unless grafana_dashboard[:dashboard][:templating] + + return if grafana_dashboard[:dashboard][:templating][:list].all? do |variable| + query_params[:"var-#{variable[:name]}"].present? + end + + raise_error 'All Grafana variables must be defined in the query parameters.' + end + + # We may choose to support further Grafana variables in future. + def validate_global_variables! + return unless panel_contains_unsupported_vars? + + raise_error "Prometheus must not include #{UNSUPPORTED_GRAFANA_GLOBAL_VARS}" + end + + # We may choose to support additional datasources in future. + def validate_datasource! + return if datasource[:access] == 'proxy' && datasource[:type] == 'prometheus' + + raise_error 'Only Prometheus datasources with proxy access in Grafana are supported.' + end + + def panel_contains_unsupported_vars? + panel[:targets].any? do |target| + UNSUPPORTED_GRAFANA_GLOBAL_VARS.any? do |variable| + target[:expr].include?(variable) + end + end + end + + def raise_error(message) + raise Validator::Error, message + end + end +end diff --git a/lib/omni_auth/strategies/saml.rb b/lib/omni_auth/strategies/saml.rb deleted file mode 100644 index ebe062f17e0..00000000000 --- a/lib/omni_auth/strategies/saml.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module OmniAuth - module Strategies - class SAML - extend ::Gitlab::Utils::Override - - # NOTE: This method duplicates code from omniauth-saml - # so that we can access authn_request to store it - # See: https://github.com/omniauth/omniauth-saml/issues/172 - override :request_phase - def request_phase - authn_request = OneLogin::RubySaml::Authrequest.new - - store_authn_request_id(authn_request) - - with_settings do |settings| - redirect(authn_request.create(settings, additional_params_for_authn_request)) - end - end - - private - - def store_authn_request_id(authn_request) - Gitlab::Auth::Saml::OriginValidator.new(session).store_origin(authn_request) - end - end - end -end diff --git a/lib/quality/kubernetes_client.rb b/lib/quality/kubernetes_client.rb index 453b9d21adb..f83652e117f 100644 --- a/lib/quality/kubernetes_client.rb +++ b/lib/quality/kubernetes_client.rb @@ -48,7 +48,8 @@ module Quality resource_names = raw_resource_names command = [ 'delete', - %(--namespace "#{namespace}") + %(--namespace "#{namespace}"), + '--ignore-not-found' ] Array(release_name).each do |release| diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb index 85e89059dbb..bbd8b4dcc3f 100644 --- a/lib/quality/test_level.rb +++ b/lib/quality/test_level.rb @@ -7,7 +7,10 @@ module Quality TEST_LEVEL_FOLDERS = { migration: %w[ migrations + ], + background_migration: %w[ lib/gitlab/background_migration + lib/ee/gitlab/background_migration ], unit: %w[ bin @@ -69,7 +72,7 @@ module Quality case file_path # Detect migration first since some background migration tests are under # spec/lib/gitlab/background_migration and tests under spec/lib are unit by default - when regexp(:migration) + when regexp(:migration), regexp(:background_migration) :migration when regexp(:unit) :unit @@ -82,6 +85,10 @@ module Quality end end + def background_migration?(file_path) + !!(file_path =~ regexp(:background_migration)) + end + private def folders_pattern(level) diff --git a/lib/sentry/client/issue.rb b/lib/sentry/client/issue.rb index 1c5d88e8862..986311ab62a 100644 --- a/lib/sentry/client/issue.rb +++ b/lib/sentry/client/issue.rb @@ -75,7 +75,21 @@ module Sentry http_get(api_urls.issue_url(issue_id))[:body] end - def parse_gitlab_issue(plugin_issues) + def parse_gitlab_issue(issue) + parse_issue_annotations(issue) || parse_plugin_issue(issue) + end + + def parse_issue_annotations(issue) + issue + .fetch('annotations', []) + .reject(&:blank?) + .map { |annotation| Nokogiri.make(annotation) } + .find { |html| html['href']&.starts_with?(Gitlab.config.gitlab.url) } + .try(:[], 'href') + end + + def parse_plugin_issue(issue) + plugin_issues = issue.fetch('pluginIssues', nil) return unless plugin_issues gitlab_plugin = plugin_issues.detect { |item| item['id'] == 'gitlab' } @@ -145,7 +159,7 @@ module Sentry short_id: issue.fetch('shortId', nil), status: issue.fetch('status', nil), frequency: issue.dig('stats', '24h'), - gitlab_issue: parse_gitlab_issue(issue.fetch('pluginIssues', nil)), + gitlab_issue: parse_gitlab_issue(issue), project_id: issue.dig('project', 'id'), project_name: issue.dig('project', 'name'), project_slug: issue.dig('project', 'slug'), diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 1c51288adf6..982c1dc8866 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -26,7 +26,7 @@ ### Environment variables RAILS_ENV="production" -EXPERIMENTAL_PUMA="" +USE_UNICORN="" # Script variable names should be lower-case not to conflict with # internal /bin/sh variables such as PATH, EDITOR or SHELL. @@ -68,10 +68,10 @@ if ! cd "$app_root" ; then fi # Select the web server to use -if [ -z "$EXPERIMENTAL_PUMA" ]; then - use_web_server="unicorn" -else +if [ -z "$USE_UNICORN" ]; then use_web_server="puma" +else + use_web_server="unicorn" fi diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index ab41dba3017..bb271b16836 100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -5,8 +5,8 @@ # Normal values are "production", "test" and "development". RAILS_ENV="production" -# Uncomment the line below to enable Puma web server instead of Unicorn. -# EXPERIMENTAL_PUMA=1 +# Uncomment the line below to enable the Unicorn web server instead of Puma. +# USE_UNICORN=1 # app_user defines the user that GitLab is run as. # The default is "git". diff --git a/lib/system_check/gitlab_shell_check.rb b/lib/system_check/gitlab_shell_check.rb index 31c4ec33247..f539719ce87 100644 --- a/lib/system_check/gitlab_shell_check.rb +++ b/lib/system_check/gitlab_shell_check.rb @@ -50,7 +50,7 @@ module SystemCheck end def gitlab_shell_version - Gitlab::Shell.new.version + Gitlab::Shell.version end end end diff --git a/lib/system_check/ldap_check.rb b/lib/system_check/ldap_check.rb index 938026424ed..3d71edbc256 100644 --- a/lib/system_check/ldap_check.rb +++ b/lib/system_check/ldap_check.rb @@ -6,7 +6,7 @@ module SystemCheck set_name 'LDAP:' def multi_check - if Gitlab::Auth::LDAP::Config.enabled? + if Gitlab::Auth::Ldap::Config.enabled? # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. limit = ENV['LDAP_CHECK_LIMIT'] @@ -21,13 +21,13 @@ module SystemCheck private def check_ldap(limit) - servers = Gitlab::Auth::LDAP::Config.providers + servers = Gitlab::Auth::Ldap::Config.providers servers.each do |server| $stdout.puts "Server: #{server}" begin - Gitlab::Auth::LDAP::Adapter.open(server) do |adapter| + Gitlab::Auth::Ldap::Adapter.open(server) do |adapter| check_ldap_auth(adapter) $stdout.puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" diff --git a/lib/tasks/cleanup.rake b/lib/tasks/cleanup.rake new file mode 100644 index 00000000000..8574f26dbdc --- /dev/null +++ b/lib/tasks/cleanup.rake @@ -0,0 +1,33 @@ +namespace :gitlab do + namespace :cleanup do + desc "GitLab | Cleanup | Delete moved repositories" + task moved: :gitlab_environment do + warn_user_is_not_gitlab + remove_flag = ENV['REMOVE'] + + Gitlab.config.repositories.storages.each do |name, repository_storage| + repo_root = repository_storage.legacy_disk_path.chomp('/') + # Look for global repos (legacy, depth 1) and normal repos (depth 2) + IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *+moved*.git)) do |find| + find.each_line do |path| + path.chomp! + + if remove_flag + if FileUtils.rm_rf(path) + puts "Removed...#{path}".color(:green) + else + puts "Cannot remove #{path}".color(:red) + end + else + puts "Can be removed: #{path}".color(:green) + end + end + end + end + + unless remove_flag + puts "To cleanup these repositories run this command with REMOVE=true".color(:yellow) + end + end + end +end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 8f34101ea15..e5e2faaa7df 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -17,9 +17,16 @@ namespace :gitlab do Rake::Task['gitlab:backup:registry:create'].invoke backup = Backup::Manager.new(progress) - backup.pack - backup.cleanup - backup.remove_old + backup.write_info + + if ENV['SKIP'] && ENV['SKIP'].include?('tar') + backup.upload + else + backup.pack + backup.upload + backup.cleanup + backup.remove_old + end progress.puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ "and are not included in this backup. You will need these files to restore a backup.\n" \ @@ -33,7 +40,8 @@ namespace :gitlab do warn_user_is_not_gitlab backup = Backup::Manager.new(progress) - backup.unpack + cleanup_required = backup.unpack + backup.verify_backup_version unless backup.skipped?('db') begin @@ -72,7 +80,10 @@ namespace :gitlab do Rake::Task['gitlab:shell:setup'].invoke Rake::Task['cache:clear'].invoke - backup.cleanup + if cleanup_required + backup.cleanup + end + puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ "and are not included in this backup. You will need to restore these files manually.".color(:red) puts "Restore task is done." diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 63f5d7f2740..c26aa848d5a 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -13,7 +13,7 @@ namespace :gitlab do print "#{user.name} (#{user.ldap_identity.extern_uid}) ..." - if Gitlab::Auth::LDAP::Access.allowed?(user) + if Gitlab::Auth::Ldap::Access.allowed?(user) puts " [OK]".color(:green) else if block_flag diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index c73691f3d45..5a583183924 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -8,13 +8,25 @@ namespace :gitlab do OUTPUT_DIR = Rails.root.join("doc/api/graphql/reference") TEMPLATES_DIR = 'lib/gitlab/graphql/docs/templates/' + # Make all feature flags enabled so that all feature flag + # controlled fields are considered visible and are output. + # Also avoids pipeline failures in case developer + # dumps schema with flags disabled locally before pushing + task enable_feature_flags: :environment do + class Feature + def self.enabled?(*args) + true + end + end + end + # Defines tasks for dumping the GraphQL schema: # - gitlab:graphql:schema:dump # - gitlab:graphql:schema:idl # - gitlab:graphql:schema:json GraphQL::RakeTask.new( schema_name: 'GitlabSchema', - dependencies: [:environment], + dependencies: [:environment, :enable_feature_flags], directory: OUTPUT_DIR, idl_outfile: "gitlab_schema.graphql", json_outfile: "gitlab_schema.json" @@ -22,7 +34,7 @@ namespace :gitlab do namespace :graphql do desc 'GitLab | GraphQL | Generate GraphQL docs' - task compile_docs: :environment do + task compile_docs: [:environment, :enable_feature_flags] do renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options) renderer.write @@ -31,7 +43,7 @@ namespace :gitlab do end desc 'GitLab | GraphQL | Check if GraphQL docs are up to date' - task check_docs: :environment do + task check_docs: [:environment, :enable_feature_flags] do renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options) doc = File.read(Rails.root.join(OUTPUT_DIR, 'index.md')) @@ -45,7 +57,7 @@ namespace :gitlab do end desc 'GitLab | GraphQL | Check if GraphQL schemas are up to date' - task check_schema: :environment do + task check_schema: [:environment, :enable_feature_flags] do idl_doc = File.read(Rails.root.join(OUTPUT_DIR, 'gitlab_schema.graphql')) json_doc = File.read(Rails.root.join(OUTPUT_DIR, 'gitlab_schema.json')) diff --git a/lib/tasks/gitlab/import_export/export.rake b/lib/tasks/gitlab/import_export/export.rake new file mode 100644 index 00000000000..c9c212fbe4d --- /dev/null +++ b/lib/tasks/gitlab/import_export/export.rake @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +# Export project to archive +# +# @example +# bundle exec rake "gitlab:import_export:export[root, root, project_to_export, /path/to/file.tar.gz, true]" +# +namespace :gitlab do + namespace :import_export do + desc 'GitLab | Import/Export | EXPERIMENTAL | Export large project archives' + task :export, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args| + # Load it here to avoid polluting Rake tasks with Sidekiq test warnings + require 'sidekiq/testing' + + logger = Logger.new($stdout) + + begin + warn_user_is_not_gitlab + + if ENV['EXPORT_DEBUG'].present? + ActiveRecord::Base.logger = logger + logger.level = Logger::DEBUG + else + logger.level = Logger::INFO + end + + task = Gitlab::ImportExport::Project::ExportTask.new( + namespace_path: args.namespace_path, + project_path: args.project_path, + username: args.username, + file_path: args.archive_path, + measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled), + logger: logger + ) + + success = task.export + + exit(success) + rescue StandardError => e + logger.error "Exception: #{e.message}" + logger.debug e.backtrace + exit 1 + end + end + end +end diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake index c832cba0287..7e2162a7774 100644 --- a/lib/tasks/gitlab/import_export/import.rake +++ b/lib/tasks/gitlab/import_export/import.rake @@ -16,195 +16,35 @@ namespace :gitlab do # Load it here to avoid polluting Rake tasks with Sidekiq test warnings require 'sidekiq/testing' - warn_user_is_not_gitlab + logger = Logger.new($stdout) - if ENV['IMPORT_DEBUG'].present? - ActiveRecord::Base.logger = Logger.new(STDOUT) - end - - GitlabProjectImport.new( - namespace_path: args.namespace_path, - project_path: args.project_path, - username: args.username, - file_path: args.archive_path, - measurement_enabled: args.measurement_enabled == 'true' - ).import - end - end -end - -class GitlabProjectImport - def initialize(opts) - @project_path = opts.fetch(:project_path) - @file_path = opts.fetch(:file_path) - @namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path)) - @current_user = User.find_by_username(opts.fetch(:username)) - @measurement_enabled = opts.fetch(:measurement_enabled) - end - - def import - show_import_start_message - - run_isolated_sidekiq_job - - show_import_failures_count - - if @project&.import_state&.last_error - puts "ERROR: #{@project.import_state.last_error}" - exit 1 - elsif @project.errors.any? - puts "ERROR: #{@project.errors.full_messages.join(', ')}" - exit 1 - else - puts 'Done!' - end - rescue StandardError => e - puts "Exception: #{e.message}" - puts e.backtrace - exit 1 - end - - private - - def with_request_store - RequestStore.begin! - yield - ensure - RequestStore.end! - RequestStore.clear! - end - - def with_count_queries(&block) - count = 0 - - counter_f = ->(name, started, finished, unique_id, payload) { - unless payload[:name].in? %w[CACHE SCHEMA] - count += 1 - end - } - - ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block) - - puts "Number of sql calls: #{count}" - end - - def with_gc_counter - gc_counts_before = GC.stat.select { |k, v| k =~ /count/ } - yield - gc_counts_after = GC.stat.select { |k, v| k =~ /count/ } - stats = gc_counts_before.merge(gc_counts_after) { |k, vb, va| va - vb } - puts "Total GC count: #{stats[:count]}" - puts "Minor GC count: #{stats[:minor_gc_count]}" - puts "Major GC count: #{stats[:major_gc_count]}" - end - - def with_measure_time - timing = Benchmark.realtime do - yield - end - - time = Time.at(timing).utc.strftime("%H:%M:%S") - puts "Time to finish: #{time}" - end - - def with_measuring - puts "Measuring enabled..." - with_gc_counter do - with_count_queries do - with_measure_time do - yield - end - end - end - end - - def measurement_enabled? - @measurement_enabled != false - end + begin + warn_user_is_not_gitlab - # We want to ensure that all Sidekiq jobs are executed - # synchronously as part of that process. - # This ensures that all expensive operations do not escape - # to general Sidekiq clusters/nodes. - def with_isolated_sidekiq_job - Sidekiq::Testing.fake! do - with_request_store do - # If you are attempting to import a large project into a development environment, - # you may see Gitaly throw an error about too many calls or invocations. - # This is due to a n+1 calls limit being set for development setups (not enforced in production) - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635 - # For development setups, this code-path will be excluded from n+1 detection. - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - measurement_enabled? ? with_measuring { yield } : yield + if ENV['IMPORT_DEBUG'].present? + ActiveRecord::Base.logger = logger + logger.level = Logger::DEBUG + else + logger.level = Logger::INFO end - end - - true - end - end - - def run_isolated_sidekiq_job - with_isolated_sidekiq_job do - @project = create_project - - execute_sidekiq_job - end - end - - def create_project - # We are disabling ObjectStorage for `import` - # as it is too slow to handle big archives: - # 1. DB transaction timeouts on upload - # 2. Download of archive before unpacking - disable_upload_object_storage do - service = Projects::GitlabProjectsImportService.new( - @current_user, - { - namespace_id: @namespace.id, - path: @project_path, - file: File.open(@file_path) - } - ) - - service.execute - end - end - - def execute_sidekiq_job - Sidekiq::Worker.drain_all - end - def disable_upload_object_storage - overwrite_uploads_setting('background_upload', false) do - overwrite_uploads_setting('direct_upload', false) do - yield + task = Gitlab::ImportExport::Project::ImportTask.new( + namespace_path: args.namespace_path, + project_path: args.project_path, + username: args.username, + file_path: args.archive_path, + measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled), + logger: logger + ) + + success = task.import + + exit(success) + rescue StandardError => e + logger.error "Exception: #{e.message}" + logger.debug e.backtrace + exit 1 end end end - - def overwrite_uploads_setting(key, value) - old_value = Settings.uploads.object_store[key] - Settings.uploads.object_store[key] = value - - yield - - ensure - Settings.uploads.object_store[key] = old_value - end - - def full_path - "#{@namespace.full_path}/#{@project_path}" - end - - def show_import_start_message - puts "Importing GitLab export: #{@file_path} into GitLab" \ - " #{full_path}" \ - " as #{@current_user.name}" - end - - def show_import_failures_count - return unless @project.import_failures.exists? - - puts "Total number of not imported relations: #{@project.import_failures.count}" - end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 5809f632c5a..d85c8fc7949 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -82,15 +82,10 @@ namespace :gitlab do puts "Using Omniauth:\t#{Gitlab::Auth.omniauth_enabled? ? "yes".color(:green) : "no"}" puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab::Auth.omniauth_enabled? - # check Gitolite version - gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.path}/VERSION" - if File.readable?(gitlab_shell_version_file) - gitlab_shell_version = File.read(gitlab_shell_version_file) - end - + # check Gitlab Shell version puts "" puts "GitLab Shell".color(:yellow) - puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" + puts "Version:\t#{Gitlab::Shell.version || "unknown".color(:red)}" puts "Repository storage paths:" Gitlab::GitalyClient::StorageSettings.allow_disk_access do Gitlab.config.repositories.storages.each do |name, repository_storage| diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index ba3e19caf3b..6586699f8ba 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -89,10 +89,12 @@ namespace :gitlab do puts "" end - Gitlab::Shell.new.remove_all_keys + authorized_keys = Gitlab::AuthorizedKeys.new + + authorized_keys.clear Key.find_in_batches(batch_size: 1000) do |keys| - unless Gitlab::Shell.new.batch_add_keys(keys) + unless authorized_keys.batch_add_keys(keys) puts "Failed to add keys...".color(:red) exit 1 end @@ -103,7 +105,7 @@ namespace :gitlab do end def ensure_write_to_authorized_keys_is_enabled - return if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled + return if Gitlab::CurrentSettings.authorized_keys_enabled? puts authorized_keys_is_disabled_warning @@ -113,7 +115,7 @@ namespace :gitlab do end puts 'Enabling the "Write to authorized_keys file" setting...' - Gitlab::CurrentSettings.current_application_settings.update!(authorized_keys_enabled: true) + Gitlab::CurrentSettings.update!(authorized_keys_enabled: true) puts 'Successfully enabled "Write to authorized_keys file"!' puts '' diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 44536a447c7..879b07da1df 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -3,7 +3,7 @@ namespace :gitlab do namespace :migrate do desc "GitLab | Uploads | Migrate all uploaded files to object storage" task all: :environment do - Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| + Gitlab::Uploads::MigrationHelper.categories.each do |args| Rake::Task["gitlab:uploads:migrate"].invoke(*args) Rake::Task["gitlab:uploads:migrate"].reenable end @@ -20,7 +20,7 @@ namespace :gitlab do namespace :migrate_to_local do desc "GitLab | Uploads | Migrate all uploaded files to local storage" task all: :environment do - Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| + Gitlab::Uploads::MigrationHelper.categories.each do |args| Rake::Task["gitlab:uploads:migrate_to_local"].invoke(*args) Rake::Task["gitlab:uploads:migrate_to_local"].reenable end diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake index e281ebd5d60..d74878835fd 100644 --- a/lib/tasks/sidekiq.rake +++ b/lib/tasks/sidekiq.rake @@ -33,6 +33,6 @@ namespace :sidekiq do task :launchd do deprecation_warning! - system(*%w(bin/background_jobs start_no_deamonize)) + system(*%w(bin/background_jobs start_silent)) end end |