diff options
Diffstat (limited to 'lib')
203 files changed, 3337 insertions, 1747 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index cecff6d3b81..ee8dc822098 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Gets a list of access requests for a #{source_type}." do detail 'This feature was introduced in GitLab 8.11.' success Entities::AccessRequester diff --git a/lib/api/api.rb b/lib/api/api.rb index 449faf5f8da..8abb24e6f69 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -7,8 +7,8 @@ module API LOG_FILENAME = Rails.root.join("log", "api_json.log") NO_SLASH_URL_PART_REGEX = %r{[^/]+} - PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze - COMMIT_ENDPOINT_REQUIREMENTS = PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze + NAMESPACE_OR_PROJECT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze + COMMIT_ENDPOINT_REQUIREMENTS = NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze insert_before Grape::Middleware::Error, GrapeLogging::Middleware::RequestLogger, @@ -20,7 +20,8 @@ module API Gitlab::GrapeLogging::Loggers::RouteLogger.new, Gitlab::GrapeLogging::Loggers::UserLogger.new, Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new, - Gitlab::GrapeLogging::Loggers::PerfLogger.new + Gitlab::GrapeLogging::Loggers::PerfLogger.new, + Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new ] allow_access_with_scope :api @@ -84,7 +85,6 @@ module API content_type :txt, "text/plain" # Ensure the namespace is right, otherwise we might load Grape::API::Helpers - helpers ::SentryHelper helpers ::API::Helpers helpers ::API::Helpers::CommonHelpers diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 61357b3f1d6..af9b519ed9e 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -94,6 +94,7 @@ module API Gitlab::Auth::TokenNotFoundError, Gitlab::Auth::ExpiredError, Gitlab::Auth::RevokedError, + Gitlab::Auth::ImpersonationDisabled, Gitlab::Auth::InsufficientScopeError] base.__send__(:rescue_from, *error_classes, oauth2_bearer_token_error_handler) # rubocop:disable GitlabSecurity/PublicSend @@ -121,6 +122,11 @@ module API :invalid_token, "Token was revoked. You have to re-authorize from the user.") + when Gitlab::Auth::ImpersonationDisabled + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token is an impersonation token but impersonation was disabled.") + when Gitlab::Auth::InsufficientScopeError # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) # does not include WWW-Authenticate header, which breaks the standard. diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index c2abf9155f3..a1851ba3627 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -14,7 +14,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do AWARDABLES.each do |awardable_params| awardable_string = awardable_params[:type].pluralize awardable_id_string = "#{awardable_params[:type]}_#{awardable_params[:find_by]}" diff --git a/lib/api/badges.rb b/lib/api/badges.rb index ab670988f47..ba554e00a16 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -22,7 +22,7 @@ module API params do requires :id, type: String, desc: "The ID of a #{source_type}" end - resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Gets a list of #{source_type} badges viewable by the authenticated user." do detail 'This feature was introduced in GitLab 10.6.' success Entities::Badge diff --git a/lib/api/boards.rb b/lib/api/boards.rb index c80e1c57864..b7c77730afb 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -16,7 +16,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do segment ':id/boards' do desc 'Get all project boards' do detail 'This feature was introduced in 8.13' diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 2735d410c8e..e7e58ad0e66 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -6,7 +6,7 @@ module API class Branches < Grape::API include PaginationParams - BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX) + BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX) before { authorize! :download_code, user_project } @@ -20,7 +20,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a project repository branches' do success Entities::Branch end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 99553d993ca..62c966e06b4 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do include PaginationParams before { authenticate! } @@ -29,7 +29,7 @@ module API not_found!('Commit') unless user_project.commit(params[:sha]) - pipelines = user_project.pipelines.where(sha: params[:sha]) + pipelines = user_project.ci_pipelines.where(sha: params[:sha]) statuses = ::CommitStatus.where(pipeline: pipelines) statuses = statuses.latest unless to_boolean(params[:all]) statuses = statuses.where(ref: params[:ref]) if params[:ref].present? @@ -75,7 +75,7 @@ module API pipeline = @project.pipeline_for(ref, commit.sha) unless pipeline - pipeline = @project.pipelines.create!( + pipeline = @project.ci_pipelines.create!( source: :external, sha: commit.sha, ref: ref, diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 337b92a6183..9d23daafe95 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -23,7 +23,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a project repository commits' do success Entities::Commit end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index ce35720d408..df6d2721977 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -31,7 +31,7 @@ module API params do requires :id, type: String, desc: 'The ID of the project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authorize_admin_project } desc "Get a specific project's deploy keys" do diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 6747e2e5005..8706a971a1a 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all deployments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Deployment diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 39c6d28391d..91eb6a23701 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -17,7 +17,7 @@ module API params do requires :id, type: String, desc: "The ID of a #{parent_type}" end - resource parent_type.pluralize.to_sym, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Get a list of #{noteable_type.to_s.downcase} discussions" do success Entities::Discussion end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cff05643f3b..5dbfbb85e9e 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -145,7 +145,9 @@ module API expose :import_status # TODO: Use `expose_nil` once we upgrade the grape-entity gem - expose :import_error, if: lambda { |status, _ops| status.import_error } + expose :import_error, if: lambda { |project, _ops| project.import_state&.last_error } do |project| + project.import_state.last_error + end end class BasicProjectDetails < ProjectIdentity @@ -248,7 +250,10 @@ module API expose :creator_id expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status - expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } + + expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } do |project| + project.import_state&.last_error + end expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -710,6 +715,10 @@ module API expose :diff_refs, using: Entities::DiffRefs + # Allow the status of a rebase to be determined + expose :merge_error + expose :rebase_in_progress?, as: :rebase_in_progress, if: -> (_, options) { options[:include_rebase_in_progress] } + expose :diverged_commits_count, as: :diverged_commits_count, if: -> (_, options) { options[:include_diverged_commits_count] } def build_available?(options) diff --git a/lib/api/environments.rb b/lib/api/environments.rb index c64217a6977..633f24d3c9a 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -11,7 +11,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all environments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Environment diff --git a/lib/api/events.rb b/lib/api/events.rb index 6e0b508be19..44dae57770d 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -97,7 +97,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "List a Project's visible events" do success Entities::Event end diff --git a/lib/api/files.rb b/lib/api/files.rb index bcd2cd48a45..ca59d330e1c 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -2,7 +2,9 @@ module API class Files < Grape::API - FILE_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX) + include APIGuard + + FILE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX) # Prevents returning plain/text responses for files with .txt extension after_validation { content_type "application/json" } @@ -79,6 +81,8 @@ module API requires :id, type: String, desc: 'The project ID' end resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do + allow_access_with_scope :read_repository, if: -> (request) { request.get? || request.head? } + desc 'Get raw file metadata from repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index dc30e868e2e..9a20ee8c8b9 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -19,7 +19,7 @@ module API requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do segment ':id/boards' do desc 'Find a group board' do detail 'This feature was introduced in 10.6' diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index b36436dbf43..d4287e4a7c4 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of group milestones' do success Entities::Milestone end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index ae7241e9a30..3f048e0dc56 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -11,7 +11,7 @@ module API requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get group-level variables' do success Entities::Variable end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index b3d10721692..626a2008dee 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -140,7 +140,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Update a group. Available only for users who can administrate groups.' do success Entities::Group end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 9fda73d5b92..2cceb2ec798 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -368,10 +368,10 @@ module API end def handle_api_exception(exception) - if sentry_enabled? && report_exception?(exception) + if report_exception?(exception) define_params_for_grape_middleware - sentry_context - Raven.capture_exception(exception, extra: params) + Gitlab::Sentry.context(current_user) + Gitlab::Sentry.track_acceptable_exception(exception, extra: params) end # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 491b5085bb8..dac700482b4 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -101,7 +101,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of group issues' do success Entities::IssueBasic end @@ -128,7 +128,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do include TimeTrackingEndpoints desc 'Get a list of project issues' do diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 2229cbcd9d4..7c2d8ff11bf 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -14,7 +14,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Download the artifacts archive from a job' do detail 'This feature was introduced in GitLab 8.10' end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 697555c9605..80a5cbd6b19 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do params :optional_scope do optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', @@ -56,7 +56,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get ':id/pipelines/:pipeline_id/jobs' do - pipeline = user_project.pipelines.find(params[:pipeline_id]) + pipeline = user_project.ci_pipelines.find(params[:pipeline_id]) builds = pipeline.builds builds = filter_builds(builds, params[:scope]) builds = builds.preload(:job_artifacts_archive, :job_artifacts, project: [:namespace]) diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 28555454307..2e676b0aa6b 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all labels of the project' do success Entities::Label end diff --git a/lib/api/members.rb b/lib/api/members.rb index a8f67be3463..461ffe71a62 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Gets a list of group or project members viewable by the authenticated user.' do success Entities::Member end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index e4fb890960a..6ad30aa56e0 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of merge request diff versions' do detail 'This feature was introduced in GitLab 8.12.' success Entities::MergeRequestDiff diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 16f07f16387..8c1951cc535 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -74,6 +74,19 @@ module API options end + def authorize_push_to_merge_request!(merge_request) + forbidden!('Source branch does not exist') unless + merge_request.source_branch_exists? + + user_access = Gitlab::UserAccess.new( + current_user, + project: merge_request.source_project + ) + + forbidden!('Cannot push to source branch') unless + user_access.can_push_to_branch?(merge_request.source_branch) + end + params :merge_requests_params do optional :state, type: String, values: %w[opened closed locked merged all], default: 'all', desc: 'Return opened, closed, locked, merged, or all merge requests' @@ -122,7 +135,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of group merge requests' do success Entities::MergeRequestBasic end @@ -141,7 +154,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do include TimeTrackingEndpoints helpers do @@ -239,6 +252,7 @@ module API requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' optional :render_html, type: Boolean, desc: 'Returns the description and title rendered HTML' optional :include_diverged_commits_count, type: Boolean, desc: 'Returns the commits count behind the target branch' + optional :include_rebase_in_progress, type: Boolean, desc: 'Returns whether a rebase operation is ongoing ' end desc 'Get a single merge request' do success Entities::MergeRequest @@ -246,7 +260,13 @@ module API get ':id/merge_requests/:merge_request_iid' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project, render_html: params[:render_html], include_diverged_commits_count: params[:include_diverged_commits_count] + present merge_request, + with: Entities::MergeRequest, + current_user: current_user, + project: user_project, + render_html: params[:render_html], + include_diverged_commits_count: params[:include_diverged_commits_count], + include_rebase_in_progress: params[:include_rebase_in_progress] end desc 'Get the participants of a merge request' do @@ -378,6 +398,19 @@ module API .cancel(merge_request) end + desc 'Rebase the merge request against its target branch' do + detail 'This feature was added in GitLab 11.6' + end + put ':id/merge_requests/:merge_request_iid/rebase' do + merge_request = find_project_merge_request(params[:merge_request_iid]) + + authorize_push_to_merge_request!(merge_request) + + RebaseWorker.perform_async(merge_request.id, current_user.id) + + status :accepted + end + desc 'List issues that will be closed on merge' do success Entities::MRNote end diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 76639fbb031..3cc09f6ac3f 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -6,20 +6,35 @@ module API before { authenticate! } + helpers do + params :optional_list_params_ee do + # EE::API::Namespaces would override this helper + end + + # EE::API::Namespaces would override this method + def custom_namespace_present_options + {} + end + end + resource :namespaces do desc 'Get a namespaces list' do success Entities::Namespace end params do optional :search, type: String, desc: "Search query for namespaces" + use :pagination + use :optional_list_params_ee end get do namespaces = current_user.admin ? Namespace.all : current_user.namespaces namespaces = namespaces.search(params[:search]) if params[:search].present? - present paginate(namespaces), with: Entities::Namespace, current_user: current_user + options = { with: Entities::Namespace, current_user: current_user } + + present paginate(namespaces), options.reverse_merge(custom_namespace_present_options) end desc 'Get a namespace by ID' do @@ -28,7 +43,7 @@ module API params do requires :id, type: String, desc: "Namespace's ID or path" end - get ':id' do + get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do present user_namespace, with: Entities::Namespace, current_user: current_user end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 9f323b87baf..1bdf7aeb119 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -16,7 +16,7 @@ module API params do requires :id, type: String, desc: "The ID of a #{parent_type}" end - resource parent_type.pluralize.to_sym, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do noteables_str = noteable_type.to_s.underscore.pluralize desc "Get a list of #{noteable_type.to_s.downcase} notes" do diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index 4d9a4629268..8cb46bd3ad6 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -58,7 +58,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Get #{source_type} level notification level settings, defaults to Global" do detail 'This feature was introduced in GitLab 8.12' success Entities::NotificationSetting diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index c9ad47e0f0d..78442f465bd 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -4,7 +4,7 @@ module API class PagesDomains < Grape::API include PaginationParams - PAGES_DOMAINS_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(domain: API::NO_SLASH_URL_PART_REGEX) + PAGES_DOMAINS_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(domain: API::NO_SLASH_URL_PART_REGEX) before do authenticate! @@ -54,7 +54,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before do require_pages_enabled! end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index ed0a38b9d70..47b711917e2 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all pipeline schedules' do success Entities::PipelineSchedule end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index cba1e3a6684..7a7b23d2bbb 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all Pipelines of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::PipelineBasic @@ -130,7 +130,7 @@ module API helpers do def pipeline - @pipeline ||= user_project.pipelines.find(params[:pipeline_id]) + @pipeline ||= user_project.ci_pipelines.find(params[:pipeline_id]) end end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 4af4c6ac593..0e7576c9243 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -29,7 +29,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get project hooks' do success Entities::ProjectHook end diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index cbfa0c5bc1c..c64ec2fcc95 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -23,7 +23,7 @@ module API forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project') end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do requires :path, type: String, desc: 'The new project path and name' requires :file, type: File, desc: 'The project export file to be imported' diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index c7137ba5217..da31bcb8dac 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of project milestones' do success Entities::Milestone end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index f3a1b73b153..a607df411a6 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0a914f9012e..f5d21d8923f 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -128,7 +128,7 @@ module API end end - resource :users, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :users, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a user projects' do success Entities::BasicProjectDetails end @@ -224,7 +224,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a single project' do success Entities::ProjectWithAccess end diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index 47752f40e58..5af43448727 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -4,14 +4,14 @@ module API class ProtectedBranches < Grape::API include PaginationParams - BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX) + BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX) before { authorize_admin_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Get a project's protected branches" do success Entities::ProtectedBranch end diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb index ed1c5f0cc05..ee13473c848 100644 --- a/lib/api/protected_tags.rb +++ b/lib/api/protected_tags.rb @@ -4,14 +4,14 @@ module API class ProtectedTags < Grape::API include PaginationParams - TAG_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX) + TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX) before { authorize_admin_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Get a project's protected tags" do detail 'This feature was introduced in GitLab 11.3.' success Entities::ProtectedTag diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index dc844c0bd27..32e05d84491 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -11,7 +11,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb index b6fbe8c0235..0c328f7268e 100644 --- a/lib/api/resource_label_events.rb +++ b/lib/api/resource_label_events.rb @@ -16,7 +16,7 @@ module API params do requires :id, type: String, desc: "The ID of a #{parent_type}" end - resource parent_type.pluralize.to_sym, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Get a list of #{eventable_type.to_s.downcase} resource label events" do success Entities::ResourceLabelEvent detail 'This feature was introduced in 11.3' diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 2f15f3a7d76..c60d25b88cb 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -19,7 +19,6 @@ module API optional :tag_list, type: Array[String], desc: %q(List of Runner's tags) optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' end - # rubocop: disable CodeReuse/ActiveRecord post '/' do attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :maximum_timeout]) .merge(get_runner_details_from_request) @@ -28,10 +27,10 @@ module API if runner_registration_token_valid? # Create shared runner. Requires admin access attributes.merge(runner_type: :instance_type) - elsif project = Project.find_by(runners_token: params[:token]) + elsif project = Project.find_by_runners_token(params[:token]) # Create a specific runner for the project attributes.merge(runner_type: :project_type, projects: [project]) - elsif group = Group.find_by(runners_token: params[:token]) + elsif group = Group.find_by_runners_token(params[:token]) # Create a specific runner for the group attributes.merge(runner_type: :group_type, groups: [group]) else @@ -46,7 +45,6 @@ module API render_validation_error!(runner) end end - # rubocop: enable CodeReuse/ActiveRecord desc 'Deletes a registered Runner' do http_codes [[204, 'Runner was deleted'], [403, 'Forbidden']] diff --git a/lib/api/runners.rb b/lib/api/runners.rb index ce70460af11..f72b33605a7 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -126,7 +126,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authorize_admin_project } desc 'Get runners available for project' do diff --git a/lib/api/search.rb b/lib/api/search.rb index 12d97dcfe7f..f5db692afe5 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -35,12 +35,7 @@ module API end def process_results(results) - case params[:scope] - when 'blobs', 'wiki_blobs' - paginate(results).map { |blob| blob[1] } - else - paginate(results) - end + paginate(results) end def snippets? @@ -70,7 +65,7 @@ module API end end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Search on GitLab' do detail 'This feature was introduced in GitLab 10.5.' end @@ -89,7 +84,7 @@ module API end end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Search on GitLab' do detail 'This feature was introduced in GitLab 10.5.' end diff --git a/lib/api/services.rb b/lib/api/services.rb index 1cb3b8a7277..d60f0f5f08d 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -763,7 +763,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authenticate! } before { authorize_admin_project } @@ -842,7 +842,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Trigger a slash command for #{service_slug}" do detail 'Added in GitLab 8.13' end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 077e9373ac4..74ad3c35a61 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -14,7 +14,7 @@ module API requires :id, type: String, desc: 'The ID of a project' requires :subscribable_id, type: String, desc: 'The ID of a resource' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do subscribable_types.each do |type, finder| type_singularized = type.singularize entity_class = Entities.const_get(type_singularized.camelcase) diff --git a/lib/api/tags.rb b/lib/api/tags.rb index f739eacf9ba..b18eec7d796 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -4,14 +4,14 @@ module API class Tags < Grape::API include PaginationParams - TAG_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX) + TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX) before { authorize! :download_code, user_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a project repository tags' do success Entities::Tag end diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 8dab19d50c2..51f357d9477 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -82,7 +82,7 @@ module API params do requires :name, type: String, desc: 'The name of the template' end - get "templates/#{template_type}/:name" do + get "templates/#{template_type}/:name", requirements: { name: /[\w\.-]+/ } do finder = TemplateFinder.build(template_type, nil, name: declared(params)[:name]) new_template = finder.execute diff --git a/lib/api/todos.rb b/lib/api/todos.rb index ed2cf2cc31b..d2c8cf7c1aa 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -14,7 +14,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do ISSUABLE_TYPES.each do |type, finder| type_id_str = "#{type.singularize}_iid".to_sym diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index f784c857883..3ce1529f259 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Trigger a GitLab project pipeline' do success Entities::Pipeline end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index c844ba321ed..f7cae2251c2 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -11,7 +11,7 @@ module API requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get project variables' do success Entities::Variable end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 24746f4efc6..302b2797a34 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -22,7 +22,7 @@ module API end end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of wiki pages' do success Entities::WikiPageBasic end @@ -103,7 +103,7 @@ module API requires :file, type: ::API::Validations::Types::SafeFile, desc: 'The attachment file to be uploaded' optional :branch, type: String, desc: 'The name of the branch' end - post ":id/wikis/attachments", requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + post ":id/wikis/attachments", requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do authorize! :create_wiki, user_project result = ::Wikis::CreateAttachmentService.new(user_project, diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb index a27f1d46863..c6a3a763c23 100644 --- a/lib/banzai/filter/spaced_link_filter.rb +++ b/lib/banzai/filter/spaced_link_filter.rb @@ -17,6 +17,9 @@ module Banzai # This is a small extension to the CommonMark spec. If they start allowing # spaces in urls, we could then remove this filter. # + # Note: Filter::SanitizationFilter should always be run sometime after this filter + # to prevent XSS attacks + # class SpacedLinkFilter < HTML::Pipeline::Filter include ActionView::Helpers::TagHelper diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index be75e34a673..96bea7ca935 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -12,13 +12,16 @@ module Banzai def self.filters @filters ||= FilterArray[ Filter::PlantumlFilter, + + # Must always be before the SanitizationFilter to prevent XSS attacks + Filter::SpacedLinkFilter, + Filter::SanitizationFilter, Filter::SyntaxHighlightFilter, Filter::MathFilter, Filter::ColorFilter, Filter::MermaidFilter, - Filter::SpacedLinkFilter, Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 655278da711..b2c8d46ede1 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -110,11 +110,6 @@ module ExtractsPath # resolved (e.g., when a user inserts an invalid path or ref). # rubocop:disable Gitlab/ModuleWithInstanceVariables def assign_ref_vars - # assign allowed options - allowed_options = ["filter_ref"] - @options = params.select {|key, value| allowed_options.include?(key) && !value.blank? } - @options = HashWithIndifferentAccess.new(@options) - @id = get_id @ref, @path = extract_ref(@id) @repo = @project.repository diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index cb9f2582936..176766d1a8b 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -13,12 +13,18 @@ module Gitlab @request = request end - def user - find_sessionless_user || find_user_from_warden + def user(request_formats) + request_formats.each do |format| + user = find_sessionless_user(format) + + return user if user + end + + find_user_from_warden end - def find_sessionless_user - find_user_from_access_token || find_user_from_feed_token + def find_sessionless_user(request_format) + find_user_from_web_access_token(request_format) || find_user_from_feed_token(request_format) rescue Gitlab::Auth::AuthenticationError nil end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index c304adc64db..a5efe33bdc6 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -7,6 +7,7 @@ module Gitlab TokenNotFoundError = Class.new(AuthenticationError) ExpiredError = Class.new(AuthenticationError) RevokedError = Class.new(AuthenticationError) + ImpersonationDisabled = Class.new(AuthenticationError) UnauthorizedError = Class.new(AuthenticationError) class InsufficientScopeError < AuthenticationError @@ -27,8 +28,8 @@ module Gitlab current_request.env['warden']&.authenticate if verified_request? end - def find_user_from_feed_token - return unless rss_request? || ics_request? + def find_user_from_feed_token(request_format) + return unless valid_rss_format?(request_format) # NOTE: feed_token was renamed from rss_token but both needs to be supported because # users might have already added the feed to their RSS reader before the rename @@ -38,6 +39,17 @@ module Gitlab User.find_by_feed_token(token) || raise(UnauthorizedError) end + # We only allow Private Access Tokens with `api` scope to be used by web + # requests on RSS feeds or ICS files for backwards compatibility. + # It is also used by GraphQL/API requests. + def find_user_from_web_access_token(request_format) + return unless access_token && valid_web_access_format?(request_format) + + validate_access_token!(scopes: [:api]) + + access_token.user || raise(UnauthorizedError) + end + def find_user_from_access_token return unless access_token @@ -56,6 +68,8 @@ module Gitlab raise ExpiredError when AccessTokenValidationService::REVOKED raise RevokedError + when AccessTokenValidationService::IMPERSONATION_DISABLED + raise ImpersonationDisabled end end @@ -109,6 +123,26 @@ module Gitlab @current_request ||= ensure_action_dispatch_request(request) end + def valid_web_access_format?(request_format) + case request_format + when :rss + rss_request? + when :ics + ics_request? + when :api + api_request? + end + end + + def valid_rss_format?(request_format) + case request_format + when :rss + rss_request? + when :ics + ics_request? + end + end + def rss_request? current_request.path.ends_with?('.atom') || current_request.format.atom? end @@ -116,6 +150,10 @@ module Gitlab def ics_request? current_request.path.ends_with?('.ics') || current_request.format.ics? end + + def api_request? + current_request.path.starts_with?("/api/") + end end end end diff --git a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb new file mode 100644 index 00000000000..29fa0f18448 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb @@ -0,0 +1,209 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This module is used to write the full path of all projects to + # the git repository config file. + # Storing the full project path in the git config allows admins to + # easily identify a project when it is using hashed storage. + module BackfillProjectFullpathInRepoConfig + OrphanedNamespaceError = Class.new(StandardError) + + module Storage + # Class that returns the disk path for a project using hashed storage + class HashedProject + attr_accessor :project + + ROOT_PATH_PREFIX = '@hashed' + + def initialize(project) + @project = project + end + + def disk_path + "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}" + end + + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id + end + end + + # Class that returns the disk path for a project using legacy storage + class LegacyProject + attr_accessor :project + + def initialize(project) + @project = project + end + + def disk_path + project.full_path + end + end + end + + # Concern used by Project and Namespace to determine the full + # route to the project + module Routable + extend ActiveSupport::Concern + + def full_path + @full_path ||= build_full_path + end + + def build_full_path + return path unless has_parent? + + raise OrphanedNamespaceError if parent.nil? + + parent.full_path + '/' + path + end + + def has_parent? + read_attribute(association(:parent).reflection.foreign_key) + end + end + + # Class used to interact with repository using Gitaly + class Repository + attr_reader :storage + + def initialize(storage, relative_path) + @storage = storage + @relative_path = relative_path + end + + def gitaly_repository + Gitaly::Repository.new(storage_name: @storage, relative_path: @relative_path) + end + end + + # Namespace can be a user or group. It can be the root or a + # child of another namespace. + class Namespace < ActiveRecord::Base + self.table_name = 'namespaces' + self.inheritance_column = nil + + include Routable + + belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces' + has_many :projects, inverse_of: :parent + has_many :namespaces, inverse_of: :parent + end + + # Project is where the repository (etc.) is stored + class Project < ActiveRecord::Base + self.table_name = 'projects' + + include Routable + include EachBatch + + FULLPATH_CONFIG_KEY = 'gitlab.fullpath' + + belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects' + delegate :disk_path, to: :storage + + def add_fullpath_config + entries = { FULLPATH_CONFIG_KEY => full_path } + + repository_service.set_config(entries) + end + + def remove_fullpath_config + repository_service.delete_config([FULLPATH_CONFIG_KEY]) + end + + def cleanup_repository + repository_service.cleanup + end + + def storage + @storage ||= + if hashed_storage? + Storage::HashedProject.new(self) + else + Storage::LegacyProject.new(self) + end + end + + def hashed_storage? + self.storage_version && self.storage_version >= 1 + end + + def repository + @repository ||= Repository.new(repository_storage, disk_path + '.git') + end + + def repository_service + @repository_service ||= Gitlab::GitalyClient::RepositoryService.new(repository) + end + end + + # Base class for Up and Down migration classes + class BackfillFullpathMigration + RETRY_DELAY = 15.minutes + MAX_RETRIES = 2 + + # Base class for retrying one project + class BaseRetryOne + def perform(project_id, retry_count) + project = Project.find(project_id) + + return unless project + + migration_class.new.safe_perform_one(project, retry_count) + end + end + + def perform(start_id, end_id) + Project.includes(:parent).where(id: start_id..end_id).each do |project| + safe_perform_one(project) + end + end + + def safe_perform_one(project, retry_count = 0) + perform_one(project) + rescue GRPC::NotFound, GRPC::InvalidArgument, OrphanedNamespaceError + nil + rescue GRPC::BadStatus + schedule_retry(project, retry_count + 1) if retry_count < MAX_RETRIES + end + + def schedule_retry(project, retry_count) + BackgroundMigrationWorker.perform_in(RETRY_DELAY, self.class::RetryOne.name, [project.id, retry_count]) + end + end + + # Class to add the fullpath to the git repo config + class Up < BackfillFullpathMigration + # Class used to retry + class RetryOne < BaseRetryOne + def migration_class + Up + end + end + + def perform_one(project) + project.cleanup_repository + project.add_fullpath_config + end + end + + # Class to rollback adding the fullpath to the git repo config + class Down < BackfillFullpathMigration + # Class used to retry + class RetryOne < BaseRetryOne + def migration_class + Down + end + end + + def perform_one(project) + project.cleanup_repository + project.remove_fullpath_config + end + end + end + end +end diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb index bd5f12276ab..b9ad8267e37 100644 --- a/lib/gitlab/background_migration/encrypt_columns.rb +++ b/lib/gitlab/background_migration/encrypt_columns.rb @@ -5,15 +5,17 @@ module Gitlab # EncryptColumn migrates data from an unencrypted column - `foo`, say - to # an encrypted column - `encrypted_foo`, say. # + # To avoid depending on a particular version of the model in app/, add a + # model to `lib/gitlab/background_migration/models/encrypt_columns` and use + # it in the migration that enqueues the jobs, so code can be shared. + # # For this background migration to work, the table that is migrated _has_ to # have an `id` column as the primary key. Additionally, the encrypted column # should be managed by attr_encrypted, and map to an attribute with the same # name as the unencrypted column (i.e., the unencrypted column should be - # shadowed). + # shadowed), unless you want to define specific methods / accessors in the + # temporary model in `/models/encrypt_columns/your_model.rb`. # - # To avoid depending on a particular version of the model in app/, add a - # model to `lib/gitlab/background_migration/models/encrypt_columns` and use - # it in the migration that enqueues the jobs, so code can be shared. class EncryptColumns def perform(model, attributes, from, to) model = model.constantize if model.is_a?(String) @@ -36,6 +38,10 @@ module Gitlab end end + def clear_migrated_values? + true + end + private # Build a hash of { attribute => encrypted column name } @@ -72,7 +78,10 @@ module Gitlab if instance.changed? instance.save! - instance.update_columns(to_clear) + + if clear_migrated_values? + instance.update_columns(to_clear) + end end end diff --git a/lib/gitlab/background_migration/encrypt_runners_tokens.rb b/lib/gitlab/background_migration/encrypt_runners_tokens.rb new file mode 100644 index 00000000000..91e559a8765 --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_runners_tokens.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # EncryptColumn migrates data from an unencrypted column - `foo`, say - to + # an encrypted column - `encrypted_foo`, say. + # + # We only create a subclass here because we want to isolate this migration + # (migrating unencrypted runner registration tokens to encrypted columns) + # from other `EncryptColumns` migration. This class name is going to be + # serialized and stored in Redis and later picked by Sidekiq, so we need to + # create a separate class name in order to isolate these migration tasks. + # + # We can solve this differently, see tech debt issue: + # + # https://gitlab.com/gitlab-org/gitlab-ce/issues/54328 + # + class EncryptRunnersTokens < EncryptColumns + def perform(model, from, to) + resource = "::Gitlab::BackgroundMigration::Models::EncryptColumns::#{model.to_s.capitalize}" + model = resource.constantize + attributes = model.encrypted_attributes.keys + + super(model, attributes, from, to) + end + + def clear_migrated_values? + false + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb new file mode 100644 index 00000000000..41f18979d76 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `namespaces` table. + # + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + def runners_token=(value) + self.runners_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { runners_token: { attribute: :runners_token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/project.rb b/lib/gitlab/background_migration/models/encrypt_columns/project.rb new file mode 100644 index 00000000000..bfeae14584d --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/project.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `projects` table. + # + class Project < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'projects' + self.inheritance_column = :_type_disabled + + def runners_token=(value) + self.runners_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { runners_token: { attribute: :runners_token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/runner.rb b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb new file mode 100644 index 00000000000..14ddce4b147 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `token` column in `ci_runners` table. + # + class Runner < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'ci_runners' + self.inheritance_column = :_type_disabled + + def token=(value) + self.token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { token: { attribute: :token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/settings.rb b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb new file mode 100644 index 00000000000..08ae35c0671 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `application_settings` table. + # + class Settings < ActiveRecord::Base + include ::EachBatch + include ::CacheableAttributes + + self.table_name = 'application_settings' + self.inheritance_column = :_type_disabled + + after_commit do + ::ApplicationSetting.expire + end + + def runners_registration_token=(value) + self.runners_registration_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { + runners_registration_token: { + attribute: :runners_registration_token_encrypted + } + } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb index bb76eb8ed48..34e72fd9f34 100644 --- a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb +++ b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb @@ -15,12 +15,12 @@ module Gitlab attr_encrypted :token, mode: :per_attribute_iv, algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_truncated + key: ::Settings.attr_encrypted_db_key_base_32 attr_encrypted :url, mode: :per_attribute_iv, algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_truncated + key: ::Settings.attr_encrypted_db_key_base_32 end end end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index a7fcb6b0fca..7f7cc62c8ef 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -14,7 +14,7 @@ module Gitlab @ref = ref @job = job - @pipeline = @project.pipelines.latest_successful_for(@ref) + @pipeline = @project.ci_pipelines.latest_successful_for(@ref) end def entity diff --git a/lib/gitlab/badge/pipeline/status.rb b/lib/gitlab/badge/pipeline/status.rb index 37e61f07e5b..a403d839517 100644 --- a/lib/gitlab/badge/pipeline/status.rb +++ b/lib/gitlab/badge/pipeline/status.rb @@ -22,7 +22,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def status - @project.pipelines + @project.ci_pipelines .where(sha: @sha) .latest_status(@ref) || 'unknown' end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 45e550b3450..eaead41a720 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -35,7 +35,7 @@ module Gitlab def handle_errors return unless errors.any? - project.update_column(:import_error, { + project.import_state.update_column(:last_error, { message: 'The remote data could not be fully imported.', errors: errors }.to_json) diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 15aa4739ee9..d4080536d81 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -56,7 +56,7 @@ module Gitlab def handle_errors return unless errors.any? - project.update_column(:import_error, { + project.import_state.update_column(:last_error, { message: 'The remote data could not be fully imported.', errors: errors }.to_json) diff --git a/lib/gitlab/branch_push_merge_commit_analyzer.rb b/lib/gitlab/branch_push_merge_commit_analyzer.rb new file mode 100644 index 00000000000..a8f601f2451 --- /dev/null +++ b/lib/gitlab/branch_push_merge_commit_analyzer.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +module Gitlab + # Analyse a graph of commits from a push to a branch, + # for each commit, analyze that if it is the head of a merge request, + # then what should its merge_commit be, relative to the branch. + # + # A----->B----->C----->D target branch + # | ^ + # | | + # +-->E----->F--+ merged branch + # | ^ + # | | + # +->G--+ + # + # (See merge-commit-analyze-after branch in gitlab-test) + # + # Assuming + # - A is already in remote + # - B~D are all in its own branch with its own merge request, targeting the target branch + # + # When D is finally pushed to the target branch, + # what are the merge commits for all the other merge requests? + # + # We can walk backwards from the HEAD commit D, + # and find status of its parents. + # First we determine if commit belongs to the target branch (i.e. A, B, C, D), + # and then determine its merge commit. + # + # +--------+-----------------+--------------+ + # | Commit | Direct ancestor | Merge commit | + # +--------+-----------------+--------------+ + # | D | Y | D | + # +--------+-----------------+--------------+ + # | C | Y | C | + # +--------+-----------------+--------------+ + # | F | | C | + # +--------+-----------------+--------------+ + # | B | Y | B | + # +--------+-----------------+--------------+ + # | E | | C | + # +--------+-----------------+--------------+ + # | G | | C | + # +--------+-----------------+--------------+ + # + # By examining the result, it can be said that + # + # - If commit is direct ancestor of HEAD, its merge commit is itself. + # - Otherwise, the merge commit is the same as its child's merge commit. + # + class BranchPushMergeCommitAnalyzer + class CommitDecorator < SimpleDelegator + attr_accessor :merge_commit + attr_writer :direct_ancestor # boolean + + def direct_ancestor? + @direct_ancestor + end + + # @param child_commit [CommitDecorator] + # @param first_parent [Boolean] whether `self` is the first parent of `child_commit` + def set_merge_commit(child_commit:) + @merge_commit ||= direct_ancestor? ? self : child_commit.merge_commit + end + end + + # @param commits [Array] list of commits, must be ordered from the child (tip) of the graph back to the ancestors + def initialize(commits, relevant_commit_ids: nil) + @commits = commits + @id_to_commit = {} + @commits.each do |commit| + @id_to_commit[commit.id] = CommitDecorator.new(commit) + + if relevant_commit_ids + relevant_commit_ids.delete(commit.id) + break if relevant_commit_ids.empty? # Only limit the analyze up to relevant_commit_ids + end + end + + analyze + end + + def get_merge_commit(id) + get_commit(id).merge_commit.id + end + + private + + def analyze + head_commit = get_commit(@commits.first.id) + head_commit.direct_ancestor = true + head_commit.merge_commit = head_commit + + mark_all_direct_ancestors(head_commit) + + # Analyzing a commit requires its child commit be analyzed first, + # which is the case here since commits are ordered from child to parent. + @id_to_commit.each_value do |commit| + analyze_parents(commit) + end + end + + def analyze_parents(commit) + commit.parent_ids.each do |parent_commit_id| + parent_commit = get_commit(parent_commit_id) + + next unless parent_commit # parent commit may not be part of new commits + + parent_commit.set_merge_commit(child_commit: commit) + end + end + + # Mark all direct ancestors. + # If child commit is a direct ancestor, its first parent is also a direct ancestor. + # We assume direct ancestors matches the trail of the target branch over time, + # This assumption is correct most of the time, especially for gitlab managed merges, + # but there are exception cases which can't be solved (https://stackoverflow.com/a/49754723/474597) + def mark_all_direct_ancestors(commit) + loop do + commit = get_commit(commit.parent_ids.first) + + break unless commit + + commit.direct_ancestor = true + end + end + + def get_commit(id) + @id_to_commit[id] + end + end +end diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb new file mode 100644 index 00000000000..f8cda0382fe --- /dev/null +++ b/lib/gitlab/checks/base_checker.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class BaseChecker + include Gitlab::Utils::StrongMemoize + + attr_reader :change_access + delegate(*ChangeAccess::ATTRIBUTES, to: :change_access) + + def initialize(change_access) + @change_access = change_access + end + + def validate! + raise NotImplementedError + end + + private + + def deletion? + Gitlab::Git.blank_ref?(newrev) + end + + def update? + !Gitlab::Git.blank_ref?(oldrev) && !deletion? + end + + def updated_from_web? + protocol == 'web' + end + + def tag_exists? + project.repository.tag_exists?(tag_name) + end + end + end +end diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb new file mode 100644 index 00000000000..d06b2df36f2 --- /dev/null +++ b/lib/gitlab/checks/branch_check.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class BranchCheck < BaseChecker + ERROR_MESSAGES = { + delete_default_branch: 'The default branch of a project cannot be deleted.', + force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', + non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.', + non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', + merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', + push_protected_branch: 'You are not allowed to push code to protected branches on this project.' + }.freeze + + LOG_MESSAGES = { + delete_default_branch_check: "Checking if default branch is being deleted...", + protected_branch_checks: "Checking if you are force pushing to a protected branch...", + protected_branch_push_checks: "Checking if you are allowed to push to the protected branch...", + protected_branch_deletion_checks: "Checking if you are allowed to delete the protected branch..." + }.freeze + + def validate! + return unless branch_name + + logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do + if deletion? && branch_name == project.default_branch + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] + end + end + + protected_branch_checks + end + + private + + def protected_branch_checks + logger.log_timed(LOG_MESSAGES[:protected_branch_checks]) do + return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks + + if forced_push? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] + end + end + + if deletion? + protected_branch_deletion_checks + else + protected_branch_push_checks + end + end + + def protected_branch_deletion_checks + logger.log_timed(LOG_MESSAGES[:protected_branch_deletion_checks]) do + unless user_access.can_delete_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] + end + + unless updated_from_web? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] + end + end + end + + def protected_branch_push_checks + logger.log_timed(LOG_MESSAGES[:protected_branch_push_checks]) do + if matching_merge_request? + unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] + end + else + unless user_access.can_push_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message + end + end + end + end + + def push_to_protected_branch_rejected_message + if project.empty_repo? + empty_project_push_message + else + ERROR_MESSAGES[:push_protected_branch] + end + end + + def empty_project_push_message + <<~MESSAGE + + A default branch (e.g. master) does not yet exist for #{project.full_path} + Ask a project Owner or Maintainer to create a default branch: + + #{project_members_url} + + MESSAGE + end + + def project_members_url + Gitlab::Routing.url_helpers.project_project_members_url(project) + end + + def matching_merge_request? + Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? + end + + def forced_push? + Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev) + end + end + end +end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 074afe9c412..7778d3068cc 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -3,35 +3,11 @@ module Gitlab module Checks class ChangeAccess - ERROR_MESSAGES = { - push_code: 'You are not allowed to push code to this project.', - delete_default_branch: 'The default branch of a project cannot be deleted.', - force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', - non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.', - non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', - merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', - push_protected_branch: 'You are not allowed to push code to protected branches on this project.', - change_existing_tags: 'You are not allowed to change existing tags on this project.', - update_protected_tag: 'Protected tags cannot be updated.', - delete_protected_tag: 'Protected tags cannot be deleted.', - create_protected_tag: 'You are not allowed to create this tag as it is protected.', - lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' - }.freeze + ATTRIBUTES = %i[user_access project skip_authorization + skip_lfs_integrity_check protocol oldrev newrev ref + branch_name tag_name logger commits].freeze - LOG_MESSAGES = { - push_checks: "Checking if you are allowed to push...", - delete_default_branch_check: "Checking if default branch is being deleted...", - protected_branch_checks: "Checking if you are force pushing to a protected branch...", - protected_branch_push_checks: "Checking if you are allowed to push to the protected branch...", - protected_branch_deletion_checks: "Checking if you are allowed to delete the protected branch...", - tag_checks: "Checking if you are allowed to change existing tags...", - protected_tag_checks: "Checking if you are creating, updating or deleting a protected tag...", - lfs_objects_exist_check: "Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...", - commits_check_file_paths_validation: "Validating commits' file paths...", - commits_check: "Validating commit contents..." - }.freeze - - attr_reader :user_access, :project, :skip_authorization, :skip_lfs_integrity_check, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name, :logger + attr_reader(*ATTRIBUTES) def initialize( change, user_access:, project:, skip_authorization: false, @@ -50,206 +26,32 @@ module Gitlab @logger.append_message("Running checks for ref: #{@branch_name || @tag_name}") end - def exec(skip_commits_check: false) + def exec return true if skip_authorization - push_checks - branch_checks - tag_checks - lfs_objects_exist_check unless skip_lfs_integrity_check - commits_check unless skip_commits_check + ref_level_checks + # Check of commits should happen as the last step + # given they're expensive in terms of performance + commits_check true end - protected - - def push_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - unless can_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] - end - end - end - - def branch_checks - return unless branch_name - - logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do - if deletion? && branch_name == project.default_branch - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] - end - end - - protected_branch_checks - end - - def protected_branch_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks - - if forced_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] - end - end - - if deletion? - protected_branch_deletion_checks - else - protected_branch_push_checks - end - end - - def protected_branch_deletion_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - unless user_access.can_delete_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] - end - - unless updated_from_web? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] - end - end - end - - def protected_branch_push_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - if matching_merge_request? - unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] - end - else - unless user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message - end - end - end - end - - def tag_checks - return unless tag_name - - logger.log_timed(LOG_MESSAGES[__method__]) do - if tag_exists? && user_access.cannot_do_action?(:admin_project) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] - end - end - - protected_tag_checks + def commits + @commits ||= project.repository.new_commits(newrev) end - def protected_tag_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks - - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + protected - unless user_access.can_create_tag?(tag_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] - end - end + def ref_level_checks + Gitlab::Checks::PushCheck.new(self).validate! + Gitlab::Checks::BranchCheck.new(self).validate! + Gitlab::Checks::TagCheck.new(self).validate! + Gitlab::Checks::LfsCheck.new(self).validate! end def commits_check - return if deletion? || newrev.nil? - return unless should_run_commit_validations? - - logger.log_timed(LOG_MESSAGES[__method__]) do - # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - commits.each do |commit| - logger.check_timeout_reached - - commit_check.validate(commit, validations_for_commit(commit)) - end - end - end - - logger.log_timed(LOG_MESSAGES[:commits_check_file_paths_validation]) do - commit_check.validate_file_paths - end - end - - # Method overwritten in EE to inject custom validations - def validations_for_commit(_) - [] - end - - private - - def push_to_protected_branch_rejected_message - if project.empty_repo? - empty_project_push_message - else - ERROR_MESSAGES[:push_protected_branch] - end - end - - def empty_project_push_message - <<~MESSAGE - - A default branch (e.g. master) does not yet exist for #{project.full_path} - Ask a project Owner or Maintainer to create a default branch: - - #{project_members_url} - - MESSAGE - end - - def project_members_url - Gitlab::Routing.url_helpers.project_project_members_url(project) - end - - def should_run_commit_validations? - commit_check.validate_lfs_file_locks? - end - - def updated_from_web? - protocol == 'web' - end - - def tag_exists? - project.repository.tag_exists?(tag_name) - end - - def forced_push? - Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev) - end - - def update? - !Gitlab::Git.blank_ref?(oldrev) && !deletion? - end - - def deletion? - Gitlab::Git.blank_ref?(newrev) - end - - def matching_merge_request? - Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? - end - - def lfs_objects_exist_check - logger.log_timed(LOG_MESSAGES[__method__]) do - lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) - - if lfs_check.objects_missing? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] - end - end - end - - def commit_check - @commit_check ||= Gitlab::Checks::CommitCheck.new(project, user_access.user, newrev, oldrev) - end - - def commits - @commits ||= project.repository.new_commits(newrev) - end - - def can_push? - user_access.can_do_action?(:push_code) || - user_access.can_push_to_branch?(branch_name) + Gitlab::Checks::DiffCheck.new(self).validate! end end end diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb deleted file mode 100644 index 58267b6752f..00000000000 --- a/lib/gitlab/checks/commit_check.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Checks - class CommitCheck - include Gitlab::Utils::StrongMemoize - - attr_reader :project, :user, :newrev, :oldrev - - def initialize(project, user, newrev, oldrev) - @project = project - @user = user - @newrev = newrev - @oldrev = oldrev - @file_paths = [] - end - - def validate(commit, validations) - return if validations.empty? && path_validations.empty? - - commit.raw_deltas.each do |diff| - @file_paths << (diff.new_path || diff.old_path) - - validations.each do |validation| - if error = validation.call(diff) - raise ::Gitlab::GitAccess::UnauthorizedError, error - end - end - end - end - - def validate_file_paths - path_validations.each do |validation| - if error = validation.call(@file_paths) - raise ::Gitlab::GitAccess::UnauthorizedError, error - end - end - end - - def validate_lfs_file_locks? - strong_memoize(:validate_lfs_file_locks) do - project.lfs_enabled? && newrev && oldrev && project.any_lfs_file_locks? - end - end - - private - - # rubocop: disable CodeReuse/ActiveRecord - def lfs_file_locks_validation - lambda do |paths| - lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first - - if lfs_lock - return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" - end - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def path_validations - validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] - end - end - end -end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb new file mode 100644 index 00000000000..49d361fcef7 --- /dev/null +++ b/lib/gitlab/checks/diff_check.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class DiffCheck < BaseChecker + include Gitlab::Utils::StrongMemoize + + LOG_MESSAGES = { + validate_file_paths: "Validating diffs' file paths...", + diff_content_check: "Validating diff contents..." + }.freeze + + def validate! + return unless should_run_diff_validations? + return if commits.empty? + return unless uses_raw_delta_validations? + + file_paths = [] + process_raw_deltas do |diff| + file_paths << (diff.new_path || diff.old_path) + + validate_diff(diff) + end + + validate_file_paths(file_paths) + end + + private + + def should_run_diff_validations? + newrev && oldrev && !deletion? && validate_lfs_file_locks? + end + + def validate_lfs_file_locks? + strong_memoize(:validate_lfs_file_locks) do + project.lfs_enabled? && project.any_lfs_file_locks? + end + end + + def uses_raw_delta_validations? + validations_for_diff.present? || path_validations.present? + end + + def validate_diff(diff) + validations_for_diff.each do |validation| + if error = validation.call(diff) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + + # Method overwritten in EE to inject custom validations + def validations_for_diff + [] + end + + def path_validations + validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] + end + + def process_raw_deltas + logger.log_timed(LOG_MESSAGES[:diff_content_check]) do + # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + commits.each do |commit| + logger.check_timeout_reached + + commit.raw_deltas.each do |diff| + yield(diff) + end + end + end + end + end + + def validate_file_paths(file_paths) + logger.log_timed(LOG_MESSAGES[__method__]) do + path_validations.each do |validation| + if error = validation.call(file_paths) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def lfs_file_locks_validation + lambda do |paths| + lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user_access.user.id).take + + if lfs_lock + return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb new file mode 100644 index 00000000000..e42684e679a --- /dev/null +++ b/lib/gitlab/checks/lfs_check.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class LfsCheck < BaseChecker + LOG_MESSAGE = "Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...".freeze + ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'.freeze + + def validate! + return if skip_lfs_integrity_check + + logger.log_timed(LOG_MESSAGE) do + lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) + + if lfs_check.objects_missing? + raise GitAccess::UnauthorizedError, ERROR_MESSAGE + end + end + end + end + end +end diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb new file mode 100644 index 00000000000..f3a52f09868 --- /dev/null +++ b/lib/gitlab/checks/push_check.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class PushCheck < BaseChecker + def validate! + logger.log_timed("Checking if you are allowed to push...") do + unless can_push? + raise GitAccess::UnauthorizedError, 'You are not allowed to push code to this project.' + end + end + end + + private + + def can_push? + user_access.can_do_action?(:push_code) || + user_access.can_push_to_branch?(branch_name) + end + end + end +end diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb new file mode 100644 index 00000000000..2a75c8059bd --- /dev/null +++ b/lib/gitlab/checks/tag_check.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class TagCheck < BaseChecker + ERROR_MESSAGES = { + change_existing_tags: 'You are not allowed to change existing tags on this project.', + update_protected_tag: 'Protected tags cannot be updated.', + delete_protected_tag: 'Protected tags cannot be deleted.', + create_protected_tag: 'You are not allowed to create this tag as it is protected.' + }.freeze + + LOG_MESSAGES = { + tag_checks: "Checking if you are allowed to change existing tags...", + protected_tag_checks: "Checking if you are creating, updating or deleting a protected tag..." + }.freeze + + def validate! + return unless tag_name + + logger.log_timed(LOG_MESSAGES[:tag_checks]) do + if tag_exists? && user_access.cannot_do_action?(:admin_project) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] + end + end + + protected_tag_checks + end + + private + + def protected_tag_checks + logger.log_timed(LOG_MESSAGES[__method__]) do + return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks + + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + + unless user_access.can_create_tag?(tag_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index a4f01468e8e..7cabaadb122 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -54,7 +54,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def collect - query = project.pipelines + query = project.all_pipelines .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection totals_count = grouped_count(query) @@ -115,7 +115,7 @@ module Gitlab class PipelineTime < Chart def collect - commits = project.pipelines.last(30) + commits = project.all_pipelines.last(30) commits.each do |commit| @labels << commit.short_sha diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 2fb3c4582e7..6333799a491 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -15,7 +15,7 @@ module Gitlab @global = Entry::Global.new(@config) @global.compose! - rescue Loader::FormatError, + rescue Gitlab::Config::Loader::FormatError, Extendable::ExtensionError, External::Processor::IncludeError => e raise Config::ConfigError, e.message @@ -71,7 +71,7 @@ module Gitlab private def build_config(config, opts = {}) - initial_config = Loader.new(config).load! + initial_config = Gitlab::Config::Loader::Yaml.new(config).load! project = opts.fetch(:project, nil) if project diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index ef5f25b42c0..41613369ca2 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -7,10 +7,10 @@ module Gitlab ## # Entry that represents a configuration of job artifacts. # - class Artifacts < Node - include Configurable - include Validatable - include Attributable + class Artifacts < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = %i[name untracked paths reports when expire_in].freeze diff --git a/lib/gitlab/ci/config/entry/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb deleted file mode 100644 index 3c2e1df9b83..00000000000 --- a/lib/gitlab/ci/config/entry/attributable.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - module Attributable - extend ActiveSupport::Concern - - class_methods do - def attributes(*attributes) - attributes.flatten.each do |attribute| - if method_defined?(attribute) - raise ArgumentError, 'Method already defined!' - end - - define_method(attribute) do - return unless config.is_a?(Hash) - - config[attribute] - end - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/boolean.rb b/lib/gitlab/ci/config/entry/boolean.rb deleted file mode 100644 index b9639c83075..00000000000 --- a/lib/gitlab/ci/config/entry/boolean.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - ## - # Entry that represents a boolean value. - # - class Boolean < Node - include Validatable - - validations do - validates :config, boolean: true - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index 0a25057f482..7b94af24c09 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -7,9 +7,9 @@ module Gitlab ## # Entry that represents a cache configuration # - class Cache < Node - include Configurable - include Attributable + class Cache < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = %i[key untracked paths policy].freeze DEFAULT_POLICY = 'pull-push'.freeze @@ -22,7 +22,7 @@ module Gitlab entry :key, Entry::Key, description: 'Cache key used to define a cache affinity.' - entry :untracked, Entry::Boolean, + entry :untracked, ::Gitlab::Config::Entry::Boolean, description: 'Cache all untracked files.' entry :paths, Entry::Paths, diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index d9658291ebe..02e368c1813 100644 --- a/lib/gitlab/ci/config/entry/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents a job script. # - class Commands < Node - include Validatable + class Commands < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, array_of_strings_or_string: true diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb deleted file mode 100644 index 4aabf0cfa31..00000000000 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - ## - # This mixin is responsible for adding DSL, which purpose is to - # simplifly process of adding child nodes. - # - # This can be used only if parent node is a configuration entry that - # holds a hash as a configuration value, for example: - # - # job: - # script: ... - # artifacts: ... - # - module Configurable - extend ActiveSupport::Concern - - included do - include Validatable - - validations do - validates :config, type: Hash - end - end - - # rubocop: disable CodeReuse/ActiveRecord - def compose!(deps = nil) - return unless valid? - - self.class.nodes.each do |key, factory| - factory - .value(config[key]) - .with(key: key, parent: self) - - entries[key] = factory.create! - end - - yield if block_given? - - entries.each_value do |entry| - entry.compose!(deps) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - class_methods do - def nodes - Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] - end - - private - - # rubocop: disable CodeReuse/ActiveRecord - def entry(key, entry, metadata) - factory = Entry::Factory.new(entry) - .with(description: metadata[:description]) - - (@nodes ||= {}).merge!(key.to_sym => factory) - end - # rubocop: enable CodeReuse/ActiveRecord - - def helpers(*nodes) - nodes.each do |symbol| - define_method("#{symbol}_defined?") do - entries[symbol]&.specified? - end - - define_method("#{symbol}_value") do - return unless entries[symbol] && entries[symbol].valid? - - entries[symbol].value - end - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/coverage.rb b/lib/gitlab/ci/config/entry/coverage.rb index 690409ccf77..89545158bed 100644 --- a/lib/gitlab/ci/config/entry/coverage.rb +++ b/lib/gitlab/ci/config/entry/coverage.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents Coverage settings. # - class Coverage < Node - include Validatable + class Coverage < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, regexp: true diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 07e9e1d3f67..69a3a1aedef 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents an environment. # - class Environment < Node - include Validatable + class Environment < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable ALLOWED_KEYS = %i[name url action on_stop].freeze diff --git a/lib/gitlab/ci/config/entry/except_policy.rb b/lib/gitlab/ci/config/entry/except_policy.rb new file mode 100644 index 00000000000..46ded35325d --- /dev/null +++ b/lib/gitlab/ci/config/entry/except_policy.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents an only/except trigger policy for the job. + # + class ExceptPolicy < Policy + def self.default + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/factory.rb b/lib/gitlab/ci/config/entry/factory.rb deleted file mode 100644 index 85c9c3511a4..00000000000 --- a/lib/gitlab/ci/config/entry/factory.rb +++ /dev/null @@ -1,75 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - ## - # Factory class responsible for fabricating entry objects. - # - class Factory - InvalidFactory = Class.new(StandardError) - - def initialize(entry) - @entry = entry - @metadata = {} - @attributes = {} - end - - def value(value) - @value = value - self - end - - def metadata(metadata) - @metadata.merge!(metadata) - self - end - - def with(attributes) - @attributes.merge!(attributes) - self - end - - def create! - raise InvalidFactory unless defined?(@value) - - ## - # We assume that unspecified entry is undefined. - # See issue #18775. - # - if @value.nil? - Entry::Unspecified.new( - fabricate_unspecified - ) - else - fabricate(@entry, @value) - end - end - - private - - def fabricate_unspecified - ## - # If entry has a default value we fabricate concrete node - # with default value. - # - if @entry.default.nil? - fabricate(Entry::Undefined) - else - fabricate(@entry, @entry.default) - end - end - - def fabricate(entry, value = nil) - entry.new(value, @metadata).tap do |node| - node.key = @attributes[:key] - node.parent = @attributes[:parent] - node.description = @attributes[:description] - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb index eba203d9d06..09ecb5fdb99 100644 --- a/lib/gitlab/ci/config/entry/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -8,8 +8,8 @@ module Gitlab # This class represents a global entry - root Entry for entire # GitLab CI Configuration file. # - class Global < Node - include Configurable + class Global < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable entry :before_script, Entry::Script, description: 'Script that will be executed before each job.' @@ -49,7 +49,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def compose_jobs! - factory = Entry::Factory.new(Entry::Jobs) + factory = ::Gitlab::Config::Entry::Factory.new(Entry::Jobs) .value(@config.except(*self.class.nodes.keys)) .with(key: :jobs, parent: self, description: 'Jobs definition for this pipeline') diff --git a/lib/gitlab/ci/config/entry/hidden.rb b/lib/gitlab/ci/config/entry/hidden.rb index dc0ede2a25f..76e5d05639f 100644 --- a/lib/gitlab/ci/config/entry/hidden.rb +++ b/lib/gitlab/ci/config/entry/hidden.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents a hidden CI/CD key. # - class Hidden < Node - include Validatable + class Hidden < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, presence: true diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index fc453b72fa5..a13a0625e90 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents a Docker image. # - class Image < Node - include Validatable + class Image < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable ALLOWED_KEYS = %i[name entrypoint].freeze diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index c8cb3248fa7..085be5da08d 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -7,9 +7,9 @@ module Gitlab ## # Entry that represents a concrete CI/CD job. # - class Job < Node - include Configurable - include Attributable + class Job < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when start_in artifacts cache @@ -65,10 +65,10 @@ module Gitlab entry :services, Entry::Services, description: 'Services that will be used to execute this job.' - entry :only, Entry::Policy, + entry :only, Entry::OnlyPolicy, description: 'Refs policy this job will be executed for.' - entry :except, Entry::Policy, + entry :except, Entry::ExceptPolicy, description: 'Refs policy this job will be executed for.' entry :variables, Entry::Variables, diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 1535b108000..82b72e40404 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents a set of jobs. # - class Jobs < Node - include Validatable + class Jobs < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, type: Hash @@ -34,7 +34,7 @@ module Gitlab @config.each do |name, config| node = hidden?(name) ? Entry::Hidden : Entry::Job - factory = Entry::Factory.new(node) + factory = ::Gitlab::Config::Entry::Factory.new(node) .value(config || {}) .metadata(name: name) .with(key: name, parent: self, diff --git a/lib/gitlab/ci/config/entry/key.rb b/lib/gitlab/ci/config/entry/key.rb index 963b200c7bb..0c10967e629 100644 --- a/lib/gitlab/ci/config/entry/key.rb +++ b/lib/gitlab/ci/config/entry/key.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents a key. # - class Key < Node - include Validatable + class Key < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, key: true diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb deleted file mode 100644 index 4043629dea9..00000000000 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - module LegacyValidationHelpers - private - - def validate_duration(value) - value.is_a?(String) && ChronicDuration.parse(value) - rescue ChronicDuration::DurationParseError - false - end - - def validate_duration_limit(value, limit) - return false unless value.is_a?(String) - - ChronicDuration.parse(value).second.from_now < - ChronicDuration.parse(limit).second.from_now - rescue ChronicDuration::DurationParseError - false - end - - def validate_array_of_strings(values) - values.is_a?(Array) && values.all? { |value| validate_string(value) } - end - - def validate_array_of_strings_or_regexps(values) - values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) } - end - - def validate_variables(variables) - variables.is_a?(Hash) && - variables.flatten.all? do |value| - validate_string(value) || validate_integer(value) - end - end - - def validate_integer(value) - value.is_a?(Integer) - end - - def validate_string(value) - value.is_a?(String) || value.is_a?(Symbol) - end - - def validate_regexp(value) - !value.nil? && Regexp.new(value.to_s) && true - rescue RegexpError, TypeError - false - end - - def validate_string_or_regexp(value) - return true if value.is_a?(Symbol) - return false unless value.is_a?(String) - - if value.first == '/' && value.last == '/' - validate_regexp(value[1...-1]) - else - true - end - end - - def validate_boolean(value) - value.in?([true, false]) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb deleted file mode 100644 index 347089722e4..00000000000 --- a/lib/gitlab/ci/config/entry/node.rb +++ /dev/null @@ -1,103 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - ## - # Base abstract class for each configuration entry node. - # - class Node - InvalidError = Class.new(StandardError) - - attr_reader :config, :metadata - attr_accessor :key, :parent, :description - - def initialize(config, **metadata) - @config = config - @metadata = metadata - @entries = {} - - self.class.aspects.to_a.each do |aspect| - instance_exec(&aspect) - end - end - - def [](key) - @entries[key] || Entry::Undefined.new - end - - def compose!(deps = nil) - return unless valid? - - yield if block_given? - end - - def leaf? - @entries.none? - end - - def descendants - @entries.values - end - - def ancestors - @parent ? @parent.ancestors + [@parent] : [] - end - - def valid? - errors.none? - end - - def errors - [] - end - - def value - if leaf? - @config - else - meaningful = @entries.select do |_key, value| - value.specified? && value.relevant? - end - - Hash[meaningful.map { |key, entry| [key, entry.value] }] - end - end - - def specified? - true - end - - def relevant? - true - end - - def location - name = @key.presence || self.class.name.to_s.demodulize - .underscore.humanize.downcase - - ancestors.map(&:key).append(name).compact.join(':') - end - - def inspect - val = leaf? ? config : descendants - unspecified = specified? ? '' : '(unspecified) ' - "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>" - end - - def self.default - end - - def self.aspects - @aspects ||= [] - end - - private - - attr_reader :entries - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/only_policy.rb b/lib/gitlab/ci/config/entry/only_policy.rb new file mode 100644 index 00000000000..9a581b8e97e --- /dev/null +++ b/lib/gitlab/ci/config/entry/only_policy.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents an only/except trigger policy for the job. + # + class OnlyPolicy < Policy + def self.default + { refs: %w[branches tags] } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/paths.rb b/lib/gitlab/ci/config/entry/paths.rb index 9580b5e2e7f..d6f287c6552 100644 --- a/lib/gitlab/ci/config/entry/paths.rb +++ b/lib/gitlab/ci/config/entry/paths.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents an array of paths. # - class Paths < Node - include Validatable + class Paths < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, array_of_strings: true diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 0535d7c1a1a..81e74a639fc 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -5,14 +5,11 @@ module Gitlab class Config module Entry ## - # Entry that represents an only/except trigger policy for the job. + # Base class for OnlyPolicy and ExceptPolicy # - class Policy < Simplifiable - strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } - strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) } - - class RefsPolicy < Entry::Node - include Entry::Validatable + class Policy < ::Gitlab::Config::Entry::Simplifiable + class RefsPolicy < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, array_of_strings_or_regexps: true @@ -23,9 +20,9 @@ module Gitlab end end - class ComplexPolicy < Entry::Node - include Entry::Validatable - include Entry::Attributable + class ComplexPolicy < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = %i[refs kubernetes variables changes].freeze attributes :refs, :kubernetes, :variables, :changes @@ -58,7 +55,7 @@ module Gitlab end end - class UnknownStrategy < Entry::Node + class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors ["#{location} has to be either an array of conditions or a hash"] end @@ -66,6 +63,16 @@ module Gitlab def self.default end + + ## + # Class-level execution won't be inherited by subclasses by default. + # Therefore, we need to explicitly execute that for OnlyPolicy and ExceptPolicy + def self.inherited(klass) + super + + klass.strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } + klass.strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) } + end end end end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 3ac2a6fa777..a3f6cc31321 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -7,9 +7,9 @@ module Gitlab ## # Entry that represents a configuration of job artifacts. # - class Reports < Node - include Validatable - include Attributable + class Reports < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management].freeze diff --git a/lib/gitlab/ci/config/entry/retry.rb b/lib/gitlab/ci/config/entry/retry.rb index ee82ab10f9c..eaf8b38aa3c 100644 --- a/lib/gitlab/ci/config/entry/retry.rb +++ b/lib/gitlab/ci/config/entry/retry.rb @@ -7,12 +7,12 @@ module Gitlab ## # Entry that represents a retry config for a job. # - class Retry < Simplifiable + class Retry < ::Gitlab::Config::Entry::Simplifiable strategy :SimpleRetry, if: -> (config) { config.is_a?(Integer) } strategy :FullRetry, if: -> (config) { config.is_a?(Hash) } - class SimpleRetry < Entry::Node - include Entry::Validatable + class SimpleRetry < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, numericality: { only_integer: true, @@ -31,9 +31,9 @@ module Gitlab end end - class FullRetry < Entry::Node - include Entry::Validatable - include Entry::Attributable + class FullRetry < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = %i[max when].freeze attributes :max, :when @@ -73,7 +73,7 @@ module Gitlab end end - class UnknownStrategy < Entry::Node + class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors ["#{location} has to be either an integer or a hash"] end diff --git a/lib/gitlab/ci/config/entry/script.rb b/lib/gitlab/ci/config/entry/script.rb index f7d39e5cf55..9d25a82b521 100644 --- a/lib/gitlab/ci/config/entry/script.rb +++ b/lib/gitlab/ci/config/entry/script.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents a script. # - class Script < Node - include Validatable + class Script < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, array_of_strings: true diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 47bf9205147..6df67083310 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -8,7 +8,7 @@ module Gitlab # Entry that represents a configuration of Docker service. # class Service < Image - include Validatable + include ::Gitlab::Config::Entry::Validatable ALLOWED_KEYS = %i[name entrypoint command alias].freeze diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb index bdf7f80f382..71475f69218 100644 --- a/lib/gitlab/ci/config/entry/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents a configuration of Docker services. # - class Services < Node - include Validatable + class Services < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, type: Array @@ -18,7 +18,7 @@ module Gitlab super do @entries = [] @config.each do |config| - @entries << Entry::Factory.new(Entry::Service) + @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Service) .value(config || {}) .create! end diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb deleted file mode 100644 index 9961bbfaa40..00000000000 --- a/lib/gitlab/ci/config/entry/simplifiable.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - class Simplifiable < SimpleDelegator - EntryStrategy = Struct.new(:name, :condition) - - def initialize(config, **metadata) - unless self.class.const_defined?(:UnknownStrategy) - raise ArgumentError, 'UndefinedStrategy not available!' - end - - strategy = self.class.strategies.find do |variant| - variant.condition.call(config) - end - - entry = self.class.entry_class(strategy) - - super(entry.new(config, metadata)) - end - - def self.strategy(name, **opts) - EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy| - strategies.append(strategy) - end - end - - def self.strategies - @strategies ||= [] - end - - def self.entry_class(strategy) - if strategy.present? - self.const_get(strategy.name) - else - self::UnknownStrategy - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/stage.rb b/lib/gitlab/ci/config/entry/stage.rb index 65ab5953131..d6d576a3139 100644 --- a/lib/gitlab/ci/config/entry/stage.rb +++ b/lib/gitlab/ci/config/entry/stage.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents a stage for a job. # - class Stage < Node - include Validatable + class Stage < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, type: String diff --git a/lib/gitlab/ci/config/entry/stages.rb b/lib/gitlab/ci/config/entry/stages.rb index ab184246d29..2d715cbc6bb 100644 --- a/lib/gitlab/ci/config/entry/stages.rb +++ b/lib/gitlab/ci/config/entry/stages.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents a configuration for pipeline stages. # - class Stages < Node - include Validatable + class Stages < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, array_of_strings: true diff --git a/lib/gitlab/ci/config/entry/undefined.rb b/lib/gitlab/ci/config/entry/undefined.rb deleted file mode 100644 index 77dcfa88170..00000000000 --- a/lib/gitlab/ci/config/entry/undefined.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - ## - # This class represents an undefined entry. - # - class Undefined < Node - def initialize(*) - super(nil) - end - - def value - nil - end - - def valid? - true - end - - def errors - [] - end - - def specified? - false - end - - def relevant? - false - end - - def inspect - "#<#{self.class.name}>" - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/unspecified.rb b/lib/gitlab/ci/config/entry/unspecified.rb deleted file mode 100644 index bab32489d2f..00000000000 --- a/lib/gitlab/ci/config/entry/unspecified.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - ## - # This class represents an unspecified entry. - # - # It decorates original entry adding method that indicates it is - # unspecified. - # - class Unspecified < SimpleDelegator - def specified? - false - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb deleted file mode 100644 index 08a6593c980..00000000000 --- a/lib/gitlab/ci/config/entry/validatable.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - module Validatable - extend ActiveSupport::Concern - - def self.included(node) - node.aspects.append -> do - @validator = self.class.validator.new(self) - @validator.validate(:new) - end - end - - def errors - @validator.messages + descendants.flat_map(&:errors) # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - - class_methods do - def validator - @validator ||= Class.new(Entry::Validator).tap do |validator| - if defined?(@validations) - @validations.each { |rules| validator.class_eval(&rules) } - end - end - end - - private - - def validations(&block) - (@validations ||= []).append(block) - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/validator.rb b/lib/gitlab/ci/config/entry/validator.rb deleted file mode 100644 index 33ffdd3a95d..00000000000 --- a/lib/gitlab/ci/config/entry/validator.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - class Validator < SimpleDelegator - include ActiveModel::Validations - include Entry::Validators - - def initialize(entry) - super(entry) - end - - def messages - errors.full_messages.map do |error| - "#{location} #{error}".downcase - end - end - - def self.name - 'Validator' - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb deleted file mode 100644 index a1d552fb2e5..00000000000 --- a/lib/gitlab/ci/config/entry/validators.rb +++ /dev/null @@ -1,198 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - module Validators - class AllowedKeysValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - unknown_keys = value.try(:keys).to_a - options[:in] - - if unknown_keys.any? - record.errors.add(attribute, "contains unknown keys: " + - unknown_keys.join(', ')) - end - end - end - - class AllowedValuesValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - unless options[:in].include?(value.to_s) - record.errors.add(attribute, "unknown value: #{value}") - end - end - end - - class AllowedArrayValuesValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - unkown_values = value - options[:in] - unless unkown_values.empty? - record.errors.add(attribute, "contains unknown values: " + - unkown_values.join(', ')) - end - end - end - - class ArrayOfStringsValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - unless validate_array_of_strings(value) - record.errors.add(attribute, 'should be an array of strings') - end - end - end - - class BooleanValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - unless validate_boolean(value) - record.errors.add(attribute, 'should be a boolean value') - end - end - end - - class DurationValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - unless validate_duration(value) - record.errors.add(attribute, 'should be a duration') - end - - if options[:limit] - unless validate_duration_limit(value, options[:limit]) - record.errors.add(attribute, 'should not exceed the limit') - end - end - end - end - - class HashOrStringValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - unless value.is_a?(Hash) || value.is_a?(String) - record.errors.add(attribute, 'should be a hash or a string') - end - end - end - - class HashOrIntegerValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - unless value.is_a?(Hash) || value.is_a?(Integer) - record.errors.add(attribute, 'should be a hash or an integer') - end - end - end - - class KeyValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - if validate_string(value) - validate_path(record, attribute, value) - else - record.errors.add(attribute, 'should be a string or symbol') - end - end - - private - - def validate_path(record, attribute, value) - path = CGI.unescape(value.to_s) - - if path.include?('/') - record.errors.add(attribute, 'cannot contain the "/" character') - elsif path == '.' || path == '..' - record.errors.add(attribute, 'cannot be "." or ".."') - end - end - end - - class RegexpValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - unless validate_regexp(value) - record.errors.add(attribute, 'must be a regular expression') - end - end - - private - - def look_like_regexp?(value) - value.is_a?(String) && value.start_with?('/') && - value.end_with?('/') - end - - def validate_regexp(value) - look_like_regexp?(value) && - Regexp.new(value.to_s[1...-1]) && - true - rescue RegexpError - false - end - end - - class ArrayOfStringsOrRegexpsValidator < RegexpValidator - def validate_each(record, attribute, value) - unless validate_array_of_strings_or_regexps(value) - record.errors.add(attribute, 'should be an array of strings or regexps') - end - end - - private - - def validate_array_of_strings_or_regexps(values) - values.is_a?(Array) && values.all?(&method(:validate_string_or_regexp)) - end - - def validate_string_or_regexp(value) - return false unless value.is_a?(String) - return validate_regexp(value) if look_like_regexp?(value) - - true - end - end - - class ArrayOfStringsOrStringValidator < RegexpValidator - def validate_each(record, attribute, value) - unless validate_array_of_strings_or_string(value) - record.errors.add(attribute, 'should be an array of strings or a string') - end - end - - private - - def validate_array_of_strings_or_string(values) - validate_array_of_strings(values) || validate_string(values) - end - end - - class TypeValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - type = options[:with] - raise unless type.is_a?(Class) - - unless value.is_a?(type) - message = options[:message] || "should be a #{type.name}" - record.errors.add(attribute, message) - end - end - end - - class VariablesValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - unless validate_variables(value) - record.errors.add(attribute, 'should be a hash of key value pairs') - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 6fd3cec2f5f..89d790ebfa6 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -7,8 +7,8 @@ module Gitlab ## # Entry that represents environment variables. # - class Variables < Node - include Validatable + class Variables < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, variables: true diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 15ca47ef60e..ee4ea9bbb1d 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -37,8 +37,8 @@ module Gitlab end def to_hash - @hash ||= Ci::Config::Loader.new(content).load! - rescue Ci::Config::Loader::FormatError + @hash ||= Gitlab::Config::Loader::Yaml.new(content).load! + rescue Gitlab::Config::Loader::FormatError nil end diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index b445a872b3d..d33d1edfe35 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -16,6 +16,7 @@ module Gitlab trigger_requests: Array(@command.trigger_request), user: @command.current_user, pipeline_schedule: @command.schedule, + merge_request: @command.merge_request, protected: @command.protected_ref?, variables_attributes: Array(@command.variables_attributes) ) diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 05978804d92..100b9521412 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -8,7 +8,7 @@ module Gitlab Command = Struct.new( :source, :project, :current_user, :origin_ref, :checkout_sha, :after_sha, :before_sha, - :trigger_request, :schedule, + :trigger_request, :schedule, :merge_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes ) do diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index c90976b2040..3b2cae07c12 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -19,15 +19,6 @@ # * review: REVIEW_DISABLED # * stop_review: REVIEW_DISABLED # -# The sast and sast_dashboard jobs are executed to guarantee full compatibility -# with the group security dashboard and the security reports with old runners. -# If you use only runners with version 11.5 or above, you can disable the sast -# job by setting the OLD_REPORTS_DISABLED environment variable. If you use only -# runners with version below 11.5, you can disable the sast_dashboard job by -# setting the NEW_REPORTS_DISABLED environment variable. -# The sast_dashboard job will be removed in the future, when the sast job will -# use the new reports syntax. -# # In order to deploy, you must have a Kubernetes cluster configured either # via a project integration, or via group/project variables. # AUTO_DEVOPS_DOMAIN must also be set as a variable at the group or project @@ -182,29 +173,6 @@ sast: except: variables: - $SAST_DISABLED - - $OLD_REPORTS_DISABLED - -sast_dashboard: - stage: test - image: docker:stable - allow_failure: true - services: - - docker:stable-dind - script: - - setup_docker - - sast - artifacts: - reports: - sast: gl-sast-report.json - only: - refs: - - branches - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ - except: - variables: - - $SAST_DISABLED - - $NEW_REPORTS_DISABLED dependency_scanning: stage: test @@ -658,6 +626,7 @@ rollout 100%: fi if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then + echo "Deploying first release with database initialization..." helm upgrade --install \ --wait \ --set service.enabled="$service_enabled" \ @@ -680,6 +649,7 @@ rollout 100%: "$name" \ chart/ + echo "Deploying second release..." helm upgrade --reuse-values \ --wait \ --set application.initializeCommand="" \ @@ -688,6 +658,7 @@ rollout 100%: "$name" \ chart/ else + echo "Deploying new release..." helm upgrade --install \ --wait \ --set service.enabled="$service_enabled" \ diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index e6ec400e476..172926b8ab0 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -5,7 +5,7 @@ module Gitlab class YamlProcessor ValidationError = Class.new(StandardError) - include Gitlab::Ci::Config::Entry::LegacyValidationHelpers + include Gitlab::Config::Entry::LegacyValidationHelpers attr_reader :cache, :stages, :jobs diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb new file mode 100644 index 00000000000..560fe63df0e --- /dev/null +++ b/lib/gitlab/config/entry/attributable.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + module Attributable + extend ActiveSupport::Concern + + class_methods do + def attributes(*attributes) + attributes.flatten.each do |attribute| + if method_defined?(attribute) + raise ArgumentError, 'Method already defined!' + end + + define_method(attribute) do + return unless config.is_a?(Hash) + + config[attribute] + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/config/entry/boolean.rb b/lib/gitlab/config/entry/boolean.rb new file mode 100644 index 00000000000..1e8a57356e3 --- /dev/null +++ b/lib/gitlab/config/entry/boolean.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # Entry that represents a boolean value. + # + class Boolean < Node + include Validatable + + validations do + validates :config, boolean: true + end + end + end + end +end diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb new file mode 100644 index 00000000000..afdb60b2cd5 --- /dev/null +++ b/lib/gitlab/config/entry/configurable.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # This mixin is responsible for adding DSL, which purpose is to + # simplifly process of adding child nodes. + # + # This can be used only if parent node is a configuration entry that + # holds a hash as a configuration value, for example: + # + # job: + # script: ... + # artifacts: ... + # + module Configurable + extend ActiveSupport::Concern + + included do + include Validatable + + validations do + validates :config, type: Hash + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def compose!(deps = nil) + return unless valid? + + self.class.nodes.each do |key, factory| + factory + .value(config[key]) + .with(key: key, parent: self) + + entries[key] = factory.create! + end + + yield if block_given? + + entries.each_value do |entry| + entry.compose!(deps) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + class_methods do + def nodes + Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def entry(key, entry, metadata) + factory = ::Gitlab::Config::Entry::Factory.new(entry) + .with(description: metadata[:description]) + + (@nodes ||= {}).merge!(key.to_sym => factory) + end + # rubocop: enable CodeReuse/ActiveRecord + + def helpers(*nodes) + nodes.each do |symbol| + define_method("#{symbol}_defined?") do + entries[symbol]&.specified? + end + + define_method("#{symbol}_value") do + return unless entries[symbol] && entries[symbol].valid? + + entries[symbol].value + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb new file mode 100644 index 00000000000..30d43c9f9a1 --- /dev/null +++ b/lib/gitlab/config/entry/factory.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # Factory class responsible for fabricating entry objects. + # + class Factory + InvalidFactory = Class.new(StandardError) + + def initialize(entry) + @entry = entry + @metadata = {} + @attributes = {} + end + + def value(value) + @value = value + self + end + + def metadata(metadata) + @metadata.merge!(metadata) + self + end + + def with(attributes) + @attributes.merge!(attributes) + self + end + + def create! + raise InvalidFactory unless defined?(@value) + + ## + # We assume that unspecified entry is undefined. + # See issue #18775. + # + if @value.nil? + Entry::Unspecified.new( + fabricate_unspecified + ) + else + fabricate(@entry, @value) + end + end + + private + + def fabricate_unspecified + ## + # If entry has a default value we fabricate concrete node + # with default value. + # + if @entry.default.nil? + fabricate(Entry::Undefined) + else + fabricate(@entry, @entry.default) + end + end + + def fabricate(entry, value = nil) + entry.new(value, @metadata).tap do |node| + node.key = @attributes[:key] + node.parent = @attributes[:parent] + node.description = @attributes[:description] + end + end + end + end + end +end diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb new file mode 100644 index 00000000000..d3ab5625743 --- /dev/null +++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + module LegacyValidationHelpers + private + + def validate_duration(value) + value.is_a?(String) && ChronicDuration.parse(value) + rescue ChronicDuration::DurationParseError + false + end + + def validate_duration_limit(value, limit) + return false unless value.is_a?(String) + + ChronicDuration.parse(value).second.from_now < + ChronicDuration.parse(limit).second.from_now + rescue ChronicDuration::DurationParseError + false + end + + def validate_array_of_strings(values) + values.is_a?(Array) && values.all? { |value| validate_string(value) } + end + + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) } + end + + def validate_variables(variables) + variables.is_a?(Hash) && + variables.flatten.all? do |value| + validate_string(value) || validate_integer(value) + end + end + + def validate_integer(value) + value.is_a?(Integer) + end + + def validate_string(value) + value.is_a?(String) || value.is_a?(Symbol) + end + + def validate_regexp(value) + !value.nil? && Regexp.new(value.to_s) && true + rescue RegexpError, TypeError + false + end + + def validate_string_or_regexp(value) + return true if value.is_a?(Symbol) + return false unless value.is_a?(String) + + if value.first == '/' && value.last == '/' + validate_regexp(value[1...-1]) + else + true + end + end + + def validate_boolean(value) + value.in?([true, false]) + end + end + end + end +end diff --git a/lib/gitlab/config/entry/node.rb b/lib/gitlab/config/entry/node.rb new file mode 100644 index 00000000000..30357b2c95b --- /dev/null +++ b/lib/gitlab/config/entry/node.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # Base abstract class for each configuration entry node. + # + class Node + InvalidError = Class.new(StandardError) + + attr_reader :config, :metadata + attr_accessor :key, :parent, :description + + def initialize(config, **metadata) + @config = config + @metadata = metadata + @entries = {} + + self.class.aspects.to_a.each do |aspect| + instance_exec(&aspect) + end + end + + def [](key) + @entries[key] || Entry::Undefined.new + end + + def compose!(deps = nil) + return unless valid? + + yield if block_given? + end + + def leaf? + @entries.none? + end + + def descendants + @entries.values + end + + def ancestors + @parent ? @parent.ancestors + [@parent] : [] + end + + def valid? + errors.none? + end + + def errors + [] + end + + def value + if leaf? + @config + else + meaningful = @entries.select do |_key, value| + value.specified? && value.relevant? + end + + Hash[meaningful.map { |key, entry| [key, entry.value] }] + end + end + + def specified? + true + end + + def relevant? + true + end + + def location + name = @key.presence || self.class.name.to_s.demodulize + .underscore.humanize.downcase + + ancestors.map(&:key).append(name).compact.join(':') + end + + def inspect + val = leaf? ? config : descendants + unspecified = specified? ? '' : '(unspecified) ' + "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>" + end + + def self.default + end + + def self.aspects + @aspects ||= [] + end + + private + + attr_reader :entries + end + end + end +end diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb new file mode 100644 index 00000000000..3e148fe2e91 --- /dev/null +++ b/lib/gitlab/config/entry/simplifiable.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + class Simplifiable < SimpleDelegator + EntryStrategy = Struct.new(:name, :condition) + + def initialize(config, **metadata) + unless self.class.const_defined?(:UnknownStrategy) + raise ArgumentError, 'UndefinedStrategy not available!' + end + + strategy = self.class.strategies.find do |variant| + variant.condition.call(config) + end + + entry = self.class.entry_class(strategy) + + super(entry.new(config, metadata)) + end + + def self.strategy(name, **opts) + EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy| + strategies.append(strategy) + end + end + + def self.strategies + @strategies ||= [] + end + + def self.entry_class(strategy) + if strategy.present? + self.const_get(strategy.name) + else + self::UnknownStrategy + end + end + end + end + end +end diff --git a/lib/gitlab/config/entry/undefined.rb b/lib/gitlab/config/entry/undefined.rb new file mode 100644 index 00000000000..5f708abc80c --- /dev/null +++ b/lib/gitlab/config/entry/undefined.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # This class represents an undefined entry. + # + class Undefined < Node + def initialize(*) + super(nil) + end + + def value + nil + end + + def valid? + true + end + + def errors + [] + end + + def specified? + false + end + + def relevant? + false + end + + def inspect + "#<#{self.class.name}>" + end + end + end + end +end diff --git a/lib/gitlab/config/entry/unspecified.rb b/lib/gitlab/config/entry/unspecified.rb new file mode 100644 index 00000000000..c096180d0f8 --- /dev/null +++ b/lib/gitlab/config/entry/unspecified.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # This class represents an unspecified entry. + # + # It decorates original entry adding method that indicates it is + # unspecified. + # + class Unspecified < SimpleDelegator + def specified? + false + end + end + end + end +end diff --git a/lib/gitlab/config/entry/validatable.rb b/lib/gitlab/config/entry/validatable.rb new file mode 100644 index 00000000000..1c88c68c11c --- /dev/null +++ b/lib/gitlab/config/entry/validatable.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + module Validatable + extend ActiveSupport::Concern + + def self.included(node) + node.aspects.append -> do + @validator = self.class.validator.new(self) + @validator.validate(:new) + end + end + + def errors + @validator.messages + descendants.flat_map(&:errors) # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + class_methods do + def validator + @validator ||= Class.new(Entry::Validator).tap do |validator| + if defined?(@validations) + @validations.each { |rules| validator.class_eval(&rules) } + end + end + end + + private + + def validations(&block) + (@validations ||= []).append(block) + end + end + end + end + end +end diff --git a/lib/gitlab/config/entry/validator.rb b/lib/gitlab/config/entry/validator.rb new file mode 100644 index 00000000000..e5efd4a7b0a --- /dev/null +++ b/lib/gitlab/config/entry/validator.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + class Validator < SimpleDelegator + include ActiveModel::Validations + include Entry::Validators + + def initialize(entry) + super(entry) + end + + def messages + errors.full_messages.map do |error| + "#{location} #{error}".downcase + end + end + + def self.name + 'Validator' + end + end + end + end +end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb new file mode 100644 index 00000000000..25bfa50f829 --- /dev/null +++ b/lib/gitlab/config/entry/validators.rb @@ -0,0 +1,196 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + module Validators + class AllowedKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unknown_keys = value.try(:keys).to_a - options[:in] + + if unknown_keys.any? + record.errors.add(attribute, "contains unknown keys: " + + unknown_keys.join(', ')) + end + end + end + + class AllowedValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless options[:in].include?(value.to_s) + record.errors.add(attribute, "unknown value: #{value}") + end + end + end + + class AllowedArrayValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unkown_values = value - options[:in] + unless unkown_values.empty? + record.errors.add(attribute, "contains unknown values: " + + unkown_values.join(', ')) + end + end + end + + class ArrayOfStringsValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_array_of_strings(value) + record.errors.add(attribute, 'should be an array of strings') + end + end + end + + class BooleanValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_boolean(value) + record.errors.add(attribute, 'should be a boolean value') + end + end + end + + class DurationValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_duration(value) + record.errors.add(attribute, 'should be a duration') + end + + if options[:limit] + unless validate_duration_limit(value, options[:limit]) + record.errors.add(attribute, 'should not exceed the limit') + end + end + end + end + + class HashOrStringValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(Hash) || value.is_a?(String) + record.errors.add(attribute, 'should be a hash or a string') + end + end + end + + class HashOrIntegerValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(Hash) || value.is_a?(Integer) + record.errors.add(attribute, 'should be a hash or an integer') + end + end + end + + class KeyValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + if validate_string(value) + validate_path(record, attribute, value) + else + record.errors.add(attribute, 'should be a string or symbol') + end + end + + private + + def validate_path(record, attribute, value) + path = CGI.unescape(value.to_s) + + if path.include?('/') + record.errors.add(attribute, 'cannot contain the "/" character') + elsif path == '.' || path == '..' + record.errors.add(attribute, 'cannot be "." or ".."') + end + end + end + + class RegexpValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_regexp(value) + record.errors.add(attribute, 'must be a regular expression') + end + end + + private + + def look_like_regexp?(value) + value.is_a?(String) && value.start_with?('/') && + value.end_with?('/') + end + + def validate_regexp(value) + look_like_regexp?(value) && + Regexp.new(value.to_s[1...-1]) && + true + rescue RegexpError + false + end + end + + class ArrayOfStringsOrRegexpsValidator < RegexpValidator + def validate_each(record, attribute, value) + unless validate_array_of_strings_or_regexps(value) + record.errors.add(attribute, 'should be an array of strings or regexps') + end + end + + private + + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all?(&method(:validate_string_or_regexp)) + end + + def validate_string_or_regexp(value) + return false unless value.is_a?(String) + return validate_regexp(value) if look_like_regexp?(value) + + true + end + end + + class ArrayOfStringsOrStringValidator < RegexpValidator + def validate_each(record, attribute, value) + unless validate_array_of_strings_or_string(value) + record.errors.add(attribute, 'should be an array of strings or a string') + end + end + + private + + def validate_array_of_strings_or_string(values) + validate_array_of_strings(values) || validate_string(values) + end + end + + class TypeValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + type = options[:with] + raise unless type.is_a?(Class) + + unless value.is_a?(type) + message = options[:message] || "should be a #{type.name}" + record.errors.add(attribute, message) + end + end + end + + class VariablesValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_variables(value) + record.errors.add(attribute, 'should be a hash of key value pairs') + end + end + end + end + end + end +end diff --git a/lib/gitlab/config/loader/format_error.rb b/lib/gitlab/config/loader/format_error.rb new file mode 100644 index 00000000000..848ff96d201 --- /dev/null +++ b/lib/gitlab/config/loader/format_error.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Loader + FormatError = Class.new(StandardError) + end + end +end diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/config/loader/yaml.rb index b4c491e84a6..8159f8b8026 100644 --- a/lib/gitlab/ci/config/loader.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -1,15 +1,13 @@ # frozen_string_literal: true module Gitlab - module Ci - class Config - class Loader - FormatError = Class.new(StandardError) - + module Config + module Loader + class Yaml def initialize(config) @config = YAML.safe_load(config, [Symbol], [], true) rescue Psych::Exception => e - raise FormatError, e.message + raise Loader::FormatError, e.message end def valid? @@ -18,7 +16,7 @@ module Gitlab def load! unless valid? - raise FormatError, 'Invalid configuration format' + raise Loader::FormatError, 'Invalid configuration format' end @config.deep_symbolize_keys diff --git a/lib/gitlab/correlation_id.rb b/lib/gitlab/correlation_id.rb new file mode 100644 index 00000000000..0f9bde4390e --- /dev/null +++ b/lib/gitlab/correlation_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module CorrelationId + LOG_KEY = 'correlation_id'.freeze + + class << self + def use_id(correlation_id, &blk) + # always generate a id if null is passed + correlation_id ||= new_id + + ids.push(correlation_id || new_id) + + begin + yield(current_id) + ensure + ids.pop + end + end + + def current_id + ids.last + end + + def current_or_new_id + current_id || new_id + end + + private + + def ids + Thread.current[:correlation_id] ||= [] + end + + def new_id + SecureRandom.uuid + end + end + end +end diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb index 68d0b5d8f8a..87a03d9c58f 100644 --- a/lib/gitlab/crypto_helper.rb +++ b/lib/gitlab/crypto_helper.rb @@ -6,8 +6,8 @@ module Gitlab AES256_GCM_OPTIONS = { algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_truncated, - iv: Settings.attr_encrypted_db_key_base_truncated[0..11] + key: Settings.attr_encrypted_db_key_base_32, + iv: Settings.attr_encrypted_db_key_base_12 }.freeze def sha256(value) @@ -17,7 +17,7 @@ module Gitlab def aes256_gcm_encrypt(value) encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value)) - Base64.encode64(encrypted_token) + Base64.strict_encode64(encrypted_token) end def aes256_gcm_decrypt(value) diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index ea6529e2dc4..f3d37ccd72a 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -1,7 +1,12 @@ # frozen_string_literal: true # For large tables, PostgreSQL can take a long time to count rows due to MVCC. -# We can optimize this by using the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting. +# We can optimize this by using various strategies for approximate counting. +# +# For example, we can use the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting. +# +# However, since statistics are not always up to date, we also implement a table sampling strategy +# that performs an exact count but only on a sample of the table. See TablesampleCountStrategy. module Gitlab module Database module Count @@ -20,68 +25,30 @@ module Gitlab end # Takes in an array of models and returns a Hash for the approximate - # counts for them. If the model's table has not been vacuumed or - # analyzed recently, simply run the Model.count to get the data. + # counts for them. + # + # Various count strategies can be specified that are executed in + # sequence until all tables have an approximate count attached + # or we run out of strategies. + # + # Note that not all strategies are available on all supported RDBMS. # # @param [Array] # @return [Hash] of Model -> count mapping - def self.approximate_counts(models) - table_to_model_map = models.each_with_object({}) do |model, hash| - hash[model.table_name] = model - end - - table_names = table_to_model_map.keys - counts_by_table_name = Gitlab::Database.postgresql? ? reltuples_from_recently_updated(table_names) : {} + def self.approximate_counts(models, strategies: [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy]) + strategies.each_with_object({}) do |strategy, counts_by_model| + if strategy.enabled? + models_with_missing_counts = models - counts_by_model.keys - # Convert table -> count to Model -> count - counts_by_model = counts_by_table_name.each_with_object({}) do |pair, hash| - model = table_to_model_map[pair.first] - hash[model] = pair.second - end + break counts_by_model if models_with_missing_counts.empty? - missing_tables = table_names - counts_by_table_name.keys + counts = strategy.new(models_with_missing_counts).count - missing_tables.each do |table| - model = table_to_model_map[table] - counts_by_model[model] = model.count + counts.each do |model, count| + counts_by_model[model] = count + end + end end - - counts_by_model - end - - # Returns a hash of the table names that have recently updated tuples. - # - # @param [Array] table names - # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) - def self.reltuples_from_recently_updated(table_names) - query = postgresql_estimate_query(table_names) - rows = [] - - # Querying tuple stats only works on the primary. Due to load - # balancing, we need to ensure this query hits the load balancer. The - # easiest way to do this is to start a transaction. - ActiveRecord::Base.transaction do - rows = ActiveRecord::Base.connection.select_all(query) - end - - rows.each_with_object({}) { |row, data| data[row['table_name']] = row['estimate'].to_i } - rescue *CONNECTION_ERRORS - {} - end - - # Generates the PostgreSQL query to return the tuples for tables - # that have been vacuumed or analyzed in the last hour. - # - # @param [Array] table names - # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) - def self.postgresql_estimate_query(table_names) - time = "to_timestamp(#{1.hour.ago.to_i})" - <<~SQL - SELECT pg_class.relname AS table_name, reltuples::bigint AS estimate FROM pg_class - LEFT JOIN pg_stat_user_tables ON pg_class.relname = pg_stat_user_tables.relname - WHERE pg_class.relname IN (#{table_names.map { |table| "'#{table}'" }.join(',')}) - AND (last_vacuum > #{time} OR last_autovacuum > #{time} OR last_analyze > #{time} OR last_autoanalyze > #{time}) - SQL end end end diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb new file mode 100644 index 00000000000..fa6951eda22 --- /dev/null +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Count + # This strategy performs an exact count on the model. + # + # This is guaranteed to be accurate, however it also scans the + # whole table. Hence, there are no guarantees with respect + # to runtime. + # + # Note that for very large tables, this may even timeout. + class ExactCountStrategy + attr_reader :models + def initialize(models) + @models = models + end + + def count + models.each_with_object({}) do |model, data| + data[model] = model.count + end + rescue *CONNECTION_ERRORS + {} + end + + def self.enabled? + true + end + end + end + end +end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb new file mode 100644 index 00000000000..c3a674aeb7e --- /dev/null +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Count + class PgClass < ActiveRecord::Base + self.table_name = 'pg_class' + end + + # This strategy counts based on PostgreSQL's statistics in pg_stat_user_tables. + # + # Specifically, it relies on the column reltuples in said table. An additional + # check is performed to make sure statistics were updated within the last hour. + # + # Otherwise, this strategy skips tables with outdated statistics. + # + # There are no guarantees with respect to the accuracy of this strategy. Runtime + # however is guaranteed to be "fast", because it only looks up statistics. + class ReltuplesCountStrategy + attr_reader :models + def initialize(models) + @models = models + end + + # Returns a hash of the table names that have recently updated tuples. + # + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def count + size_estimates + rescue *CONNECTION_ERRORS + {} + end + + def self.enabled? + Gitlab::Database.postgresql? + end + + private + + def table_names + models.map(&:table_name) + end + + def size_estimates(check_statistics: true) + table_to_model = models.each_with_object({}) { |model, h| h[model.table_name] = model } + + # Querying tuple stats only works on the primary. Due to load balancing, the + # easiest way to do this is to start a transaction. + ActiveRecord::Base.transaction do + get_statistics(table_names, check_statistics: check_statistics).each_with_object({}) do |row, data| + model = table_to_model[row.table_name] + data[model] = row.estimate + end + end + end + + # Generates the PostgreSQL query to return the tuples for tables + # that have been vacuumed or analyzed in the last hour. + # + # @param [Array] table names + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def get_statistics(table_names, check_statistics: true) + time = 1.hour.ago + + query = PgClass.joins("LEFT JOIN pg_stat_user_tables USING (relname)") + .where(relname: table_names) + .select('pg_class.relname AS table_name, reltuples::bigint AS estimate') + + if check_statistics + query = query.where('last_vacuum > ? OR last_autovacuum > ? OR last_analyze > ? OR last_autoanalyze > ?', + time, time, time, time) + end + + query + end + end + end + end +end diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb new file mode 100644 index 00000000000..cf1cf054dbf --- /dev/null +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Count + # A tablesample count executes in two phases: + # * Estimate table sizes based on reltuples. + # * Based on the estimate: + # * If the table is considered 'small', execute an exact relation count. + # * Otherwise, count on a sample of the table using TABLESAMPLE. + # + # The size of the sample is chosen in a way that we always roughly scan + # the same amount of rows (see TABLESAMPLE_ROW_TARGET). + # + # There are no guarantees with respect to the accuracy of the result or runtime. + class TablesampleCountStrategy < ReltuplesCountStrategy + EXACT_COUNT_THRESHOLD = 10_000 + TABLESAMPLE_ROW_TARGET = 10_000 + + def count + estimates = size_estimates(check_statistics: false) + + models.each_with_object({}) do |model, count_by_model| + count = perform_count(model, estimates[model]) + count_by_model[model] = count if count + end + rescue *CONNECTION_ERRORS + {} + end + + def self.enabled? + Gitlab::Database.postgresql? && Feature.enabled?(:tablesample_counts) + end + + private + + def perform_count(model, estimate) + # If we estimate 0, we may not have statistics at all. Don't use them. + return nil unless estimate && estimate > 0 + + if estimate < EXACT_COUNT_THRESHOLD + # The table is considered small, the assumption here is that + # the exact count will be fast anyways. + model.count + else + # The table is considered large, let's only count on a sample. + tablesample_count(model, estimate) + end + end + + def tablesample_count(model, estimate) + portion = (TABLESAMPLE_ROW_TARGET.to_f / estimate).round(4) + inverse = 1 / portion + query = <<~SQL + SELECT (COUNT(*)*#{inverse})::integer AS count + FROM #{model.table_name} TABLESAMPLE SYSTEM (#{portion * 100}) + SQL + + rows = ActiveRecord::Base.connection.select_all(query) + + Integer(rows.first['count']) + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 134d1e7a724..d9578852db6 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -975,9 +975,10 @@ into similar problems in the future (e.g. when new tables are created). raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') jobs = [] + table_name = model_class.quoted_table_name model_class.each_batch(of: batch_size) do |relation| - start_id, end_id = relation.pluck('MIN(id), MAX(id)').first + start_id, end_id = relation.pluck("MIN(#{table_name}.id), MAX(#{table_name}.id)").first if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE # Note: This code path generally only helps with many millions of rows diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 10df037a0dd..c5bbf522f7c 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -34,6 +34,16 @@ module Gitlab @diff_files ||= diffs.decorate! { |diff| decorate_diff!(diff) } end + # This mutates `diff_files` lines. + def unfold_diff_files(positions) + positions_grouped_by_path = positions.group_by { |position| position.file_path } + + diff_files.each do |diff_file| + positions = positions_grouped_by_path.fetch(diff_file.file_path, []) + positions.each { |position| diff_file.unfold_diff_lines(position) } + end + end + def diff_file_with_old_path(old_path) diff_files.find { |diff_file| diff_file.old_path == old_path } end diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb index 586c5cf87af..663bad95db7 100644 --- a/lib/gitlab/diff/file_collection/compare.rb +++ b/lib/gitlab/diff/file_collection/compare.rb @@ -10,6 +10,10 @@ module Gitlab diff_options: diff_options, diff_refs: diff_refs) end + + def unfold_diff_lines(positions) + # no-op + end end end end diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index b4db3f93c9c..3958814208c 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -4,8 +4,6 @@ # the result is joined and sorted by file name module Gitlab class FileFinder - BATCH_SIZE = 100 - attr_reader :project, :ref delegate :repository, to: :project @@ -16,60 +14,35 @@ module Gitlab end def find(query) - query = Gitlab::Search::Query.new(query) do - filter :filename, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}$/i } - filter :path, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}/i } - filter :extension, matcher: ->(filter, blob) { blob.filename =~ /\.#{filter[:regex_value]}$/i } + query = Gitlab::Search::Query.new(query, encode_binary: true) do + filter :filename, matcher: ->(filter, blob) { blob.binary_filename =~ /#{filter[:regex_value]}$/i } + filter :path, matcher: ->(filter, blob) { blob.binary_filename =~ /#{filter[:regex_value]}/i } + filter :extension, matcher: ->(filter, blob) { blob.binary_filename =~ /\.#{filter[:regex_value]}$/i } end - by_content = find_by_content(query.term) - - already_found = Set.new(by_content.map(&:filename)) - by_filename = find_by_filename(query.term, except: already_found) + files = find_by_filename(query.term) + find_by_content(query.term) - files = (by_content + by_filename) - .sort_by(&:filename) + files = query.filter_results(files) if query.filters.any? - query.filter_results(files).map { |blob| [blob.filename, blob] } + files end private def find_by_content(query) - results = repository.search_files_by_content(query, ref).first(BATCH_SIZE) - results.map { |result| Gitlab::ProjectSearchResults.parse_search_result(result, project) } - end - - def find_by_filename(query, except: []) - filenames = search_filenames(query, except) - - blobs(filenames).map do |blob| - Gitlab::SearchResults::FoundBlob.new( - id: blob.id, - filename: blob.path, - basename: File.basename(blob.path, File.extname(blob.path)), - ref: ref, - startline: 1, - data: blob.data, - project: project - ) + repository.search_files_by_content(query, ref).map do |result| + Gitlab::Search::FoundBlob.new(content_match: result, project: project, ref: ref, repository: repository) end end - def search_filenames(query, except) - filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE) - - filenames.delete_if { |filename| except.include?(filename) } unless except.empty? - - filenames - end - - def blob_refs(filenames) - filenames.map { |filename| [ref, filename] } + def find_by_filename(query) + search_filenames(query).map do |filename| + Gitlab::Search::FoundBlob.new(blob_filename: filename, project: project, ref: ref, repository: repository) + end end - def blobs(filenames) - Gitlab::Git::Blob.batch(repository, blob_refs(filenames), blob_size_limit: 1024) + def search_filenames(query) + repository.search_files_by_name(query, ref) end end end diff --git a/lib/gitlab/git/repository_cleaner.rb b/lib/gitlab/git/repository_cleaner.rb new file mode 100644 index 00000000000..2d1d8435cf3 --- /dev/null +++ b/lib/gitlab/git/repository_cleaner.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class RepositoryCleaner + include Gitlab::Git::WrapsGitalyErrors + + attr_reader :repository + + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @repository = repository + end + + def apply_bfg_object_map(io) + wrapped_gitaly_errors do + gitaly_cleanup_client.apply_bfg_object_map(io) + end + end + + private + + def gitaly_cleanup_client + @gitaly_cleanup_client ||= Gitlab::GitalyClient::CleanupService.new(repository) + end + end + end +end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 9be553a8b86..255601382b1 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -193,6 +193,7 @@ module Gitlab feature = feature_stack && feature_stack[0] metadata['call_site'] = feature.to_s if feature metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage + metadata['correlation_id'] = Gitlab::CorrelationId.current_id if Gitlab::CorrelationId.current_id metadata.merge!(server_feature_flags) diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb new file mode 100644 index 00000000000..8e412a9b3ef --- /dev/null +++ b/lib/gitlab/gitaly_client/cleanup_service.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module GitalyClient + class CleanupService + attr_reader :repository, :gitaly_repo, :storage + + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def apply_bfg_object_map(io) + first_request = Gitaly::ApplyBfgObjectMapRequest.new(repository: gitaly_repo) + + enum = Enumerator.new do |y| + y.yield first_request + + while data = io.read(RepositoryService::MAX_MSG_SIZE) + y.yield Gitaly::ApplyBfgObjectMapRequest.new(object_map: data) + end + end + + GitalyClient.call( + storage, + :cleanup_service, + :apply_bfg_object_map, + enum, + timeout: GitalyClient.no_timeout + ) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index c32c2c0b2fb..22d2d149e65 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -385,7 +385,8 @@ module Gitlab file_path: encode_binary(action[:file_path]), previous_path: encode_binary(action[:previous_path]), base64_content: action[:encoding] == 'base64', - execute_filemode: !!action[:execute_filemode] + execute_filemode: !!action[:execute_filemode], + infer_content: !!action[:infer_content] ) rescue RangeError raise ArgumentError, "Unknown action '#{action[:action]}'" diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 374dc9d3c00..bc3ea9e9226 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -80,7 +80,7 @@ module Gitlab end def fail_import(message) - project.mark_import_as_failed(message) + project.import_state.mark_as_failed(message) false end end diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index a77ac1e4fa6..9d81441d96e 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -41,8 +41,7 @@ module Gitlab Gitlab::SidekiqStatus .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) - project.ensure_import_state - project.import_state&.update_column(:jid, jid) + project.import_state.update_column(:jid, jid) Stage::ImportRepositoryWorker .perform_async(project.id) diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 31bab20b044..4fbb87385c3 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -44,9 +44,8 @@ module Gitlab def update_signature!(cached_signature) using_keychain do |gpg_key| cached_signature.update!(attributes(gpg_key)) + @signature = cached_signature end - - @signature = cached_signature end private @@ -59,11 +58,15 @@ module Gitlab # the proper signature. # NOTE: the invoked method is #fingerprint but it's only returning # 16 characters (the format used by keyid) instead of 40. - gpg_key = find_gpg_key(verified_signature.fingerprint) + fingerprint = verified_signature&.fingerprint + + break unless fingerprint + + gpg_key = find_gpg_key(fingerprint) if gpg_key Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key) - @verified_signature = nil + clear_memoization(:verified_signature) end yield gpg_key @@ -71,9 +74,16 @@ module Gitlab end def verified_signature - @verified_signature ||= GPGME::Crypto.new.verify(signature_text, signed_text: signed_text) do |verified_signature| + strong_memoize(:verified_signature) { gpgme_signature } + end + + def gpgme_signature + GPGME::Crypto.new.verify(signature_text, signed_text: signed_text) do |verified_signature| + # Return the first signature for now: https://gitlab.com/gitlab-org/gitlab-ce/issues/54932 break verified_signature end + rescue GPGME::Error + nil end def create_cached_signature! @@ -92,7 +102,7 @@ module Gitlab commit_sha: @commit.sha, project: @commit.project, gpg_key: gpg_key, - gpg_key_primary_keyid: gpg_key&.keyid || verified_signature.fingerprint, + gpg_key_primary_keyid: gpg_key&.keyid || verified_signature&.fingerprint, gpg_key_user_name: user_infos[:name], gpg_key_user_email: user_infos[:email], verification_status: verification_status @@ -102,7 +112,7 @@ module Gitlab def verification_status(gpg_key) return :unknown_key unless gpg_key return :unverified_key unless gpg_key.verified? - return :unverified unless verified_signature.valid? + return :unverified unless verified_signature&.valid? if gpg_key.verified_and_belongs_to_email?(@commit.committer_email) :verified diff --git a/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb new file mode 100644 index 00000000000..fa4c5d86d44 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# This module adds additional correlation id the grape logger +module Gitlab + module GrapeLogging + module Loggers + class CorrelationIdLogger < ::GrapeLogging::Loggers::Base + def parameters(_, _) + { Gitlab::CorrelationId::LOG_KEY => Gitlab::CorrelationId.current_id } + end + end + end + end +end diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb index c940ea7305e..97cbdc6cb39 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/group_hierarchy.rb @@ -34,8 +34,8 @@ module Gitlab # reached. So all ancestors *lower* than the specified ancestor will be # included. # rubocop: disable CodeReuse/ActiveRecord - def ancestors(upto: nil) - base_and_ancestors(upto: upto).where.not(id: ancestors_base.select(:id)) + def ancestors(upto: nil, hierarchy_order: nil) + base_and_ancestors(upto: upto, hierarchy_order: hierarchy_order).where.not(id: ancestors_base.select(:id)) end # rubocop: enable CodeReuse/ActiveRecord @@ -45,11 +45,22 @@ module Gitlab # Passing an `upto` will stop the recursion once the specified parent_id is # reached. So all ancestors *lower* than the specified acestor will be # included. - def base_and_ancestors(upto: nil) + # + # Passing a `hierarchy_order` with either `:asc` or `:desc` will cause the + # recursive query order from most nested group to root or from the root + # ancestor to most nested group respectively. This uses a `depth` column + # where `1` is defined as the depth for the base and increment as we go up + # each parent. + # rubocop: disable CodeReuse/ActiveRecord + def base_and_ancestors(upto: nil, hierarchy_order: nil) return ancestors_base unless Group.supports_nested_groups? - read_only(base_and_ancestors_cte(upto).apply_to(model.all)) + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) + recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order + + read_only(recursive_query) end + # rubocop: enable CodeReuse/ActiveRecord # Returns a relation that includes the descendants_base set of groups # and all their descendants (recursively). @@ -107,16 +118,22 @@ module Gitlab private # rubocop: disable CodeReuse/ActiveRecord - def base_and_ancestors_cte(stop_id = nil) + def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) + depth_column = :depth + + base_query = ancestors_base.except(:order) + base_query = base_query.select("1 as #{depth_column}", groups_table[Arel.star]) if hierarchy_order - cte << ancestors_base.except(:order) + cte << base_query # Recursively get all the ancestors of the base set. parent_query = model .from([groups_table, cte.table]) .where(groups_table[:id].eq(cte.table[:parent_id])) .except(:order) + + parent_query = parent_query.select(cte.table[depth_column] + 1, groups_table[Arel.star]) if hierarchy_order parent_query = parent_query.where(cte.table[:parent_id].not_eq(stop_id)) if stop_id cte << parent_query diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index b40eac3de9a..d10d4f2f746 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -51,7 +51,7 @@ project_tree: - resource_label_events: - label: :priorities - - pipelines: + - ci_pipelines: - notes: - :author - events: @@ -98,13 +98,12 @@ excluded_attributes: - :avatar - :import_type - :import_source - - :import_error - :mirror - :runners_token + - :runners_token_encrypted - :repository_storage - :repository_read_only - :lfs_enabled - - :import_jid - :created_at - :updated_at - :id @@ -116,6 +115,13 @@ excluded_attributes: - :remote_mirror_available_overridden - :description_html - :repository_languages + - :bfg_object_map + namespaces: + - :runners_token + - :runners_token_encrypted + project_import_state: + - :last_error + - :jid prometheus_metrics: - :common - :identifier @@ -137,6 +143,7 @@ excluded_attributes: statuses: - :trace - :token + - :token_encrypted - :when - :artifacts_file - :artifacts_metadata @@ -154,6 +161,9 @@ excluded_attributes: - :encrypted_token_iv - :encrypted_url - :encrypted_url_iv + runners: + - :token + - :token_encrypted services: - :template diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 8cd4efd91cc..a56ec65b9f1 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -26,6 +26,8 @@ module Gitlab @project_members = @tree_hash.delete('project_members') + RelationRenameService.rename(@tree_hash) + ActiveRecord::Base.uncached do ActiveRecord::Base.no_touching do create_relations @@ -214,7 +216,7 @@ module Gitlab end def nil_iid_pipeline?(relation_key, relation_item) - relation_key == 'pipelines' && relation_item['iid'].nil? + relation_key == 'ci_pipelines' && relation_item['iid'].nil? end end end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 29f2dc80813..2255635acdf 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -34,6 +34,8 @@ module Gitlab project_json['project_members'] += group_members_json + RelationRenameService.add_new_associations(project_json) + project_json.to_json end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 097c7653754..a4902e2104f 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -4,12 +4,14 @@ module Gitlab module ImportExport class RelationFactory OVERRIDES = { snippets: :project_snippets, + ci_pipelines: 'Ci::Pipeline', pipelines: 'Ci::Pipeline', stages: 'Ci::Stage', statuses: 'commit_status', triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', + runners: 'Ci::Runner', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', @@ -33,7 +35,7 @@ module Gitlab EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze - TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze + TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze def self.create(*args) new(*args).create diff --git a/lib/gitlab/import_export/relation_rename_service.rb b/lib/gitlab/import_export/relation_rename_service.rb new file mode 100644 index 00000000000..179bde5e21e --- /dev/null +++ b/lib/gitlab/import_export/relation_rename_service.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# This class is intended to help with relation renames within Gitlab versions +# and allow compatibility between versions. +# If you have to change one relationship name that is imported/exported, +# you should add it to the RENAMES constant indicating the old name and the +# new one. +# The behavior of these renamed relationships should be transient and it should +# only last one release until you completely remove the renaming from the list. +# +# When importing, this class will check the project hash and: +# - if only the old relationship name is found, it will rename it with the new one +# - if only the new relationship name is found, it will do nothing +# - if it finds both, it will use the new relationship data +# +# When exporting, this class will duplicate the keys in the resulting file. +# This way, if we open the file in an old version of the exporter it will work +# and also it will with the newer versions. +module Gitlab + module ImportExport + class RelationRenameService + RENAMES = { + 'pipelines' => 'ci_pipelines' # Added in 11.6, remove in 11.7 + }.freeze + + def self.rename(tree_hash) + return unless tree_hash&.present? + + RENAMES.each do |old_name, new_name| + old_entry = tree_hash.delete(old_name) + + next if tree_hash[new_name] + next unless old_entry + + tree_hash[new_name] = old_entry + end + end + + def self.add_new_associations(tree_hash) + RENAMES.each do |old_name, new_name| + next if tree_hash.key?(old_name) + + tree_hash[old_name] = tree_hash[new_name] + end + end + end + end +end diff --git a/lib/gitlab/json_logger.rb b/lib/gitlab/json_logger.rb index 3bff77731f6..a5a5759cc89 100644 --- a/lib/gitlab/json_logger.rb +++ b/lib/gitlab/json_logger.rb @@ -10,6 +10,7 @@ module Gitlab data = {} data[:severity] = severity data[:time] = timestamp.utc.iso8601(3) + data[Gitlab::CorrelationId::LOG_KEY] = Gitlab::CorrelationId.current_id case message when String diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index 3748fd6b5ef..a9957a85d48 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -85,6 +85,8 @@ module Gitlab end def to_kubeconfig(url:, namespace:, token:, ca_pem: nil) + return unless token.present? + config = { apiVersion: 'v1', clusters: [ @@ -113,7 +115,7 @@ module Gitlab kubeconfig_embed_ca_pem(config, ca_pem) if ca_pem - config.deep_stringify_keys + YAML.dump(config.deep_stringify_keys) end private diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index b947f6b551e..fe839940f74 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -46,6 +46,7 @@ module Gitlab :create_secret, :create_service_account, :update_config_map, + :update_secret, :update_service_account, to: :core_client @@ -80,8 +81,64 @@ module Gitlab @kubeclient_options = kubeclient_options end + def create_or_update_cluster_role_binding(resource) + if cluster_role_binding_exists?(resource) + update_cluster_role_binding(resource) + else + create_cluster_role_binding(resource) + end + end + + def create_or_update_role_binding(resource) + if role_binding_exists?(resource) + update_role_binding(resource) + else + create_role_binding(resource) + end + end + + def create_or_update_service_account(resource) + if service_account_exists?(resource) + update_service_account(resource) + else + create_service_account(resource) + end + end + + def create_or_update_secret(resource) + if secret_exists?(resource) + update_secret(resource) + else + create_secret(resource) + end + end + private + def cluster_role_binding_exists?(resource) + get_cluster_role_binding(resource.metadata.name) + rescue ::Kubeclient::ResourceNotFoundError + false + end + + def role_binding_exists?(resource) + get_role_binding(resource.metadata.name, resource.metadata.namespace) + rescue ::Kubeclient::ResourceNotFoundError + false + end + + def service_account_exists?(resource) + get_service_account(resource.metadata.name, resource.metadata.namespace) + rescue ::Kubeclient::ResourceNotFoundError + false + end + + def secret_exists?(resource) + get_secret(resource.metadata.name, resource.metadata.namespace) + rescue ::Kubeclient::ResourceNotFoundError + false + end + def build_kubeclient(api_group, api_version) ::Kubeclient::Client.new( join_api_url(api_prefix, api_group), diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 43695451b87..c526d31a591 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -80,8 +80,7 @@ module Gitlab def handle_errors return unless errors.any? - project.ensure_import_state - project.import_state&.update_column(:last_error, { + project.import_state.update_column(:last_error, { message: 'The remote data could not be fully imported.', errors: errors }.to_json) diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index fa44bd842b2..05d3096a208 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -38,7 +38,7 @@ module Gitlab end def type - actor.is_a?(User) ? :lfs_token : :lfs_deploy_token + user? ? :lfs_token : :lfs_deploy_token end def actor_name diff --git a/lib/gitlab/middleware/correlation_id.rb b/lib/gitlab/middleware/correlation_id.rb new file mode 100644 index 00000000000..73542dd422e --- /dev/null +++ b/lib/gitlab/middleware/correlation_id.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +# A dumb middleware that steals correlation id +# and sets it as a global context for the request +module Gitlab + module Middleware + class CorrelationId + include ActionView::Helpers::TagHelper + + def initialize(app) + @app = app + end + + def call(env) + ::Gitlab::CorrelationId.use_id(correlation_id(env)) do + @app.call(env) + end + end + + private + + def correlation_id(env) + if Gitlab.rails5? + request(env).request_id + else + request(env).uuid + end + end + + def request(env) + ActionDispatch::Request.new(env) + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 04df881bf03..a68f8801c2a 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -17,9 +17,9 @@ module Gitlab when 'notes' notes.page(page).per(per_page) when 'blobs' - Kaminari.paginate_array(blobs).page(page).per(per_page) + paginated_blobs(blobs, page) when 'wiki_blobs' - Kaminari.paginate_array(wiki_blobs).page(page).per(per_page) + paginated_blobs(wiki_blobs, page) when 'commits' Kaminari.paginate_array(commits).page(page).per(per_page) else @@ -55,37 +55,6 @@ module Gitlab @commits_count ||= commits.count end - def self.parse_search_result(result, project = nil) - ref = nil - filename = nil - basename = nil - - data = [] - startline = 0 - - result.each_line.each_with_index do |line, index| - prefix ||= line.match(/^(?<ref>[^:]*):(?<filename>[^\x00]*)\x00(?<startline>\d+)\x00/)&.tap do |matches| - ref = matches[:ref] - filename = matches[:filename] - startline = matches[:startline] - startline = startline.to_i - index - extname = Regexp.escape(File.extname(filename)) - basename = filename.sub(/#{extname}$/, '') - end - - data << line.sub(prefix.to_s, '') - end - - FoundBlob.new( - filename: filename, - basename: basename, - ref: ref, - startline: startline, - data: data.join, - project: project - ) - end - def single_commit_result? return false if commits_count != 1 @@ -97,6 +66,14 @@ module Gitlab private + def paginated_blobs(blobs, page) + results = Kaminari.paginate_array(blobs).page(page).per(per_page) + + Gitlab::Search::FoundBlob.preload_blobs(results) + + results + end + def blobs return [] unless Ability.allowed?(@current_user, :download_code, @project) diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb new file mode 100644 index 00000000000..a62ab1521a7 --- /dev/null +++ b/lib/gitlab/search/found_blob.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +module Gitlab + module Search + class FoundBlob + include EncodingHelper + include Presentable + include BlobLanguageFromGitAttributes + include Gitlab::Utils::StrongMemoize + + attr_reader :project, :content_match, :blob_filename + + FILENAME_REGEXP = /\A(?<ref>[^:]*):(?<filename>[^\x00]*)\x00/.freeze + CONTENT_REGEXP = /^(?<ref>[^:]*):(?<filename>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze + + def self.preload_blobs(blobs) + to_fetch = blobs.select { |blob| blob.is_a?(self) && blob.blob_filename } + + to_fetch.each { |blob| blob.fetch_blob } + end + + def initialize(opts = {}) + @id = opts.fetch(:id, nil) + @binary_filename = opts.fetch(:filename, nil) + @binary_basename = opts.fetch(:basename, nil) + @ref = opts.fetch(:ref, nil) + @startline = opts.fetch(:startline, nil) + @binary_data = opts.fetch(:data, nil) + @per_page = opts.fetch(:per_page, 20) + @project = opts.fetch(:project, nil) + # Some caller does not have project object (e.g. elastic search), + # yet they can trigger many calls in one go, + # causing duplicated queries. + # Allow those to just pass project_id instead. + @project_id = opts.fetch(:project_id, nil) + @content_match = opts.fetch(:content_match, nil) + @blob_filename = opts.fetch(:blob_filename, nil) + @repository = opts.fetch(:repository, nil) + end + + def id + @id ||= parsed_content[:id] + end + + def ref + @ref ||= parsed_content[:ref] + end + + def startline + @startline ||= parsed_content[:startline] + end + + # binary_filename is used for running filters on all matches, + # for grepped results (which use content_match), we get + # filename from the beginning of the grepped result which is faster + # then parsing whole snippet + def binary_filename + @binary_filename ||= content_match ? search_result_filename : parsed_content[:binary_filename] + end + + def filename + @filename ||= encode_utf8(@binary_filename || parsed_content[:binary_filename]) + end + + def basename + @basename ||= encode_utf8(@binary_basename || parsed_content[:binary_basename]) + end + + def data + @data ||= encode_utf8(@binary_data || parsed_content[:binary_data]) + end + + def path + filename + end + + def project_id + @project_id || @project&.id + end + + def present + super(presenter_class: BlobPresenter) + end + + def fetch_blob + path = [ref, blob_filename] + missing_blob = { binary_filename: blob_filename } + + BatchLoader.for(path).batch(default_value: missing_blob) do |refs, loader| + Gitlab::Git::Blob.batch(repository, refs, blob_size_limit: 1024).each do |blob| + # if the blob couldn't be fetched for some reason, + # show at least the blob filename + data = { + id: blob.id, + binary_filename: blob.path, + binary_basename: File.basename(blob.path, File.extname(blob.path)), + ref: ref, + startline: 1, + binary_data: blob.data, + project: project + } + + loader.call([ref, blob.path], data) + end + end + end + + private + + def search_result_filename + content_match.match(FILENAME_REGEXP) { |matches| matches[:filename] } + end + + def parsed_content + strong_memoize(:parsed_content) do + if content_match + parse_search_result + elsif blob_filename + fetch_blob + else + {} + end + end + end + + def parse_search_result + ref = nil + filename = nil + basename = nil + + data = [] + startline = 0 + + content_match.each_line.each_with_index do |line, index| + prefix ||= line.match(CONTENT_REGEXP)&.tap do |matches| + ref = matches[:ref] + filename = matches[:filename] + startline = matches[:startline] + startline = startline.to_i - index + extname = Regexp.escape(File.extname(filename)) + basename = filename.sub(/#{extname}$/, '') + end + + data << line.sub(prefix.to_s, '') + end + + { + binary_filename: filename, + binary_basename: basename, + ref: ref, + startline: startline, + binary_data: data.join, + project: project + } + end + + def repository + @repository ||= project.repository + end + end + end +end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb index 7f69083a492..ba0e16607a6 100644 --- a/lib/gitlab/search/query.rb +++ b/lib/gitlab/search/query.rb @@ -3,6 +3,8 @@ module Gitlab module Search class Query < SimpleDelegator + include EncodingHelper + def initialize(query, filter_opts = {}, &block) @raw_query = query.dup @filters = [] @@ -50,7 +52,9 @@ module Gitlab end def parse_filter(filter, input) - filter[:parser].call(input) + result = filter[:parser].call(input) + + @filter_options[:encode_binary] ? encode_binary(result) : result end end end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 458737f31eb..491148ec1a6 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -2,42 +2,6 @@ module Gitlab class SearchResults - class FoundBlob - include EncodingHelper - include Presentable - include BlobLanguageFromGitAttributes - - attr_reader :id, :filename, :basename, :ref, :startline, :data, :project - - def initialize(opts = {}) - @id = opts.fetch(:id, nil) - @filename = encode_utf8(opts.fetch(:filename, nil)) - @basename = encode_utf8(opts.fetch(:basename, nil)) - @ref = opts.fetch(:ref, nil) - @startline = opts.fetch(:startline, nil) - @data = encode_utf8(opts.fetch(:data, nil)) - @per_page = opts.fetch(:per_page, 20) - @project = opts.fetch(:project, nil) - # Some caller does not have project object (e.g. elastic search), - # yet they can trigger many calls in one go, - # causing duplicated queries. - # Allow those to just pass project_id instead. - @project_id = opts.fetch(:project_id, nil) - end - - def path - filename - end - - def project_id - @project_id || @project&.id - end - - def present - super(presenter_class: BlobPresenter) - end - end - attr_reader :current_user, :query, :per_page # Limit search results by passed projects diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 8079c5882c4..46d01964eac 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -3,7 +3,8 @@ module Gitlab module Sentry def self.enabled? - Rails.env.production? && Gitlab::CurrentSettings.sentry_enabled? + (Rails.env.production? || Rails.env.development?) && + Gitlab::CurrentSettings.sentry_enabled? end def self.context(current_user = nil) @@ -31,7 +32,7 @@ module Gitlab def self.track_exception(exception, issue_url: nil, extra: {}) track_acceptable_exception(exception, issue_url: issue_url, extra: extra) - raise exception if should_raise? + raise exception if should_raise_for_dev? end # This should be used when you do not want to raise an exception in @@ -43,7 +44,11 @@ module Gitlab extra[:issue_url] = issue_url if issue_url context # Make sure we've set everything we know in the context - Raven.capture_exception(exception, extra: extra) + tags = { + Gitlab::CorrelationId::LOG_KEY.to_sym => Gitlab::CorrelationId.current_id + } + + Raven.capture_exception(exception, tags: tags, extra: extra) end end @@ -55,7 +60,7 @@ module Gitlab end end - def self.should_raise? + def self.should_raise_for_dev? Rails.env.development? || Rails.env.test? end end diff --git a/lib/gitlab/sidekiq_middleware/correlation_injector.rb b/lib/gitlab/sidekiq_middleware/correlation_injector.rb new file mode 100644 index 00000000000..b807b3a03ed --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/correlation_injector.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class CorrelationInjector + def call(worker_class, job, queue, redis_pool) + job[Gitlab::CorrelationId::LOG_KEY] ||= + Gitlab::CorrelationId.current_or_new_id + + yield + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/correlation_logger.rb b/lib/gitlab/sidekiq_middleware/correlation_logger.rb new file mode 100644 index 00000000000..cb8ff4a6284 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/correlation_logger.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class CorrelationLogger + def call(worker, job, queue) + correlation_id = job[Gitlab::CorrelationId::LOG_KEY] + + Gitlab::CorrelationId.use_id(correlation_id) do + yield + end + end + end + end +end diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 76bb9eb611e..2dd4b7a4092 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -18,6 +18,10 @@ module Gitlab def find(key) file_name = "#{key}#{@extension}" + # The key is untrusted input, so ensure we can't be directed outside + # of base_dir + Gitlab::Utils.check_path_traversal!(file_name) + directory = select_directory(file_name) directory ? File.join(category_directory(directory), file_name) : nil end diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index b92cefefb8f..8e234148a63 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -26,6 +26,11 @@ module Gitlab def find(key) file_name = "#{key}#{@extension}" + + # The key is untrusted input, so ensure we can't be directed outside + # of base_dir inside the repository + Gitlab::Utils.check_path_traversal!(file_name) + directory = select_directory(file_name) raise FileNotFoundError if directory.nil? diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 86efe8ad114..b8040f73cee 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'resolv' +require 'ipaddress' module Gitlab class UrlBlocker @@ -10,11 +11,8 @@ module Gitlab def validate!(url, allow_localhost: false, allow_local_network: true, enforce_user: false, ports: [], protocols: []) return true if url.nil? - begin - uri = Addressable::URI.parse(url) - rescue Addressable::URI::InvalidURIError - raise BlockedUrlError, "URI is invalid" - end + # Param url can be a string, URI or Addressable::URI + uri = parse_url(url) # Allow imports from the GitLab instance itself but only from the configured ports return true if internal?(uri) @@ -26,7 +24,9 @@ module Gitlab validate_hostname!(uri.hostname) begin - addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM) + addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM).map do |addr| + addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr + end rescue SocketError return true end @@ -49,6 +49,18 @@ module Gitlab private + def parse_url(url) + raise Addressable::URI::InvalidURIError if multiline?(url) + + Addressable::URI.parse(url) + rescue Addressable::URI::InvalidURIError, URI::InvalidURIError + raise BlockedUrlError, 'URI is invalid' + end + + def multiline?(url) + CGI.unescape(url.to_s) =~ /\n|\r/ + end + def validate_port!(port, ports) return if port.blank? # Only ports under 1024 are restricted @@ -73,13 +85,14 @@ module Gitlab def validate_hostname!(value) return if value.blank? + return if IPAddress.valid?(value) return if value =~ /\A\p{Alnum}/ - raise BlockedUrlError, "Hostname needs to start with an alphanumeric character" + raise BlockedUrlError, "Hostname or IP address invalid" end def validate_localhost!(addrs_info) - local_ips = ["127.0.0.1", "::1", "0.0.0.0"] + local_ips = ["::", "0.0.0.0"] local_ips.concat(Socket.ip_address_list.map(&:ip_address)) return if (local_ips & addrs_info.map(&:ip_address)).empty? @@ -94,7 +107,7 @@ module Gitlab end def validate_local_network!(addrs_info) - return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? } + return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? || addr.ipv6_unique_local? } raise BlockedUrlError, "Requests to the local network are not allowed" end @@ -111,12 +124,14 @@ module Gitlab end def internal_web?(uri) - uri.hostname == config.gitlab.host && + uri.scheme == config.gitlab.protocol && + uri.hostname == config.gitlab.host && (uri.port.blank? || uri.port == config.gitlab.port) end def internal_shell?(uri) - uri.hostname == config.gitlab_shell.ssh_host && + uri.scheme == 'ssh' && + uri.hostname == config.gitlab_shell.ssh_host && (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 9bceec749fc..008e9cd1d24 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -2,6 +2,8 @@ module Gitlab class UsageData + APPROXIMATE_COUNT_MODELS = [Label, MergeRequest, Note, Todo].freeze + class << self def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } @@ -55,7 +57,11 @@ module Gitlab environments: count(::Environment), clusters: count(::Clusters::Cluster), clusters_enabled: count(::Clusters::Cluster.enabled), + project_clusters_enabled: count(::Clusters::Cluster.enabled.project_type), + group_clusters_enabled: count(::Clusters::Cluster.enabled.group_type), clusters_disabled: count(::Clusters::Cluster.disabled), + project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type), + group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type), clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), clusters_applications_helm: count(::Clusters::Applications::Helm.installed), @@ -69,12 +75,9 @@ module Gitlab issues: count(Issue), keys: count(Key), label_lists: count(List.label), - labels: count(Label), lfs_objects: count(LfsObject), - merge_requests: count(MergeRequest), milestone_lists: count(List.milestone), milestones: count(Milestone), - notes: count(Note), pages_domains: count(PagesDomain), projects: count(Project), projects_imported_from_github: count(Project.where(import_type: 'github')), @@ -82,10 +85,9 @@ module Gitlab releases: count(Release), remote_mirrors: count(RemoteMirror), snippets: count(Snippet), - todos: count(Todo), uploads: count(Upload), web_hooks: count(WebHook) - }.merge(services_usage) + }.merge(services_usage).merge(approximate_counts) } end # rubocop: enable CodeReuse/ActiveRecord @@ -160,6 +162,16 @@ module Gitlab fallback end # rubocop: enable CodeReuse/ActiveRecord + + def approximate_counts + approx_counts = Gitlab::Database::Count.approximate_counts(APPROXIMATE_COUNT_MODELS) + + APPROXIMATE_COUNT_MODELS.each_with_object({}) do |model, result| + key = model.name.underscore.pluralize.to_sym + + result[key] = approx_counts[model] || -1 + end + end end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 9e59137a2c0..26fc56227a2 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -4,6 +4,15 @@ module Gitlab module Utils extend self + # Ensure that the relative path will not traverse outside the base directory + def check_path_traversal!(path) + raise StandardError.new("Invalid path") if path.start_with?("..#{File::SEPARATOR}") || + path.include?("#{File::SEPARATOR}..#{File::SEPARATOR}") || + path.end_with?("#{File::SEPARATOR}..") + + path + end + # Run system command without outputting to stdout. # # @param cmd [Array<String>] @@ -16,6 +25,21 @@ module Gitlab str.force_encoding(Encoding::UTF_8) end + def ensure_utf8_size(str, bytes:) + raise ArgumentError, 'Empty string provided!' if str.empty? + raise ArgumentError, 'Negative string size provided!' if bytes.negative? + + truncated = str.each_char.each_with_object(+'') do |char, object| + if object.bytesize + char.bytesize > bytes + break object + else + object.concat(char) + end + end + + truncated + ('0' * (bytes - truncated.bytesize)) + end + # Append path to host, making sure there's one single / in between def append_path(host, path) "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}" diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb index a00cd65594c..5303b3582ab 100644 --- a/lib/gitlab/wiki_file_finder.rb +++ b/lib/gitlab/wiki_file_finder.rb @@ -2,6 +2,8 @@ module Gitlab class WikiFileFinder < FileFinder + BATCH_SIZE = 100 + attr_reader :repository def initialize(project, ref) @@ -12,13 +14,11 @@ module Gitlab private - def search_filenames(query, except) + def search_filenames(query) safe_query = Regexp.escape(query.tr(' ', '-')) safe_query = Regexp.new(safe_query, Regexp::IGNORECASE) filenames = repository.ls_files(ref) - filenames.delete_if { |filename| except.include?(filename) } unless except.empty? - filenames.grep(safe_query).first(BATCH_SIZE) end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e1f777e9cd1..da22ea9cf5c 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -13,6 +13,7 @@ module Gitlab INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'.freeze NOTIFICATION_CHANNEL = 'workhorse:notifications'.freeze ALLOWED_GIT_HTTP_ACTIONS = %w[git_receive_pack git_upload_pack info_refs].freeze + DETECT_HEADER = 'Gitlab-Workhorse-Detect-Content-Type'.freeze # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 # bytes https://tools.ietf.org/html/rfc4868#section-2.6 diff --git a/lib/omni_auth/strategies/jwt.rb b/lib/omni_auth/strategies/jwt.rb index a792903fde7..2f3d477a591 100644 --- a/lib/omni_auth/strategies/jwt.rb +++ b/lib/omni_auth/strategies/jwt.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'omniauth' +require 'openssl' require 'jwt' module OmniAuth @@ -37,7 +38,19 @@ module OmniAuth end def decoded - @decoded ||= ::JWT.decode(request.params['jwt'], options.secret, options.algorithm).first + secret = + case options.algorithm + when *%w[RS256 RS384 RS512] + OpenSSL::PKey::RSA.new(options.secret).public_key + when *%w[ES256 ES384 ES512] + OpenSSL::PKey::EC.new(options.secret).tap { |key| key.private_key = nil } + when *%w(HS256 HS384 HS512) + options.secret + else + raise NotImplementedError, "Unsupported algorithm: #{options.algorithm}" + end + + @decoded ||= ::JWT.decode(request.params['jwt'], secret, true, { algorithm: options.algorithm }).first (options.required_claims || []).each do |field| raise ClaimInvalid, "Missing required '#{field}' claim" unless @decoded.key?(field.to_s) @@ -45,7 +58,7 @@ module OmniAuth raise ClaimInvalid, "Missing required 'iat' claim" if options.valid_within && !@decoded["iat"] - if options.valid_within && (Time.now.to_i - @decoded["iat"]).abs > options.valid_within + if options.valid_within && (Time.now.to_i - @decoded["iat"]).abs > options.valid_within.to_i raise ClaimInvalid, "'iat' timestamp claim is too skewed from present" end diff --git a/lib/system_check/gitaly_check.rb b/lib/system_check/gitaly_check.rb new file mode 100644 index 00000000000..3d2517a7aca --- /dev/null +++ b/lib/system_check/gitaly_check.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module SystemCheck + class GitalyCheck < BaseCheck + set_name 'Gitaly:' + + def multi_check + Gitlab::HealthChecks::GitalyCheck.readiness.each do |result| + $stdout.print "#{result.labels[:shard]} ... " + + if result.success + $stdout.puts 'OK'.color(:green) + else + $stdout.puts "FAIL: #{result.message}".color(:red) + end + end + end + end +end diff --git a/lib/system_check/gitlab_shell_check.rb b/lib/system_check/gitlab_shell_check.rb new file mode 100644 index 00000000000..31c4ec33247 --- /dev/null +++ b/lib/system_check/gitlab_shell_check.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module SystemCheck + # Used by gitlab:gitlab_shell:check rake task + class GitlabShellCheck < BaseCheck + set_name 'GitLab Shell:' + + def multi_check + check_gitlab_shell + check_gitlab_shell_self_test + end + + private + + def check_gitlab_shell + required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required) + current_version = Gitlab::VersionInfo.parse(gitlab_shell_version) + + $stdout.print "GitLab Shell version >= #{required_version} ? ... " + if current_version.valid? && required_version <= current_version + $stdout.puts "OK (#{current_version})".color(:green) + else + $stdout.puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red) + end + end + + def check_gitlab_shell_self_test + gitlab_shell_repo_base = gitlab_shell_path + check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base) + $stdout.puts "Running #{check_cmd}" + + if system(check_cmd, chdir: gitlab_shell_repo_base) + $stdout.puts 'gitlab-shell self-check successful'.color(:green) + else + $stdout.puts 'gitlab-shell self-check failed'.color(:red) + try_fixing_it( + 'Make sure GitLab is running;', + 'Check the gitlab-shell configuration file:', + sudo_gitlab("editor #{File.expand_path('config.yml', gitlab_shell_repo_base)}") + ) + fix_and_rerun + end + end + + # Helper methods + ######################## + + def gitlab_shell_path + Gitlab.config.gitlab_shell.path + end + + def gitlab_shell_version + Gitlab::Shell.new.version + end + end +end diff --git a/lib/system_check/incoming_email_check.rb b/lib/system_check/incoming_email_check.rb new file mode 100644 index 00000000000..155b6547595 --- /dev/null +++ b/lib/system_check/incoming_email_check.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module SystemCheck + # Used by gitlab:incoming_email:check rake task + class IncomingEmailCheck < BaseCheck + set_name 'Incoming Email:' + + def multi_check + if Gitlab.config.incoming_email.enabled + checks = [ + SystemCheck::IncomingEmail::ImapAuthenticationCheck + ] + + if Rails.env.production? + checks << SystemCheck::IncomingEmail::InitdConfiguredCheck + checks << SystemCheck::IncomingEmail::MailRoomRunningCheck + else + checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck + end + + SystemCheck.run('Reply by email', checks) + else + $stdout.puts 'Reply by email is disabled in config/gitlab.yml' + end + end + end +end diff --git a/lib/system_check/ldap_check.rb b/lib/system_check/ldap_check.rb new file mode 100644 index 00000000000..619fb3cccb8 --- /dev/null +++ b/lib/system_check/ldap_check.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module SystemCheck + # Used by gitlab:ldap:check rake task + class LdapCheck < BaseCheck + set_name 'LDAP:' + + def multi_check + if Gitlab::Auth::LDAP::Config.enabled? + # Only show up to 100 results because LDAP directories can be very big. + # This setting only affects the `rake gitlab:check` script. + limit = ENV['LDAP_CHECK_LIMIT'] + limit = 100 if limit.blank? + + check_ldap(limit) + else + $stdout.puts 'LDAP is disabled in config/gitlab.yml' + end + end + + private + + def check_ldap(limit) + servers = Gitlab::Auth::LDAP::Config.providers + + servers.each do |server| + $stdout.puts "Server: #{server}" + + begin + Gitlab::Auth::LDAP::Adapter.open(server) do |adapter| + check_ldap_auth(adapter) + + $stdout.puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" + + users = adapter.users(adapter.config.uid, '*', limit) + users.each do |user| + $stdout.puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" + end + end + rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e + $stdout.puts "Could not connect to the LDAP server: #{e.message}".color(:red) + end + end + end + + def check_ldap_auth(adapter) + auth = adapter.config.has_auth? + + message = if auth && adapter.ldap.bind + 'Success'.color(:green) + elsif auth + 'Failed. Check `bind_dn` and `password` configuration values'.color(:red) + else + 'Anonymous. No `bind_dn` or `password` configured'.color(:yellow) + end + + $stdout.puts "LDAP authentication... #{message}" + end + end +end diff --git a/lib/system_check/orphans/repository_check.rb b/lib/system_check/orphans/repository_check.rb index ef8fe945f61..33020417e95 100644 --- a/lib/system_check/orphans/repository_check.rb +++ b/lib/system_check/orphans/repository_check.rb @@ -4,7 +4,6 @@ module SystemCheck module Orphans class RepositoryCheck < SystemCheck::BaseCheck set_name 'Orphaned repositories:' - attr_accessor :orphans def multi_check Gitlab::GitalyClient::StorageSettings.allow_disk_access do diff --git a/lib/system_check/rake_task/app_task.rb b/lib/system_check/rake_task/app_task.rb new file mode 100644 index 00000000000..cc32feb8604 --- /dev/null +++ b/lib/system_check/rake_task/app_task.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:app:check rake task + module AppTask + extend RakeTaskHelpers + + def self.name + 'GitLab App' + end + + def self.checks + [ + SystemCheck::App::GitConfigCheck, + SystemCheck::App::DatabaseConfigExistsCheck, + SystemCheck::App::MigrationsAreUpCheck, + SystemCheck::App::OrphanedGroupMembersCheck, + SystemCheck::App::GitlabConfigExistsCheck, + SystemCheck::App::GitlabConfigUpToDateCheck, + SystemCheck::App::LogWritableCheck, + SystemCheck::App::TmpWritableCheck, + SystemCheck::App::UploadsDirectoryExistsCheck, + SystemCheck::App::UploadsPathPermissionCheck, + SystemCheck::App::UploadsPathTmpPermissionCheck, + SystemCheck::App::InitScriptExistsCheck, + SystemCheck::App::InitScriptUpToDateCheck, + SystemCheck::App::ProjectsHaveNamespaceCheck, + SystemCheck::App::RedisVersionCheck, + SystemCheck::App::RubyVersionCheck, + SystemCheck::App::GitVersionCheck, + SystemCheck::App::GitUserDefaultSSHConfigCheck, + SystemCheck::App::ActiveUsersCheck + ] + end + end + end +end diff --git a/lib/system_check/rake_task/gitaly_task.rb b/lib/system_check/rake_task/gitaly_task.rb new file mode 100644 index 00000000000..0c3f694f98a --- /dev/null +++ b/lib/system_check/rake_task/gitaly_task.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:gitaly:check rake task + class GitalyTask + extend RakeTaskHelpers + + def self.name + 'Gitaly' + end + + def self.checks + [SystemCheck::GitalyCheck] + end + end + end +end diff --git a/lib/system_check/rake_task/gitlab_shell_task.rb b/lib/system_check/rake_task/gitlab_shell_task.rb new file mode 100644 index 00000000000..120e984c68b --- /dev/null +++ b/lib/system_check/rake_task/gitlab_shell_task.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:gitlab_shell:check rake task + class GitlabShellTask + extend RakeTaskHelpers + + def self.name + 'GitLab Shell' + end + + def self.checks + [SystemCheck::GitlabShellCheck] + end + end + end +end diff --git a/lib/system_check/rake_task/gitlab_task.rb b/lib/system_check/rake_task/gitlab_task.rb new file mode 100644 index 00000000000..7ff36fd6eb5 --- /dev/null +++ b/lib/system_check/rake_task/gitlab_task.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:check rake task + class GitlabTask + extend RakeTaskHelpers + + def self.name + 'GitLab' + end + + def self.manual_run_checks! + start_checking("#{name} subtasks") + + subtasks.each(&:run_checks!) + + finished_checking("#{name} subtasks") + end + + def self.subtasks + [ + SystemCheck::RakeTask::GitlabShellTask, + SystemCheck::RakeTask::GitalyTask, + SystemCheck::RakeTask::SidekiqTask, + SystemCheck::RakeTask::IncomingEmailTask, + SystemCheck::RakeTask::LdapTask, + SystemCheck::RakeTask::AppTask + ] + end + end + end +end diff --git a/lib/system_check/rake_task/incoming_email_task.rb b/lib/system_check/rake_task/incoming_email_task.rb new file mode 100644 index 00000000000..c296c46feab --- /dev/null +++ b/lib/system_check/rake_task/incoming_email_task.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:incoming_email:check rake task + class IncomingEmailTask + extend RakeTaskHelpers + + def self.name + 'Incoming Email' + end + + def self.checks + [SystemCheck::IncomingEmailCheck] + end + end + end +end diff --git a/lib/system_check/rake_task/ldap_task.rb b/lib/system_check/rake_task/ldap_task.rb new file mode 100644 index 00000000000..03a180b9dfb --- /dev/null +++ b/lib/system_check/rake_task/ldap_task.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:ldap:check rake task + class LdapTask + extend RakeTaskHelpers + + def self.name + 'LDAP' + end + + def self.checks + [SystemCheck::LdapCheck] + end + end + end +end diff --git a/lib/system_check/rake_task/orphans/namespace_task.rb b/lib/system_check/rake_task/orphans/namespace_task.rb new file mode 100644 index 00000000000..2822da45bc1 --- /dev/null +++ b/lib/system_check/rake_task/orphans/namespace_task.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + module Orphans + # Used by gitlab:orphans:check_namespaces rake task + class NamespaceTask + extend RakeTaskHelpers + + def self.name + 'Orphans' + end + + def self.checks + [SystemCheck::Orphans::NamespaceCheck] + end + end + end + end +end diff --git a/lib/system_check/rake_task/orphans/repository_task.rb b/lib/system_check/rake_task/orphans/repository_task.rb new file mode 100644 index 00000000000..f14b3af22e8 --- /dev/null +++ b/lib/system_check/rake_task/orphans/repository_task.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + module Orphans + # Used by gitlab:orphans:check_repositories rake task + class RepositoryTask + extend RakeTaskHelpers + + def self.name + 'Orphans' + end + + def self.checks + [SystemCheck::Orphans::RepositoryCheck] + end + end + end + end +end diff --git a/lib/system_check/rake_task/orphans_task.rb b/lib/system_check/rake_task/orphans_task.rb new file mode 100644 index 00000000000..31f8ede25e0 --- /dev/null +++ b/lib/system_check/rake_task/orphans_task.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:orphans:check rake task + class OrphansTask + extend RakeTaskHelpers + + def self.name + 'Orphans' + end + + def self.checks + [ + SystemCheck::Orphans::NamespaceCheck, + SystemCheck::Orphans::RepositoryCheck + ] + end + end + end +end diff --git a/lib/system_check/rake_task/rake_task_helpers.rb b/lib/system_check/rake_task/rake_task_helpers.rb new file mode 100644 index 00000000000..95f2a34a719 --- /dev/null +++ b/lib/system_check/rake_task/rake_task_helpers.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Provides the run! method intended to be called from system check rake tasks + module RakeTaskHelpers + include ::SystemCheck::Helpers + + def run! + warn_user_is_not_gitlab + + if self.respond_to?(:manual_run_checks!) + manual_run_checks! + else + run_checks! + end + end + + def run_checks! + SystemCheck.run(name, checks) + end + + def name + raise NotImplementedError + end + + def checks + raise NotImplementedError + end + end + end +end diff --git a/lib/system_check/rake_task/sidekiq_task.rb b/lib/system_check/rake_task/sidekiq_task.rb new file mode 100644 index 00000000000..3ccf009d4b9 --- /dev/null +++ b/lib/system_check/rake_task/sidekiq_task.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:sidekiq:check rake task + class SidekiqTask + extend RakeTaskHelpers + + def self.name + 'Sidekiq' + end + + def self.checks + [SystemCheck::SidekiqCheck] + end + end + end +end diff --git a/lib/system_check/sidekiq_check.rb b/lib/system_check/sidekiq_check.rb new file mode 100644 index 00000000000..2f5fc2cea89 --- /dev/null +++ b/lib/system_check/sidekiq_check.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module SystemCheck + # Used by gitlab:sidekiq:check rake task + class SidekiqCheck < BaseCheck + set_name 'Sidekiq:' + + def multi_check + check_sidekiq_running + only_one_sidekiq_running + end + + private + + def check_sidekiq_running + $stdout.print "Running? ... " + + if sidekiq_process_count > 0 + $stdout.puts "yes".color(:green) + else + $stdout.puts "no".color(:red) + try_fixing_it( + sudo_gitlab("RAILS_ENV=production bin/background_jobs start") + ) + for_more_information( + see_installation_guide_section("Install Init Script"), + "see log/sidekiq.log for possible errors" + ) + fix_and_rerun + end + end + + def only_one_sidekiq_running + process_count = sidekiq_process_count + return if process_count.zero? + + $stdout.print 'Number of Sidekiq processes ... ' + + if process_count == 1 + $stdout.puts '1'.color(:green) + else + $stdout.puts "#{process_count}".color(:red) + try_fixing_it( + 'sudo service gitlab stop', + "sudo pkill -u #{gitlab_user} -f sidekiq", + "sleep 10 && sudo pkill -9 -u #{gitlab_user} -f sidekiq", + 'sudo service gitlab start' + ) + fix_and_rerun + end + end + + def sidekiq_process_count + ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) + ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count + end + end +end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index a497d26312e..2235a6ba194 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -82,7 +82,7 @@ namespace :gettext do # `gettext:find` writes touches to temp files to `stderr` which would cause # `static-analysis` to report failures. We can ignore these. - silence_sdterr do + silence_stderr do Rake::Task['gettext:find'].invoke end @@ -119,7 +119,7 @@ namespace :gettext do end end - def silence_sdterr(&block) + def silence_stderr(&block) old_stderr = $stderr.dup $stderr.reopen(File::NULL) $stderr.sync = true diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index a2c3e32948f..b594f150c3b 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -1,299 +1,66 @@ namespace :gitlab do desc 'GitLab | Check the configuration of GitLab and its environment' - task check: %w{gitlab:gitlab_shell:check - gitlab:gitaly:check - gitlab:sidekiq:check - gitlab:incoming_email:check - gitlab:ldap:check - gitlab:app:check} + task check: :gitlab_environment do + SystemCheck::RakeTask::GitlabTask.run! + end namespace :app do desc 'GitLab | Check the configuration of the GitLab Rails app' task check: :gitlab_environment do - warn_user_is_not_gitlab - - checks = [ - SystemCheck::App::GitConfigCheck, - SystemCheck::App::DatabaseConfigExistsCheck, - SystemCheck::App::MigrationsAreUpCheck, - SystemCheck::App::OrphanedGroupMembersCheck, - SystemCheck::App::GitlabConfigExistsCheck, - SystemCheck::App::GitlabConfigUpToDateCheck, - SystemCheck::App::LogWritableCheck, - SystemCheck::App::TmpWritableCheck, - SystemCheck::App::UploadsDirectoryExistsCheck, - SystemCheck::App::UploadsPathPermissionCheck, - SystemCheck::App::UploadsPathTmpPermissionCheck, - SystemCheck::App::InitScriptExistsCheck, - SystemCheck::App::InitScriptUpToDateCheck, - SystemCheck::App::ProjectsHaveNamespaceCheck, - SystemCheck::App::RedisVersionCheck, - SystemCheck::App::RubyVersionCheck, - SystemCheck::App::GitVersionCheck, - SystemCheck::App::GitUserDefaultSSHConfigCheck, - SystemCheck::App::ActiveUsersCheck - ] - - SystemCheck.run('GitLab', checks) + SystemCheck::RakeTask::AppTask.run! end end namespace :gitlab_shell do desc "GitLab | Check the configuration of GitLab Shell" task check: :gitlab_environment do - warn_user_is_not_gitlab - start_checking "GitLab Shell" - - check_gitlab_shell - check_gitlab_shell_self_test - - finished_checking "GitLab Shell" - end - - # Checks - ######################## - - def check_gitlab_shell_self_test - gitlab_shell_repo_base = gitlab_shell_path - check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base) - puts "Running #{check_cmd}" - - if system(check_cmd, chdir: gitlab_shell_repo_base) - puts 'gitlab-shell self-check successful'.color(:green) - else - puts 'gitlab-shell self-check failed'.color(:red) - try_fixing_it( - 'Make sure GitLab is running;', - 'Check the gitlab-shell configuration file:', - sudo_gitlab("editor #{File.expand_path('config.yml', gitlab_shell_repo_base)}") - ) - fix_and_rerun - end - end - - # Helper methods - ######################## - - def gitlab_shell_path - Gitlab.config.gitlab_shell.path - end - - def gitlab_shell_version - Gitlab::Shell.new.version - end - - def gitlab_shell_major_version - Gitlab::Shell.version_required.split('.')[0].to_i - end - - def gitlab_shell_minor_version - Gitlab::Shell.version_required.split('.')[1].to_i - end - - def gitlab_shell_patch_version - Gitlab::Shell.version_required.split('.')[2].to_i + SystemCheck::RakeTask::GitlabShellTask.run! end end namespace :gitaly do desc 'GitLab | Check the health of Gitaly' task check: :gitlab_environment do - warn_user_is_not_gitlab - start_checking 'Gitaly' - - Gitlab::HealthChecks::GitalyCheck.readiness.each do |result| - print "#{result.labels[:shard]} ... " - - if result.success - puts 'OK'.color(:green) - else - puts "FAIL: #{result.message}".color(:red) - end - end - - finished_checking 'Gitaly' + SystemCheck::RakeTask::GitalyTask.run! end end namespace :sidekiq do desc "GitLab | Check the configuration of Sidekiq" task check: :gitlab_environment do - warn_user_is_not_gitlab - start_checking "Sidekiq" - - check_sidekiq_running - only_one_sidekiq_running - - finished_checking "Sidekiq" - end - - # Checks - ######################## - - def check_sidekiq_running - print "Running? ... " - - if sidekiq_process_count > 0 - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - sudo_gitlab("RAILS_ENV=production bin/background_jobs start") - ) - for_more_information( - see_installation_guide_section("Install Init Script"), - "see log/sidekiq.log for possible errors" - ) - fix_and_rerun - end - end - - def only_one_sidekiq_running - process_count = sidekiq_process_count - return if process_count.zero? - - print 'Number of Sidekiq processes ... ' - - if process_count == 1 - puts '1'.color(:green) - else - puts "#{process_count}".color(:red) - try_fixing_it( - 'sudo service gitlab stop', - "sudo pkill -u #{gitlab_user} -f sidekiq", - "sleep 10 && sudo pkill -9 -u #{gitlab_user} -f sidekiq", - 'sudo service gitlab start' - ) - fix_and_rerun - end - end - - def sidekiq_process_count - ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) - ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count + SystemCheck::RakeTask::SidekiqTask.run! end end namespace :incoming_email do desc "GitLab | Check the configuration of Reply by email" task check: :gitlab_environment do - warn_user_is_not_gitlab - - if Gitlab.config.incoming_email.enabled - checks = [ - SystemCheck::IncomingEmail::ImapAuthenticationCheck - ] - - if Rails.env.production? - checks << SystemCheck::IncomingEmail::InitdConfiguredCheck - checks << SystemCheck::IncomingEmail::MailRoomRunningCheck - else - checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck - end - - SystemCheck.run('Reply by email', checks) - else - puts 'Reply by email is disabled in config/gitlab.yml' - end + SystemCheck::RakeTask::IncomingEmailTask.run! end end namespace :ldap do task :check, [:limit] => :gitlab_environment do |_, args| - # Only show up to 100 results because LDAP directories can be very big. - # This setting only affects the `rake gitlab:check` script. - args.with_defaults(limit: 100) - warn_user_is_not_gitlab - start_checking "LDAP" - - if Gitlab::Auth::LDAP::Config.enabled? - check_ldap(args.limit) - else - puts 'LDAP is disabled in config/gitlab.yml' - end - - finished_checking "LDAP" - end - - def check_ldap(limit) - servers = Gitlab::Auth::LDAP::Config.providers - - servers.each do |server| - puts "Server: #{server}" + ENV['LDAP_CHECK_LIMIT'] = args.limit if args.limit.present? - begin - Gitlab::Auth::LDAP::Adapter.open(server) do |adapter| - check_ldap_auth(adapter) - - puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" - - users = adapter.users(adapter.config.uid, '*', limit) - users.each do |user| - puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" - end - end - rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e - puts "Could not connect to the LDAP server: #{e.message}".color(:red) - end - end - end - - def check_ldap_auth(adapter) - auth = adapter.config.has_auth? - - message = if auth && adapter.ldap.bind - 'Success'.color(:green) - elsif auth - 'Failed. Check `bind_dn` and `password` configuration values'.color(:red) - else - 'Anonymous. No `bind_dn` or `password` configured'.color(:yellow) - end - - puts "LDAP authentication... #{message}" + SystemCheck::RakeTask::LdapTask.run! end end namespace :orphans do desc 'Gitlab | Check for orphaned namespaces and repositories' task check: :gitlab_environment do - warn_user_is_not_gitlab - checks = [ - SystemCheck::Orphans::NamespaceCheck, - SystemCheck::Orphans::RepositoryCheck - ] - - SystemCheck.run('Orphans', checks) + SystemCheck::RakeTask::OrphansTask.run! end desc 'GitLab | Check for orphaned namespaces in the repositories path' task check_namespaces: :gitlab_environment do - warn_user_is_not_gitlab - checks = [SystemCheck::Orphans::NamespaceCheck] - - SystemCheck.run('Orphans', checks) + SystemCheck::RakeTask::Orphans::NamespaceTask.run! end desc 'GitLab | Check for orphaned repositories in the repositories path' task check_repositories: :gitlab_environment do - warn_user_is_not_gitlab - checks = [SystemCheck::Orphans::RepositoryCheck] - - SystemCheck.run('Orphans', checks) - end - end - - # Helper methods - ########################## - - def check_gitlab_shell - required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version) - current_version = Gitlab::VersionInfo.parse(gitlab_shell_version) - - print "GitLab Shell version >= #{required_version} ? ... " - if current_version.valid? && required_version <= current_version - puts "OK (#{current_version})".color(:green) - else - puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red) + SystemCheck::RakeTask::Orphans::RepositoryTask.run! end end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index e8ae5dfa540..451ba651674 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -6,7 +6,7 @@ namespace :gitlab do desc "GitLab | Cleanup | Clean namespaces" task dirs: :gitlab_environment do namespaces = Set.new(Namespace.pluck(:path)) - namespaces << Storage::HashedProject::ROOT_PATH_PREFIX + namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX Gitaly::Server.all.each do |server| all_dirs = Gitlab::GitalyClient::StorageService @@ -49,7 +49,7 @@ namespace :gitlab do # TODO ignoring hashed repositories for now. But revisit to fully support # possible orphaned hashed repos - next if repo_with_namespace.start_with?(Storage::HashedProject::ROOT_PATH_PREFIX) + next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX) next if Project.find_by_full_path(repo_with_namespace) new_path = path + move_suffix diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake index 5a1c8006052..15cec80b6a6 100644 --- a/lib/tasks/gitlab/web_hook.rake +++ b/lib/tasks/gitlab/web_hook.rake @@ -25,11 +25,22 @@ namespace :gitlab do web_hook_url = ENV['URL'] namespace_path = ENV['NAMESPACE'] - projects = find_projects(namespace_path) - project_ids = projects.pluck(:id) + web_hooks = find_web_hooks(namespace_path) puts "Removing webhooks with the url '#{web_hook_url}' ... " - count = WebHook.where(url: web_hook_url, project_id: project_ids, type: 'ProjectHook').delete_all + + # FIXME: Hook URLs are now encrypted, so there is no way to efficiently + # find them all in SQL. For now, check them in Ruby. If this is too slow, + # we could consider storing a hash of the URL alongside the encrypted + # value to speed up searches + count = 0 + web_hooks.find_each do |hook| + next unless hook.url == web_hook_url + + hook.destroy! + count += 1 + end + puts "#{count} webhooks were removed." end @@ -37,29 +48,37 @@ namespace :gitlab do task list: :environment do namespace_path = ENV['NAMESPACE'] - projects = find_projects(namespace_path) - web_hooks = projects.all.map(&:hooks).flatten - web_hooks.each do |hook| + web_hooks = find_web_hooks(namespace_path) + web_hooks.find_each do |hook| puts "#{hook.project.name.truncate(20).ljust(20)} -> #{hook.url}" end - puts "\n#{web_hooks.size} webhooks found." + puts "\n#{web_hooks.count} webhooks found." end end def find_projects(namespace_path) if namespace_path.blank? Project - elsif namespace_path == '/' - Project.in_namespace(nil) else - namespace = Namespace.where(path: namespace_path).first - if namespace - Project.in_namespace(namespace.id) - else + namespace = Namespace.find_by_full_path(namespace_path) + + unless namespace puts "Namespace not found: #{namespace_path}".color(:red) exit 2 end + + Project.in_namespace(namespace.id) + end + end + + def find_web_hooks(namespace_path) + if namespace_path.blank? + ProjectHook + else + project_ids = find_projects(namespace_path).select(:id) + + ProjectHook.where(project_id: project_ids) end end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index a16d4c47273..f912f521dfb 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -42,7 +42,7 @@ class GithubImport end def import! - @project.force_import_start + @project.import_state.force_start import_success = false @@ -57,7 +57,7 @@ class GithubImport puts "Import finished. Timings: #{timings}".color(:green) else puts "Import was not successful. Errors were as follows:" - puts @project.import_error + puts @project.import_state.last_error end end |