diff options
| author | Filipa Lacerda <filipa@gitlab.com> | 2018-10-08 10:40:10 +0100 |
|---|---|---|
| committer | Filipa Lacerda <filipa@gitlab.com> | 2018-10-08 10:40:10 +0100 |
| commit | fa875ba7a9441df6827ef1d6b05405c66ee0c579 (patch) | |
| tree | 23d0cf911c9bf6a73fec9bb1f3de1bf61bedeacd /lib | |
| parent | ecefe090460687a078e3d1aacf621fd5bff07fb5 (diff) | |
| parent | 838c1076694d24d180e19625d663749c8b5c1a1c (diff) | |
| download | gitlab-ce-fa875ba7a9441df6827ef1d6b05405c66ee0c579.tar.gz | |
Merge branch 'master' into 42611-removed-branch-link
* master: (1252 commits)
Render log artifact files in GitLab
Check disabled_services when finding a service
Fix invalid parent path on group settings page
Backport CE changes for: [Frontend only] Batch comments on merge requests
Add button to insert table in markdown editor
Update GITALY_SERVER_VERSION
Updates Laravel.gitlab-ci.yml template
Update operations metrics empty state
Fix LFS uploaded images not being rendered
Prepare admin/projects/show view to allow EE specific feature
Add timed incremental rollout to Auto DevOps
Update spec comment to point to correct issue
Fix documentation for variables
Document Security and Licence Management features permissions
Fix time dependent jobs spec
Use a CTE to remove the query timeout
Backport changes from gitlab-ee!7538
Fix CE to EE merge (backport)
Add changelog entry
Refactor Feature.flipper method
...
Diffstat (limited to 'lib')
418 files changed, 6857 insertions, 2234 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index ae13c248171..cecff6d3b81 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class AccessRequests < Grape::API include PaginationParams @@ -18,6 +20,7 @@ module API params do use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/access_requests" do source = find_source(source_type, params[:id]) @@ -26,6 +29,7 @@ module API present access_requesters, with: Entities::AccessRequester end + # rubocop: enable CodeReuse/ActiveRecord desc "Requests access for the authenticated user to a #{source_type}." do detail 'This feature was introduced in GitLab 8.11.' @@ -50,6 +54,7 @@ module API requires :user_id, type: Integer, desc: 'The user ID of the access requester' optional :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' end + # rubocop: disable CodeReuse/ActiveRecord put ':id/access_requests/:user_id/approve' do source = find_source(source_type, params[:id]) @@ -61,6 +66,7 @@ module API status :created present member, with: Entities::Member end + # rubocop: enable CodeReuse/ActiveRecord desc 'Denies an access request for the given user.' do detail 'This feature was introduced in GitLab 8.11.' @@ -68,6 +74,7 @@ module API params do requires :user_id, type: Integer, desc: 'The user ID of the access requester' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/access_requests/:user_id" do source = find_source(source_type, params[:id]) member = source.requesters.find_by!(user_id: params[:user_id]) @@ -76,6 +83,7 @@ module API ::Members::DestroyService.new(current_user).execute(member) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/api.rb b/lib/api/api.rb index c000666d992..c49c52213bf 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class API < Grape::API include APIGuard @@ -15,8 +17,10 @@ module API include: [ GrapeLogging::Loggers::FilterParameters.new, GrapeLogging::Loggers::ClientEnv.new, + Gitlab::GrapeLogging::Loggers::RouteLogger.new, Gitlab::GrapeLogging::Loggers::UserLogger.new, - Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new + Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new, + Gitlab::GrapeLogging::Loggers::PerfLogger.new ] allow_access_with_scope :api @@ -116,6 +120,7 @@ module API mount ::API::Namespaces mount ::API::Notes mount ::API::Discussions + mount ::API::ResourceLabelEvents mount ::API::NotificationSettings mount ::API::PagesDomains mount ::API::Pipelines @@ -127,6 +132,7 @@ module API mount ::API::Projects mount ::API::ProjectSnapshots mount ::API::ProjectSnippets + mount ::API::ProjectTemplates mount ::API::ProtectedBranches mount ::API::ProtectedTags mount ::API::Repositories diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 8ee7987cfff..61357b3f1d6 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Guard API with OAuth 2.0 Access Token require 'rack/oauth2' diff --git a/lib/api/applications.rb b/lib/api/applications.rb index b122cdefe4e..f29cd7fc003 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # External applications API class Applications < Grape::API diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb index 70219bc8ea0..0f14d003065 100644 --- a/lib/api/avatar.rb +++ b/lib/api/avatar.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Avatar < Grape::API resource :avatar do diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index bde4b3ff4f6..c2abf9155f3 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class AwardEmoji < Grape::API include PaginationParams @@ -103,6 +105,7 @@ module API awardable.user_can_award?(current_user) end + # rubocop: disable CodeReuse/ActiveRecord def awardable @awardable ||= begin @@ -119,6 +122,7 @@ module API end end end + # rubocop: enable CodeReuse/ActiveRecord def read_ability(awardable) case awardable diff --git a/lib/api/badges.rb b/lib/api/badges.rb index 8ceffe9c5ef..ab670988f47 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Badges < Grape::API include PaginationParams diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 0f89414148b..c80e1c57864 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Boards < Grape::API include BoardsResponses diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb index 7e873012efe..86d9b24802f 100644 --- a/lib/api/boards_responses.rb +++ b/lib/api/boards_responses.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module BoardsResponses extend ActiveSupport::Concern @@ -49,11 +51,13 @@ module API end end + # rubocop: disable CodeReuse/ActiveRecord def authorize_list_type_resource! unless available_labels_for(board_parent).exists?(params[:label_id]) render_api_error!({ error: 'Label not found!' }, 400) end end + # rubocop: enable CodeReuse/ActiveRecord params :list_creation_params do requires :label_id, type: Integer, desc: 'The ID of an existing label' diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 3e445e6b1fa..2735d410c8e 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mime/types' module API @@ -9,14 +11,6 @@ module API before { authorize! :download_code, user_project } helpers do - def find_branch!(branch_name) - begin - user_project.repository.find_branch(branch_name) || not_found!('Branch') - rescue Gitlab::Git::CommandError - render_api_error!('The branch refname is invalid', 400) - end - end - params :filter_params do optional :search, type: String, desc: 'Return list of branches matching the search criteria' optional :sort, type: String, desc: 'Return list of branches sorted by the given field' @@ -77,10 +71,11 @@ module API success Entities::Branch end params do - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch' optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch' end + # rubocop: disable CodeReuse/ActiveRecord put ':id/repository/branches/:branch/protect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project @@ -108,14 +103,16 @@ module API render_api_error!(protected_branch.errors.full_messages, 422) end end + # rubocop: enable CodeReuse/ActiveRecord # Note: This API will be deprecated in favor of the protected branches API. desc 'Unprotect a single branch' do success Entities::Branch end params do - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false end + # rubocop: disable CodeReuse/ActiveRecord put ':id/repository/branches/:branch/unprotect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project @@ -125,13 +122,14 @@ module API present branch, with: Entities::Branch, current_user: current_user, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord desc 'Create branch' do success Entities::Branch end params do - requires :branch, type: String, desc: 'The name of the branch' - requires :ref, type: String, desc: 'Create branch from commit sha or existing branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false + requires :ref, type: String, desc: 'Create branch from commit sha or existing branch', allow_blank: false end post ':id/repository/branches' do authorize_push_project @@ -151,7 +149,7 @@ module API desc 'Delete a branch' params do - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false end delete ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_push_project diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index d7138b2f2fe..19148758fc5 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class BroadcastMessages < Grape::API include PaginationParams diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb index c13154dc0ec..6eddc5e5b61 100644 --- a/lib/api/circuit_breakers.rb +++ b/lib/api/circuit_breakers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class CircuitBreakers < Grape::API before { authenticated_as_admin! } diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 829eef18795..99553d993ca 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mime/types' module API @@ -21,6 +23,7 @@ module API optional :all, type: String, desc: 'Show all statuses, default: false' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/repository/commits/:sha/statuses' do authorize!(:read_commit_status, user_project) @@ -34,6 +37,7 @@ module API statuses = statuses.where(name: params[:name]) if params[:name].present? present paginate(statuses), with: Entities::CommitStatus end + # rubocop: enable CodeReuse/ActiveRecord desc 'Post status to a commit' do success Entities::CommitStatus @@ -49,6 +53,7 @@ module API optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :coverage, type: Float, desc: 'The total code coverage' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/statuses/:sha' do authorize! :create_commit_status, user_project @@ -118,6 +123,7 @@ module API render_api_error!(e.message, 400) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 92329465b2c..e59abd3e3d0 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mime/types' module API @@ -71,12 +73,32 @@ module API detail 'This feature was introduced in GitLab 8.13' end params do - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.' + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false requires :commit_message, type: String, desc: 'Commit message' - requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' + requires :actions, type: Array, desc: 'Actions to perform in commit' do + requires :action, type: String, desc: 'The action to perform, `create`, `delete`, `move`, `update`, `chmod`', values: %w[create update move delete chmod].freeze + requires :file_path, type: String, desc: 'Full path to the file. Ex. `lib/class.rb`' + given action: ->(action) { action == 'move' } do + requires :previous_path, type: String, desc: 'Original full path to the file being moved. Ex. `lib/class1.rb`' + end + given action: ->(action) { %w[create move].include? action } do + optional :content, type: String, desc: 'File content' + end + given action: ->(action) { action == 'update' } do + requires :content, type: String, desc: 'File content' + end + optional :encoding, type: String, desc: '`text` or `base64`', default: 'text', values: %w[text base64] + given action: ->(action) { %w[update move delete].include? action } do + optional :last_commit_id, type: String, desc: 'Last known file commit id' + end + given action: ->(action) { action == 'chmod' } do + requires :execute_filemode, type: Boolean, desc: 'When `true/false` enables/disables the execute flag on the file.' + end + end optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' + optional :stats, type: Boolean, default: true, desc: 'Include commit stats' end post ':id/repository/commits' do authorize_push_to_branch!(params[:branch]) @@ -89,7 +111,10 @@ module API if result[:status] == :success commit_detail = user_project.repository.commit(result[:result]) - present commit_detail, with: Entities::CommitDetail + + Gitlab::WebIdeCommitsCounter.increment if find_user_from_warden + + present commit_detail, with: Entities::CommitDetail, stats: params[:stats] else render_api_error!(result[:message], 400) end @@ -136,6 +161,7 @@ module API use :pagination requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) @@ -144,6 +170,7 @@ module API present paginate(notes), with: Entities::CommitNote end + # rubocop: enable CodeReuse/ActiveRecord desc 'Cherry pick commit into a branch' do detail 'This feature was introduced in GitLab 8.15' @@ -151,7 +178,7 @@ module API end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked' - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false end post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do authorize_push_to_branch!(params[:branch]) @@ -159,8 +186,7 @@ module API commit = user_project.commit(params[:sha]) not_found!('Commit') unless commit - branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless branch + find_branch!(params[:branch]) commit_params = { commit: commit, @@ -171,7 +197,7 @@ module API result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute if result[:status] == :success - branch = user_project.repository.find_branch(params[:branch]) + branch = find_branch!(params[:branch]) present user_project.repository.commit(branch.dereferenced_target), with: Entities::Commit else render_api_error!(result[:message], 400) diff --git a/lib/api/custom_attributes_endpoints.rb b/lib/api/custom_attributes_endpoints.rb index 5000aa0d9ac..2149e04451e 100644 --- a/lib/api/custom_attributes_endpoints.rb +++ b/lib/api/custom_attributes_endpoints.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module CustomAttributesEndpoints extend ActiveSupport::Concern @@ -30,6 +32,7 @@ module API params do use :custom_attributes_key end + # rubocop: disable CodeReuse/ActiveRecord get ':id/custom_attributes/:key' do resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend authorize! :read_custom_attribute @@ -38,12 +41,14 @@ module API present custom_attribute, with: Entities::CustomAttribute end + # rubocop: enable CodeReuse/ActiveRecord desc "Set a custom attribute on a #{attributable_name}" params do use :custom_attributes_key requires :value, type: String, desc: 'The value of the custom attribute' end + # rubocop: disable CodeReuse/ActiveRecord put ':id/custom_attributes/:key' do resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend authorize! :update_custom_attribute @@ -59,11 +64,13 @@ module API render_validation_error!(custom_attribute) end end + # rubocop: enable CodeReuse/ActiveRecord desc "Delete a custom attribute on a #{attributable_name}" params do use :custom_attributes_key end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/custom_attributes/:key' do resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend authorize! :update_custom_attribute @@ -72,6 +79,7 @@ module API status 204 end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 6769855b899..ce35720d408 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class DeployKeys < Grape::API include PaginationParams @@ -9,9 +11,11 @@ module API project.deploy_keys_projects.create(attrs) end + # rubocop: disable CodeReuse/ActiveRecord def find_by_deploy_key(project, key_id) project.deploy_keys_projects.find_by!(deploy_key: key_id) end + # rubocop: enable CodeReuse/ActiveRecord end desc 'Return all deploy keys' @@ -36,11 +40,13 @@ module API params do use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/deploy_keys" do keys = user_project.deploy_keys_projects.preload(:deploy_key) present paginate(keys), with: Entities::DeployKeysProject end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get single deploy key' do success Entities::DeployKeysProject @@ -62,6 +68,7 @@ module API requires :title, type: String, desc: 'The name of the deploy key' optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" end + # rubocop: disable CodeReuse/ActiveRecord post ":id/deploy_keys" do params[:key].strip! @@ -94,6 +101,7 @@ module API render_validation_error!(deploy_key_project) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Update an existing deploy key for a project' do success Entities::SSHKey @@ -147,12 +155,14 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the deploy key' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/deploy_keys/:key_id" do deploy_key_project = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) not_found!('Deploy Key') unless deploy_key_project destroy_conditionally!(deploy_key_project) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 184fae0eb76..6747e2e5005 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Deployments RESTful API endpoints class Deployments < Grape::API @@ -18,11 +20,13 @@ module API optional :order_by, type: String, values: %w[id iid created_at ref], default: 'id', desc: 'Return deployments ordered by `id` or `iid` or `created_at` or `ref`' optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/deployments' do authorize! :read_deployment, user_project present paginate(user_project.deployments.order(params[:order_by] => params[:sort])), with: Entities::Deployment end + # rubocop: enable CodeReuse/ActiveRecord desc 'Gets a specific deployment' do detail 'This feature was introduced in GitLab 8.11.' diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 13c34e3473a..39c6d28391d 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Discussions < Grape::API include PaginationParams @@ -23,6 +25,7 @@ module API requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/#{noteables_path}/:noteable_id/discussions" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) @@ -36,6 +39,7 @@ module API present paginate(discussions), with: Entities::Discussion end + # rubocop: enable CodeReuse/ActiveRecord desc "Get a single #{noteable_type.to_s.downcase} discussion" do success Entities::Discussion @@ -219,6 +223,7 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def readable_discussion_notes(noteable, discussion_id) notes = noteable.notes .where(discussion_id: discussion_id) @@ -228,6 +233,7 @@ module API notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 95b25d7351a..120545792f2 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Entities class WikiPageBasic < Grape::Entity @@ -10,6 +12,28 @@ module API expose :content end + class WikiAttachment < Grape::Entity + include Gitlab::FileMarkdownLinkBuilder + + expose :file_name + expose :file_path + expose :branch + expose :link do + expose :file_path, as: :url + expose :markdown do |_entity| + self.markdown_link + end + end + + def filename + object.file_name + end + + def secure_url + object.file_path + end + end + class UserSafe < Grape::Entity expose :id, :name, :username end @@ -31,7 +55,7 @@ module API class User < UserBasic expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } - expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization + expose :bio, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization end class UserActivity < Grape::Entity @@ -83,6 +107,7 @@ module API expose :project_id, :issues_events, :confidential_issues_events expose :note_events, :confidential_note_events, :pipeline_events, :wiki_page_events expose :job_events + expose :push_events_branch_filter end class SharedGroup < Grape::Entity @@ -145,6 +170,7 @@ module API expose :namespace, using: 'API::Entities::NamespaceBasic' expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes + # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) # Preloading tags, should be done with using only `:tags`, # as `:tags` are defined as: `has_many :tags, through: :taggings` @@ -154,6 +180,7 @@ module API .preload(:import_state, :tags) .preload(namespace: [:route, :owner]) end + # rubocop: enable CodeReuse/ActiveRecord end class Project < BasicProjectDetails @@ -224,6 +251,7 @@ module API expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics + # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) # Preloading tags, should be done with using only `:tags`, # as `:tags` are defined as: `has_many :tags, through: :taggings` @@ -235,6 +263,7 @@ module API forked_project_link: :forked_from_project, forked_from_project: [:route, :forks, :tags, namespace: :route]) end + # rubocop: enable CodeReuse/ActiveRecord def self.forks_counting_projects(projects_relation) projects_relation + projects_relation.map(&:forked_from_project).compact @@ -370,6 +399,10 @@ module API expose :can_push do |repo_branch, options| Gitlab::UserAccess.new(options[:current_user], project: options[:project]).can_push_to_branch?(repo_branch.name) end + + expose :default do |repo_branch, options| + options[:project].default_branch == repo_branch.name + end end class TreeObject < Grape::Entity @@ -531,10 +564,12 @@ module API expose :total_time_spent, as: :human_total_time_spent end + # rubocop: disable CodeReuse/ActiveRecord def total_time_spent # Avoids an N+1 query since timelogs are preloaded object.timelogs.map(&:time_spent).sum end + # rubocop: enable CodeReuse/ActiveRecord end class ExternalIssue < Grape::Entity @@ -661,6 +696,8 @@ module API expose :diff_refs, using: Entities::DiffRefs + expose :diverged_commits_count, as: :diverged_commits_count, if: -> (_, options) { options[:include_diverged_commits_count] } + def build_available?(options) options[:project]&.feature_available?(:builds, options[:current_user]) end @@ -907,6 +944,7 @@ module API end end + # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) relation = super(projects_relation, options) @@ -931,6 +969,7 @@ module API relation end + # rubocop: enable CodeReuse/ActiveRecord end class LabelBasic < Grape::Entity @@ -1037,9 +1076,11 @@ module API options[:project].repository.commit(repo_tag.dereferenced_target) end + # rubocop: disable CodeReuse/ActiveRecord expose :release, using: Entities::Release do |repo_tag, options| options[:project].releases.find_by(tag: repo_tag.name) end + # rubocop: enable CodeReuse/ActiveRecord end class Runner < Grape::Entity @@ -1062,6 +1103,7 @@ module API expose :version, :revision, :platform, :architecture expose :contacted_at expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.instance_type? } + # rubocop: disable CodeReuse/ActiveRecord expose :projects, with: Entities::BasicProjectDetails do |runner, options| if options[:current_user].admin? runner.projects @@ -1069,6 +1111,8 @@ module API options[:current_user].authorized_projects.where(id: runner.projects) end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord expose :groups, with: Entities::BasicGroupDetails do |runner, options| if options[:current_user].admin? runner.groups @@ -1076,6 +1120,7 @@ module API options[:current_user].authorized_groups.where(id: runner.groups) end end + # rubocop: enable CodeReuse/ActiveRecord end class RunnerRegistrationDetails < Grape::Entity @@ -1176,6 +1221,7 @@ module API end class TemplatesList < Grape::Entity + expose :key expose :name end @@ -1408,5 +1454,19 @@ module API badge.type == 'ProjectBadge' ? 'project' : 'group' end end + + class ResourceLabelEvent < Grape::Entity + expose :id + expose :user, using: Entities::UserBasic + expose :created_at + expose :resource_type do |event, options| + event.issuable.class.name + end + expose :resource_id do |event, options| + event.issuable.id + end + expose :label, using: Entities::LabelBasic + expose :action + end end end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index fa828f43001..c64217a6977 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Environments RESTfull API endpoints class Environments < Grape::API diff --git a/lib/api/events.rb b/lib/api/events.rb index a415508a632..6e0b508be19 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Events < Grape::API include PaginationParams @@ -16,12 +18,29 @@ module API desc: 'Return events sorted in ascending and descending order' end - def present_events(events) + RedactedEvent = OpenStruct.new(target_title: 'Confidential event').freeze + + def redact_events(events) + events.map do |event| + if event.visible_to_user?(current_user) + event + else + RedactedEvent + end + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def present_events(events, redact: true) events = events.reorder(created_at: params[:sort]) .with_associations - present paginate(events), with: Entities::Event + events = paginate(events) + events = redact_events(events) if redact + + present events, with: Entities::Event end + # rubocop: enable CodeReuse/ActiveRecord end resource :events do @@ -36,13 +55,16 @@ module API use :event_filter_params use :sort_params end + # rubocop: disable CodeReuse/ActiveRecord get do authenticate! events = EventsFinder.new(params.merge(source: current_user, current_user: current_user)).execute.preload(:author, :target) - present_events(events) + # Since we're viewing our own events, redaction is unnecessary + present_events(events, redact: false) end + # rubocop: enable CodeReuse/ActiveRecord end params do @@ -60,6 +82,7 @@ module API use :event_filter_params use :sort_params end + # rubocop: disable CodeReuse/ActiveRecord get ':id/events' do user = find_user(params[:id]) not_found!('User') unless user @@ -68,6 +91,7 @@ module API present_events(events) end + # rubocop: enable CodeReuse/ActiveRecord end params do @@ -82,11 +106,13 @@ module API use :event_filter_params use :sort_params end + # rubocop: disable CodeReuse/ActiveRecord get ":id/events" do events = EventsFinder.new(params.merge(source: user_project, current_user: current_user)).execute.preload(:author, :target) present_events(events) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/features.rb b/lib/api/features.rb index 11d848584d9..6f2422af13a 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Features < Grape::API before { authenticated_as_admin! } @@ -14,6 +16,7 @@ module API end end + # rubocop: disable CodeReuse/ActiveRecord def gate_targets(params) targets = [] targets << Feature.group(params[:feature_group]) if params[:feature_group] @@ -21,6 +24,7 @@ module API targets end + # rubocop: enable CodeReuse/ActiveRecord end resource :features do diff --git a/lib/api/files.rb b/lib/api/files.rb index ff4f75c12df..bcd2cd48a45 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Files < Grape::API FILE_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX) @@ -58,7 +60,7 @@ module API params :simple_file_params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.' + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false requires :commit_message, type: String, allow_blank: false, desc: 'Commit message' optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' optional :author_email, type: String, desc: 'The email of the author' @@ -80,7 +82,7 @@ module API desc 'Get raw file metadata from repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! @@ -91,7 +93,7 @@ module API desc 'Get raw file contents from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag commit' + requires :ref, type: String, desc: 'The name of branch, tag commit', allow_blank: false end get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! @@ -104,7 +106,7 @@ module API desc 'Get file metadata from repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end head ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! @@ -115,7 +117,7 @@ module API desc 'Get a file from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index 3832cdc10a8..dc30e868e2e 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class GroupBoards < Grape::API include BoardsResponses diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index 4b4352c2b27..b36436dbf43 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class GroupMilestones < Grape::API include MilestoneResponses diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 55d5c7f1606..ae7241e9a30 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class GroupVariables < Grape::API include PaginationParams @@ -27,6 +29,7 @@ module API params do requires :key, type: String, desc: 'The key of the variable' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/variables/:key' do key = params[:key] variable = user_group.variables.find_by(key: key) @@ -35,6 +38,7 @@ module API present variable, with: Entities::Variable end + # rubocop: enable CodeReuse/ActiveRecord desc 'Create a new variable in a group' do success Entities::Variable @@ -64,6 +68,7 @@ module API optional :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' end + # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do variable = user_group.variables.find_by(key: params[:key]) @@ -77,6 +82,7 @@ module API render_validation_error!(variable) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an existing variable from a group' do success Entities::Variable @@ -84,12 +90,14 @@ module API params do requires :key, type: String, desc: 'The key of the variable' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/variables/:key' do variable = user_group.variables.find_by(key: params[:key]) not_found!('GroupVariable') unless variable destroy_conditionally!(variable) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index b4f441f6a4f..64b998ab455 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Groups < Grape::API include PaginationParams @@ -38,6 +40,7 @@ module API use :pagination end + # rubocop: disable CodeReuse/ActiveRecord def find_groups(params, parent_id = nil) find_params = params.slice(:all_available, :custom_attributes, :owned, :min_access_level) find_params[:parent] = find_group!(parent_id) if parent_id @@ -53,6 +56,7 @@ module API groups end + # rubocop: enable CodeReuse/ActiveRecord def find_group_projects(params) group = find_group!(params[:id]) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index be17653dbb2..a7ba8066233 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers include Gitlab::Utils @@ -94,6 +96,7 @@ module API LabelsFinder.new(current_user, search_params).execute end + # rubocop: disable CodeReuse/ActiveRecord def find_user(id) if id =~ /^\d+$/ User.find_by(id: id) @@ -101,14 +104,19 @@ module API User.find_by(username: id) end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def find_project(id) + projects = Project.without_deleted + if id.is_a?(Integer) || id =~ /^\d+$/ - Project.find_by(id: id) + projects.find_by(id: id) elsif id.include?("/") - Project.find_by_full_path(id) + projects.find_by_full_path(id) end end + # rubocop: enable CodeReuse/ActiveRecord def find_project!(id) project = find_project(id) @@ -120,6 +128,7 @@ module API end end + # rubocop: disable CodeReuse/ActiveRecord def find_group(id) if id.to_s =~ /^\d+$/ Group.find_by(id: id) @@ -127,6 +136,7 @@ module API Group.find_by_full_path(id) end end + # rubocop: enable CodeReuse/ActiveRecord def find_group!(id) group = find_group(id) @@ -138,6 +148,7 @@ module API end end + # rubocop: disable CodeReuse/ActiveRecord def find_namespace(id) if id.to_s =~ /^\d+$/ Namespace.find_by(id: id) @@ -145,6 +156,7 @@ module API Namespace.find_by_full_path(id) end end + # rubocop: enable CodeReuse/ActiveRecord def find_namespace!(id) namespace = find_namespace(id) @@ -156,6 +168,12 @@ module API end end + def find_branch!(branch_name) + user_project.repository.find_branch(branch_name) || not_found!('Branch') + rescue Gitlab::Git::CommandError + render_api_error!('The branch refname is invalid', 400) + end + def find_project_label(id) labels = available_labels_for(user_project) label = labels.find_by_id(id) || labels.find_by_title(id) @@ -163,13 +181,17 @@ module API label || not_found!('Label') end + # rubocop: disable CodeReuse/ActiveRecord def find_project_issue(iid) IssuesFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid) end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def find_project_merge_request(iid) MergeRequestsFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid) end + # rubocop: enable CodeReuse/ActiveRecord def find_project_commit(id) user_project.commit_by(oid: id) @@ -180,11 +202,13 @@ module API SnippetsFinder.new(current_user, finder_params).find(id) end + # rubocop: disable CodeReuse/ActiveRecord def find_merge_request_with_access(iid, access_level = :read_merge_request) merge_request = user_project.merge_requests.find_by!(iid: iid) authorize! access_level, merge_request merge_request end + # rubocop: enable CodeReuse/ActiveRecord def find_build!(id) user_project.builds.find(id.to_i) @@ -276,9 +300,11 @@ module API Gitlab.rails5? ? permitted_attrs.to_h : permitted_attrs end + # rubocop: disable CodeReuse/ActiveRecord def filter_by_iid(items, iid) items.where(iid: iid) end + # rubocop: enable CodeReuse/ActiveRecord def filter_by_search(items, text) items.search(text) @@ -357,9 +383,10 @@ module API # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 trace = exception.backtrace - message = "\n#{exception.class} (#{exception.message}):\n" + message = ["\n#{exception.class} (#{exception.message}):\n"] message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) message << " " << trace.join("\n ") + message = message.join API.logger.add Logger::FATAL, message @@ -375,12 +402,14 @@ module API # project helpers + # rubocop: disable CodeReuse/ActiveRecord def reorder_projects(projects) projects.reorder(params[:order_by] => params[:sort]) end + # rubocop: enable CodeReuse/ActiveRecord def project_finder_params - finder_params = {} + finder_params = { without_deleted: true } finder_params[:owned] = true if params[:owned].present? finder_params[:non_public] = true if params[:membership].present? finder_params[:starred] = true if params[:starred].present? diff --git a/lib/api/helpers/badges_helpers.rb b/lib/api/helpers/badges_helpers.rb index 1f8afbf3c90..46ce5b4e7b5 100644 --- a/lib/api/helpers/badges_helpers.rb +++ b/lib/api/helpers/badges_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module BadgesHelpers diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb index 9993caa5249..7551ca50a7f 100644 --- a/lib/api/helpers/common_helpers.rb +++ b/lib/api/helpers/common_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module CommonHelpers diff --git a/lib/api/helpers/custom_attributes.rb b/lib/api/helpers/custom_attributes.rb index 10d652e33f5..88208226c40 100644 --- a/lib/api/helpers/custom_attributes.rb +++ b/lib/api/helpers/custom_attributes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module CustomAttributes @@ -12,6 +14,7 @@ module API desc: 'Filter with custom attributes' end + # rubocop: disable CodeReuse/ActiveRecord def with_custom_attributes(collection_or_resource, options = {}) options = options.merge( with_custom_attributes: params[:with_custom_attributes] && @@ -24,6 +27,7 @@ module API [collection_or_resource, options] end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb index dd4f6c41131..23b1cd1ad45 100644 --- a/lib/api/helpers/custom_validators.rb +++ b/lib/api/helpers/custom_validators.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module CustomValidators diff --git a/lib/api/helpers/headers_helpers.rb b/lib/api/helpers/headers_helpers.rb index c9c44e3c218..7553af9d156 100644 --- a/lib/api/helpers/headers_helpers.rb +++ b/lib/api/helpers/headers_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module HeadersHelpers diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 83151be82ad..4eaaca96b49 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module InternalHelpers diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index fed8846e505..73d58ee7f37 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # rubocop:disable GitlabSecurity/PublicSend module API @@ -17,6 +19,7 @@ module API .non_request end + # rubocop: disable CodeReuse/ActiveRecord def find_all_members_for_project(project) shared_group_ids = project.project_group_links.pluck(:group_id) project_group_ids = project.group&.self_and_ancestors&.pluck(:id) @@ -28,13 +31,16 @@ module API .where(project_authorizations: { project_id: project.id }) .where(source_id: source_ids) end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def find_all_members_for_group(group) source_ids = group.self_and_ancestors.pluck(:id) Member.includes(:user) .where(source_id: source_ids) .where(source_type: 'Namespace') end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 7b1f5c2584b..216b2c45741 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module NotesHelpers diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 3308212216e..d311cbb5f7e 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module Pagination @@ -91,6 +93,7 @@ module API @request_context = request_context end + # rubocop: disable CodeReuse/ActiveRecord def paginate(relation) pagination = KeysetPaginationInfo.new(relation, request_context) @@ -112,6 +115,7 @@ module API paged_relation end + # rubocop: enable CodeReuse/ActiveRecord private @@ -183,6 +187,7 @@ module API private + # rubocop: disable CodeReuse/ActiveRecord def add_default_order(relation) if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? relation = relation.order(:id) @@ -190,6 +195,7 @@ module API relation end + # rubocop: enable CodeReuse/ActiveRecord def add_pagination_headers(paginated_data) header 'X-Per-Page', paginated_data.limit_value.to_s diff --git a/lib/api/helpers/project_snapshots_helpers.rb b/lib/api/helpers/project_snapshots_helpers.rb index 94798a8cb51..1b5dc281e38 100644 --- a/lib/api/helpers/project_snapshots_helpers.rb +++ b/lib/api/helpers/project_snapshots_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module ProjectSnapshotsHelpers diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 381d5e8968c..e6a72b949f9 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module ProjectsHelpers @@ -26,6 +28,7 @@ module API optional :avatar, type: File, desc: 'Avatar image for project' optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' + optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" end params :optional_project_params do diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb index bc7333ca4b3..793ae11b41d 100644 --- a/lib/api/helpers/related_resources_helpers.rb +++ b/lib/api/helpers/related_resources_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module RelatedResourcesHelpers diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 61eb88d3331..45d0343bc89 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module Runner diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 516f25db15b..6a264c4cc6d 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Internal access API class Internal < Grape::API @@ -6,8 +8,17 @@ module API helpers ::API::Helpers::InternalHelpers helpers ::Gitlab::Identifier + UNKNOWN_CHECK_RESULT_ERROR = 'Unknown check result'.freeze + + helpers do + def response_with_status(code: 200, success: true, message: nil, **extra_options) + status code + { status: success, message: message }.merge(extra_options).compact + end + end + namespace 'internal' do - # Check if git command is allowed to project + # Check if git command is allowed for project # # Params: # key_id - ssh key id for Git over SSH @@ -17,9 +28,8 @@ module API # project - project full_path (not path on disk) # action - git action (git-upload-pack or git-receive-pack) # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList + # rubocop: disable CodeReuse/ActiveRecord post "/allowed" do - status 200 - # Stores some Git-specific env thread-safely env = parse_env Gitlab::Git::HookEnv.set(gl_repository, env) if project @@ -49,29 +59,49 @@ module API namespace_path: namespace_path, project_path: project_path, redirected_path: redirected_path) - begin - access_checker.check(params[:action], params[:changes]) - @project ||= access_checker.project - rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e - break { status: false, message: e.message } - end + check_result = begin + result = access_checker.check(params[:action], params[:changes]) + @project ||= access_checker.project + result + rescue Gitlab::GitAccess::UnauthorizedError => e + break response_with_status(code: 401, success: false, message: e.message) + rescue Gitlab::GitAccess::NotFoundError => e + break response_with_status(code: 404, success: false, message: e.message) + end log_user_activity(actor) - { - status: true, - gl_repository: gl_repository, - gl_id: Gitlab::GlId.gl_id(user), - gl_username: user&.username, - - # This repository_path is a bogus value but gitlab-shell still requires - # its presence. https://gitlab.com/gitlab-org/gitlab-shell/issues/135 - repository_path: '/', + case check_result + when ::Gitlab::GitAccessResult::Success + payload = { + gl_repository: gl_repository, + gl_id: Gitlab::GlId.gl_id(user), + gl_username: user&.username, + git_config_options: [], + + # This repository_path is a bogus value but gitlab-shell still requires + # its presence. https://gitlab.com/gitlab-org/gitlab-shell/issues/135 + repository_path: '/', + + gitaly: gitaly_payload(params[:action]) + } + + # Custom option for git-receive-pack command + receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i + if receive_max_input_size > 0 + payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" + end - gitaly: gitaly_payload(params[:action]) - } + response_with_status(**payload) + when ::Gitlab::GitAccessResult::CustomAction + response_with_status(code: 300, message: check_result.message, payload: check_result.payload) + else + response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR) + end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord post "/lfs_authenticate" do status 200 @@ -93,6 +123,7 @@ module API repository_http_path: project.http_url_to_repo } end + # rubocop: enable CodeReuse/ActiveRecord get "/merge_request_urls" do merge_request_urls @@ -101,6 +132,7 @@ module API # # Get a ssh key using the fingerprint # + # rubocop: disable CodeReuse/ActiveRecord get "/authorized_keys" do fingerprint = params.fetch(:fingerprint) do Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint @@ -109,10 +141,12 @@ module API not_found!("Key") if key.nil? present key, with: Entities::SSHKey end + # rubocop: enable CodeReuse/ActiveRecord # # Discover user by ssh key, user id or username # + # rubocop: disable CodeReuse/ActiveRecord get "/discover" do if params[:key_id] key = Key.find(params[:key_id]) @@ -125,6 +159,7 @@ module API present user, with: Entities::UserSafe end + # rubocop: enable CodeReuse/ActiveRecord get "/check" do { @@ -151,6 +186,7 @@ module API end end + # rubocop: disable CodeReuse/ActiveRecord post '/two_factor_recovery_codes' do status 200 @@ -192,6 +228,7 @@ module API { success: true, recovery_codes: codes } end + # rubocop: enable CodeReuse/ActiveRecord post '/pre_receive' do status 200 diff --git a/lib/api/issues.rb b/lib/api/issues.rb index cedfd2fbaa0..25d78053c88 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Issues < Grape::API include PaginationParams @@ -7,6 +9,7 @@ module API helpers ::Gitlab::IssuableMetadata helpers do + # rubocop: disable CodeReuse/ActiveRecord def find_issues(args = {}) args = declared_params.merge(args) @@ -20,6 +23,7 @@ module API issues.reorder(args[:order_by] => args[:sort]) end + # rubocop: enable CodeReuse/ActiveRecord params :issues_params do optional :labels, type: String, desc: 'Comma-separated list of label names' @@ -207,6 +211,7 @@ module API at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, :discussion_locked, :labels, :created_at, :due_date, :confidential, :state_event end + # rubocop: disable CodeReuse/ActiveRecord put ':id/issues/:issue_iid' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42322') @@ -234,6 +239,7 @@ module API render_validation_error!(issue) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Move an existing issue' do success Entities::Issue @@ -242,6 +248,7 @@ module API requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' requires :to_project_id, type: Integer, desc: 'The ID of the new project' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/issues/:issue_iid/move' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42323') @@ -258,11 +265,13 @@ module API render_api_error!(error.message, 400) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete a project issue' params do requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/issues/:issue_iid" do issue = user_project.issues.find_by(iid: params[:issue_iid]) not_found!('Issue') unless issue @@ -273,6 +282,7 @@ module API Issuable::DestroyService.new(user_project, current_user).execute(issue) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'List merge requests closing issue' do success Entities::MergeRequestBasic @@ -280,6 +290,7 @@ module API params do requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/issues/:issue_iid/closed_by' do issue = find_project_issue(params[:issue_iid]) @@ -288,6 +299,7 @@ module API present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord desc 'List participants for an issue' do success Entities::UserBasic diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 32379d7c8ab..2229cbcd9d4 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class JobArtifacts < Grape::API before { authenticate_non_get! } @@ -21,6 +23,7 @@ module API requires :job, type: String, desc: 'The name for the job' end route_setting :authentication, job_token_allowed: true + # rubocop: disable CodeReuse/ActiveRecord get ':id/jobs/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do authorize_download_artifacts! @@ -30,6 +33,7 @@ module API present_carrierwave_file!(latest_build.artifacts_file) end + # rubocop: enable CodeReuse/ActiveRecord desc 'Download the artifacts archive from a job' do detail 'This feature was introduced in GitLab 8.5' diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index fc8c52085ab..fa992b9a440 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Jobs < Grape::API include PaginationParams @@ -34,6 +36,7 @@ module API use :optional_scope use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/jobs' do builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) @@ -41,6 +44,7 @@ module API builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, pipeline: :project) present paginate(builds), with: Entities::Job end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get pipeline jobs' do success Entities::Job @@ -50,6 +54,7 @@ module API use :optional_scope use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/pipelines/:pipeline_id/jobs' do pipeline = user_project.pipelines.find(params[:pipeline_id]) builds = pipeline.builds @@ -58,6 +63,7 @@ module API present paginate(builds), with: Entities::Job end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a specific job of a project' do success Entities::Job @@ -145,7 +151,7 @@ module API present build, with: Entities::Job end - desc 'Trigger a manual job' do + desc 'Trigger a actionable job (manual, scheduled, etc)' do success Entities::Job detail 'This feature was added in GitLab 8.11' end @@ -168,6 +174,7 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? @@ -178,6 +185,7 @@ module API builds.where(status: available_statuses && scope) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/keys.rb b/lib/api/keys.rb index fd93f797f72..d5280a0035d 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Keys API class Keys < Grape::API diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 81eaf56e48e..28555454307 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Labels < Grape::API include PaginationParams @@ -27,6 +29,7 @@ module API optional :description, type: String, desc: 'The description of label to be created' optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true end + # rubocop: disable CodeReuse/ActiveRecord post ':id/labels' do authorize! :admin_label, user_project @@ -43,6 +46,7 @@ module API render_validation_error!(label) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an existing label' do success Entities::Label @@ -50,6 +54,7 @@ module API params do requires :name, type: String, desc: 'The name of the label to be deleted' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/labels' do authorize! :admin_label, user_project @@ -58,6 +63,7 @@ module API destroy_conditionally!(label) end + # rubocop: enable CodeReuse/ActiveRecord desc 'Update an existing label. At least one optional parameter is required.' do success Entities::Label @@ -70,6 +76,7 @@ module API optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true at_least_one_of :new_name, :color, :description, :priority end + # rubocop: disable CodeReuse/ActiveRecord put ':id/labels' do authorize! :admin_label, user_project @@ -95,6 +102,7 @@ module API present label, with: Entities::Label, current_user: current_user, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/lint.rb b/lib/api/lint.rb index d202eaa4c49..0342a4b6654 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Lint < Grape::API namespace :ci do diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb index 5d55224c1a7..de77bef43ce 100644 --- a/lib/api/markdown.rb +++ b/lib/api/markdown.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Markdown < Grape::API params do @@ -10,7 +12,8 @@ module API detail "This feature was introduced in GitLab 11.0." end post do - context = { only_path: false } + context = { only_path: false, current_user: current_user } + context[:pipeline] = params[:gfm] ? :full : :plain_markdown if params[:project] project = Project.find_by_full_path(params[:project]) @@ -22,9 +25,7 @@ module API context[:skip_project_check] = true end - context[:pipeline] = params[:gfm] ? :full : :plain_markdown - - { html: Banzai.render(params[:text], context) } + { html: Banzai.render_and_post_process(params[:text], context) } end end end diff --git a/lib/api/members.rb b/lib/api/members.rb index d23dd834c69..a8f67be3463 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Members < Grape::API include PaginationParams @@ -18,6 +20,7 @@ module API optional :query, type: String, desc: 'A query string to search for members' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/members" do source = find_source(source_type, params[:id]) @@ -27,6 +30,7 @@ module API present members, with: Entities::Member end + # rubocop: enable CodeReuse/ActiveRecord desc 'Gets a list of group or project members viewable by the authenticated user, including those who gained membership through ancestor group.' do success Entities::Member @@ -35,6 +39,7 @@ module API optional :query, type: String, desc: 'A query string to search for members' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/members/all" do source = find_source(source_type, params[:id]) @@ -44,6 +49,7 @@ module API present members, with: Entities::Member end + # rubocop: enable CodeReuse/ActiveRecord desc 'Gets a member of a group or project.' do success Entities::Member @@ -51,6 +57,7 @@ module API params do requires :user_id, type: Integer, desc: 'The user ID of the member' end + # rubocop: disable CodeReuse/ActiveRecord get ":id/members/:user_id" do source = find_source(source_type, params[:id]) @@ -59,6 +66,7 @@ module API present member, with: Entities::Member end + # rubocop: enable CodeReuse/ActiveRecord desc 'Adds a member to a group or project.' do success Entities::Member @@ -68,6 +76,7 @@ module API requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' end + # rubocop: disable CodeReuse/ActiveRecord post ":id/members" do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) @@ -88,6 +97,7 @@ module API render_validation_error!(member) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Updates a member of a group or project.' do success Entities::Member @@ -97,6 +107,7 @@ module API requires :access_level, type: Integer, desc: 'A valid access level' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' end + # rubocop: disable CodeReuse/ActiveRecord put ":id/members/:user_id" do source = find_source(source_type, params.delete(:id)) authorize_admin_source!(source_type, source) @@ -113,11 +124,13 @@ module API render_validation_error!(updated_member) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Removes a user from a group or project.' params do requires :user_id, type: Integer, desc: 'The user ID of the member' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) member = source.members.find_by!(user_id: params[:user_id]) @@ -126,6 +139,7 @@ module API ::Members::DestroyService.new(current_user).execute(member) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 95ef8f42954..e4fb890960a 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # MergeRequestDiff API class MergeRequestDiffs < Grape::API diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index abad418771c..440d94ae186 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class MergeRequests < Grape::API include PaginationParams @@ -28,9 +30,9 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def find_merge_requests(args = {}) args = declared_params.merge(args) - args[:milestone_title] = args.delete(:milestone) args[:label_name] = args.delete(:labels) args[:scope] = args[:scope].underscore if args[:scope] @@ -45,6 +47,7 @@ module API merge_requests .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs) end + # rubocop: enable CodeReuse/ActiveRecord def merge_request_pipelines_with_access authorize! :read_pipeline, user_project @@ -93,6 +96,7 @@ module API optional :source_branch, type: String, desc: 'Return merge requests with the given source branch' optional :target_branch, type: String, desc: 'Return merge requests with the given target branch' optional :search, type: String, desc: 'Search merge requests for text present in the title or description' + optional :wip, type: String, values: %w[yes no], desc: 'Search merge requests for WIP in the title' use :pagination end end @@ -233,6 +237,7 @@ module API params do requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' optional :render_html, type: Boolean, desc: 'Returns the description and title rendered HTML' + optional :include_diverged_commits_count, type: Boolean, desc: 'Returns the commits count behind the target branch' end desc 'Get a single merge request' do success Entities::MergeRequest @@ -240,7 +245,7 @@ module API get ':id/merge_requests/:merge_request_iid' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project, render_html: params[:render_html] + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project, render_html: params[:render_html], include_diverged_commits_count: params[:include_diverged_commits_count] end desc 'Get the participants of a merge request' do diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb index a8eb137e46a..a0ca39b69d4 100644 --- a/lib/api/milestone_responses.rb +++ b/lib/api/milestone_responses.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module MilestoneResponses extend ActiveSupport::Concern diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 32b77aedba8..76639fbb031 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Namespaces < Grape::API include PaginationParams diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 39923e6d5b5..9f323b87baf 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Notes < Grape::API include PaginationParams @@ -28,6 +30,7 @@ module API desc: 'Return notes sorted in `asc` or `desc` order.' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/#{noteables_str}/:noteable_id/notes" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) @@ -45,6 +48,7 @@ module API .reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: Entities::Note end + # rubocop: enable CodeReuse/ActiveRecord desc "Get a single #{noteable_type.to_s.downcase} note" do success Entities::Note diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index bf0d6b9e434..4d9a4629268 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # notification_settings API class NotificationSettings < Grape::API diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index ba33993d852..c9ad47e0f0d 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class PagesDomains < Grape::API include PaginationParams @@ -13,9 +15,11 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def find_pages_domain! user_project.pages_domains.find_by(domain: params[:domain]) || not_found!('PagesDomain') end + # rubocop: enable CodeReuse/ActiveRecord def pages_domain @pages_domain ||= find_pages_domain! @@ -61,11 +65,13 @@ module API params do use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/pages/domains" do authorize! :read_pages, user_project present paginate(user_project.pages_domains.order(:domain)), with: Entities::PagesDomain end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a single pages domain' do success Entities::PagesDomain diff --git a/lib/api/pagination_params.rb b/lib/api/pagination_params.rb index f566eb3ed2b..ae03595eb25 100644 --- a/lib/api/pagination_params.rb +++ b/lib/api/pagination_params.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Concern for declare pagination params. # diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 37f32411296..ed0a38b9d70 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class PipelineSchedules < Grape::API include PaginationParams @@ -16,6 +18,7 @@ module API optional :scope, type: String, values: %w[active inactive], desc: 'The scope of pipeline schedules' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/pipeline_schedules' do authorize! :read_pipeline_schedule, user_project @@ -23,6 +26,7 @@ module API .preload([:owner, :last_pipeline]) present paginate(schedules), with: Entities::PipelineSchedule end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a single pipeline schedule' do success Entities::PipelineScheduleDetails @@ -39,7 +43,7 @@ module API end params do requires :description, type: String, desc: 'The description of pipeline schedule' - requires :ref, type: String, desc: 'The branch/tag name will be triggered' + requires :ref, type: String, desc: 'The branch/tag name will be triggered', allow_blank: false requires :cron, type: String, desc: 'The cron' optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone' optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule' @@ -161,6 +165,7 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def pipeline_schedule @pipeline_schedule ||= user_project @@ -172,7 +177,9 @@ module API end end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def pipeline_schedule_variable @pipeline_schedule_variable ||= pipeline_schedule.variables.find_by(key: params[:key]).tap do |pipeline_schedule_variable| @@ -181,6 +188,7 @@ module API end end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 5d33a13d035..1cfb982c04b 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Pipelines < Grape::API include PaginationParams @@ -43,6 +45,7 @@ module API requires :ref, type: String, desc: 'Reference' optional :variables, Array, desc: 'Array of variables available in the pipeline' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/pipeline' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42124') @@ -63,6 +66,7 @@ module API render_validation_error!(new_pipeline) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Gets a specific pipeline for the project' do detail 'This feature was introduced in GitLab 8.11' diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 15c57a2fc02..e34ed0bdb44 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectExport < Grape::API before do @@ -21,12 +23,8 @@ module API detail 'This feature was introduced in GitLab 10.6.' end get ':id/export/download' do - path = user_project.export_project_path - - if path - present_disk_file!(path, File.basename(path), 'application/gzip') - elsif user_project.export_project_object_exists? - present_carrierwave_file!(user_project.import_export_upload.export_file) + if user_project.export_file_exists? + present_carrierwave_file!(user_project.export_file) else render_api_error!('404 Not found or has expired', 404) end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 4760a1c08d7..4af4c6ac593 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectHooks < Grape::API include PaginationParams @@ -20,6 +22,7 @@ module API optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" + optional :push_events_branch_filter, type: String, desc: "Trigger hook on specified branch only" end end @@ -63,6 +66,7 @@ module API present hook, with: Entities::ProjectHook else error!("Invalid url given", 422) if hook.errors[:url].present? + error!("Invalid branch filter given", 422) if hook.errors[:push_events_branch_filter].present? not_found!("Project hook #{hook.errors.messages}") end @@ -84,6 +88,7 @@ module API present hook, with: Entities::ProjectHook else error!("Invalid url given", 422) if hook.errors[:url].present? + error!("Invalid branch filter given", 422) if hook.errors[:push_events_branch_filter].present? not_found!("Project hook #{hook.errors.messages}") end diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index bc5152e539f..cbfa0c5bc1c 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectImport < Grape::API include PaginationParams diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 72cf32d7717..c7137ba5217 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectMilestones < Grape::API include PaginationParams diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb index 71005acc587..175fbb2ce92 100644 --- a/lib/api/project_snapshots.rb +++ b/lib/api/project_snapshots.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectSnapshots < Grape::API helpers ::API::Helpers::ProjectSnapshotsHelpers diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 0ada0ef4708..f3a1b73b153 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectSnippets < Grape::API include PaginationParams @@ -85,6 +87,7 @@ module API desc: 'The visibility of the snippet' at_least_one_of :title, :file_name, :code, :visibility_level end + # rubocop: disable CodeReuse/ActiveRecord put ":id/snippets/:snippet_id" do snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id)) not_found!('Snippet') unless snippet @@ -107,11 +110,13 @@ module API render_validation_error!(snippet) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete a project snippet' params do requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/snippets/:snippet_id" do snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) not_found!('Snippet') unless snippet @@ -120,11 +125,13 @@ module API destroy_conditionally!(snippet) end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a raw project snippet' params do requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' end + # rubocop: disable CodeReuse/ActiveRecord get ":id/snippets/:snippet_id/raw" do snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) not_found!('Snippet') unless snippet @@ -133,6 +140,7 @@ module API content_type 'text/plain' present snippet.content end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get the user agent details for a project snippet' do success Entities::UserAgentDetail @@ -140,6 +148,7 @@ module API params do requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' end + # rubocop: disable CodeReuse/ActiveRecord get ":id/snippets/:snippet_id/user_agent_detail" do authenticated_as_admin! @@ -149,6 +158,7 @@ module API present snippet.user_agent_detail, with: Entities::UserAgentDetail end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb new file mode 100644 index 00000000000..d05ddad7466 --- /dev/null +++ b/lib/api/project_templates.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module API + class ProjectTemplates < Grape::API + include PaginationParams + + TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses].freeze + + before { authenticate_non_get! } + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses) of the template' + end + resource :projects do + desc 'Get a list of templates available to this project' do + detail 'This endpoint was introduced in GitLab 11.4' + end + params do + use :pagination + end + get ':id/templates/:type' do + templates = TemplateFinder + .build(params[:type], user_project) + .execute + + present paginate(::Kaminari.paginate_array(templates)), with: Entities::TemplatesList + end + + desc 'Download a template available to this project' do + detail 'This endpoint was introduced in GitLab 11.4' + end + params do + requires :name, type: String, desc: 'The name of the template' + + optional :project, type: String, desc: 'The project name to use when expanding placeholders in the template. Only affects licenses' + optional :fullname, type: String, desc: 'The full name of the copyright holder to use when expanding placeholders in the template. Only affects licenses' + end + get ':id/templates/:type/:name', requirements: { name: /[\w\.-]+/ } do + template = TemplateFinder + .build(params[:type], user_project, name: params[:name]) + .execute + + not_found!('Template') unless template.present? + + template.resolve!( + project_name: params[:project].presence, + fullname: params[:fullname].presence || current_user&.name + ) + + if template.is_a?(::LicenseTemplate) + present template, with: Entities::License + else + present template, with: Entities::Template + end + end + end + end +end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 2801ae918c6..ae2d327e45b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_dependency 'declarative_policy' module API @@ -198,6 +200,7 @@ module API use :optional_project_params use :create_params end + # rubocop: disable CodeReuse/ActiveRecord post "user/:user_id" do authenticated_as_admin! user = User.find_by(id: params.delete(:user_id)) @@ -214,6 +217,7 @@ module API render_validation_error!(project) end end + # rubocop: enable CodeReuse/ActiveRecord end params do @@ -283,6 +287,12 @@ module API present_projects forks end + desc 'Check pages access of this project' + get ':id/pages_access' do + authorize! :read_pages_content, user_project unless user_project.public_pages? + status 200 + end + desc 'Update an existing project' do success Entities::Project end @@ -444,6 +454,7 @@ module API params do requires :group_id, type: Integer, desc: 'The ID of the group' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/share/:group_id" do authorize! :admin_project, user_project @@ -452,6 +463,7 @@ module API destroy_conditionally!(link) end + # rubocop: enable CodeReuse/ActiveRecord desc 'Upload a file' params do diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb index 9fd79c491c2..8edcfea7c93 100644 --- a/lib/api/projects_relation_builder.rb +++ b/lib/api/projects_relation_builder.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module ProjectsRelationBuilder extend ActiveSupport::Concern diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index a30eb46c220..47752f40e58 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProtectedBranches < Grape::API include PaginationParams @@ -16,11 +18,13 @@ module API params do use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/protected_branches' do protected_branches = user_project.protected_branches.preload(:push_access_levels, :merge_access_levels) present paginate(protected_branches), with: Entities::ProtectedBranch, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a single protected branch' do success Entities::ProtectedBranch @@ -28,11 +32,13 @@ module API params do requires :name, type: String, desc: 'The name of the branch or wildcard' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do protected_branch = user_project.protected_branches.find_by!(name: params[:name]) present protected_branch, with: Entities::ProtectedBranch, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord desc 'Protect a single branch or wildcard' do success Entities::ProtectedBranch @@ -40,12 +46,13 @@ module API params do requires :name, type: String, desc: 'The name of the protected branch' optional :push_access_level, type: Integer, - values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, + values: ProtectedBranch::PushAccessLevel.allowed_access_levels, desc: 'Access levels allowed to push (defaults: `40`, maintainer access level)' optional :merge_access_level, type: Integer, - values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, + values: ProtectedBranch::MergeAccessLevel.allowed_access_levels, desc: 'Access levels allowed to merge (defaults: `40`, maintainer access level)' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/protected_branches' do protected_branch = user_project.protected_branches.find_by(name: params[:name]) if protected_branch @@ -62,11 +69,13 @@ module API render_api_error!(protected_branch.errors.full_messages, 422) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Unprotect a single branch' params do requires :name, type: String, desc: 'The name of the protected branch' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do protected_branch = user_project.protected_branches.find_by!(name: params[:name]) @@ -75,6 +84,7 @@ module API destroy_service.execute(protected_branch) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb index bf0a7184e1c..ed1c5f0cc05 100644 --- a/lib/api/protected_tags.rb +++ b/lib/api/protected_tags.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProtectedTags < Grape::API include PaginationParams @@ -17,11 +19,13 @@ module API params do use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/protected_tags' do protected_tags = user_project.protected_tags.preload(:create_access_levels) present paginate(protected_tags), with: Entities::ProtectedTag, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a single protected tag' do detail 'This feature was introduced in GitLab 11.3.' @@ -30,11 +34,13 @@ module API params do requires :name, type: String, desc: 'The name of the tag or wildcard' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/protected_tags/:name', requirements: TAG_ENDPOINT_REQUIREMENTS do protected_tag = user_project.protected_tags.find_by!(name: params[:name]) present protected_tag, with: Entities::ProtectedTag, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord desc 'Protect a single tag or wildcard' do detail 'This feature was introduced in GitLab 11.3.' @@ -43,7 +49,7 @@ module API params do requires :name, type: String, desc: 'The name of the protected tag' optional :create_access_level, type: Integer, default: Gitlab::Access::MAINTAINER, - values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, + values: ProtectedTag::CreateAccessLevel.allowed_access_levels, desc: 'Access levels allowed to create (defaults: `40`, maintainer access level)' end post ':id/protected_tags' do @@ -69,11 +75,13 @@ module API params do requires :name, type: String, desc: 'The name of the protected tag' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/protected_tags/:name', requirements: TAG_ENDPOINT_REQUIREMENTS do protected_tag = user_project.protected_tags.find_by!(name: params[:name]) destroy_conditionally!(protected_tag) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 79736107bbb..5125f302fbb 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mime/types' module API diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb new file mode 100644 index 00000000000..b6fbe8c0235 --- /dev/null +++ b/lib/api/resource_label_events.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module API + class ResourceLabelEvents < Grape::API + include PaginationParams + helpers ::API::Helpers::NotesHelpers + + before { authenticate! } + + EVENTABLE_TYPES = [Issue, MergeRequest].freeze + + EVENTABLE_TYPES.each do |eventable_type| + parent_type = eventable_type.parent_class.to_s.underscore + eventables_str = eventable_type.to_s.underscore.pluralize + + params do + requires :id, type: String, desc: "The ID of a #{parent_type}" + end + resource parent_type.pluralize.to_sym, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc "Get a list of #{eventable_type.to_s.downcase} resource label events" do + success Entities::ResourceLabelEvent + detail 'This feature was introduced in 11.3' + end + params do + requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' + use :pagination + end + + # rubocop: disable CodeReuse/ActiveRecord + get ":id/#{eventables_str}/:eventable_id/resource_label_events" do + eventable = find_noteable(parent_type, eventables_str, params[:eventable_id]) + events = eventable.resource_label_events.includes(:label, :user) + + present paginate(events), with: Entities::ResourceLabelEvent + end + # rubocop: enable CodeReuse/ActiveRecord + + desc "Get a single #{eventable_type.to_s.downcase} resource label event" do + success Entities::ResourceLabelEvent + detail 'This feature was introduced in 11.3' + end + params do + requires :event_id, type: String, desc: 'The ID of a resource label event' + requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' + end + get ":id/#{eventables_str}/:eventable_id/resource_label_events/:event_id" do + eventable = find_noteable(parent_type, eventables_str, params[:eventable_id]) + event = eventable.resource_label_events.find(params[:event_id]) + + present event, with: Entities::ResourceLabelEvent + end + end + end + end +end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index c9931c2d603..d8768a54986 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Runner < Grape::API helpers ::API::Helpers::Runner @@ -17,6 +19,7 @@ module API optional :tag_list, type: Array[String], desc: %q(List of Runner's tags) optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' end + # rubocop: disable CodeReuse/ActiveRecord post '/' do attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :maximum_timeout]) .merge(get_runner_details_from_request) @@ -43,6 +46,7 @@ module API render_validation_error!(runner) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Deletes a registered Runner' do http_codes [[204, 'Runner was deleted'], [403, 'Forbidden']] diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 51242341dba..ce70460af11 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Runners < Grape::API include PaginationParams @@ -9,12 +11,20 @@ module API success Entities::Runner end params do - optional :scope, type: String, values: %w[active paused online], + optional :scope, type: String, values: Ci::Runner::AVAILABLE_STATUSES, desc: 'The scope of specific runners to show' + optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES, + desc: 'The type of the runners to show' + optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, + desc: 'The status of the runners to show' use :pagination end get do - runners = filter_runners(current_user.ci_owned_runners, params[:scope], without: %w(specific shared)) + runners = current_user.ci_owned_runners + runners = filter_runners(runners, params[:scope], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) + runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + present paginate(runners), with: Entities::Runner end @@ -22,13 +32,22 @@ module API success Entities::Runner end params do - optional :scope, type: String, values: %w[active paused online specific shared], + optional :scope, type: String, values: Ci::Runner::AVAILABLE_SCOPES, desc: 'The scope of specific runners to show' + optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES, + desc: 'The type of the runners to show' + optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, + desc: 'The status of the runners to show' use :pagination end get 'all' do authenticated_as_admin! - runners = filter_runners(Ci::Runner.all, params[:scope]) + + runners = Ci::Runner.all + runners = filter_runners(runners, params[:scope]) + runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) + runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + present paginate(runners), with: Entities::Runner end @@ -94,7 +113,7 @@ module API optional :status, type: String, desc: 'Status of the job', values: Ci::Build::AVAILABLE_STATUSES use :pagination end - get ':id/jobs' do + get ':id/jobs' do runner = get_runner(params[:id]) authenticate_list_runners_jobs!(runner) @@ -114,12 +133,20 @@ module API success Entities::Runner end params do - optional :scope, type: String, values: %w[active paused online specific shared], + optional :scope, type: String, values: Ci::Runner::AVAILABLE_SCOPES, desc: 'The scope of specific runners to show' + optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES, + desc: 'The type of the runners to show' + optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, + desc: 'The status of the runners to show' use :pagination end get ':id/runners' do - runners = filter_runners(Ci::Runner.owned_or_instance_wide(user_project.id), params[:scope]) + runners = Ci::Runner.owned_or_instance_wide(user_project.id) + runners = filter_runners(runners, params[:scope]) + runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) + runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + present paginate(runners), with: Entities::Runner end @@ -146,6 +173,7 @@ module API params do requires :runner_id, type: Integer, desc: 'The ID of the runner' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/runners/:runner_id' do runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id]) not_found!('Runner') unless runner_project @@ -155,18 +183,14 @@ module API destroy_conditionally!(runner_project) end + # rubocop: enable CodeReuse/ActiveRecord end helpers do - def filter_runners(runners, scope, options = {}) + def filter_runners(runners, scope, allowed_scopes: ::Ci::Runner::AVAILABLE_SCOPES) return runners unless scope.present? - available_scopes = ::Ci::Runner::AVAILABLE_SCOPES - if options[:without] - available_scopes = available_scopes - options[:without] - end - - if (available_scopes & [scope]).empty? + unless allowed_scopes.include?(scope) render_api_error!('Scope contains invalid value', 400) end diff --git a/lib/api/scope.rb b/lib/api/scope.rb index d5165b2e482..707775e5d15 100644 --- a/lib/api/scope.rb +++ b/lib/api/scope.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Encapsulate a scope used for authorization, such as `api`, or `read_user` module API class Scope diff --git a/lib/api/search.rb b/lib/api/search.rb index 37fbabe419c..12d97dcfe7f 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Search < Grape::API include PaginationParams diff --git a/lib/api/services.rb b/lib/api/services.rb index d1a5ee7db35..0ae05ce08f1 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -821,11 +821,13 @@ module API TRIGGER_SERVICES.each do |service_slug, settings| helpers do + # rubocop: disable CodeReuse/ActiveRecord def slash_command_service(project, service_slug, params) project.services.active.where(template: false).find do |service| service.try(:token) == params[:token] && service.to_param == service_slug.underscore end end + # rubocop: enable CodeReuse/ActiveRecord end params do diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 897010217dc..edbd134822c 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Settings < Grape::API before { authenticated_as_admin! } @@ -102,7 +104,7 @@ module API end optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects' - optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' + optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' end @@ -117,11 +119,6 @@ module API given shared_runners_enabled: ->(val) { val } do requires :shared_runners_text, type: String, desc: 'Shared runners text ' end - optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling' - given sidekiq_throttling_enabled: ->(val) { val } do - requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.' - requires :sidekiq_throttling_queues, type: Array[String], desc: 'Choose which queues you wish to throttle' - end optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb index 11f2b40269a..daa9598a204 100644 --- a/lib/api/sidekiq_metrics.rb +++ b/lib/api/sidekiq_metrics.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'sidekiq/api' module API diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index b30305b4bc9..f1786c15f4f 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Snippets API class Snippets < Grape::API @@ -92,6 +94,7 @@ module API desc: 'The visibility of the snippet' at_least_one_of :title, :file_name, :content, :visibility end + # rubocop: disable CodeReuse/ActiveRecord put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) break not_found!('Snippet') unless snippet @@ -110,6 +113,7 @@ module API render_validation_error!(snippet) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Remove snippet' do detail 'This feature was introduced in GitLab 8.15.' @@ -118,6 +122,7 @@ module API params do requires :id, type: Integer, desc: 'The ID of a snippet' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) break not_found!('Snippet') unless snippet @@ -126,6 +131,7 @@ module API destroy_conditionally!(snippet) end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a raw snippet' do detail 'This feature was introduced in GitLab 8.15.' @@ -133,6 +139,7 @@ module API params do requires :id, type: Integer, desc: 'The ID of a snippet' end + # rubocop: disable CodeReuse/ActiveRecord get ":id/raw" do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) break not_found!('Snippet') unless snippet @@ -141,6 +148,7 @@ module API content_type 'text/plain' present snippet.content end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get the user agent details for a snippet' do success Entities::UserAgentDetail @@ -148,6 +156,7 @@ module API params do requires :id, type: Integer, desc: 'The ID of a snippet' end + # rubocop: disable CodeReuse/ActiveRecord get ":id/user_agent_detail" do authenticated_as_admin! @@ -157,6 +166,7 @@ module API present snippet.user_agent_detail, with: Entities::UserAgentDetail end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index b3e1e23031a..077e9373ac4 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Subscriptions < Grape::API before { authenticate! } diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index c7a460df46a..51fae0e54aa 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class SystemHooks < Grape::API include PaginationParams @@ -63,12 +65,14 @@ module API params do requires :id, type: Integer, desc: 'The ID of the system hook' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id" do hook = SystemHook.find_by(id: params[:id]) not_found!('System hook') unless hook destroy_conditionally!(hook) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 5e0afc6a7e4..f739eacf9ba 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Tags < Grape::API include PaginationParams diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 927baaea652..8dab19d50c2 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -1,18 +1,17 @@ +# frozen_string_literal: true + module API class Templates < Grape::API include PaginationParams GLOBAL_TEMPLATE_TYPES = { gitignores: { - klass: Gitlab::Template::GitignoreTemplate, gitlab_version: 8.8 }, gitlab_ci_ymls: { - klass: Gitlab::Template::GitlabCiYmlTemplate, gitlab_version: 8.9 }, dockerfiles: { - klass: Gitlab::Template::DockerfileTemplate, gitlab_version: 8.15 } }.freeze @@ -36,7 +35,7 @@ module API popular = declared(params)[:popular] popular = to_boolean(popular) if popular.present? - templates = LicenseTemplateFinder.new(popular: popular).execute + templates = TemplateFinder.build(:licenses, nil, popular: popular).execute present paginate(::Kaminari.paginate_array(templates)), with: ::API::Entities::License end @@ -49,8 +48,7 @@ module API requires :name, type: String, desc: 'The name of the template' end get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ } do - templates = LicenseTemplateFinder.new.execute - template = templates.find { |template| template.key == params[:name] } + template = TemplateFinder.build(:licenses, nil, name: params[:name]).execute not_found!('License') unless template.present? @@ -63,7 +61,6 @@ module API end GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| - klass = properties[:klass] gitlab_version = properties[:gitlab_version] desc 'Get the list of the available template' do @@ -74,7 +71,7 @@ module API use :pagination end get "templates/#{template_type}" do - templates = ::Kaminari.paginate_array(klass.all) + templates = ::Kaminari.paginate_array(TemplateFinder.build(template_type, nil).execute) present paginate(templates), with: Entities::TemplatesList end @@ -86,7 +83,8 @@ module API requires :name, type: String, desc: 'The name of the template' end get "templates/#{template_type}/:name" do - new_template = klass.find(declared(params)[:name]) + finder = TemplateFinder.build(template_type, nil, name: declared(params)[:name]) + new_template = finder.execute render_response(template_type, new_template) end diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index 2bb451dea89..93fe06bec27 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module TimeTrackingEndpoints extend ActiveSupport::Concern diff --git a/lib/api/todos.rb b/lib/api/todos.rb index c6dbcf84e3a..ed2cf2cc31b 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Todos < Grape::API include PaginationParams diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index b29e660c6e0..f784c857883 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Triggers < Grape::API include PaginationParams @@ -10,7 +12,7 @@ module API success Entities::Pipeline end params do - requires :ref, type: String, desc: 'The commit sha or name of a branch or tag' + requires :ref, type: String, desc: 'The commit sha or name of a branch or tag', allow_blank: false requires :token, type: String, desc: 'The unique token of trigger' optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end @@ -42,6 +44,7 @@ module API params do use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/triggers' do authenticate! authorize! :admin_build, user_project @@ -50,6 +53,7 @@ module API present paginate(triggers), with: Entities::Trigger end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get specific trigger of a project' do success Entities::Trigger diff --git a/lib/api/users.rb b/lib/api/users.rb index b0811bb4aad..501c5cf1df3 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Users < Grape::API include PaginationParams @@ -14,11 +16,14 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def find_user_by_id(params) id = params[:user_id] || params[:id] User.find_by(id: id) || not_found!('User') end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def reorder_users(users) if params[:order_by] && params[:sort] users.reorder(params[:order_by] => params[:sort]) @@ -26,6 +31,7 @@ module API users end end + # rubocop: enable CodeReuse/ActiveRecord params :optional_attributes do optional :skype, type: String, desc: 'The Skype username' @@ -38,12 +44,12 @@ module API optional :provider, type: String, desc: 'The external provider' optional :bio, type: String, desc: 'The biography of the user' optional :location, type: String, desc: 'The location of the user' + optional :public_email, type: String, desc: 'The public email of the user' optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' optional :avatar, type: File, desc: 'Avatar image for user' optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' - optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit by minimum access level of authenticated user' all_or_none_of :extern_uid, :provider end @@ -75,6 +81,7 @@ module API use :pagination use :with_custom_attributes end + # rubocop: disable CodeReuse/ActiveRecord get do authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) @@ -102,6 +109,7 @@ module API present paginate(users), options end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a single user' do success Entities::User @@ -111,6 +119,7 @@ module API use :with_custom_attributes end + # rubocop: disable CodeReuse/ActiveRecord get ":id" do user = User.find_by(id: params[:id]) not_found!('User') unless user && can?(current_user, :read_user, user) @@ -120,6 +129,7 @@ module API present user, opts end + # rubocop: enable CodeReuse/ActiveRecord desc "Get the status of a user" params do @@ -145,6 +155,7 @@ module API requires :username, type: String, desc: 'The username of the user' use :optional_attributes end + # rubocop: disable CodeReuse/ActiveRecord post do authenticated_as_admin! @@ -165,6 +176,7 @@ module API render_validation_error!(user) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Update a user. Available only for admins.' do success Entities::UserPublic @@ -178,6 +190,7 @@ module API optional :username, type: String, desc: 'The username of the user' use :optional_attributes end + # rubocop: disable CodeReuse/ActiveRecord put ":id" do authenticated_as_admin! @@ -216,6 +229,7 @@ module API render_validation_error!(user) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add an SSH key to a specified user. Available only for admins.' do success Entities::SSHKey @@ -225,6 +239,7 @@ module API requires :key, type: String, desc: 'The new SSH key' requires :title, type: String, desc: 'The title of the new SSH key' end + # rubocop: disable CodeReuse/ActiveRecord post ":id/keys" do authenticated_as_admin! @@ -239,22 +254,23 @@ module API render_validation_error!(key) end end + # rubocop: enable CodeReuse/ActiveRecord - desc 'Get the SSH keys of a specified user. Available only for admins.' do + desc 'Get the SSH keys of a specified user.' do success Entities::SSHKey end params do requires :id, type: Integer, desc: 'The ID of the user' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/keys' do - authenticated_as_admin! - user = User.find_by(id: params[:id]) - not_found!('User') unless user + not_found!('User') unless user && can?(current_user, :read_user, user) present paginate(user.keys), with: Entities::SSHKey end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an existing SSH key from a specified user. Available only for admins.' do success Entities::SSHKey @@ -263,6 +279,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' requires :key_id, type: Integer, desc: 'The ID of the SSH key' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/keys/:key_id' do authenticated_as_admin! @@ -274,6 +291,7 @@ module API destroy_conditionally!(key) end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add a GPG key to a specified user. Available only for admins.' do detail 'This feature was added in GitLab 10.0' @@ -283,6 +301,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' requires :key, type: String, desc: 'The new GPG key' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/gpg_keys' do authenticated_as_admin! @@ -297,6 +316,7 @@ module API render_validation_error!(key) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get the GPG keys of a specified user. Available only for admins.' do detail 'This feature was added in GitLab 10.0' @@ -306,6 +326,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/gpg_keys' do authenticated_as_admin! @@ -314,6 +335,7 @@ module API present paginate(user.gpg_keys), with: Entities::GPGKey end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an existing GPG key from a specified user. Available only for admins.' do detail 'This feature was added in GitLab 10.0' @@ -322,6 +344,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' requires :key_id, type: Integer, desc: 'The ID of the GPG key' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/gpg_keys/:key_id' do authenticated_as_admin! @@ -334,6 +357,7 @@ module API status 204 key.destroy end + # rubocop: enable CodeReuse/ActiveRecord desc 'Revokes an existing GPG key from a specified user. Available only for admins.' do detail 'This feature was added in GitLab 10.0' @@ -342,6 +366,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' requires :key_id, type: Integer, desc: 'The ID of the GPG key' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/gpg_keys/:key_id/revoke' do authenticated_as_admin! @@ -354,6 +379,7 @@ module API key.revoke status :accepted end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add an email address to a specified user. Available only for admins.' do success Entities::Email @@ -361,7 +387,9 @@ module API params do requires :id, type: Integer, desc: 'The ID of the user' requires :email, type: String, desc: 'The email of the user' + optional :skip_confirmation, type: Boolean, desc: 'Skip confirmation of email and assume it is verified' end + # rubocop: disable CodeReuse/ActiveRecord post ":id/emails" do authenticated_as_admin! @@ -376,6 +404,7 @@ module API render_validation_error!(email) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get the emails addresses of a specified user. Available only for admins.' do success Entities::Email @@ -384,6 +413,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/emails' do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -391,6 +421,7 @@ module API present paginate(user.emails), with: Entities::Email end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an email address of a specified user. Available only for admins.' do success Entities::Email @@ -399,6 +430,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' requires :email_id, type: Integer, desc: 'The ID of the email' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/emails/:email_id' do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -411,6 +443,7 @@ module API Emails::DestroyService.new(current_user, user: user).execute(email) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete a user. Available only for admins.' do success Entities::Email @@ -419,6 +452,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions" end + # rubocop: disable CodeReuse/ActiveRecord delete ":id" do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42279') @@ -431,11 +465,13 @@ module API user.delete_async(deleted_by: current_user, params: params) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Block a user. Available only for admins.' params do requires :id, type: Integer, desc: 'The ID of the user' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/block' do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -447,11 +483,13 @@ module API forbidden!('LDAP blocked users cannot be modified by the API') end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Unblock a user. Available only for admins.' params do requires :id, type: Integer, desc: 'The ID of the user' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/unblock' do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -463,6 +501,7 @@ module API user.activate end end + # rubocop: enable CodeReuse/ActiveRecord params do requires :user_id, type: Integer, desc: 'The ID of the user' @@ -475,9 +514,11 @@ module API PersonalAccessTokensFinder.new({ user: user, impersonation: true }.merge(options)) end + # rubocop: disable CodeReuse/ActiveRecord def find_impersonation_token finder.find_by(id: declared_params[:impersonation_token_id]) || not_found!('Impersonation Token') end + # rubocop: enable CodeReuse/ActiveRecord end before { authenticated_as_admin! } @@ -578,12 +619,14 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the SSH key' end + # rubocop: disable CodeReuse/ActiveRecord get "keys/:key_id" do key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key present key, with: Entities::SSHKey end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add a new SSH key to the currently authenticated user' do success Entities::SSHKey @@ -608,12 +651,14 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the SSH key' end + # rubocop: disable CodeReuse/ActiveRecord delete "keys/:key_id" do key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key destroy_conditionally!(key) end + # rubocop: enable CodeReuse/ActiveRecord desc "Get the currently authenticated user's GPG keys" do detail 'This feature was added in GitLab 10.0' @@ -633,12 +678,14 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the GPG key' end + # rubocop: disable CodeReuse/ActiveRecord get 'gpg_keys/:key_id' do key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key present key, with: Entities::GPGKey end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add a new GPG key to the currently authenticated user' do detail 'This feature was added in GitLab 10.0' @@ -663,6 +710,7 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the GPG key' end + # rubocop: disable CodeReuse/ActiveRecord post 'gpg_keys/:key_id/revoke' do key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key @@ -670,6 +718,7 @@ module API key.revoke status :accepted end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete a GPG key from the currently authenticated user' do detail 'This feature was added in GitLab 10.0' @@ -677,6 +726,7 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the SSH key' end + # rubocop: disable CodeReuse/ActiveRecord delete 'gpg_keys/:key_id' do key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key @@ -684,6 +734,7 @@ module API status 204 key.destroy end + # rubocop: enable CodeReuse/ActiveRecord desc "Get the currently authenticated user's email addresses" do success Entities::Email @@ -701,12 +752,14 @@ module API params do requires :email_id, type: Integer, desc: 'The ID of the email' end + # rubocop: disable CodeReuse/ActiveRecord get "emails/:email_id" do email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email present email, with: Entities::Email end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add new email address to the currently authenticated user' do success Entities::Email @@ -728,6 +781,7 @@ module API params do requires :email_id, type: Integer, desc: 'The ID of the email' end + # rubocop: disable CodeReuse/ActiveRecord delete "emails/:email_id" do email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email @@ -736,12 +790,14 @@ module API Emails::DestroyService.new(current_user, user: current_user).execute(email) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a list of user activities' params do optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get "activities" do authenticated_as_admin! @@ -751,6 +807,7 @@ module API present paginate(activities), with: Entities::UserActivity end + # rubocop: enable CodeReuse/ActiveRecord desc 'Set the status of the current user' do success Entities::UserStatus diff --git a/lib/api/variables.rb b/lib/api/variables.rb index a34de9410e8..c844ba321ed 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Variables < Grape::API include PaginationParams @@ -27,6 +29,7 @@ module API params do requires :key, type: String, desc: 'The key of the variable' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/variables/:key' do key = params[:key] variable = user_project.variables.find_by(key: key) @@ -35,6 +38,7 @@ module API present variable, with: Entities::Variable end + # rubocop: enable CodeReuse/ActiveRecord desc 'Create a new variable in a project' do success Entities::Variable @@ -64,6 +68,7 @@ module API optional :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' end + # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key]) @@ -77,6 +82,7 @@ module API render_validation_error!(variable) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an existing variable from a project' do success Entities::Variable @@ -84,6 +90,7 @@ module API params do requires :key, type: String, desc: 'The key of the variable' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key]) not_found!('Variable') unless variable @@ -92,6 +99,7 @@ module API status 204 variable.destroy end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/version.rb b/lib/api/version.rb index 3b10bfa6a7d..74cd857f447 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Version < Grape::API before { authenticate! } diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index b3fc4e876ad..6e1d4eb335f 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -1,6 +1,16 @@ +# frozen_string_literal: true + module API class Wikis < Grape::API helpers do + def commit_params(attrs) + { + file_name: attrs[:file][:filename], + file_content: File.read(attrs[:file][:tempfile]), + branch_name: attrs[:branch] + } + end + params :wiki_page_params do requires :content, type: String, desc: 'Content of a wiki page' requires :title, type: String, desc: 'Title of a wiki page' @@ -84,6 +94,29 @@ module API status 204 WikiPages::DestroyService.new(user_project, current_user).execute(wiki_page) end + + desc 'Upload an attachment to the wiki repository' do + detail 'This feature was introduced in GitLab 11.3.' + success Entities::WikiAttachment + end + params do + requires :file, type: File, desc: 'The attachment file to be uploaded' + optional :branch, type: String, desc: 'The name of the branch' + end + post ":id/wikis/attachments", requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + authorize! :create_wiki, user_project + + result = ::Wikis::CreateAttachmentService.new(user_project, + current_user, + commit_params(declared_params(include_missing: false))).execute + + if result[:status] == :success + status(201) + present OpenStruct.new(result[:result]), with: Entities::WikiAttachment + else + render_api_error!(result[:message], 400) + end + end end end end diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 45a935ab352..33658ae225f 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index adf85ca4719..5e795a449de 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 086ca5986bd..e6bf3d1856f 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'yaml' module Backup diff --git a/lib/backup/files.rb b/lib/backup/files.rb index e287aa1e392..0032ae8f84b 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'open3' require_relative 'helper' diff --git a/lib/backup/helper.rb b/lib/backup/helper.rb index 54b9ce10b4d..22f00aef569 100644 --- a/lib/backup/helper.rb +++ b/lib/backup/helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Backup module Helper def access_denied_error(path) diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb index 185ff8ae6bd..0dfe56e214f 100644 --- a/lib/backup/lfs.rb +++ b/lib/backup/lfs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index a3641505196..afdc6f383c1 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Backup class Manager ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze @@ -241,6 +243,7 @@ module Backup backup_created_at: Time.now, gitlab_version: Gitlab::VERSION, tar_version: tar_version, + installation_type: Gitlab::INSTALLATION_TYPE, skipped: ENV["SKIP"] } end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb index 542e35a7c7c..a4be728df08 100644 --- a/lib/backup/pages.rb +++ b/lib/backup/pages.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb index 35821805797..d16ed2facf1 100644 --- a/lib/backup/registry.rb +++ b/lib/backup/registry.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 906ed498026..c8a5377bfa0 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'yaml' module Backup diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 49b117a7ee3..9577df2634a 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/banzai.rb b/lib/banzai.rb index 5df98f66f3b..788f29a6c08 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -1,4 +1,11 @@ module Banzai + # if you need to render markdown, then you probably need to post_process as well, + # such as removing references that the current user doesn't have + # permission to make + def self.render_and_post_process(text, context = {}) + post_process(render(text, context), context) + end + def self.render(text, context = {}) Renderer.render(text, context) end diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index 3f1e95d4cc0..43f913a8859 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -13,6 +13,7 @@ module Banzai # Returns a Project, or nil if the reference can't be found def parent_from_ref(ref) return context[:project] || context[:group] unless ref + return context[:project] if context[:project]&.full_path == ref Project.find_by_full_path(ref) end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index ad0806df8e6..4764f8e1e19 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -296,7 +296,7 @@ module Banzai # Returns projects for the given paths. def find_for_paths(paths) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? cache = refs_cache to_query = paths - cache.keys @@ -340,7 +340,7 @@ module Banzai end def refs_cache - RequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} + Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} end def parent_type diff --git a/lib/banzai/filter/epic_reference_filter.rb b/lib/banzai/filter/epic_reference_filter.rb index e06e2fb3870..26bcf5c04b4 100644 --- a/lib/banzai/filter/epic_reference_filter.rb +++ b/lib/banzai/filter/epic_reference_filter.rb @@ -9,6 +9,12 @@ module Banzai def self.object_class Epic end + + private + + def group + context[:group] || context[:project]&.group + end end end end diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index b4a7a44e109..8159dcfed72 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -97,9 +97,7 @@ module Banzai private def external_issues_cached(attribute) - return project.public_send(attribute) unless RequestStore.active? # rubocop:disable GitlabSecurity/PublicSend - - cached_attributes = RequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } + cached_attributes = Gitlab::SafeRequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? # rubocop:disable GitlabSecurity/PublicSend cached_attributes[project.id][attribute] end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index b92e9e55bb9..04ec38209c7 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -48,7 +48,7 @@ module Banzai include_ancestor_groups: true, only_group_labels: true } else - { project_id: parent.id, + { project: parent, include_ancestor_groups: true } end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 8e838d04bad..7acbc933adc 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -60,7 +60,11 @@ module Banzai path_parts.unshift(relative_url_root, project.full_path) end - path = Addressable::URI.escape(File.join(*path_parts)) + begin + path = Addressable::URI.escape(File.join(*path_parts)) + rescue Addressable::URI::InvalidURIError + return + end html_attr.value = if context[:only_path] diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb index 574a8a6c7a5..a27f1d46863 100644 --- a/lib/banzai/filter/spaced_link_filter.rb +++ b/lib/banzai/filter/spaced_link_filter.rb @@ -8,22 +8,31 @@ module Banzai # # Based on Banzai::Filter::AutolinkFilter # - # CommonMark does not allow spaces in the url portion of a link. - # For example, `[example](page slug)` is not valid. However, + # CommonMark does not allow spaces in the url portion of a link/url. + # For example, `[example](page slug)` is not valid. + # Neither is ``. However, particularly # in our wikis, we support (via RedCarpet) this type of link, allowing # wiki pages to be easily linked by their title. This filter adds that functionality. - # The intent is for this to only be used in Wikis - in general, we want - # to adhere to CommonMark's spec. + # + # This is a small extension to the CommonMark spec. If they start allowing + # spaces in urls, we could then remove this filter. # class SpacedLinkFilter < HTML::Pipeline::Filter include ActionView::Helpers::TagHelper # Pattern to match a standard markdown link # - # Rubular: http://rubular.com/r/z9EAHxYmKI - LINK_PATTERN = /\[([^\]]+)\]\(([^)"]+)(?: \"([^\"]+)\")?\)/ - - # Text matching LINK_PATTERN inside these elements will not be linked + # Rubular: http://rubular.com/r/2EXEQ49rg5 + LINK_OR_IMAGE_PATTERN = %r{ + (?<preview_operator>!)? + \[(?<text>.+?)\] + \( + (?<new_link>.+?) + (?<title>\ ".+?")? + \) + }x + + # Text matching LINK_OR_IMAGE_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set # The XPath query to use for finding text nodes to parse. @@ -38,7 +47,7 @@ module Banzai doc.xpath(TEXT_QUERY).each do |node| content = node.to_html - next unless content.match(LINK_PATTERN) + next unless content.match(LINK_OR_IMAGE_PATTERN) html = spaced_link_filter(content) @@ -53,25 +62,37 @@ module Banzai private def spaced_link_match(link) - match = LINK_PATTERN.match(link) - return link unless match && match[1] && match[2] + match = LINK_OR_IMAGE_PATTERN.match(link) + return link unless match # escape the spaces in the url so that it's a valid markdown link, # then run it through the markdown processor again, let it do its magic - text = match[1] - new_link = match[2].gsub(' ', '%20') - title = match[3] ? " \"#{match[3]}\"" : '' - html = Banzai::Filter::MarkdownFilter.call("[#{text}](#{new_link}#{title})", context) + html = Banzai::Filter::MarkdownFilter.call(transform_markdown(match), context) # link is wrapped in a <p>, so strip that off html.sub('<p>', '').chomp('</p>') end def spaced_link_filter(text) - Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_PATTERN) do |link, left:, right:| + Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_OR_IMAGE_PATTERN) do |link, left:, right:| spaced_link_match(link) end end + + def transform_markdown(match) + preview_operator, text, new_link, title = process_match(match) + + "#{preview_operator}[#{text}](#{new_link}#{title})" + end + + def process_match(match) + [ + match[:preview_operator], + match[:text], + match[:new_link].gsub(' ', '%20'), + match[:title] + ] + end end end end diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 870721f895d..1728a442533 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'uri' - module Banzai module Filter # HTML filter that "fixes" links to pages/files in a wiki. @@ -13,8 +11,12 @@ module Banzai def call return doc unless project_wiki? - doc.search('a:not(.gfm)').each do |el| - process_link_attr el.attribute('href') + doc.search('a:not(.gfm)').each { |el| process_link_attr(el.attribute('href')) } + doc.search('video').each { |el| process_link_attr(el.attribute('src')) } + doc.search('img').each do |el| + attr = el.attribute('data-src') || el.attribute('src') + + process_link_attr(attr) end doc diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index 072d24e5a11..4bf80aff418 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -10,11 +10,16 @@ module Banzai def apply_rules # Special case: relative URLs beginning with `/uploads/` refer to - # user-uploaded files and will be handled elsewhere. - return @uri.to_s if @uri.relative? && @uri.path.starts_with?('/uploads/') + # user-uploaded files will be handled elsewhere. + return @uri.to_s if public_upload? + + # Special case: relative URLs beginning with Wikis::CreateAttachmentService::ATTACHMENT_PATH + # refer to user-uploaded files to the wiki repository. + unless repository_upload? + apply_file_link_rules! + apply_hierarchical_link_rules! + end - apply_file_link_rules! - apply_hierarchical_link_rules! apply_relative_link_rules! @uri.to_s end @@ -39,6 +44,14 @@ module Banzai @uri = Addressable::URI.parse(link) end end + + def public_upload? + @uri.relative? && @uri.path.starts_with?('/uploads/') + end + + def repository_upload? + @uri.relative? && @uri.path.starts_with?(Wikis::CreateAttachmentService::ATTACHMENT_PATH) + end end end end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index a176f1e261b..7137c1da57d 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -38,6 +38,7 @@ module Banzai redacted_data = redacted[index] object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html(save_options).html_safe) # rubocop:disable GitlabSecurity/PublicSend object.user_visible_reference_count = redacted_data[:visible_reference_count] if object.respond_to?(:user_visible_reference_count) + object.total_reference_count = redacted_data[:total_reference_count] if object.respond_to?(:total_reference_count) end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index e9be05e174e..bd34614f149 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -16,6 +16,7 @@ module Banzai Filter::MathFilter, Filter::ColorFilter, Filter::MermaidFilter, + Filter::SpacedLinkFilter, Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, diff --git a/lib/banzai/pipeline/label_pipeline.rb b/lib/banzai/pipeline/label_pipeline.rb new file mode 100644 index 00000000000..725cccc4b2b --- /dev/null +++ b/lib/banzai/pipeline/label_pipeline.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Banzai + module Pipeline + class LabelPipeline < BasePipeline + def self.filters + @filters ||= FilterArray[ + Filter::SanitizationFilter, + Filter::LabelReferenceFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index 737ff0cc818..c37b8e71cb0 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -5,7 +5,6 @@ module Banzai @filters ||= begin super.insert_after(Filter::TableOfContentsFilter, Filter::GollumTagsFilter) .insert_before(Filter::TaskListFilter, Filter::WikiLinkFilter) - .insert_before(Filter::WikiLinkFilter, Filter::SpacedLinkFilter) end end end diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index 28928d6f376..e77bee78496 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -37,7 +37,13 @@ module Banzai all_document_nodes.each do |entry| nodes_for_document = entry[:nodes] - doc_data = { document: entry[:document], visible_reference_count: nodes_for_document.count } + + doc_data = { + document: entry[:document], + total_reference_count: nodes_for_document.count, + visible_reference_count: nodes_for_document.count + } + metadata << doc_data nodes_for_document.each do |node| diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 68752f5bb5a..334ba97bfb3 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -166,7 +166,7 @@ module Banzai # objects that have not yet been queried. For objects that have already # been queried the object is returned from the cache. def collection_objects_for_ids(collection, ids) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? ids = ids.map(&:to_i) cache = collection_cache[collection_cache_key(collection)] to_query = ids - cache.keys @@ -215,7 +215,7 @@ module Banzai # def projects_for_nodes(nodes) @projects_for_nodes ||= - grouped_objects_for_nodes(nodes, Project, 'data-project') + grouped_objects_for_nodes(nodes, Project.includes(:project_feature), 'data-project') end def can?(user, permission, subject = :global) @@ -248,7 +248,7 @@ module Banzai end def collection_cache - RequestStore[:banzai_collection_cache] ||= Hash.new do |hash, key| + Gitlab::SafeRequestStore[:banzai_collection_cache] ||= Hash.new do |hash, key| hash[key] = {} end end diff --git a/lib/banzai/renderer/common_mark/html.rb b/lib/banzai/renderer/common_mark/html.rb index 46b609c36b0..0b27316da1b 100644 --- a/lib/banzai/renderer/common_mark/html.rb +++ b/lib/banzai/renderer/common_mark/html.rb @@ -4,15 +4,11 @@ module Banzai class HTML < CommonMarker::HtmlRenderer def code_block(node) block do - code = node.string_content - lang = node.fence_info - lang_attr = lang.present? ? %Q{ lang="#{lang}"} : '' - result = - "<pre>" \ - "<code#{lang_attr}>#{ERB::Util.html_escape(code)}</code>" \ - "</pre>" - - out(result) + out("<pre#{sourcepos(node)}><code") + out(' lang="', node.fence_info, '"') if node.fence_info.present? + out('>') + out(escape_html(node.string_content)) + out('</code></pre>') end end end diff --git a/lib/banzai/request_store_reference_cache.rb b/lib/banzai/request_store_reference_cache.rb index 426131442a2..9a9704f9837 100644 --- a/lib/banzai/request_store_reference_cache.rb +++ b/lib/banzai/request_store_reference_cache.rb @@ -1,8 +1,8 @@ module Banzai module RequestStoreReferenceCache def cached_call(request_store_key, cache_key, path: []) - if RequestStore.active? - cache = RequestStore[request_store_key] ||= Hash.new do |hash, key| + if Gitlab::SafeRequestStore.active? + cache = Gitlab::SafeRequestStore[request_store_key] ||= Hash.new do |hash, key| hash[key] = Hash.new { |h, k| h[k] = {} } end diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 61849a40383..1ab14c1c155 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -28,6 +28,7 @@ module ContainerRegistry @components ||= @path.split('/') end + # rubocop: disable CodeReuse/ActiveRecord def nodes raise InvalidRegistryPathError unless valid? @@ -35,17 +36,20 @@ module ContainerRegistry components.take(length).join('/') end end + # rubocop: enable CodeReuse/ActiveRecord def has_project? repository_project.present? end + # rubocop: disable CodeReuse/ActiveRecord def has_repository? return false unless has_project? repository_project.container_repositories .where(name: repository_name).any? end + # rubocop: enable CodeReuse/ActiveRecord def root_repository? @path == project_path diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 728deea224f..c785bca4dad 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -73,11 +73,13 @@ module ContainerRegistry end end + # rubocop: disable CodeReuse/ActiveRecord def total_size return unless layers layers.map(&:size).sum if v2? end + # rubocop: enable CodeReuse/ActiveRecord def delete return unless digest diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index dda6cd38dcd..10d34b0c6e7 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -10,8 +10,6 @@ require_dependency 'declarative_policy/step' require_dependency 'declarative_policy/base' -require 'thread' - module DeclarativePolicy CLASS_CACHE_MUTEX = Mutex.new CLASS_CACHE_IVAR = :@__DeclarativePolicy_CLASS_CACHE diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 515095af1c2..24fdcd6fbb1 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -1,74 +1,42 @@ -class EventFilter - attr_accessor :params - - class << self - def all - 'all' - end - - def push - 'push' - end - - def merged - 'merged' - end +# frozen_string_literal: true - def issue - 'issue' - end - - def comments - 'comments' - end - - def team - 'team' - end +class EventFilter + attr_accessor :filter + + ALL = 'all' + PUSH = 'push' + MERGED = 'merged' + ISSUE = 'issue' + COMMENTS = 'comments' + TEAM = 'team' + FILTERS = [ALL, PUSH, MERGED, ISSUE, COMMENTS, TEAM].freeze + + def initialize(filter) + # Split using comma to maintain backward compatibility Ex/ "filter1,filter2" + filter = filter.to_s.split(',')[0].to_s + @filter = FILTERS.include?(filter) ? filter : ALL end - def initialize(params) - @params = if params - params.dup - else - [] # EventFilter.default_filter - end + def active?(key) + filter == key.to_s end + # rubocop: disable CodeReuse/ActiveRecord def apply_filter(events) - return events if params.blank? || params == EventFilter.all - - case params - when EventFilter.push + case filter + when PUSH events.where(action: Event::PUSHED) - when EventFilter.merged + when MERGED events.where(action: Event::MERGED) - when EventFilter.comments + when COMMENTS events.where(action: Event::COMMENTED) - when EventFilter.team + when TEAM events.where(action: [Event::JOINED, Event::LEFT, Event::EXPIRED]) - when EventFilter.issue + when ISSUE events.where(action: [Event::CREATED, Event::UPDATED, Event::CLOSED, Event::REOPENED]) - end - end - - def options(key) - filter = params.dup - - if filter.include? key - filter.delete key - else - filter << key - end - - filter - end - - def active?(key) - if params.present? - params.include? key else - key == EventFilter.all + events end end + # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index e8dbde176ef..e02d403f7b1 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -54,7 +54,7 @@ module ExtractsPath valid_refs = ref_names.select { |v| id.start_with?("#{v}/") } - if valid_refs.length == 0 + if valid_refs.empty? # No exact ref match, so just try our best pair = id.match(%r{([^/]+)(.*)}).captures else diff --git a/lib/feature.rb b/lib/feature.rb index 24dbcb32fc0..a8324d99c10 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -28,11 +28,7 @@ class Feature end def persisted_names - if RequestStore.active? - RequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names - else - FlipperFeature.feature_names - end + Gitlab::SafeRequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names end def persisted?(feature) @@ -42,13 +38,21 @@ class Feature persisted_names.include?(feature.name.to_s) end - def enabled?(key, thing = nil) - get(key).enabled?(thing) + # use `default_enabled: true` to default the flag to being `enabled` + # unless set explicitly. The default is `disabled` + def enabled?(key, thing = nil, default_enabled: false) + feature = Feature.get(key) + + # If we're not default enabling the flag or the feature has been set, always evaluate. + # `persisted?` can potentially generate DB queries and also checks for inclusion + # in an array of feature names (177 at last count), possibly reducing performance by half. + # So we only perform the `persisted` check if `default_enabled: true` + !default_enabled || Feature.persisted?(feature) ? feature.enabled?(thing) : true end - def disabled?(key, thing = nil) + def disabled?(key, thing = nil, default_enabled: false) # we need to make different method calls to make it easy to mock / define expectations in test mode - thing.nil? ? !enabled?(key) : !enabled?(key, thing) + thing.nil? ? !enabled?(key, default_enabled: default_enabled) : !enabled?(key, thing, default_enabled: default_enabled) end def enable(key, thing = true) @@ -68,8 +72,8 @@ class Feature end def flipper - if RequestStore.active? - RequestStore[:flipper] ||= build_flipper_instance + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance else @flipper ||= build_flipper_instance end diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index 69d981e8be9..53aa8d04e5c 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -32,6 +32,7 @@ class FileSizeValidator < ActiveModel::EachValidator end end + # rubocop: disable CodeReuse/ActiveRecord def validate_each(record, attribute, value) raise(ArgumentError, "A CarrierWave::Uploader::Base object was expected") unless value.is_a? CarrierWave::Uploader::Base @@ -62,6 +63,7 @@ class FileSizeValidator < ActiveModel::EachValidator record.errors.add(attribute, MESSAGES[key], errors_options) end end + # rubocop: enable CodeReuse/ActiveRecord def help Helper.instance diff --git a/lib/gitlab.rb b/lib/gitlab.rb index ab6b609d099..7790534d5d7 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -47,4 +47,8 @@ module Gitlab def self.dev_env_or_com? Rails.env.development? || org? || com? end + + def self.pre_release? + VERSION.include?('pre') + end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 111e18b2076..a36d551d1d7 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -136,6 +136,7 @@ module Gitlab Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, full_authentication_abilities) end + # rubocop: disable CodeReuse/ActiveRecord def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) @@ -146,7 +147,9 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def personal_access_token_check(password) return unless password.present? @@ -156,6 +159,7 @@ module Gitlab Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end + # rubocop: enable CodeReuse/ActiveRecord def valid_oauth_token?(token) token && token.accessible? && valid_scoped_token?(token, [:api]) @@ -177,6 +181,7 @@ module Gitlab end.uniq end + # rubocop: disable CodeReuse/ActiveRecord def deploy_token_check(login, password) return unless password.present? @@ -192,6 +197,7 @@ module Gitlab Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes) end end + # rubocop: enable CodeReuse/ActiveRecord def lfs_token_check(login, password, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index eeab7791643..f323d2e0f7a 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -92,12 +92,12 @@ module Gitlab if provider Gitlab::AppLogger.info( "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" + "blocking GitLab user \"#{user.name}\" (#{user.email})" ) else Gitlab::AppLogger.info( "Account is not provided by LDAP, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" + "blocking GitLab user \"#{user.name}\" (#{user.email})" ) end end @@ -107,7 +107,7 @@ module Gitlab Gitlab::AppLogger.info( "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ - "unblocking Gitlab user \"#{user.name}\" (#{user.email})" + "unblocking GitLab user \"#{user.name}\" (#{user.email})" ) end end diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index 922d0567d99..3c21ddf3241 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -11,11 +11,13 @@ module Gitlab extend ::Gitlab::Utils::Override class << self + # rubocop: disable CodeReuse/ActiveRecord def find_by_uid_and_provider(uid, provider) identity = ::Identity.with_extern_uid(provider, uid).take identity && identity.user end + # rubocop: enable CodeReuse/ActiveRecord end def save diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 589e8062226..2b4f6ed75e5 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -112,11 +112,13 @@ module Gitlab build_new_user end + # rubocop: disable CodeReuse/ActiveRecord def find_by_email return unless auth_hash.has_attribute?(:email) ::User.find_by(email: auth_hash.email.downcase) end + # rubocop: enable CodeReuse/ActiveRecord def auto_link_ldap_user? Gitlab.config.omniauth.auto_link_ldap_user @@ -180,10 +182,12 @@ module Gitlab @auth_hash = AuthHash.new(auth_hash) end + # rubocop: disable CodeReuse/ActiveRecord def find_by_uid_and_provider identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take identity&.user end + # rubocop: enable CodeReuse/ActiveRecord def build_new_user user_params = user_attributes.merge(skip_confirmation: true) diff --git a/lib/gitlab/auth/omniauth_identity_linker_base.rb b/lib/gitlab/auth/omniauth_identity_linker_base.rb index f79ce6bb809..8ae29a02a13 100644 --- a/lib/gitlab/auth/omniauth_identity_linker_base.rb +++ b/lib/gitlab/auth/omniauth_identity_linker_base.rb @@ -33,11 +33,13 @@ module Gitlab @changed = identity.save end + # rubocop: disable CodeReuse/ActiveRecord def identity @identity ||= current_user.identities .with_extern_uid(provider, uid) .first_or_initialize(extern_uid: uid) end + # rubocop: enable CodeReuse/ActiveRecord def provider oauth['provider'] diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index c7993665421..064cba43278 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -71,6 +71,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def find_personal_access_token token = current_request.params[PRIVATE_TOKEN_PARAM].presence || @@ -81,6 +82,7 @@ module Gitlab # Expiration, revocation and scopes are verified in `validate_access_token!` PersonalAccessToken.find_by(token: token) || raise(UnauthorizedError) end + # rubocop: enable CodeReuse/ActiveRecord def find_oauth_access_token token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb new file mode 100644 index 00000000000..0d333e47e7b --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_columns.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # EncryptColumn migrates data from an unencrypted column - `foo`, say - to + # an encrypted column - `encrypted_foo`, say. + # + # For this background migration to work, the table that is migrated _has_ to + # have an `id` column as the primary key. Additionally, the encrypted column + # should be managed by attr_encrypted, and map to an attribute with the same + # name as the unencrypted column (i.e., the unencrypted column should be + # shadowed). + # + # To avoid depending on a particular version of the model in app/, add a + # model to `lib/gitlab/background_migration/models/encrypt_columns` and use + # it in the migration that enqueues the jobs, so code can be shared. + class EncryptColumns + def perform(model, attributes, from, to) + model = model.constantize if model.is_a?(String) + attributes = expand_attributes(model, Array(attributes).map(&:to_sym)) + + model.transaction do + # Use SELECT ... FOR UPDATE to prevent the value being changed while + # we are encrypting it + relation = model.where(id: from..to).lock + + relation.each do |instance| + encrypt!(instance, attributes) + end + end + end + + private + + # Build a hash of { attribute => encrypted column name } + def expand_attributes(klass, attributes) + expanded = attributes.flat_map do |attribute| + attr_config = klass.encrypted_attributes[attribute] + crypt_column_name = attr_config&.fetch(:attribute) + + raise "Couldn't determine encrypted column for #{klass}##{attribute}" if + crypt_column_name.nil? + + [attribute, crypt_column_name] + end + + Hash[*expanded] + end + + # Generate ciphertext for each column and update the database + def encrypt!(instance, attributes) + to_clear = attributes + .map { |plain, crypt| apply_attribute!(instance, plain, crypt) } + .compact + .flat_map { |plain| [plain, nil] } + + to_clear = Hash[*to_clear] + + if instance.changed? + instance.save! + instance.update_columns(to_clear) + end + end + + def apply_attribute!(instance, plain_column, crypt_column) + plaintext = instance[plain_column] + ciphertext = instance[crypt_column] + + # No need to do anything if the plaintext is nil, or an encrypted + # value already exists + return nil unless plaintext.present? && !ciphertext.present? + + # attr_encrypted will calculate and set the expected value for us + instance.public_send("#{plain_column}=", plaintext) # rubocop:disable GitlabSecurity/PublicSend + + plain_column + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb new file mode 100644 index 00000000000..5cd638083b0 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/ClassLength + +module Gitlab + module BackgroundMigration + ## + # The class to migrate job artifacts from `ci_builds` to `ci_job_artifacts` + class MigrateLegacyArtifacts + FILE_LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL + ARCHIVE_FILE_TYPE = 1 # equal to Ci::JobArtifact.file_types['archive'] + METADATA_FILE_TYPE = 2 # equal to Ci::JobArtifact.file_types['metadata'] + LEGACY_PATH_FILE_LOCATION = 1 # equal to Ci::JobArtifact.file_location['legacy_path'] + + def perform(start_id, stop_id) + ActiveRecord::Base.transaction do + insert_archives(start_id, stop_id) + insert_metadatas(start_id, stop_id) + delete_legacy_artifacts(start_id, stop_id) + end + end + + private + + def insert_archives(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO + ci_job_artifacts ( + project_id, + job_id, + expire_at, + file_location, + created_at, + updated_at, + file, + size, + file_store, + file_type + ) + SELECT + project_id, + id, + artifacts_expire_at, + #{LEGACY_PATH_FILE_LOCATION}, + created_at, + created_at, + artifacts_file, + artifacts_size, + COALESCE(artifacts_file_store, #{FILE_LOCAL_STORE}), + #{ARCHIVE_FILE_TYPE} + FROM + ci_builds + WHERE + id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND artifacts_file <> '' + AND NOT EXISTS ( + SELECT + 1 + FROM + ci_job_artifacts + WHERE + ci_builds.id = ci_job_artifacts.job_id + AND ci_job_artifacts.file_type = #{ARCHIVE_FILE_TYPE}) + SQL + end + + def insert_metadatas(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO + ci_job_artifacts ( + project_id, + job_id, + expire_at, + file_location, + created_at, + updated_at, + file, + size, + file_store, + file_type + ) + SELECT + project_id, + id, + artifacts_expire_at, + #{LEGACY_PATH_FILE_LOCATION}, + created_at, + created_at, + artifacts_metadata, + NULL, + COALESCE(artifacts_metadata_store, #{FILE_LOCAL_STORE}), + #{METADATA_FILE_TYPE} + FROM + ci_builds + WHERE + id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND artifacts_file <> '' + AND artifacts_metadata <> '' + AND NOT EXISTS ( + SELECT + 1 + FROM + ci_job_artifacts + WHERE + ci_builds.id = ci_job_artifacts.job_id + AND ci_job_artifacts.file_type = #{METADATA_FILE_TYPE}) + SQL + end + + def delete_legacy_artifacts(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + UPDATE + ci_builds + SET + artifacts_file = NULL, + artifacts_file_store = NULL, + artifacts_size = NULL, + artifacts_metadata = NULL, + artifacts_metadata_store = NULL + WHERE + id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND artifacts_file <> '' + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb new file mode 100644 index 00000000000..bb76eb8ed48 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `token` and `url` columns + class WebHook < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'web_hooks' + self.inheritance_column = :_type_disabled + + attr_encrypted :token, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_truncated + + attr_encrypted :url, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_truncated + end + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_external_pipeline_source.rb b/lib/gitlab/background_migration/populate_external_pipeline_source.rb new file mode 100644 index 00000000000..036fe641757 --- /dev/null +++ b/lib/gitlab/background_migration/populate_external_pipeline_source.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateExternalPipelineSource + module Migratable + class Pipeline < ActiveRecord::Base + self.table_name = 'ci_pipelines' + + def self.sources + { + unknown: nil, + push: 1, + web: 2, + trigger: 3, + schedule: 4, + api: 5, + external: 6 + } + end + end + + class CommitStatus < ActiveRecord::Base + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + + scope :has_pipeline, -> { where('ci_builds.commit_id=ci_pipelines.id') } + scope :of_type, -> (type) { where('type=?', type) } + end + end + + def perform(start_id, stop_id) + external_pipelines(start_id, stop_id) + .update_all(source: Migratable::Pipeline.sources[:external]) + end + + private + + def external_pipelines(start_id, stop_id) + Migratable::Pipeline.where(id: (start_id..stop_id)) + .where( + 'EXISTS (?) AND NOT EXISTS (?)', + Migratable::CommitStatus.of_type('GenericCommitStatus').has_pipeline.select(1), + Migratable::CommitStatus.of_type('Ci::Build').has_pipeline.select(1) + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb index 68f3fa62170..9941c2fe6d9 100644 --- a/lib/gitlab/background_migration/remove_restricted_todos.rb +++ b/lib/gitlab/background_migration/remove_restricted_todos.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true # rubocop:disable Style/Documentation +# rubocop:disable Metrics/ClassLength module Gitlab module BackgroundMigration @@ -49,11 +50,14 @@ module Gitlab private def remove_non_members_todos(project_id) - Todo.where(project_id: project_id) - .where('user_id NOT IN (?)', authorized_users(project_id)) - .each_batch(of: 5000) do |batch| - batch.delete_all - end + if Gitlab::Database.postgresql? + batch_remove_todos_cte(project_id) + else + unauthorized_project_todos(project_id) + .each_batch(of: 5000) do |batch| + batch.delete_all + end + end end def remove_confidential_issue_todos(project_id) @@ -86,10 +90,13 @@ module Gitlab next if target_types.empty? - Todo.where(project_id: project_id) - .where('user_id NOT IN (?)', authorized_users(project_id)) - .where(target_type: target_types) - .delete_all + if Gitlab::Database.postgresql? + batch_remove_todos_cte(project_id, target_types) + else + unauthorized_project_todos(project_id) + .where(target_type: target_types) + .delete_all + end end end @@ -100,6 +107,65 @@ module Gitlab def authorized_users(project_id) ProjectAuthorization.select(:user_id).where(project_id: project_id) end + + def unauthorized_project_todos(project_id) + Todo.where(project_id: project_id) + .where('user_id NOT IN (?)', authorized_users(project_id)) + end + + def batch_remove_todos_cte(project_id, target_types = nil) + loop do + count = remove_todos_cte(project_id, target_types) + + break if count == 0 + end + end + + def remove_todos_cte(project_id, target_types = nil) + sql = [] + sql << with_all_todos_sql(project_id, target_types) + sql << as_deleted_sql + sql << "SELECT count(*) FROM deleted" + + result = Todo.connection.exec_query(sql.join(' ')) + result.rows[0][0].to_i + end + + def with_all_todos_sql(project_id, target_types = nil) + if target_types + table = Arel::Table.new(:todos) + in_target = table[:target_type].in(target_types) + target_types_sql = " AND #{in_target.to_sql}" + end + + <<-SQL + WITH all_todos AS ( + SELECT id + FROM "todos" + WHERE "todos"."project_id" = #{project_id} + AND (user_id NOT IN ( + SELECT "project_authorizations"."user_id" + FROM "project_authorizations" + WHERE "project_authorizations"."project_id" = #{project_id}) + #{target_types_sql} + ) + ), + SQL + end + + def as_deleted_sql + <<-SQL + deleted AS ( + DELETE FROM todos + WHERE id IN ( + SELECT id + FROM all_todos + LIMIT 5000 + ) + RETURNING id + ) + SQL + end end end end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 778d78185ff..16fd6f01495 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -36,6 +36,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def raw_coverage return unless @pipeline @@ -47,6 +48,7 @@ module Gitlab .try(:coverage) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/badge/pipeline/status.rb b/lib/gitlab/badge/pipeline/status.rb index 5fee7a93475..d1d9b7949f5 100644 --- a/lib/gitlab/badge/pipeline/status.rb +++ b/lib/gitlab/badge/pipeline/status.rb @@ -18,11 +18,13 @@ module Gitlab 'pipeline' end + # rubocop: disable CodeReuse/ActiveRecord def status @project.pipelines .where(sha: @sha) .latest_status(@ref) || 'unknown' end + # rubocop: enable CodeReuse/ActiveRecord def metadata @metadata ||= Pipeline::Metadata.new(self) diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index fa0186c854c..a7dfccea2f6 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -43,6 +43,7 @@ module Gitlab find_user_id(username) || project.creator_id end + # rubocop: disable CodeReuse/ActiveRecord def find_user_id(username) return nil unless username @@ -53,6 +54,7 @@ module Gitlab .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) .try(:id) end + # rubocop: enable CodeReuse/ActiveRecord def repo @repo ||= client.repo(project.import_source) @@ -68,6 +70,7 @@ module Gitlab errors << { type: :wiki, errors: e.message } end + # rubocop: disable CodeReuse/ActiveRecord def import_issues return unless repo.issues_enabled? @@ -101,6 +104,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def import_issue_comments(issue, gitlab_issue) client.issue_comments(repo, issue.iid).each do |comment| diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index b591d94668f..15aa4739ee9 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -7,6 +7,7 @@ module Gitlab attr_reader :recover_missing_commits attr_reader :project, :project_key, :repository_slug, :client, :errors, :users + attr_accessor :logger REMOTE_NAME = 'bitbucket_server'.freeze BATCH_SIZE = 100 @@ -36,6 +37,7 @@ module Gitlab @errors = [] @users = {} @temp_branches = [] + @logger = Gitlab::Import::Logger.build end def execute @@ -44,6 +46,8 @@ module Gitlab delete_temp_branches handle_errors + log_info(stage: "complete") + true end @@ -118,15 +122,21 @@ module Gitlab client.create_branch(project_key, repository_slug, branch_name, sha) branches_created << temp_branch rescue BitbucketServer::Connection::ConnectionError => e - Rails.logger.warn("BitbucketServerImporter: Unable to recreate branch for SHA #{sha}: #{e}") + log_warn(message: "Unable to recreate branch", sha: sha, error: e.message) end end end def import_repository + log_info(stage: 'import_repository', message: 'starting import') + project.ensure_repository project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap, remote_name: REMOTE_NAME) + + log_info(stage: 'import_repository', message: 'finished import') rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e + log_error(stage: 'import_repository', message: 'failed import', error: e.message) + # Expire cache to prevent scenarios such as: # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true # 2. Retried import, repo is broken or not imported but +exists?+ still returns true @@ -157,7 +167,10 @@ module Gitlab begin import_bitbucket_pull_request(pull_request) rescue StandardError => e - errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, trace: e.backtrace.join("\n"), raw_response: pull_request.raw } + backtrace = Gitlab::Profiler.clean_backtrace(e.backtrace) + log_error(stage: 'import_pull_requests', iid: pull_request.iid, error: e.message, backtrace: backtrace) + + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, backtrace: backtrace.join("\n"), raw_response: pull_request.raw } end end end @@ -169,12 +182,15 @@ module Gitlab client.delete_branch(project_key, repository_slug, branch.name, branch.sha) project.repository.delete_branch(branch.name) rescue BitbucketServer::Connection::ConnectionError => e + log_error(stage: 'delete_temp_branches', branch: branch.name, error: e.message) @errors << { type: :delete_temp_branches, branch_name: branch.name, errors: e.message } end end end def import_bitbucket_pull_request(pull_request) + log_info(stage: 'import_bitbucket_pull_requests', message: 'starting', iid: pull_request.iid) + description = '' description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author_email) description += pull_request.description if pull_request.description @@ -201,9 +217,13 @@ module Gitlab merge_request = creator.execute(attributes) import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + + log_info(stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid) end def import_pull_request_comments(pull_request, merge_request) + log_info(stage: 'import_pull_request_comments', message: 'starting', iid: merge_request.iid) + comments, other_activities = client.activities(project_key, repository_slug, pull_request.iid).partition(&:comment?) merge_event = other_activities.find(&:merge_event?) @@ -213,9 +233,17 @@ module Gitlab import_inline_comments(inline_comments.map(&:comment), merge_request) import_standalone_pr_comments(pr_comments.map(&:comment), merge_request) + + log_info(stage: 'import_pull_request_comments', message: 'finished', iid: merge_request.iid, + merge_event_found: merge_event.present?, + inline_comments_count: inline_comments.count, + standalone_pr_comments: pr_comments.count) end + # rubocop: disable CodeReuse/ActiveRecord def import_merge_event(merge_request, merge_event) + log_info(stage: 'import_merge_event', message: 'starting', iid: merge_request.iid) + committer = merge_event.committer_email user_id = gitlab_user_id(committer) @@ -223,9 +251,14 @@ module Gitlab merge_request.update({ merge_commit_sha: merge_event.merge_commit }) metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request) metric.update(merged_by_id: user_id, merged_at: timestamp) + + log_info(stage: 'import_merge_event', message: 'finished', iid: merge_request.iid) end + # rubocop: enable CodeReuse/ActiveRecord def import_inline_comments(inline_comments, merge_request) + log_info(stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid) + inline_comments.each do |comment| position = build_position(merge_request, comment) parent = create_diff_note(merge_request, comment, position) @@ -238,6 +271,8 @@ module Gitlab create_diff_note(merge_request, reply, position, discussion_id) end end + + log_info(stage: 'import_inline_comments', message: 'finished', iid: merge_request.iid) end def create_diff_note(merge_request, comment, position, discussion_id = nil) @@ -252,11 +287,14 @@ module Gitlab return note end + log_info(stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid) + # Bitbucket Server supports the ability to comment on any line, not just the # line in the diff. If we can't add the note as a DiffNote, fallback to creating # a regular note. create_fallback_diff_note(merge_request, comment, position) rescue StandardError => e + log_error(stage: 'create_diff_note', comment_id: comment.id, error: e.message) errors << { type: :pull_request, id: comment.id, errors: e.message } nil end @@ -294,7 +332,8 @@ module Gitlab merge_request.notes.create!(pull_request_comment_attributes(replies)) end rescue StandardError => e - errors << { type: :pull_request, iid: comment.id, errors: e.message } + log_error(stage: 'import_standalone_pr_comments', merge_request_id: merge_request.id, comment_id: comment.id, error: e.message) + errors << { type: :pull_request, comment_id: comment.id, errors: e.message } end end end @@ -324,6 +363,26 @@ module Gitlab updated_at: comment.updated_at } end + + def log_info(details) + logger.info(log_base_data.merge(details)) + end + + def log_error(details) + logger.error(log_base_data.merge(details)) + end + + def log_warn(details) + logger.warn(log_base_data.merge(details)) + end + + def log_base_data + { + class: self.class.name, + project_id: project.id, + project_path: project.full_path + } + end end end end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index 671b8e7e1b1..b96e161a5b6 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -26,8 +26,8 @@ module Gitlab define_method(method_name) do |*args| store = - if RequestStore.active? - RequestStore.store + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore.store else ivar_name = # ! and ? cannot be used as ivar name "@cache_#{method_name.to_s.tr('!?', "\u2605\u2606")}" diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb index 22310e313ac..7e0c34aada3 100644 --- a/lib/gitlab/checks/commit_check.rb +++ b/lib/gitlab/checks/commit_check.rb @@ -43,6 +43,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def lfs_file_locks_validation lambda do |paths| lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first @@ -52,6 +53,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def path_validations validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index b816a8f00cd..3f7adecc621 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -6,6 +6,7 @@ module Gitlab @newrev = newrev end + # rubocop: disable CodeReuse/ActiveRecord def objects_missing? return false unless @newrev && @project.lfs_enabled? @@ -20,6 +21,7 @@ module Gitlab existing_count != new_lfs_pointers.count end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index 849848515da..86f4aaeb4d3 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -7,12 +7,14 @@ module Gitlab @project = project end + # rubocop: disable CodeReuse/ActiveRecord def match? @project.merge_requests .with_state(:locked) .where(in_progress_merge_commit_sha: @newrev, target_branch: @branch_name) .exists? end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 428c0505808..85072a072d6 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -96,12 +96,14 @@ module Gitlab blank_node? || @entries.include?(@path.to_s) end + # rubocop: disable CodeReuse/ActiveRecord def total_size descendant_pattern = /^#{Regexp.escape(@path.to_s)}/ entries.sum do |path, entry| (entry[:size] if path =~ descendant_pattern).to_i end end + # rubocop: enable CodeReuse/ActiveRecord def path @path.to_s diff --git a/lib/gitlab/ci/build/policy/changes.rb b/lib/gitlab/ci/build/policy/changes.rb new file mode 100644 index 00000000000..7bf51519752 --- /dev/null +++ b/lib/gitlab/ci/build/policy/changes.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Policy + class Changes < Policy::Specification + def initialize(globs) + @globs = Array(globs) + end + + def satisfied_by?(pipeline, seed) + return true unless pipeline.branch_updated? + + pipeline.modified_paths.any? do |path| + @globs.any? do |glob| + File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 46ed330dbbf..7b7354bce16 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -2,12 +2,14 @@ module Gitlab module Ci module Charts module DailyInterval + # rubocop: disable CodeReuse/ActiveRecord def grouped_count(query) query .group("DATE(#{::Ci::Pipeline.table_name}.created_at)") .count(:created_at) .transform_keys { |date| date.strftime(@format) } # rubocop:disable Gitlab/ModuleWithInstanceVariables end + # rubocop: enable CodeReuse/ActiveRecord def interval_step @interval_step ||= 1.day @@ -15,6 +17,7 @@ module Gitlab end module MonthlyInterval + # rubocop: disable CodeReuse/ActiveRecord def grouped_count(query) if Gitlab::Database.postgresql? query @@ -27,6 +30,7 @@ module Gitlab .count(:created_at) end end + # rubocop: enable CodeReuse/ActiveRecord def interval_step @interval_step ||= 1.month @@ -46,6 +50,7 @@ module Gitlab collect end + # rubocop: disable CodeReuse/ActiveRecord def collect query = project.pipelines .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection @@ -64,6 +69,7 @@ module Gitlab current += interval_step end end + # rubocop: enable CodeReuse/ActiveRecord end class YearChart < Chart diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 66ac4a40616..fe98d25af29 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -1,15 +1,22 @@ module Gitlab module Ci - ## + # # Base GitLab CI Configuration facade # class Config - # EE would override this and utilize opts argument + ConfigError = Class.new(StandardError) + def initialize(config, opts = {}) - @config = Loader.new(config).load! + @config = Config::Extendable + .new(build_config(config, opts)) + .to_hash @global = Entry::Global.new(@config) @global.compose! + rescue Loader::FormatError, Extendable::ExtensionError => e + raise Config::ConfigError, e.message + rescue ::Gitlab::Ci::External::Processor::FileError => e + raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message end def valid? @@ -58,6 +65,24 @@ module Gitlab def jobs @global.jobs_value end + + private + + def build_config(config, opts = {}) + initial_config = Loader.new(config).load! + project = opts.fetch(:project, nil) + + if project + process_external_files(initial_config, project, opts) + else + initial_config + end + end + + def process_external_files(config, project, opts) + sha = opts.fetch(:sha) { project.repository.root_ref_sha } + ::Gitlab::Ci::External::Processor.new(config, project, sha).perform + end end end end diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index 7cddd2c7b7e..697f622c45e 100644 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -24,6 +24,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def compose!(deps = nil) return unless valid? @@ -41,6 +42,7 @@ module Gitlab entry.compose!(deps) end end + # rubocop: enable CodeReuse/ActiveRecord class_methods do def nodes @@ -49,12 +51,14 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def entry(key, entry, metadata) factory = Entry::Factory.new(entry) .with(description: metadata[:description]) (@nodes ||= {}).merge!(key.to_sym => factory) end + # rubocop: enable CodeReuse/ActiveRecord def helpers(*nodes) nodes.each do |symbol| diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb index a4ec8f0ff2f..04077fa7a61 100644 --- a/lib/gitlab/ci/config/entry/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -45,6 +45,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def compose_jobs! factory = Entry::Factory.new(Entry::Jobs) .value(@config.except(*self.class.nodes.keys)) @@ -53,6 +54,7 @@ module Gitlab @entries[:jobs] = factory.create! end + # rubocop: enable CodeReuse/ActiveRecord def compose_deprecated_entries! ## diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 91aac6df4b1..f290ff3a565 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -9,9 +9,10 @@ module Gitlab include Configurable include Attributable - ALLOWED_KEYS = %i[tags script only except type image services allow_failure - type stage when artifacts cache dependencies before_script - after_script variables environment coverage retry].freeze + ALLOWED_KEYS = %i[tags script only except type image services + allow_failure type stage when start_in artifacts cache + dependencies before_script after_script variables + environment coverage retry extends].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -27,12 +28,16 @@ module Gitlab greater_than_or_equal_to: 0, less_than_or_equal_to: 2 } validates :when, - inclusion: { in: %w[on_success on_failure always manual], + inclusion: { in: %w[on_success on_failure always manual delayed], message: 'should be on_success, on_failure, ' \ - 'always or manual' } + 'always, manual or delayed' } validates :dependencies, array_of_strings: true + validates :extends, type: String end + + validates :start_in, duration: { limit: '1 day' }, if: :delayed? + validates :start_in, absence: true, unless: :delayed? end entry :before_script, Entry::Script, @@ -81,7 +86,8 @@ module Gitlab :cache, :image, :services, :only, :except, :variables, :artifacts, :commands, :environment, :coverage, :retry - attributes :script, :tags, :allow_failure, :when, :dependencies, :retry + attributes :script, :tags, :allow_failure, :when, :dependencies, + :retry, :extends, :start_in def compose!(deps = nil) super do @@ -111,6 +117,10 @@ module Gitlab self.when == 'manual' end + def delayed? + self.when == 'delayed' + end + def ignored? allow_failure.nil? ? manual_action? : allow_failure end diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 5671a09480b..96b6f2e5d6c 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -26,6 +26,7 @@ module Gitlab name.to_s.start_with?('.') end + # rubocop: disable CodeReuse/ActiveRecord def compose!(deps = nil) super do @config.each do |name, config| @@ -45,6 +46,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index a78a85397bd..a3d4432be82 100644 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -11,6 +11,15 @@ module Gitlab false end + def validate_duration_limit(value, limit) + return false unless value.is_a?(String) + + ChronicDuration.parse(value).second.from_now < + ChronicDuration.parse(limit).second.from_now + rescue ChronicDuration::DurationParseError + false + end + def validate_array_of_strings(values) values.is_a?(Array) && values.all? { |value| validate_string(value) } end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 09e8e52b60f..c92562f8c85 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -25,17 +25,19 @@ module Gitlab include Entry::Validatable include Entry::Attributable - attributes :refs, :kubernetes, :variables + ALLOWED_KEYS = %i[refs kubernetes variables changes].freeze + attributes :refs, :kubernetes, :variables, :changes validations do validates :config, presence: true - validates :config, allowed_keys: %i[refs kubernetes variables] + validates :config, allowed_keys: ALLOWED_KEYS validate :variables_expressions_syntax with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true validates :kubernetes, allowed_values: %w[active] validates :variables, array_of_strings: true + validates :changes, array_of_strings: true end def variables_expressions_syntax diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 5963f3eb90c..98f12c226b3 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -9,7 +11,7 @@ module Gitlab include Validatable include Attributable - ALLOWED_KEYS = %i[junit].freeze + ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast].freeze attributes ALLOWED_KEYS @@ -19,6 +21,11 @@ module Gitlab with_options allow_nil: true do validates :junit, array_of_strings_or_string: true + validates :codequality, array_of_strings_or_string: true + validates :sast, array_of_strings_or_string: true + validates :dependency_scanning, array_of_strings_or_string: true + validates :container_scanning, array_of_strings_or_string: true + validates :dast, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index b3c889ee92f..f6b4ba7843e 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -49,6 +49,12 @@ module Gitlab unless validate_duration(value) record.errors.add(attribute, 'should be a duration') end + + if options[:limit] + unless validate_duration_limit(value, options[:limit]) + record.errors.add(attribute, 'should not exceed the limit') + end + end end end diff --git a/lib/gitlab/ci/config/extendable.rb b/lib/gitlab/ci/config/extendable.rb new file mode 100644 index 00000000000..a43901c69fe --- /dev/null +++ b/lib/gitlab/ci/config/extendable.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Extendable + include Enumerable + + ExtensionError = Class.new(StandardError) + + def initialize(hash) + @hash = hash.to_h.deep_dup + + each { |entry| entry.extend! if entry.extensible? } + end + + def each + @hash.each_key do |key| + yield Extendable::Entry.new(key, @hash) + end + end + + def to_hash + @hash.to_h + end + end + end + end +end diff --git a/lib/gitlab/ci/config/extendable/entry.rb b/lib/gitlab/ci/config/extendable/entry.rb new file mode 100644 index 00000000000..7793db09d33 --- /dev/null +++ b/lib/gitlab/ci/config/extendable/entry.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Extendable + class Entry + InvalidExtensionError = Class.new(Extendable::ExtensionError) + CircularDependencyError = Class.new(Extendable::ExtensionError) + NestingTooDeepError = Class.new(Extendable::ExtensionError) + + MAX_NESTING_LEVELS = 10 + + attr_reader :key + + def initialize(key, context, parent = nil) + @key = key + @context = context + @parent = parent + + unless @context.key?(@key) + raise StandardError, 'Invalid entry key!' + end + end + + def extensible? + value.is_a?(Hash) && value.key?(:extends) + end + + def value + @value ||= @context.fetch(@key) + end + + def base_hash! + @base ||= Extendable::Entry + .new(extends_key, @context, self) + .extend! + end + + def extends_key + value.fetch(:extends).to_s.to_sym if extensible? + end + + def ancestors + @ancestors ||= Array(@parent&.ancestors) + Array(@parent&.key) + end + + def extend! + return value unless extensible? + + if unknown_extension? + raise Entry::InvalidExtensionError, + "#{key}: unknown key in `extends`" + end + + if invalid_base? + raise Entry::InvalidExtensionError, + "#{key}: invalid base hash in `extends`" + end + + if nesting_too_deep? + raise Entry::NestingTooDeepError, + "#{key}: nesting too deep in `extends`" + end + + if circular_dependency? + raise Entry::CircularDependencyError, + "#{key}: circular dependency detected in `extends`" + end + + @context[key] = base_hash!.deep_merge(value) + end + + private + + def nesting_too_deep? + ancestors.count > MAX_NESTING_LEVELS + end + + def circular_dependency? + ancestors.include?(key) + end + + def unknown_extension? + !@context.key?(extends_key) + end + + def invalid_base? + !@context[extends_key].is_a?(Hash) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/file/base.rb b/lib/gitlab/ci/external/file/base.rb new file mode 100644 index 00000000000..f4da07b0b02 --- /dev/null +++ b/lib/gitlab/ci/external/file/base.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Base + YAML_WHITELIST_EXTENSION = /(yml|yaml)$/i.freeze + + def initialize(location, opts = {}) + @location = location + end + + def valid? + location.match(YAML_WHITELIST_EXTENSION) && content + end + + def content + raise NotImplementedError, 'content must be implemented and return a string or nil' + end + + def error_message + raise NotImplementedError, 'error_message must be implemented and return a string' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/file/local.rb b/lib/gitlab/ci/external/file/local.rb new file mode 100644 index 00000000000..1aa7f687507 --- /dev/null +++ b/lib/gitlab/ci/external/file/local.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Local < Base + attr_reader :location, :project, :sha + + def initialize(location, opts = {}) + super + + @project = opts.fetch(:project) + @sha = opts.fetch(:sha) + end + + def content + @content ||= fetch_local_content + end + + def error_message + "Local file '#{location}' is not valid." + end + + private + + def fetch_local_content + project.repository.blob_data_at(sha, location) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb new file mode 100644 index 00000000000..59bb3e8999e --- /dev/null +++ b/lib/gitlab/ci/external/file/remote.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Remote < Base + include Gitlab::Utils::StrongMemoize + attr_reader :location + + def content + return @content if defined?(@content) + + @content = strong_memoize(:content) do + begin + Gitlab::HTTP.get(location) + rescue Gitlab::HTTP::Error, Timeout::Error, SocketError, Gitlab::HTTP::BlockedUrlError + nil + end + end + end + + def error_message + "Remote file '#{location}' is not valid." + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/mapper.rb b/lib/gitlab/ci/external/mapper.rb new file mode 100644 index 00000000000..58bd6a19acf --- /dev/null +++ b/lib/gitlab/ci/external/mapper.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + class Mapper + def initialize(values, project, sha) + @locations = Array(values.fetch(:include, [])) + @project = project + @sha = sha + end + + def process + locations.map { |location| build_external_file(location) } + end + + private + + attr_reader :locations, :project, :sha + + def build_external_file(location) + if ::Gitlab::UrlSanitizer.valid?(location) + Gitlab::Ci::External::File::Remote.new(location) + else + options = { project: project, sha: sha } + Gitlab::Ci::External::File::Local.new(location, options) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/processor.rb b/lib/gitlab/ci/external/processor.rb new file mode 100644 index 00000000000..76cf3ce89f9 --- /dev/null +++ b/lib/gitlab/ci/external/processor.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + class Processor + FileError = Class.new(StandardError) + + def initialize(values, project, sha) + @values = values + @external_files = Gitlab::Ci::External::Mapper.new(values, project, sha).process + @content = {} + end + + def perform + return values if external_files.empty? + + external_files.each do |external_file| + validate_external_file(external_file) + @content.deep_merge!(content_of(external_file)) + end + + append_inline_content + remove_include_keyword + end + + private + + attr_reader :values, :external_files, :content + + def validate_external_file(external_file) + unless external_file.valid? + raise FileError, external_file.error_message + end + end + + def content_of(external_file) + Gitlab::Ci::Config::Loader.new(external_file.content).load! + end + + def append_inline_content + @content.deep_merge!(@values) + end + + def remove_include_keyword + content.delete(:include) + content + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb deleted file mode 100644 index a4eccc08dfc..00000000000 --- a/lib/gitlab/ci/parsers.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Gitlab - module Ci - module Parsers - def self.fabricate!(file_type) - "Gitlab::Ci::Parsers::#{file_type.classify}".constantize.new - end - end - end -end diff --git a/lib/gitlab/ci/parsers/junit.rb b/lib/gitlab/ci/parsers/junit.rb deleted file mode 100644 index 3c4668ec13b..00000000000 --- a/lib/gitlab/ci/parsers/junit.rb +++ /dev/null @@ -1,69 +0,0 @@ -module Gitlab - module Ci - module Parsers - class Junit - attr_reader :data - - JunitParserError = Class.new(StandardError) - - def parse!(xml_data, test_suite) - @data = Hash.from_xml(xml_data) - - each_suite do |testcases| - testcases.each do |testcase| - test_case = create_test_case(testcase) - test_suite.add_test_case(test_case) - end - end - rescue REXML::ParseException => e - raise JunitParserError, "XML parsing failed: #{e.message}" - rescue => e - raise JunitParserError, "JUnit parsing failed: #{e.message}" - end - - private - - def each_suite - testsuites.each do |testsuite| - yield testcases(testsuite) - end - end - - def testsuites - if data['testsuites'] - data['testsuites']['testsuite'] - else - [data['testsuite']] - end - end - - def testcases(testsuite) - if testsuite['testcase'].is_a?(Array) - testsuite['testcase'] - else - [testsuite['testcase']] - end - end - - def create_test_case(data) - if data['failure'] - status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED - system_output = data['failure'] - else - status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS - system_output = nil - end - - ::Gitlab::Ci::Reports::TestCase.new( - classname: data['classname'], - name: data['name'], - file: data['file'], - execution_time: data['time'], - status: status, - system_output: system_output - ) - end - end - end - end -end diff --git a/lib/gitlab/ci/parsers/test.rb b/lib/gitlab/ci/parsers/test.rb new file mode 100644 index 00000000000..c6bc9662b07 --- /dev/null +++ b/lib/gitlab/ci/parsers/test.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Test + ParserNotFoundError = Class.new(StandardError) + + PARSERS = { + junit: ::Gitlab::Ci::Parsers::Test::Junit + }.freeze + + def self.fabricate!(file_type) + PARSERS.fetch(file_type.to_sym).new + rescue KeyError + raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb new file mode 100644 index 00000000000..5d7d9a751d8 --- /dev/null +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Test + class Junit + JunitParserError = Class.new(StandardError) + + def parse!(xml_data, test_suite) + root = Hash.from_xml(xml_data) + + all_cases(root) do |test_case| + test_case = create_test_case(test_case) + test_suite.add_test_case(test_case) + end + rescue REXML::ParseException => e + raise JunitParserError, "XML parsing failed: #{e.message}" + rescue => e + raise JunitParserError, "JUnit parsing failed: #{e.message}" + end + + private + + def all_cases(root, parent = nil, &blk) + return unless root.present? + + [root].flatten.compact.map do |node| + next unless node.is_a?(Hash) + + # we allow only one top-level 'testsuites' + all_cases(node['testsuites'], root, &blk) unless parent + + # we require at least one level of testsuites or testsuite + each_case(node['testcase'], &blk) if parent + + # we allow multiple nested 'testsuite' (eg. PHPUnit) + all_cases(node['testsuite'], root, &blk) + end + end + + def each_case(testcase, &blk) + return unless testcase.present? + + [testcase].flatten.compact.map(&blk) + end + + def create_test_case(data) + if data['failure'] + status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED + system_output = data['failure'] + else + status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS + system_output = nil + end + + ::Gitlab::Ci::Reports::TestCase.new( + classname: data['classname'], + name: data['name'], + file: data['file'], + execution_time: data['time'], + status: status, + system_output: system_output + ) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index f4c8d5342c1..02493c7fe02 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -5,6 +5,7 @@ module Gitlab class Create < Chain::Base include Chain::Helpers + # rubocop: disable CodeReuse/ActiveRecord def perform! ::Ci::Pipeline.transaction do pipeline.save! @@ -23,6 +24,7 @@ module Gitlab rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") end + # rubocop: enable CodeReuse/ActiveRecord def break? !pipeline.persisted? diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb index 469fc094cc8..30701e1de1b 100644 --- a/lib/gitlab/ci/pipeline/duration.rb +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -86,6 +86,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def from_pipeline(pipeline) status = %w[success failed running canceled] builds = pipeline.builds.latest @@ -93,6 +94,7 @@ module Gitlab from_builds(builds) end + # rubocop: enable CodeReuse/ActiveRecord def from_builds(builds) now = Time.now @@ -134,9 +136,11 @@ module Gitlab Period.new(previous.first, [previous.last, current.last].max) end + # rubocop: disable CodeReuse/ActiveRecord def process_duration(periods) periods.sum(&:duration) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb index c6e732e68eb..c87bdb4a8a2 100644 --- a/lib/gitlab/ci/reports/test_reports.rb +++ b/lib/gitlab/ci/reports/test_reports.rb @@ -12,13 +12,17 @@ module Gitlab test_suites[suite_name] ||= TestSuite.new(suite_name) end + # rubocop: disable CodeReuse/ActiveRecord def total_time test_suites.values.sum(&:total_time) end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def total_count test_suites.values.sum(&:total_count) end + # rubocop: enable CodeReuse/ActiveRecord def total_status if failed_count > 0 || error_count > 0 @@ -30,7 +34,9 @@ module Gitlab TestCase::STATUS_TYPES.each do |status_type| define_method("#{status_type}_count") do + # rubocop: disable CodeReuse/ActiveRecord test_suites.values.sum { |suite| suite.public_send("#{status_type}_count") } # rubocop:disable GitlabSecurity/PublicSend + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb index c0943f5a51a..726c6a11a81 100644 --- a/lib/gitlab/ci/reports/test_reports_comparer.rb +++ b/lib/gitlab/ci/reports/test_reports_comparer.rb @@ -29,7 +29,9 @@ module Gitlab %w(total_count resolved_count failed_count).each do |method| define_method(method) do + # rubocop: disable CodeReuse/ActiveRecord suite_comparers.sum { |suite| suite.public_send(method) } # rubocop:disable GitlabSecurity/PublicSend + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index b722d0ba735..b5f15397c0f 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -21,9 +21,11 @@ module Gitlab @total_time += test_case.execution_time end + # rubocop: disable CodeReuse/ActiveRecord def total_count test_cases.values.sum(&:count) end + # rubocop: enable CodeReuse/ActiveRecord def total_status if failed_count > 0 || error_count > 0 diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 2b26ebb45a1..4a74d6d6ed1 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -5,6 +5,7 @@ module Gitlab class Factory < Status::Factory def self.extended_statuses [[Status::Build::Erased, + Status::Build::Scheduled, Status::Build::Manual, Status::Build::Canceled, Status::Build::Created, @@ -14,6 +15,7 @@ module Gitlab Status::Build::Retryable], [Status::Build::Failed], [Status::Build::FailedAllowed, + Status::Build::Unschedule, Status::Build::Play, Status::Build::Stop], [Status::Build::Action], diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 508b4814631..50b0d044265 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -10,9 +10,12 @@ module Gitlab stuck_or_timeout_failure: 'stuck or timeout failure', runner_system_failure: 'runner system failure', missing_dependency_failure: 'missing dependency failure', - runner_unsupported: 'unsupported runner' + runner_unsupported: 'unsupported runner', + stale_schedule: 'stale schedule' }.freeze + private_constant :REASONS + def status_tooltip base_message end @@ -25,6 +28,10 @@ module Gitlab build.failed? end + def self.reasons + REASONS + end + private def base_message @@ -36,7 +43,7 @@ module Gitlab end def failure_reason_message - REASONS.fetch(subject.failure_reason.to_sym) + self.class.reasons.fetch(subject.failure_reason.to_sym) end end end diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb new file mode 100644 index 00000000000..eebb3f761c5 --- /dev/null +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -0,0 +1,38 @@ +module Gitlab + module Ci + module Status + module Build + class Scheduled < Status::Extended + def illustration + { + image: 'illustrations/illustrations_scheduled-job_countdown.svg', + size: 'svg-394', + title: _("This is a scheduled to run in ") + " #{execute_in}", + content: _("This job will automatically run after it's timer finishes. " \ + "Often they are used for incremental roll-out deploys " \ + "to production environments. When unscheduled it converts " \ + "into a manual action.") + } + end + + def status_tooltip + "scheduled manual action (#{execute_in})" + end + + def self.matches?(build, user) + build.scheduled? && build.scheduled_at + end + + private + + include TimeHelper + + def execute_in + remaining_seconds = [0, subject.scheduled_at - Time.now].max + duration_in_numbers(remaining_seconds, true) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/unschedule.rb b/lib/gitlab/ci/status/build/unschedule.rb new file mode 100644 index 00000000000..e1b7b83428c --- /dev/null +++ b/lib/gitlab/ci/status/build/unschedule.rb @@ -0,0 +1,41 @@ +module Gitlab + module Ci + module Status + module Build + class Unschedule < Status::Extended + def label + 'unschedule action' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'time-out' + end + + def action_title + 'Unschedule' + end + + def action_button_title + _('Unschedule job') + end + + def action_path + unschedule_project_job_path(subject.project, subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.scheduled? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 17f9a75f436..00d8f01cbdc 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -5,6 +5,7 @@ module Gitlab class Factory < Status::Factory def self.extended_statuses [[Status::SuccessWarning, + Status::Pipeline::Scheduled, Status::Pipeline::Blocked]] end diff --git a/lib/gitlab/ci/status/pipeline/scheduled.rb b/lib/gitlab/ci/status/pipeline/scheduled.rb new file mode 100644 index 00000000000..9ec6994bd2f --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/scheduled.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Pipeline + class Scheduled < Status::Extended + def text + s_('CiStatusText|scheduled') + end + + def label + s_('CiStatusLabel|waiting for delayed job') + end + + def self.matches?(pipeline, user) + pipeline.scheduled? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb new file mode 100644 index 00000000000..542100e41da --- /dev/null +++ b/lib/gitlab/ci/status/scheduled.rb @@ -0,0 +1,23 @@ +module Gitlab + module Ci + module Status + class Scheduled < Status::Core + def text + s_('CiStatusText|scheduled') + end + + def label + s_('CiStatusLabel|scheduled') + end + + def icon + 'status_scheduled' + end + + def favicon + 'favicon_status_scheduled' + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml new file mode 100644 index 00000000000..5f9d54ff574 --- /dev/null +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -0,0 +1,51 @@ +# Read more about this script on this blog post https://about.gitlab.com/2016/11/30/setting-up-gitlab-ci-for-android-projects/, by Greyson Parrelli +image: openjdk:8-jdk + +variables: + ANDROID_COMPILE_SDK: "25" + ANDROID_BUILD_TOOLS: "24.0.0" + ANDROID_SDK_TOOLS: "24.4.1" + +before_script: + - apt-get --quiet update --yes + - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 + - wget --quiet --output-document=android-sdk.tgz https://dl.google.com/android/android-sdk_r${ANDROID_SDK_TOOLS}-linux.tgz + - tar --extract --gzip --file=android-sdk.tgz + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter android-${ANDROID_COMPILE_SDK} + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter platform-tools + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter build-tools-${ANDROID_BUILD_TOOLS} + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter extra-android-m2repository + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter extra-google-google_play_services + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter extra-google-m2repository + - export ANDROID_HOME=$PWD/android-sdk-linux + - export PATH=$PATH:$PWD/android-sdk-linux/platform-tools/ + - chmod +x ./gradlew + +stages: + - build + - test + +build: + stage: build + script: + - ./gradlew assembleDebug + artifacts: + paths: + - app/build/outputs/ + +unitTests: + stage: test + script: + - ./gradlew test + +functionalTests: + stage: test + script: + - wget --quiet --output-document=android-wait-for-emulator https://raw.githubusercontent.com/travis-ci/travis-cookbooks/0f497eb71291b52a703143c5cd63a217c8766dc9/community-cookbooks/android-sdk/files/default/android-wait-for-emulator + - chmod +x android-wait-for-emulator + - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter sys-img-x86-google_apis-${ANDROID_COMPILE_SDK} + - echo no | android-sdk-linux/tools/android create avd -n test -t android-${ANDROID_COMPILE_SDK} --abi google_apis/x86 + - android-sdk-linux/tools/emulator64-x86 -avd test -no-window -no-audio & + - ./android-wait-for-emulator + - adb shell input keyevent 82 + - ./gradlew cAT diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml new file mode 100644 index 00000000000..72547c1b407 --- /dev/null +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -0,0 +1,897 @@ +# Auto DevOps +# This CI/CD configuration provides a standard pipeline for +# * building a Docker image (using a buildpack if necessary), +# * storing the image in the container registry, +# * running tests from a buildpack, +# * running code quality analysis, +# * creating a review app for each topic branch, +# * and continuous deployment to production +# +# Test jobs may be disabled by setting environment variables: +# * test: TEST_DISABLED +# * code_quality: CODE_QUALITY_DISABLED +# * license_management: LICENSE_MANAGEMENT_DISABLED +# * performance: PERFORMANCE_DISABLED +# * sast: SAST_DISABLED +# * dependency_scanning: DEPENDENCY_SCANNING_DISABLED +# * container_scanning: CONTAINER_SCANNING_DISABLED +# * dast: DAST_DISABLED +# * review: REVIEW_DISABLED +# * stop_review: REVIEW_DISABLED +# +# In order to deploy, you must have a Kubernetes cluster configured either +# via a project integration, or via group/project variables. +# AUTO_DEVOPS_DOMAIN must also be set as a variable at the group or project +# level, or manually added below. +# +# Continuous deployment to production is enabled by default. +# If you want to deploy to staging first, set STAGING_ENABLED environment variable. +# If you want to enable incremental rollout, either manual or time based, +# set INCREMENTAL_ROLLOUT_TYPE environment variable to "manual" or "timed". +# If you want to use canary deployments, set CANARY_ENABLED environment variable. +# +# If Auto DevOps fails to detect the proper buildpack, or if you want to +# specify a custom buildpack, set a project variable `BUILDPACK_URL` to the +# repository URL of the buildpack. +# e.g. BUILDPACK_URL=https://github.com/heroku/heroku-buildpack-ruby.git#v142 +# If you need multiple buildpacks, add a file to your project called +# `.buildpacks` that contains the URLs, one on each line, in order. +# Note: Auto CI does not work with multiple buildpacks yet + +image: alpine:latest + +variables: + # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. + # AUTO_DEVOPS_DOMAIN: domain.example.com + + POSTGRES_USER: user + POSTGRES_PASSWORD: testing-password + POSTGRES_ENABLED: "true" + POSTGRES_DB: $CI_ENVIRONMENT_SLUG + + KUBERNETES_VERSION: 1.8.6 + HELM_VERSION: 2.11.0 + + DOCKER_DRIVER: overlay2 + +stages: + - build + - test + - review + - dast + - staging + - canary + - production + - incremental rollout 10% + - incremental rollout 25% + - incremental rollout 50% + - incremental rollout 100% + - performance + - cleanup + +build: + stage: build + image: docker:stable-git + services: + - docker:stable-dind + script: + - setup_docker + - build + only: + - branches + +test: + services: + - postgres:latest + variables: + POSTGRES_DB: test + stage: test + image: gliderlabs/herokuish:latest + script: + - setup_test_db + - cp -R . /tmp/app + - /bin/herokuish buildpack test + only: + - branches + except: + variables: + - $TEST_DISABLED + +code_quality: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - code_quality + artifacts: + paths: [gl-code-quality-report.json] + only: + - branches + except: + variables: + - $CODE_QUALITY_DISABLED + +license_management: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - license_management + artifacts: + paths: [gl-license-management-report.json] + only: + - branches + only: + variables: + - $GITLAB_FEATURES =~ /\blicense_management\b/ + except: + variables: + - $LICENSE_MANAGEMENT_DISABLED + +performance: + stage: performance + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - performance + artifacts: + paths: + - performance.json + - sitespeed-results/ + only: + refs: + - branches + kubernetes: active + except: + variables: + - $PERFORMANCE_DISABLED + +sast: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - sast + artifacts: + paths: [gl-sast-report.json] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bsast\b/ + except: + variables: + - $SAST_DISABLED + +dependency_scanning: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - dependency_scanning + artifacts: + paths: [gl-dependency-scanning-report.json] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + except: + variables: + - $DEPENDENCY_SCANNING_DISABLED + +container_scanning: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - container_scanning + artifacts: + paths: [gl-container-scanning-report.json] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bsast_container\b/ + except: + variables: + - $CONTAINER_SCANNING_DISABLED + +dast: + stage: dast + allow_failure: true + image: registry.gitlab.com/gitlab-org/security-products/zaproxy + variables: + POSTGRES_DB: "false" + script: + - dast + artifacts: + paths: [gl-dast-report.json] + only: + refs: + - branches + kubernetes: active + variables: + - $GITLAB_FEATURES =~ /\bdast\b/ + except: + refs: + - master + variables: + - $DAST_DISABLED + +review: + stage: review + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + - persist_environment_url + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN + on_stop: stop_review + artifacts: + paths: [environment_url.txt] + only: + refs: + - branches + kubernetes: active + except: + refs: + - master + variables: + - $REVIEW_DISABLED + +stop_review: + stage: cleanup + variables: + GIT_STRATEGY: none + script: + - install_dependencies + - initialize_tiller + - delete + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + when: manual + allow_failure: true + only: + refs: + - branches + kubernetes: active + except: + refs: + - master + variables: + - $REVIEW_DISABLED + +# Staging deploys are disabled by default since +# continuous deployment to production is enabled by default +# If you prefer to automatically deploy to staging and +# only manually promote to production, enable this job by setting +# STAGING_ENABLED. + +staging: + stage: staging + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + environment: + name: staging + url: http://$CI_PROJECT_PATH_SLUG-staging.$AUTO_DEVOPS_DOMAIN + only: + refs: + - master + kubernetes: active + variables: + - $STAGING_ENABLED + +# Canaries are also disabled by default, but if you want them, +# and know what the downsides are, you can enable this by setting +# CANARY_ENABLED. + +canary: + stage: canary + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy canary + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + when: manual + only: + refs: + - master + kubernetes: active + variables: + - $CANARY_ENABLED + +.production: &production_template + stage: production + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + - delete canary + - delete rollout + - persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + artifacts: + paths: [environment_url.txt] + +production: + <<: *production_template + only: + refs: + - master + kubernetes: active + except: + variables: + - $STAGING_ENABLED + - $CANARY_ENABLED + - $INCREMENTAL_ROLLOUT_ENABLED + - $INCREMENTAL_ROLLOUT_MODE + +production_manual: + <<: *production_template + when: manual + allow_failure: false + only: + refs: + - master + kubernetes: active + variables: + - $STAGING_ENABLED + - $CANARY_ENABLED + except: + variables: + - $INCREMENTAL_ROLLOUT_ENABLED + - $INCREMENTAL_ROLLOUT_MODE + +# This job implements incremental rollout on for every push to `master`. + +.rollout: &rollout_template + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy rollout $ROLLOUT_PERCENTAGE + - scale stable $((100-ROLLOUT_PERCENTAGE)) + - delete canary + - persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + artifacts: + paths: [environment_url.txt] + +.manual_rollout_template: &manual_rollout_template + <<: *rollout_template + stage: production + when: manual + # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4) + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_MODE == "manual" + - $INCREMENTAL_ROLLOUT_ENABLED + except: + variables: + - $INCREMENTAL_ROLLOUT_MODE == "timed" + +.timed_rollout_template: &timed_rollout_template + <<: *rollout_template + when: delayed + start_in: 5 minutes + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_MODE == "timed" + +timed rollout 10%: + <<: *timed_rollout_template + stage: incremental rollout 10% + variables: + ROLLOUT_PERCENTAGE: 10 + +timed rollout 25%: + <<: *timed_rollout_template + stage: incremental rollout 25% + variables: + ROLLOUT_PERCENTAGE: 25 + +timed rollout 50%: + <<: *timed_rollout_template + stage: incremental rollout 50% + variables: + ROLLOUT_PERCENTAGE: 50 + +timed rollout 100%: + <<: *timed_rollout_template + <<: *production_template + stage: incremental rollout 100% + variables: + ROLLOUT_PERCENTAGE: 100 + +rollout 10%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 10 + +rollout 25%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 25 + +rollout 50%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 50 + +rollout 100%: + <<: *manual_rollout_template + <<: *production_template + allow_failure: false + +# --------------------------------------------------------------------------- + +.auto_devops: &auto_devops | + # Auto DevOps variables and functions + [[ "$TRACE" ]] && set -x + auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} + export DATABASE_URL=${DATABASE_URL-$auto_database_url} + export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG + export CI_APPLICATION_TAG=$CI_COMMIT_SHA + export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID} + export TILLER_NAMESPACE=$KUBE_NAMESPACE + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products + export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + + function registry_login() { + if [[ -n "$CI_REGISTRY_USER" ]]; then + echo "Logging to GitLab Container Registry with CI credentials..." + docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + echo "" + fi + } + + function container_scanning() { + registry_login + + docker run -d --name db arminc/clair-db:latest + docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.1 + apk add -U wget ca-certificates + docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} + wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64 + mv clair-scanner_linux_amd64 clair-scanner + chmod +x clair-scanner + touch clair-whitelist.yml + retries=0 + echo "Waiting for clair daemon to start" + while( ! wget -T 10 -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done + ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-container-scanning-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true + } + + function code_quality() { + docker run --env SOURCE_CODE="$PWD" \ + --volume "$PWD":/code \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code + } + + function license_management() { + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" + LICENSE_MANAGEMENT_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + + docker run --volume "$PWD:/code" \ + "registry.gitlab.com/gitlab-org/security-products/license-management:$LICENSE_MANAGEMENT_VERSION" analyze /code + } + + function sast() { + case "$CI_SERVER_VERSION" in + *-ee) + + # Deprecation notice for CONFIDENCE_LEVEL variable + if [ -z "$SAST_CONFIDENCE_LEVEL" -a "$CONFIDENCE_LEVEL" ]; then + SAST_CONFIDENCE_LEVEL="$CONFIDENCE_LEVEL" + echo "WARNING: CONFIDENCE_LEVEL is deprecated and MUST be replaced with SAST_CONFIDENCE_LEVEL" + fi + + docker run --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}" \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code + ;; + *) + echo "GitLab EE is required" + ;; + esac + } + + function dependency_scanning() { + case "$CI_SERVER_VERSION" in + *-ee) + docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code + ;; + *) + echo "GitLab EE is required" + ;; + esac + } + + function get_replicas() { + track="${1:-stable}" + percentage="${2:-100}" + + env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) + env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) + + if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then + # for stable track get number of replicas from `PRODUCTION_REPLICAS` + eval new_replicas=\$${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + new_replicas=$REPLICAS + fi + else + # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` + eval new_replicas=\$${env_track}_${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + eval new_replicas=\${env_track}_REPLICAS + fi + fi + + replicas="${new_replicas:-1}" + replicas="$(($replicas * $percentage / 100))" + + # always return at least one replicas + if [[ $replicas -gt 0 ]]; then + echo "$replicas" + else + echo 1 + fi + } + + function deploy() { + track="${1-stable}" + percentage="${2:-100}" + name="$CI_ENVIRONMENT_SLUG" + + replicas="1" + service_enabled="true" + postgres_enabled="$POSTGRES_ENABLED" + + # if track is different than stable, + # re-use all attached resources + if [[ "$track" != "stable" ]]; then + name="$name-$track" + service_enabled="false" + postgres_enabled="false" + fi + + replicas=$(get_replicas "$track" "$percentage") + + if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then + secret_name='gitlab-registry' + else + secret_name='' + fi + + if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="$CI_APPLICATION_REPOSITORY" \ + --set image.tag="$CI_APPLICATION_TAG" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set application.initializeCommand="$DB_INITIALIZE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + + helm upgrade --reuse-values \ + --wait \ + --set application.initializeCommand="" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + else + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="$CI_APPLICATION_REPOSITORY" \ + --set image.tag="$CI_APPLICATION_TAG" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + fi + + kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name" + } + + function scale() { + track="${1-stable}" + percentage="${2-100}" + name="$CI_ENVIRONMENT_SLUG" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + replicas=$(get_replicas "$track" "$percentage") + + if [[ -n "$(helm ls -q "^$name$")" ]]; then + helm upgrade --reuse-values \ + --wait \ + --set replicaCount="$replicas" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + fi + } + + function install_dependencies() { + apk add -U openssl curl tar gzip bash ca-certificates git + curl -L -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub + curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk + apk add glibc-2.28-r0.apk + rm glibc-2.28-r0.apk + + curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx + mv linux-amd64/helm /usr/bin/ + mv linux-amd64/tiller /usr/bin/ + helm version --client + tiller -version + + curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" + chmod +x /usr/bin/kubectl + kubectl version --client + } + + function setup_docker() { + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + } + + function setup_test_db() { + if [ -z ${KUBERNETES_PORT+x} ]; then + DB_HOST=postgres + else + DB_HOST=localhost + fi + export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}" + } + + function download_chart() { + if [[ ! -d chart ]]; then + auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} + auto_chart_name=$(basename $auto_chart) + auto_chart_name=${auto_chart_name%.tgz} + auto_chart_name=${auto_chart_name%.tar.gz} + else + auto_chart="chart" + auto_chart_name="chart" + fi + + helm init --client-only + helm repo add gitlab https://charts.gitlab.io + if [[ ! -d "$auto_chart" ]]; then + helm fetch ${auto_chart} --untar + fi + if [ "$auto_chart_name" != "chart" ]; then + mv ${auto_chart_name} chart + fi + + helm dependency update chart/ + helm dependency build chart/ + } + + function ensure_namespace() { + kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" + } + + function check_kube_domain() { + if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then + echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set" + echo "You can do it in Auto DevOps project settings or defining a variable at group or project level" + echo "You can also manually add it in .gitlab-ci.yml" + false + else + true + fi + } + + function build() { + registry_login + + if [[ -f Dockerfile ]]; then + echo "Building Dockerfile-based application..." + docker build \ + --build-arg HTTP_PROXY="$HTTP_PROXY" \ + --build-arg http_proxy="$http_proxy" \ + --build-arg HTTPS_PROXY="$HTTPS_PROXY" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg FTP_PROXY="$FTP_PROXY" \ + --build-arg ftp_proxy="$ftp_proxy" \ + --build-arg NO_PROXY="$NO_PROXY" \ + --build-arg no_proxy="$no_proxy" \ + -t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" . + else + echo "Building Heroku-based application using gliderlabs/herokuish docker image..." + docker run -i \ + -e BUILDPACK_URL \ + -e HTTP_PROXY \ + -e http_proxy \ + -e HTTPS_PROXY \ + -e https_proxy \ + -e FTP_PROXY \ + -e ftp_proxy \ + -e NO_PROXY \ + -e no_proxy \ + --name="$CI_CONTAINER_NAME" -v "$(pwd):/tmp/app:ro" gliderlabs/herokuish /bin/herokuish buildpack build + docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + docker rm "$CI_CONTAINER_NAME" >/dev/null + echo "" + + echo "Configuring $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG docker image..." + docker create --expose 5000 --env PORT=5000 --name="$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" /bin/herokuish procfile start web + docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + docker rm "$CI_CONTAINER_NAME" >/dev/null + echo "" + fi + + echo "Pushing to GitLab Container Registry..." + docker push "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + echo "" + } + + function initialize_tiller() { + echo "Checking Tiller..." + + export HELM_HOST=":44134" + tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & + echo "Tiller is listening on ${HELM_HOST}" + + if ! helm version --debug; then + echo "Failed to init Tiller." + return 1 + fi + echo "" + } + + function create_secret() { + echo "Create secret..." + if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then + return + fi + + kubectl create secret -n "$KUBE_NAMESPACE" \ + docker-registry gitlab-registry \ + --docker-server="$CI_REGISTRY" \ + --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \ + --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \ + --docker-email="$GITLAB_USER_EMAIL" \ + -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - + } + + function dast() { + export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + + mkdir /zap/wrk/ + /zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true + cp /zap/wrk/gl-dast-report.json . + } + + function performance() { + export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + + mkdir gitlab-exporter + wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js + + mkdir sitespeed-results + + if [ -f .gitlab-urls.txt ] + then + sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt + else + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" + fi + + mv sitespeed-results/data/performance.json performance.json + } + + function persist_environment_url() { + echo $CI_ENVIRONMENT_URL > environment_url.txt + } + + function delete() { + track="${1-stable}" + name="$CI_ENVIRONMENT_SLUG" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + if [[ -n "$(helm ls -q "^$name$")" ]]; then + helm delete --purge "$name" + fi + } + +before_script: + - *auto_devops diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml new file mode 100644 index 00000000000..2d218b2e164 --- /dev/null +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -0,0 +1,35 @@ +# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options + +# you can delete this line if you're not using Docker +image: busybox:latest + +before_script: + - echo "Before script section" + - echo "For example you might run an update here or install a build dependency" + - echo "Or perhaps you might print out some debugging details" + +after_script: + - echo "After script section" + - echo "For example you might do some cleanup here" + +build1: + stage: build + script: + - echo "Do your build here" + +test1: + stage: test + script: + - echo "Do a test here" + - echo "For example run a test suite" + +test2: + stage: test + script: + - echo "Do another parallel test here" + - echo "For example run a lint test" + +deploy1: + stage: deploy + script: + - echo "Do your deploy here" diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml new file mode 100644 index 00000000000..c83c49d8c95 --- /dev/null +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -0,0 +1,26 @@ +# use the official gcc image, based on debian +# can use verions as well, like gcc:5.2 +# see https://hub.docker.com/_/gcc/ +image: gcc + +build: + stage: build + # instead of calling g++ directly you can also use some build toolkit like make + # install the necessary build tools when needed + # before_script: + # - apt update && apt -y install make autoconf + script: + - g++ helloworld.cpp -o mybinary + artifacts: + paths: + - mybinary + # depending on your build setup it's most likely a good idea to cache outputs to reduce the build time + # cache: + # paths: + # - "*.o" + +# run tests using the binary built before +test: + stage: test + script: + - ./runmytests.sh diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml new file mode 100644 index 00000000000..4d5b6484d6e --- /dev/null +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -0,0 +1,51 @@ +# This file uses Test Kitchen with the kitchen-dokken driver to +# perform functional testing. Doing so requires that your runner be a +# Docker runner configured for privileged mode. Please see +# https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode +# for help configuring your runner properly, or, if you want to switch +# to a different driver, see http://kitchen.ci/docs/drivers + +image: "chef/chefdk" +services: + - docker:dind + +variables: + DOCKER_HOST: "tcp://docker:2375" + KITCHEN_LOCAL_YAML: ".kitchen.dokken.yml" + +stages: + - lint + - unit + - functional + +foodcritic: + stage: lint + script: + - chef exec foodcritic . + +cookstyle: + stage: lint + script: + - chef exec cookstyle . + +chefspec: + stage: unit + script: + - chef exec rspec spec + +# Set up your test matrix here. Example: +#verify-centos-6: +# stage: functional +# before_script: +# - apt-get update +# - apt-get -y install rsync +# script: +# - kitchen verify default-centos-6 --destroy=always +# +#verify-centos-7: +# stage: functional +# before_script: +# - apt-get update +# - apt-get -y install rsync +# script: +# - kitchen verify default-centos-7 --destroy=always diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml new file mode 100644 index 00000000000..f066285b1ad --- /dev/null +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -0,0 +1,22 @@ +# Based on openjdk:8, already includes lein +image: clojure:lein-2.7.0 +# If you need to configure a database, add a `services` section here +# See https://docs.gitlab.com/ce/ci/services/postgres.html +# Make sure you configure the connection as well + +before_script: + # If you need to install any external applications, like a + # postgres client, you may want to uncomment the line below: + # + #- apt-get update -y + # + # Retrieve project dependencies + # Do this on before_script since it'll be shared between both test and + # any production sections a user adds + - lein deps + +test: + script: + # If you need to run any migrations or configure the database, this + # would be the point to do it. + - lein test diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml new file mode 100644 index 00000000000..36386a19fdc --- /dev/null +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/crystallang/crystal/ +image: "crystallang/crystal:latest" + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# services: +# - mysql:latest +# - redis:latest +# - postgres:latest + +# variables: +# POSTGRES_DB: database_name + +# Cache shards in between builds +cache: + paths: + - lib + +# This is a basic example for a shard or script which doesn't use +# services such as redis or postgres +before_script: + - apt-get update -qq && apt-get install -y -qq libxml2-dev + - crystal -v # Print out Crystal version for debugging + - shards + +# If you are using built-in Crystal Spec. +spec: + script: + - crystal spec + +# If you are using minitest.cr +minitest: + script: + - crystal test/spec_test.cr # change to the file(s) you execute for tests diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml new file mode 100644 index 00000000000..57afcbbe8b5 --- /dev/null +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -0,0 +1,49 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/python +image: python:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - postgres:latest + +variables: + POSTGRES_DB: database_name + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - ~/.cache/pip/ + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + - python -V # Print out python version for debugging + # Uncomment next line if your Django app needs a JS runtime: + # - apt-get update -q && apt-get install nodejs -yqq + - pip install -r requirements.txt + +# To get Django tests to work you may need to create a settings file using +# the following DATABASES: +# +# DATABASES = { +# 'default': { +# 'ENGINE': 'django.db.backends.postgresql_psycopg2', +# 'NAME': 'ci', +# 'USER': 'postgres', +# 'PASSWORD': 'postgres', +# 'HOST': 'postgres', +# 'PORT': '5432', +# }, +# } +# +# and then adding `--settings app.settings.ci` (or similar) to the test command + +test: + variables: + DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB" + script: + - python manage.py test diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml new file mode 100644 index 00000000000..eeefadaa019 --- /dev/null +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -0,0 +1,24 @@ +# Official docker image. +image: docker:latest + +services: + - docker:dind + +before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + +build-master: + stage: build + script: + - docker build --pull -t "$CI_REGISTRY_IMAGE" . + - docker push "$CI_REGISTRY_IMAGE" + only: + - master + +build: + stage: build + script: + - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" . + - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" + except: + - master diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml new file mode 100644 index 00000000000..cf9c731637c --- /dev/null +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -0,0 +1,18 @@ +image: elixir:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +before_script: + - mix local.rebar --force + - mix local.hex --force + - mix deps.get + +mix: + script: + - mix test diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml new file mode 100644 index 00000000000..d572d7a1edc --- /dev/null +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -0,0 +1,35 @@ +image: golang:latest + +variables: + # Please edit to your GitLab project + REPO_NAME: gitlab.com/namespace/project + +# The problem is that to be able to use go get, one needs to put +# the repository in the $GOPATH. So for example if your gitlab domain +# is gitlab.com, and that your repository is namespace/project, and +# the default GOPATH being /go, then you'd need to have your +# repository in /go/src/gitlab.com/namespace/project +# Thus, making a symbolic link corrects this. +before_script: + - mkdir -p $GOPATH/src/$(dirname $REPO_NAME) + - ln -svf $CI_PROJECT_DIR $GOPATH/src/$REPO_NAME + - cd $GOPATH/src/$REPO_NAME + +stages: + - test + - build + +format: + stage: test + script: + - go fmt $(go list ./... | grep -v /vendor/) + - go vet $(go list ./... | grep -v /vendor/) + - go test -race $(go list ./... | grep -v /vendor/) + +compile: + stage: build + script: + - go build -race -ldflags "-extldflags '-static'" -o $CI_PROJECT_DIR/mybinary + artifacts: + paths: + - mybinary diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml new file mode 100644 index 00000000000..48d98dddfad --- /dev/null +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -0,0 +1,36 @@ +# This is the Gradle build system for JVM applications +# https://gradle.org/ +# https://github.com/gradle/gradle +image: gradle:alpine + +# Disable the Gradle daemon for Continuous Integration servers as correctness +# is usually a priority over speed in CI environments. Using a fresh +# runtime for each build is more reliable since the runtime is completely +# isolated from any previous builds. +variables: + GRADLE_OPTS: "-Dorg.gradle.daemon=false" + +before_script: + - export GRADLE_USER_HOME=`pwd`/.gradle + +build: + stage: build + script: gradle --build-cache assemble + cache: + key: "$CI_COMMIT_REF_NAME" + policy: push + paths: + - build + - .gradle + + +test: + stage: test + script: gradle check + cache: + key: "$CI_COMMIT_REF_NAME" + policy: pull + paths: + - build + - .gradle + diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml new file mode 100644 index 00000000000..7fc698d50cf --- /dev/null +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -0,0 +1,40 @@ +# This template uses the java:8 docker image because there isn't any +# official Grails image at this moment +# +# Grails Framework https://grails.org/ is a powerful Groovy-based web application framework for the JVM +# +# This yml works with Grails 3.x only +# Feel free to change GRAILS_VERSION version with your project version (3.0.1, 3.1.1,...) +# Feel free to change GRADLE_VERSION version with your gradle project version (2.13, 2.14,...) +# If you use Angular profile, this yml it's prepared to work with it + +image: java:8 + +variables: + GRAILS_VERSION: "3.1.9" + GRADLE_VERSION: "2.13" + +# We use SDKMan as tool for managing versions +before_script: + - apt-get update -qq && apt-get install -y -qq unzip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install gradle $GRADLE_VERSION < /dev/null + - sdk use gradle $GRADLE_VERSION +# As it's not a good idea to version gradle.properties feel free to add your +# environments variable here + - echo grailsVersion=$GRAILS_VERSION > gradle.properties + - echo gradleWrapperVersion=2.14 >> gradle.properties +# refresh dependencies from your project + - ./gradlew --refresh-dependencies +# Be aware that if you are using Angular profile, +# Bower cannot be run as root if you don't allow it before. +# Feel free to remove next line if you are not using Bower + - echo {\"allow_root\":true} > /root/.bowerrc + +# This build job does the full grails pipeline +# (compile, test, integrationTest, war, assemble). +build: + script: + - ./gradlew build
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml new file mode 100644 index 00000000000..04c21b4725d --- /dev/null +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -0,0 +1,76 @@ +# This is an example .gitlab-ci.yml file to test (and optionally report the coverage +# results of) your [Julia][1] packages. Please refer to the [documentation][2] +# for more information about package development in Julia. +# +# Here, it is assumed that your Julia package is named `MyPackage`. Change it to +# whatever name you have given to your package. +# +# [1]: http://julialang.org/ +# [2]: https://docs.julialang.org/en/v1/manual/documentation/index.html + +# Below is the template to run your tests in Julia +.test_template: &test_definition + # Uncomment below if you would like to run the tests on specific references + # only, such as the branches `master`, `development`, etc. + # only: + # - master + # - development + script: + # Let's run the tests. Substitute `coverage = false` below, if you do not + # want coverage results. + - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage"); Pkg.test("MyPackage"; coverage = true)' + # Comment out below if you do not want coverage results. + - julia -e 'using Pkg; Pkg.add("Coverage"); + import MyPackage; cd(joinpath(dirname(pathof(MyPackage)), "..")); + using Coverage; cl, tl = get_summary(process_folder()); + println("(", cl/tl*100, "%) covered")' + +# Name a test and select an appropriate image. +# images comes from Docker hub +test:0.7: + image: julia:0.7 + <<: *test_definition + +test:1.0: + image: julia:1.0 + <<: *test_definition + +# Maybe you would like to test your package against the development branch: +# test:1.1-dev (not sure there is such an image in docker, so not tested yet): +# image: julia:v1.1-dev +# # ... allowing for failures, since we are testing against the development +# # branch: +# allow_failure: true +# <<: *test_definition + +# REMARK: Do not forget to enable the coverage feature for your project, if you +# are using code coverage reporting above. This can be done by +# +# - Navigating to the `CI/CD Pipelines` settings of your project, +# - Copying and pasting the default `Simplecov` regex example provided, i.e., +# `\(\d+.\d+\%\) covered` in the `test coverage parsing` textfield. + +# Example documentation deployment +pages: + image: julia:0.7 + stage: deploy + script: + - apt-get update -qq && apt-get install -y git # needed by Documenter + - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage");' # rebuild Julia (can be put somewhere else I'm sure + - julia -e 'using Pkg; import MyPackage; Pkg.add("Documenter")' # install Documenter + - julia --color=yes docs/make.jl # make documentation + - mv docs/build public # move to the directory picked up by Gitlab pages + artifacts: + paths: + - public + only: + - master + + +# WARNING: This template is using the `julia` images from [Docker +# Hub][3]. One can use custom Julia images and/or the official ones found +# in the same place. However, care must be taken to correctly locate the binary +# file (`/opt/julia/bin/julia` above), which is usually given on the image's +# description page. +# +# [3]: https://hub.docker.com/_/julia/ diff --git a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml new file mode 100644 index 00000000000..a4aed36889e --- /dev/null +++ b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml @@ -0,0 +1,11 @@ +# use docker image with latex preinstalled +# since there is no official latex image, use https://github.com/blang/latex-docker +# possible alternative: https://github.com/natlownes/docker-latex +image: blang/latex + +build: + script: + - latexmk -pdf + artifacts: + paths: + - "*.pdf" diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml new file mode 100644 index 00000000000..d0cad285572 --- /dev/null +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -0,0 +1,85 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/php +image: php:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + +variables: + MYSQL_DATABASE: project_name + MYSQL_ROOT_PASSWORD: secret + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - vendor/ + - node_modules/ + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + # Update packages + - apt-get update -yqq + + # Prep for Node + - apt-get install gnupg -yqq + + # Upgrade to Node 8 + - curl -sL https://deb.nodesource.com/setup_8.x | bash - + + # Install dependencies + - apt-get install git nodejs libcurl4-gnutls-dev libicu-dev libmcrypt-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libpq-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev -yqq + + # Install php extensions + - docker-php-ext-install mbstring pdo_mysql curl json intl gd xml zip bz2 opcache + + # Install & enable Xdebug for code coverage reports + - pecl install xdebug + - docker-php-ext-enable xdebug + + # Install Composer and project dependencies. + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install + + # Install Node dependencies. + # comment this out if you don't have a node dependency + - npm install + + # Copy over testing configuration. + # Don't forget to set the database config in .env.testing correctly + # DB_HOST=mysql + # DB_DATABASE=project_name + # DB_USERNAME=root + # DB_PASSWORD=secret + - cp .env.testing .env + + # Run npm build + # comment this out if you don't have a frontend build + # you can change this to to your frontend building script like + # npm run build + - npm run dev + + # Generate an application key. Re-cache. + - php artisan key:generate + - php artisan config:cache + + # Run database migrations. + - php artisan migrate + + # Run database seed + - php artisan db:seed + +test: + script: + # run laravel tests + - php vendor/bin/phpunit --coverage-text --colors=never + + # run frontend tests + # if you have any task for testing frontend + # set it in your package.json script + # comment this out if you don't have a frontend test + - npm test diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml new file mode 100644 index 00000000000..d61ff239e13 --- /dev/null +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -0,0 +1,102 @@ +--- +# Build JAVA applications using Apache Maven (http://maven.apache.org) +# For docker image tags see https://hub.docker.com/_/maven/ +# +# For general lifecycle information see https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html +# +# This template will build and test your projects as well as create the documentation. +# +# * Caches downloaded dependencies and plugins between invocation. +# * Verify but don't deploy merge requests. +# * Deploy built artifacts from master branch only. +# * Shows how to use multiple jobs in test stage for verifying functionality +# with multiple JDKs. +# * Uses site:stage to collect the documentation for multi-module projects. +# * Publishes the documentation for `master` branch. + +variables: + # This will supress any download for dependencies and plugins or upload messages which would clutter the console log. + # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work. + MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" + # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used + # when running from the command line. + # `installAtEnd` and `deployAtEnd` are only effective with recent version of the corresponding plugins. + MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true" + +# Cache downloaded dependencies and plugins between builds. +# To keep cache across branches add 'key: "$CI_JOB_NAME"' +cache: + paths: + - .m2/repository + +# This will only validate and compile stuff and run e.g. maven-enforcer-plugin. +# Because some enforcer rules might check dependency convergence and class duplications +# we use `test-compile` here instead of `validate`, so the correct classpath is picked up. +.validate: &validate + stage: build + script: + - 'mvn $MAVEN_CLI_OPTS test-compile' + +# For merge requests do not `deploy` but only run `verify`. +# See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html +.verify: &verify + stage: test + script: + - 'mvn $MAVEN_CLI_OPTS verify site site:stage' + except: + - master + +# Validate merge requests using JDK7 +validate:jdk7: + <<: *validate + image: maven:3.3.9-jdk-7 + +# Validate merge requests using JDK8 +validate:jdk8: + <<: *validate + image: maven:3.3.9-jdk-8 + +# Verify merge requests using JDK7 +verify:jdk7: + <<: *verify + image: maven:3.3.9-jdk-7 + +# Verify merge requests using JDK8 +verify:jdk8: + <<: *verify + image: maven:3.3.9-jdk-8 + + +# For `master` branch run `mvn deploy` automatically. +# Here you need to decide whether you want to use JDK7 or 8. +# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner. +# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets. +# See https://maven.apache.org/settings.html +deploy:jdk8: + # Use stage test here, so the pages job may later pickup the created site. + stage: test + script: + - 'mvn $MAVEN_CLI_OPTS deploy site site:stage' + only: + - master + # Archive up the built documentation site. + artifacts: + paths: + - target/staging + image: maven:3.3.9-jdk-8 + + +pages: + image: busybox:latest + stage: deploy + script: + # Because Maven appends the artifactId automatically to the staging path if you did define a parent pom, + # you might need to use `mv target/staging/YOUR_ARTIFACT_ID public` instead. + - mv target/staging public + dependencies: + - deploy:jdk8 + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml new file mode 100644 index 00000000000..3585f99760f --- /dev/null +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -0,0 +1,42 @@ +# This is a simple gitlab continuous integration template (compatible with the shared runner provided on gitlab.com) +# using the official mono docker image to build a visual studio project. +# +# MyProject.sln +# MyProject\ +# MyProject\ +# MyProject.csproj (console application) +# MyProject.Test\ +# MyProject.Test.csproj (test library using nuget packages "NUnit" and "NUnit.ConsoleRunner") +# +# Please find the full example project here: +# https://gitlab.com/tobiaskoch/gitlab-ci-example-mono + +# see https://hub.docker.com/_/mono/ +image: mono:latest + +stages: + - test + - deploy + +before_script: + - nuget restore -NonInteractive + +release: + stage: deploy + only: + - master + artifacts: + paths: + - build/release/MyProject.exe + script: + # The output path is relative to the position of the csproj-file + - msbuild /p:Configuration="Release" /p:Platform="Any CPU" + /p:OutputPath="./../../build/release/" "MyProject.sln" + +debug: + stage: test + script: + # The output path is relative to the position of the csproj-file + - msbuild /p:Configuration="Debug" /p:Platform="Any CPU" + /p:OutputPath="./../../build/debug/" "MyProject.sln" + - mono packages/NUnit.ConsoleRunner.3.6.0/tools/nunit3-console.exe build/debug/MyProject.Test.dll
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml new file mode 100644 index 00000000000..41de1458582 --- /dev/null +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -0,0 +1,27 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/node/tags/ +image: node:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - node_modules/ + +test_async: + script: + - npm install + - node ./specs/start.js ./specs/async.spec.js + +test_db: + script: + - npm install + - node ./specs/start.js ./specs/db-postgres.spec.js diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml new file mode 100644 index 00000000000..290b9997084 --- /dev/null +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -0,0 +1,92 @@ +image: ayufan/openshift-cli + +stages: + - test + - review + - staging + - production + - cleanup + +variables: + OPENSHIFT_SERVER: openshift.default.svc.cluster.local + # OPENSHIFT_DOMAIN: apps.example.com + # Configure this variable in Secure Variables: + # OPENSHIFT_TOKEN: my.openshift.token + +test1: + stage: test + before_script: [] + script: + - echo run tests + +test2: + stage: test + before_script: [] + script: + - echo run tests + +.deploy: &deploy + before_script: + - oc login "$OPENSHIFT_SERVER" --token="$OPENSHIFT_TOKEN" --insecure-skip-tls-verify + - oc project "$CI_PROJECT_NAME-$CI_PROJECT_ID" 2> /dev/null || oc new-project "$CI_PROJECT_NAME-$CI_PROJECT_ID" + script: + - "oc get services $APP 2> /dev/null || oc new-app . --name=$APP --strategy=docker" + - "oc start-build $APP --from-dir=. --follow || sleep 3s && oc start-build $APP --from-dir=. --follow" + - "oc get routes $APP 2> /dev/null || oc expose service $APP --hostname=$APP_HOST" + +review: + <<: *deploy + stage: review + variables: + APP: review-$CI_COMMIT_REF_NAME + APP_HOST: $CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN + on_stop: stop-review + only: + - branches + except: + - master + +stop-review: + <<: *deploy + stage: cleanup + script: + - oc delete all -l "app=$APP" + when: manual + variables: + APP: review-$CI_COMMIT_REF_NAME + GIT_STRATEGY: none + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + only: + - branches + except: + - master + +staging: + <<: *deploy + stage: staging + variables: + APP: staging + APP_HOST: $CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN + environment: + name: staging + url: http://$CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN + only: + - master + +production: + <<: *deploy + stage: production + variables: + APP: production + APP_HOST: $CI_PROJECT_NAME.$OPENSHIFT_DOMAIN + when: manual + environment: + name: production + url: http://$CI_PROJECT_NAME.$OPENSHIFT_DOMAIN + only: + - master diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml new file mode 100644 index 00000000000..33f44ee9222 --- /dev/null +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Select image from https://hub.docker.com/_/php/ +image: php:7.1.1 + +# Select what we should cache between builds +cache: + paths: + - vendor/ + +before_script: +- apt-get update -yqq +- apt-get install -yqq git libmcrypt-dev libpq-dev libcurl4-gnutls-dev libicu-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev +# Install PHP extensions +- docker-php-ext-install mbstring mcrypt pdo_pgsql curl json intl gd xml zip bz2 opcache +# Install & enable Xdebug for code coverage reports +- pecl install xdebug +- docker-php-ext-enable xdebug +# Install and run Composer +- curl -sS https://getcomposer.org/installer | php +- php composer.phar install + +# Bring in any services we need http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service +# See http://docs.gitlab.com/ce/ci/services/README.html for examples. +services: + - mysql:5.7 + +# Set any variables we need +variables: + # Configure mysql environment variables (https://hub.docker.com/r/_/mysql/) + MYSQL_DATABASE: mysql_database + MYSQL_ROOT_PASSWORD: mysql_strong_password + +# Run our tests +# If Xdebug was installed you can generate a coverage report and see code coverage metrics. +test: + script: + - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml new file mode 100644 index 00000000000..fa296057c72 --- /dev/null +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -0,0 +1,26 @@ +image: + name: hashicorp/packer:1.0.4 + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +before_script: + - packer --version + +stages: + - validate + - deploy + +validate: + stage: validate + script: + - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer validate + +build: + stage: deploy + environment: production + script: + - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer build + when: manual + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml new file mode 100644 index 00000000000..7fcc0b436b5 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/brunch +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules/ + + script: + - npm install -g brunch + - brunch build --production + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml new file mode 100644 index 00000000000..791afdd23f1 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -0,0 +1,13 @@ +# Full project: https://gitlab.com/pages/doxygen +image: alpine + +pages: + script: + - apk update && apk add doxygen + - doxygen doxygen/Doxyfile + - mv doxygen/documentation/html/ public/ + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml new file mode 100644 index 00000000000..9df2a4797b2 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -0,0 +1,17 @@ +image: node:latest + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - node_modules/ + +pages: + script: + - yarn install + - ./node_modules/.bin/gatsby build --prefix-paths + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml new file mode 100644 index 00000000000..249a168aa33 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/plain-html +pages: + stage: deploy + script: + - mkdir .public + - cp -r * .public + - mv .public public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml new file mode 100644 index 00000000000..dd3ef149668 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/harp +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules + + script: + - npm install -g harp + - harp compile ./ public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml new file mode 100644 index 00000000000..02d02250bbf --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/hexo +image: node:6.10.0 + +pages: + script: + - npm install + - ./node_modules/hexo/bin/hexo generate + artifacts: + paths: + - public + cache: + paths: + - node_modules + key: project + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml new file mode 100644 index 00000000000..b8cfb0f56f6 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -0,0 +1,17 @@ +# Full project: https://gitlab.com/pages/hugo +image: dettmering/hugo-build + +pages: + script: + - hugo + artifacts: + paths: + - public + only: + - master + +test: + script: + - hugo + except: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml new file mode 100644 index 00000000000..f5b40f2b9f1 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -0,0 +1,25 @@ +# Full project: https://gitlab.com/pages/hyde +image: python:2.7 + +cache: + paths: + - vendor/ + +test: + stage: test + script: + - pip install hyde + - hyde gen + except: + - master + +pages: + stage: deploy + script: + - pip install hyde + - hyde gen -d public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml new file mode 100644 index 00000000000..7abfaf53e8e --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -0,0 +1,32 @@ +# This template uses the java:8 docker image because there isn't any +# official JBake image at this moment +# +# JBake https://jbake.org/ is a Java based, open source, static site/blog generator for developers & designers +# +# This yml works with jBake 2.5.1 +# Feel free to change JBAKE_VERSION version +# +# HowTo at: https://jorge.aguilera.gitlab.io/howtojbake/ + +image: java:8 + +variables: + JBAKE_VERSION: 2.5.1 + + +# We use SDKMan as tool for managing versions +before_script: + - apt-get update -qq && apt-get install -y -qq unzip zip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install jbake $JBAKE_VERSION < /dev/null + - sdk use jbake $JBAKE_VERSION + +# This build job produced the output directory of your site +pages: + script: + - jbake . public + artifacts: + paths: + - public
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml new file mode 100644 index 00000000000..37f50554036 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -0,0 +1,30 @@ +# Template project: https://gitlab.com/pages/jekyll +# Docs: https://docs.gitlab.com/ce/pages/ +image: ruby:2.3 + +variables: + JEKYLL_ENV: production + +before_script: +- bundle install + +test: + stage: test + script: + - bundle exec jekyll build -d test + artifacts: + paths: + - test + except: + - master + +pages: + stage: deploy + script: + - bundle exec jekyll build -d public + artifacts: + paths: + - public + only: + - master + diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml new file mode 100644 index 00000000000..0e5fb410a4e --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -0,0 +1,42 @@ +# Jigsaw is a simple static sites generator with Laravel's Blade. +# +# Full project: https://github.com/tightenco/jigsaw + +image: php:7.2 + +# These folders are cached between builds +cache: + paths: + - vendor/ + - node_modules/ + +before_script: + # Update packages + - apt-get update -yqq + + # Install dependencies + - apt-get install -yqq gnupg zlib1g-dev libpng-dev + + # Install Node 8 + - curl -sL https://deb.nodesource.com/setup_8.x | bash - + - apt-get install -yqq nodejs + + # Install php extensions + - docker-php-ext-install zip + + # Install Composer and project dependencies. + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install + + # Install Node dependencies. + - npm install + +pages: + script: + - npm run production + - mv build_production public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml new file mode 100644 index 00000000000..c5c44a5d86c --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/hyde +image: python:2.7 + +pages: + script: + - pip install lektor + - lektor build --output-path public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml new file mode 100644 index 00000000000..50e8b7ccd46 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -0,0 +1,17 @@ +# Full project: https://gitlab.com/pages/metalsmith +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules/ + + script: + - npm install -g metalsmith + - npm install + - make build + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml new file mode 100644 index 00000000000..9f4cc0574d6 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -0,0 +1,27 @@ +# Full project: https://gitlab.com/pages/middleman +image: ruby:2.3 + +cache: + paths: + - vendor + +test: + script: + - apt-get update -yqqq + - apt-get install -y nodejs + - bundle install --path vendor + - bundle exec middleman build + except: + - master + +pages: + script: + - apt-get update -yqqq + - apt-get install -y nodejs + - bundle install --path vendor + - bundle exec middleman build + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml new file mode 100644 index 00000000000..b469b316ba5 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/nanoc +image: ruby:2.3 + +pages: + script: + - bundle install -j4 + - nanoc + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml new file mode 100644 index 00000000000..4762ec9acfd --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -0,0 +1,15 @@ +# Full project: https://gitlab.com/pages/octopress +image: ruby:2.3 + +pages: + script: + - apt-get update -qq && apt-get install -qq nodejs + - bundle install -j4 + - bundle exec rake generate + - mv public .public + - mv .public/octopress public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml new file mode 100644 index 00000000000..c5f3154f587 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml @@ -0,0 +1,10 @@ +# Full project: https://gitlab.com/pages/pelican +image: python:2.7-alpine + +pages: + script: + - pip install -r requirements.txt + - pelican -s publishconf.py + artifacts: + paths: + - public/ diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml new file mode 100644 index 00000000000..2e0589de652 --- /dev/null +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -0,0 +1,51 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/python/tags/ +image: python:latest + +# Change pip's cache directory to be inside the project directory since we can +# only cache local items. +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache" + +# Pip's cache doesn't store the python packages +# https://pip.pypa.io/en/stable/reference/pip_install/#caching +# +# If you want to also cache the installed packages, you have to install +# them in a virtualenv and cache it as well. +cache: + paths: + - .cache/pip + - venv/ + +before_script: + - python -V # Print out python version for debugging + - pip install virtualenv + - virtualenv venv + - source venv/bin/activate + +test: + script: + - python setup.py test + - pip install tox flake8 # you can also use tox + - tox -e py36,flake8 + +run: + script: + - python setup.py bdist_wheel + # an alternative approach is to install and run: + - pip install dist/* + # run the command here + artifacts: + paths: + - dist/*.whl + +pages: + script: + - pip install sphinx sphinx-rtd-theme + - cd doc ; make html + - mv build/html/ ../public/ + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml new file mode 100644 index 00000000000..93cb31f48c0 --- /dev/null +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -0,0 +1,54 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/ruby/tags/ +image: "ruby:2.5" + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +variables: + POSTGRES_DB: database_name + +# Cache gems in between builds +cache: + paths: + - vendor/ruby + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + - ruby -v # Print out ruby version for debugging + # Uncomment next line if your rails app needs a JS runtime: + # - apt-get update -q && apt-get install nodejs -yqq + - gem install bundler --no-ri --no-rdoc # Bundler is not installed with the image + - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby + +# Optional - Delete if not using `rubocop` +rubocop: + script: + - rubocop + +rspec: + script: + - rspec spec + +rails: + variables: + DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB" + script: + - rails db:migrate + - rails db:seed + - rails test + +# This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk +# are supported too: https://github.com/travis-ci/dpl +deploy: + type: deploy + environment: production + script: + - gem install dpl + - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_PRODUCTION_KEY diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml new file mode 100644 index 00000000000..cab087c48c7 --- /dev/null +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -0,0 +1,23 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/rust/tags/ +image: "rust:latest" + +# Optional: Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +#services: +# - mysql:latest +# - redis:latest +# - postgres:latest + +# Optional: Install a C compiler, cmake and git into the container. +# You will often need this when you (or any of your dependencies) depends on C code. +#before_script: +#- apt-get update -yqq +#- apt-get install -yqq --no-install-recommends build-essential + +# Use cargo to test the project +test:cargo: + script: + - rustc --version && cargo --version # Print version info for debugging + - cargo test --all --verbose diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml new file mode 100644 index 00000000000..b4208ed9d7d --- /dev/null +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -0,0 +1,22 @@ +# Official Java image. Look for the different tagged releases at +# https://hub.docker.com/r/library/java/tags/ . A Java image is not required +# but an image with a JVM speeds up the build a bit. +image: java:8 + +before_script: + # Enable the usage of sources over https + - apt-get update -yqq + - apt-get install apt-transport-https -yqq + # Add keyserver for SBT + - echo "deb http://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list + - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + # Install SBT + - apt-get update -yqq + - apt-get install sbt -yqq + # Log the sbt version + - sbt sbt-version + +test: + script: + # Execute your project's tests + - sbt clean test diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml new file mode 100644 index 00000000000..ba8a802ba4f --- /dev/null +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -0,0 +1,30 @@ +# Lifted from: https://about.gitlab.com/2016/03/10/setting-up-gitlab-ci-for-ios-projects/ +# This file assumes an own GitLab CI runner, setup on a macOS system. +stages: + - build + - archive + +build_project: + stage: build + script: + - xcodebuild clean -project ProjectName.xcodeproj -scheme SchemeName | xcpretty + - xcodebuild test -project ProjectName.xcodeproj -scheme SchemeName -destination 'platform=iOS Simulator,name=iPhone 8,OS=11.3' | xcpretty -s + tags: + - ios_11-3 + - xcode_9-3 + - macos_10-13 + +archive_project: + stage: archive + script: + - xcodebuild clean archive -archivePath build/ProjectName -scheme SchemeName + - xcodebuild -exportArchive -exportFormat ipa -archivePath "build/ProjectName.xcarchive" -exportPath "build/ProjectName.ipa" -exportProvisioningProfile "ProvisioningProfileName" + only: + - master + artifacts: + paths: + - build/ProjectName.ipa + tags: + - ios_11-3 + - xcode_9-3 + - macos_10-13 diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml new file mode 100644 index 00000000000..7160fce26a8 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -0,0 +1,55 @@ +# Official image for Hashicorp's Terraform. It uses light image which is Alpine +# based as it is much lighter. +# +# Entrypoint is also needed as image by default set `terraform` binary as an +# entrypoint. +image: + name: hashicorp/terraform:light + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +# Default output file for Terraform plan +variables: + PLAN: plan.tfplan + +cache: + paths: + - .terraform + +before_script: + - terraform --version + - terraform init + +stages: + - validate + - build + - deploy + +validate: + stage: validate + script: + - terraform validate + +plan: + stage: build + script: + - terraform plan -out=$PLAN + artifacts: + name: plan + paths: + - $PLAN + +# Separate apply job for manual launching Terraform as it can be destructive +# action. +apply: + stage: deploy + environment: + name: production + script: + - terraform apply -input=false $PLAN + dependencies: + - plan + when: manual + only: + - master diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml new file mode 100644 index 00000000000..fc3d4ecdbba --- /dev/null +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -0,0 +1,86 @@ +# The following script will work for any project that can be built from command line by msbuild +# It uses powershell shell executor, so you need to add the following line to your config.toml file +# (located in gitlab-runner.exe directory): +# shell = "powershell" +# +# The script is composed of 3 stages: build, test and deploy. +# +# The build stage restores NuGet packages and uses msbuild to build the exe and msi +# One major issue you'll find is that you can't build msi projects from command line +# if you use vdproj. There are workarounds building msi via devenv, but they rarely work +# The best solution is migrating your vdproj projects to WiX, as it can be build directly +# by msbuild. +# +# The test stage runs nunit from command line against Test project inside your solution +# It also saves the resulting TestResult.xml file +# +# The deploy stage copies the exe and msi from build stage to a network drive +# You need to have the network drive mapped as Local System user for gitlab-runner service to see it +# The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473), +# running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes + + +# place project specific paths in variables to make the rest of the script more generic +variables: + EXE_RELEASE_FOLDER: 'YourApp\bin\Release' + MSI_RELEASE_FOLDER: 'Setup\bin\Release' + TEST_FOLDER: 'Tests\bin\Release' + DEPLOY_FOLDER: 'P:\Projects\YourApp\Builds' + + NUGET_PATH: 'C:\NuGet\nuget.exe' + MSBUILD_PATH: 'C:\Program Files (x86)\MSBuild\14.0\Bin\msbuild.exe' + NUNIT_PATH: 'C:\Program Files (x86)\NUnit.org\nunit-console\nunit3-console.exe' + +stages: + - build + - test + - deploy + +build_job: + stage: build + only: + - tags # the build process will only be started by git tag commits + script: + - '& "$env:NUGET_PATH" restore' # restore Nuget dependencies + - '& "$env:MSBUILD_PATH" /p:Configuration=Release' # build the project + artifacts: + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + paths: + - '$env:EXE_RELEASE_FOLDER\YourApp.exe' # saving exe to copy to deploy folder + - '$env:MSI_RELEASE_FOLDER\YourApp Setup.msi' # saving msi to copy to deploy folder + - '$env:TEST_FOLDER\' # saving entire Test project so NUnit can run tests + +test_job: + stage: test + only: + - tags + script: + - '& "$env:NUNIT_PATH" ".\$env:TEST_FOLDER\Tests.dll"' # running NUnit tests + artifacts: + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + paths: + - '.\TestResult.xml' # saving NUnit results to copy to deploy folder + dependencies: + - build_job + +deploy_job: + stage: deploy + only: + - tags + script: + # Compose a folder for each release based on commit tag. + # Assuming your tag is Rev1.0.0.1, and your last commit message is 'First commit' + # the artifact files will be copied to: + # P:\Projects\YourApp\Builds\Rev1.0.0.1 - First commit\ + - '$commitSubject = git log -1 --pretty=%s' + - '$deployFolder = $($env:DEPLOY_FOLDER) + "\" + $($env:CI_BUILD_TAG) + " - " + $commitSubject + "\"' + + # xcopy takes care of recursively creating required folders + - 'xcopy /y ".\$env:EXE_RELEASE_FOLDER\YourApp.exe" "$deployFolder"' + - 'xcopy /y ".\$env:MSI_RELEASE_FOLDER\YourApp Setup.msi" "$deployFolder"' + - 'xcopy /y ".\TestResult.xml" "$deployFolder"' + + dependencies: + - build_job + - test_job +
\ No newline at end of file diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index bfe0c2a2c26..2147f62a84a 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -133,6 +133,7 @@ module Gitlab invalidate_chunk_cache end + # rubocop: disable CodeReuse/ActiveRecord def truncate(offset) raise ArgumentError, 'Outside of file' if offset > size || offset < 0 return if offset == size # Skip the following process as it doesn't affect anything @@ -148,6 +149,7 @@ module Gitlab ensure invalidate_chunk_cache end + # rubocop: enable CodeReuse/ActiveRecord def flush # no-op @@ -206,9 +208,11 @@ module Gitlab @chunks_cache = [] end + # rubocop: disable CodeReuse/ActiveRecord def current_chunk @chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index) end + # rubocop: enable CodeReuse/ActiveRecord def build_chunk @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index) @@ -218,13 +222,17 @@ module Gitlab current_chunk || build_chunk end + # rubocop: disable CodeReuse/ActiveRecord def trace_chunks ::Ci::BuildTraceChunk.where(build: build) end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def calculate_size trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index e829f2a95f8..a427aa30683 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -16,7 +16,7 @@ module Gitlab end initial_parsing - rescue Gitlab::Ci::Config::Loader::FormatError => e + rescue Gitlab::Ci::Config::ConfigError => e raise ValidationError, e.message end @@ -49,7 +49,8 @@ module Gitlab script: job[:script], after_script: job[:after_script], environment: job[:environment], - retry: job[:retry] + retry: job[:retry], + start_in: job[:start_in] }.compact } end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index f55ab535efe..82a405362c2 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -38,6 +38,7 @@ module Gitlab end # Accepts a path in the form of "#{hex_secret}/#{filename}" + # rubocop: disable CodeReuse/ActiveRecord def find_correct_path(upload_path) upload = Upload.find_by(uploader: 'FileUploader', path: upload_path) return unless upload && upload.local? && upload.model @@ -52,6 +53,7 @@ module Gitlab # I.e. the project record might be missing, which raises an exception. nil end + # rubocop: enable CodeReuse/ActiveRecord def move_to_lost_and_found(path, dry_run) new_path = path.sub(/\A#{ProjectUploadFileFinder::ABSOLUTE_UPLOAD_DIR}/, LOST_AND_FOUND) @@ -107,18 +109,22 @@ module Gitlab new(path_matched[1], path_matched[2]) end + # rubocop: disable CodeReuse/ActiveRecord def orphan? return true if full_path.nil? || upload_path.nil? # It's possible to reduce to one query, but `where_full_path_in` is complex !Upload.exists?(path: upload_path, model_id: project_id, model_type: 'Project', uploader: 'FileUploader') end + # rubocop: enable CodeReuse/ActiveRecord private + # rubocop: disable CodeReuse/ActiveRecord def project_id @project_id ||= Project.where_full_path_in([full_path]).pluck(:id) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb index 45a5aea4fcd..eba1faacc3a 100644 --- a/lib/gitlab/cleanup/remote_uploads.rb +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -33,6 +33,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def each_orphan_file # we want to skip files already moved to lost_and_found directory lost_dir_match = "^#{lost_and_found_dir}\/" @@ -50,6 +51,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def move_to_lost_and_found(file) new_path = "#{lost_and_found_dir}/#{file.key}" diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 4c28489f45a..1ffc2639237 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -7,9 +7,14 @@ module Gitlab def initialize(contributor, current_user = nil) @contributor = contributor @current_user = current_user - @projects = ContributedProjectsFinder.new(contributor).execute(current_user) + @projects = if @contributor.include_private_contributions? + ContributedProjectsFinder.new(@contributor).execute(@contributor) + else + ContributedProjectsFinder.new(contributor).execute(current_user) + end end + # rubocop: disable CodeReuse/ActiveRecord def activity_dates return @activity_dates if @activity_dates.present? @@ -25,25 +30,25 @@ module Gitlab note_events = event_counts(date_from, :merge_requests) .having(action: [Event::COMMENTED]) - union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) - events = Event.find_by_sql(union.to_sql).map(&:attributes) + events = Event + .from_union([repo_events, issue_events, mr_events, note_events]) + .map(&:attributes) @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| activities[event["date"]] += event["total_amount"] end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def events_by_date(date) return Event.none unless can_read_cross_project? - events = Event.contributions.where(author_id: contributor.id) + Event.contributions.where(author_id: contributor.id) .where(created_at: date.beginning_of_day..date.end_of_day) .where(project_id: projects) - - # Use visible_to_user? instead of the complicated logic in activity_dates - # because we're only viewing the events for a single day. - events.select { |event| event.visible_to_user?(current_user) } end + # rubocop: enable CodeReuse/ActiveRecord def starting_year 1.year.ago.year @@ -59,6 +64,7 @@ module Gitlab Ability.allowed?(current_user, :read_cross_project) end + # rubocop: disable CodeReuse/ActiveRecord def event_counts(date_from, feature) t = Event.arel_table @@ -87,5 +93,6 @@ module Gitlab .where(conditions) .where("events.project_id in (#{authed_projects.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 9147ef401da..de7c959e706 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -2,17 +2,17 @@ module Gitlab module CurrentSettings class << self def current_application_settings - if RequestStore.active? - RequestStore.fetch(:current_application_settings) { ensure_application_settings! } - else - ensure_application_settings! - end + Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end def fake_application_settings(attributes = {}) Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) end + def clear_in_memory_application_settings! + @in_memory_application_settings = nil + end + def method_missing(name, *args, &block) current_application_settings.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index eb246d393a1..f382992cb0a 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -26,7 +26,8 @@ module Gitlab stages: pipeline.stages_names, created_at: pipeline.created_at, finished_at: pipeline.finished_at, - duration: pipeline.duration + duration: pipeline.duration, + variables: pipeline.variables.map(&:hook_attrs) } end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index c169c8fe135..b498f113859 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -97,11 +97,15 @@ module Gitlab } end - # This method provide a sample data generated with + # This method provides a sample data generated with # existing project and commits to test webhooks def build_sample(project, user) + # Use sample data if repo has no commit + # (expect the case of test service configuration settings) + return sample_data if project.empty_repo? + ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" - commits = project.repository.commits(project.default_branch.to_s, limit: 3) rescue [] + commits = project.repository.commits(project.default_branch.to_s, limit: 3) build(project, user, commits.last&.id, commits.first&.id, ref, commits) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 8eacad078c8..42f9605f5ac 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -249,5 +249,21 @@ module Gitlab end private_class_method :database_version + + def self.add_post_migrate_path_to_rails(force: false) + return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force + + Rails.application.config.paths['db'].each do |db_path| + path = Rails.root.join(db_path, 'post_migrate').to_s + + unless Rails.application.config.paths['db/migrate'].include? path + Rails.application.config.paths['db/migrate'] << path + + # Rails memoizes migrations at certain points where it won't read the above + # path just yet. As such we must also update the following list of paths. + ActiveRecord::Migrator.migrations_paths << path + end + end + end end end diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index d32837f5793..7d334a79009 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -2,6 +2,8 @@ module Gitlab module Database # Model that can be used for querying permissions of a SQL user. class Grant < ActiveRecord::Base + include FromUnion + self.table_name = if Database.postgresql? 'information_schema.role_table_grants' @@ -42,9 +44,7 @@ module Gitlab .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") ] - union = SQL::Union.new(queries).to_sql - - Grant.from("(#{union}) privs").any? + Grant.from_union(queries, alias_as: 'privs').any? end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 7f012312819..30541ee3553 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1073,6 +1073,10 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(index_sql).to_i > 0 end + + def mysql_compatible_index_length + Gitlab::Database.mysql? ? 20 : nil + end end end end diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb new file mode 100644 index 00000000000..36e4559b554 --- /dev/null +++ b/lib/gitlab/database/subquery.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Subquery + class << self + def self_join(relation) + t = relation.arel_table + t2 = if !Gitlab.rails5? + relation.arel.as('t2') + else + # Work around a bug in Rails 5, where LIMIT causes trouble + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 + r = relation.limit(nil).arel + r.take(relation.limit_value) if relation.limit_value + r.as('t2') + end + + relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first) + end + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d16a55720b7..fb117baca9e 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -20,8 +20,9 @@ module Gitlab DiffViewer::Image ].sort_by { |v| v.binary? ? 0 : 1 }.freeze - def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil) + def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil, stats: nil) @diff = diff + @stats = stats @repository = repository @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs @@ -165,11 +166,11 @@ module Gitlab end def added_lines - diff_lines.count(&:added?) + @stats&.additions || diff_lines.count(&:added?) end def removed_lines - diff_lines.count(&:removed?) + @stats&.deletions || diff_lines.count(&:removed?) end def file_identifier @@ -211,13 +212,17 @@ module Gitlab old_blob && new_blob && old_blob.binary? != new_blob.binary? end + # rubocop: disable CodeReuse/ActiveRecord def size valid_blobs.map(&:size).sum end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def raw_size valid_blobs.map(&:raw_size).sum end + # rubocop: enable CodeReuse/ActiveRecord def raw_binary? try_blobs(:raw_binary?) diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index c79d8d3cb21..b79ff771a2b 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -2,23 +2,27 @@ module Gitlab module Diff module FileCollection class Base - attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs + include Gitlab::Utils::StrongMemoize + + attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs, :diffable delegate :count, :size, :real_size, to: :diff_files def self.default_options - ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false) + ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true) end def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil) diff_options = self.class.default_options.merge(diff_options || {}) @diffable = diffable + @include_stats = diff_options.delete(:include_stats) @diffs = diffable.raw_diffs(diff_options) @project = project @diff_options = diff_options @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + @repository = project.repository end def diff_files @@ -33,12 +37,37 @@ module Gitlab diff_files.find { |diff_file| diff_file.new_path == new_path } end + def clear_cache + # No-op + end + + def write_cache + # No-op + end + private + def diff_stats_collection + strong_memoize(:diff_stats) do + # There are scenarios where we don't need to request Diff Stats, + # when caching for instance. + next unless @include_stats + next unless diff_refs + + @repository.diff_stats(diff_refs.base_sha, diff_refs.head_sha) + end + end + def decorate_diff!(diff) return diff if diff.is_a?(File) - Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs) + stats = diff_stats_collection&.find_by_path(diff.new_path) + + Gitlab::Diff::File.new(diff, + repository: project.repository, + diff_refs: diff_refs, + fallback_diff_refs: fallback_diff_refs, + stats: stats) end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index be25e1bab21..0dd073a3a8e 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -2,6 +2,8 @@ module Gitlab module Diff module FileCollection class MergeRequestDiff < Base + extend ::Gitlab::Utils::Override + def initialize(merge_request_diff, diff_options:) @merge_request_diff = merge_request_diff @@ -13,70 +15,35 @@ module Gitlab end def diff_files - # Make sure to _not_ send any method call to Gitlab::Diff::File - # _before_ all of them were collected (`super`). Premature method calls will - # trigger N+1 RPCs to Gitaly through BatchLoader records (Blob.lazy). - # diff_files = super - diff_files.each { |diff_file| cache_highlight!(diff_file) if cacheable?(diff_file) } - store_highlight_cache + diff_files.each { |diff_file| cache.decorate(diff_file) } diff_files end - def real_size - @merge_request_diff.real_size + override :write_cache + def write_cache + cache.write_if_empty end - def clear_cache! - Rails.cache.delete(cache_key) + override :clear_cache + def clear_cache + cache.clear end def cache_key - [@merge_request_diff, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] - end - - private - - def highlight_diff_file_from_cache!(diff_file, cache_diff_lines) - diff_file.highlighted_diff_lines = cache_diff_lines.map do |line| - Gitlab::Diff::Line.init_from_hash(line) - end + cache.key end - # - # If we find the highlighted diff files lines on the cache we replace existing diff_files lines (no highlighted) - # for the highlighted ones, so we just skip their execution. - # If the highlighted diff files lines are not cached we calculate and cache them. - # - # The content of the cache is a Hash where the key identifies the file and the values are Arrays of - # hashes that represent serialized diff lines. - # - def cache_highlight!(diff_file) - item_key = diff_file.file_identifier - - if highlight_cache[item_key] - highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key]) - else - highlight_cache[item_key] = diff_file.highlighted_diff_lines.map(&:to_hash) - end - end - - def highlight_cache - return @highlight_cache if defined?(@highlight_cache) - - @highlight_cache = Rails.cache.read(cache_key) || {} - @highlight_cache_was_empty = @highlight_cache.empty? - @highlight_cache + def real_size + @merge_request_diff.real_size end - def store_highlight_cache - Rails.cache.write(cache_key, highlight_cache, expires_in: 1.week) if @highlight_cache_was_empty - end + private - def cacheable?(diff_file) - @merge_request_diff.present? && diff_file.text? && diff_file.diffable? + def cache + @cache ||= Gitlab::Diff::HighlightCache.new(self) end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 1f012043e56..a605ddb5c33 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -24,7 +24,7 @@ module Gitlab # ignore highlighting for "match" lines next diff_line if diff_line.meta? - rich_line = highlight_line(diff_line) || diff_line.text + rich_line = highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text) if line_inline_diffs = inline_diffs[i] begin diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb new file mode 100644 index 00000000000..e4390771db2 --- /dev/null +++ b/lib/gitlab/diff/highlight_cache.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true +# +module Gitlab + module Diff + class HighlightCache + delegate :diffable, to: :@diff_collection + delegate :diff_options, to: :@diff_collection + + def initialize(diff_collection, backend: Rails.cache) + @backend = backend + @diff_collection = diff_collection + end + + # - Reads from cache + # - Assigns DiffFile#highlighted_diff_lines for cached files + def decorate(diff_file) + if content = read_file(diff_file) + diff_file.highlighted_diff_lines = content.map do |line| + Gitlab::Diff::Line.init_from_hash(line) + end + end + end + + # It populates a Hash in order to submit a single write to the memory + # cache. This avoids excessive IO generated by N+1's (1 writing for + # each highlighted line or file). + def write_if_empty + return if cached_content.present? + + @diff_collection.diff_files.each do |diff_file| + next unless cacheable?(diff_file) + + diff_file_id = diff_file.file_identifier + + cached_content[diff_file_id] = diff_file.highlighted_diff_lines.map(&:to_hash) + end + + cache.write(key, cached_content, expires_in: 1.week) + end + + def clear + cache.delete(key) + end + + def key + [diffable, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] + end + + private + + def read_file(diff_file) + cached_content[diff_file.file_identifier] + end + + def cache + @backend + end + + def cached_content + @cached_content ||= cache.read(key) || {} + end + + def cacheable?(diff_file) + diffable.present? && diff_file.text? && diff_file.diffable? + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 99970779c67..72d5ec547da 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -67,6 +67,7 @@ module Gitlab private # Finds pairs of old/new line pairs that represent the same line that changed + # rubocop: disable CodeReuse/ActiveRecord def find_changed_line_pairs(lines) # Prefixes of all diff lines, indicating their types # For example: `" - + -+ ---+++ --+ -++"` @@ -89,6 +90,7 @@ module Gitlab changed_line_pairs end + # rubocop: enable CodeReuse/ActiveRecord end private diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 1ab6df0b6ae..5b67cd46c48 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -79,16 +79,10 @@ module Gitlab } end + # We have to keep this here since it is still used for conflict resolution + # Conflict::File#as_json renders json diff lines in sections def as_json(opts = nil) - { - line_code: line_code, - type: type, - old_line: old_line, - new_line: new_line, - text: text, - rich_text: rich_text || CGI.escapeHTML(text), - meta_data: meta_positions - } + DiffLineSerializer.new.represent(self) end private diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 978962ab2eb..f967494199e 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -69,6 +69,10 @@ module Gitlab JSON.generate(formatter.to_h, opts) end + def as_json(opts = nil) + to_h.as_json(opts) + end + def type formatter.line_age end @@ -101,21 +105,21 @@ module Gitlab return @diff_file if defined?(@diff_file) @diff_file = begin - if RequestStore.active? - key = { - project_id: repository.project.id, - start_sha: start_sha, - head_sha: head_sha, - path: file_path - } - - RequestStore.fetch(key) { find_diff_file(repository) } - else - find_diff_file(repository) - end + key = { + project_id: repository.project.id, + start_sha: start_sha, + head_sha: head_sha, + path: file_path + } + + Gitlab::SafeRequestStore.fetch(key) { find_diff_file(repository) } end end + def diff_options + { paths: paths, expanded: true, include_stats: false } + end + def diff_line(repository) @diff_line ||= diff_file(repository)&.line_for_position(self) end @@ -130,7 +134,7 @@ module Gitlab return unless diff_refs.complete? return unless comparison = diff_refs.compare_in(repository.project) - comparison.diffs(paths: paths, expanded: true).diff_files.first + comparison.diffs(diff_options).diff_files.first end def get_formatter_class(type) diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index e08b5be8984..cebedb19dcc 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -1,20 +1,23 @@ -require 'gitlab/email/handler/create_merge_request_handler' -require 'gitlab/email/handler/create_note_handler' -require 'gitlab/email/handler/create_issue_handler' -require 'gitlab/email/handler/unsubscribe_handler' +# frozen_string_literal: true module Gitlab module Email module Handler - HANDLERS = [ - UnsubscribeHandler, - CreateNoteHandler, - CreateMergeRequestHandler, - CreateIssueHandler - ].freeze + def self.handlers + @handlers ||= load_handlers + end + + def self.load_handlers + [ + UnsubscribeHandler, + CreateNoteHandler, + CreateMergeRequestHandler, + CreateIssueHandler + ] + end def self.for(mail, mail_key) - HANDLERS.find do |klass| + handlers.find do |klass| handler = klass.new(mail, mail_key) break handler if handler.can_handle? end diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 0bba433d04b..35bb49ad19a 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email module Handler diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index fc8615afcae..69982efbbe6 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' module Gitlab @@ -28,9 +30,11 @@ module Gitlab record_name: 'issue') end + # rubocop: disable CodeReuse/ActiveRecord def author @author ||= User.find_by(incoming_email_token: incoming_email_token) end + # rubocop: enable CodeReuse/ActiveRecord def project @project ||= Project.find_by_full_path(project_path) diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index 2316e58c3fc..e68ae60ff98 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' @@ -32,9 +34,11 @@ module Gitlab record_name: 'merge_request') end + # rubocop: disable CodeReuse/ActiveRecord def author @author ||= User.find_by(incoming_email_token: incoming_email_token) end + # rubocop: enable CodeReuse/ActiveRecord def project @project ||= Project.find_by_full_path(project_path) diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 379b114e957..c7c573595fa 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index 38b1425364f..ff6b2c729b2 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email module Handler diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 56751e4e41e..d2f617b868a 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' module Gitlab diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb index bb14a8cd9e7..2c827265d8c 100644 --- a/lib/gitlab/fake_application_settings.rb +++ b/lib/gitlab/fake_application_settings.rb @@ -5,12 +5,6 @@ # column type without parsing db/schema.rb. module Gitlab class FakeApplicationSettings < OpenStruct - def initialize(options = {}) - super - - FakeApplicationSettings.define_predicate_methods(options) - end - # Mimic ActiveRecord predicate methods for boolean values def self.define_predicate_methods(options) options.each do |key, value| @@ -23,5 +17,23 @@ module Gitlab end end end + + def initialize(options = {}) + super + + FakeApplicationSettings.define_predicate_methods(options) + end + + def key_restriction_for(type) + 0 + end + + def allowed_key_types + ApplicationSetting::SUPPORTED_KEY_TYPES + end + + def pick_repository_storage + repository_storages.sample + end end end diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index 4850a6c0430..050a1ad3a0b 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -47,7 +47,7 @@ module Gitlab end def appearance - RequestStore.store[:appearance] ||= (Appearance.current || Appearance.new) + Gitlab::SafeRequestStore[:appearance] ||= (Appearance.current || Appearance.new) end def appearance_favicon diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 49bc9c0b671..8f55e94975c 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -8,7 +8,7 @@ module Gitlab # Project files readme: %r{\Areadme[^/]*\z}i, changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i, - license: %r{\A(licen[sc]e|copying)(\.[^/]+)?\z}i, + license: %r{\A((un)?licen[sc]e|copying)(\.[^/]+)?\z}i, contributing: %r{\Acontributing[^/]*\z}i, version: 'version', avatar: /\Alogo\.(png|jpg|gif)\z/, diff --git a/lib/gitlab/file_markdown_link_builder.rb b/lib/gitlab/file_markdown_link_builder.rb new file mode 100644 index 00000000000..5386656efe7 --- /dev/null +++ b/lib/gitlab/file_markdown_link_builder.rb @@ -0,0 +1,21 @@ +# Builds the markdown link of a file +# It needs the methods filename and secure_url (final destination url) to be defined. +module Gitlab + module FileMarkdownLinkBuilder + include FileTypeDetection + + def markdown_link + return unless name = markdown_name + + markdown = "[#{name.gsub(']', '\\]')}](#{secure_url})" + markdown.prepend("!") if image_or_video? || dangerous? + markdown + end + + def markdown_name + return unless filename.present? + + image_or_video? ? File.basename(filename, File.extname(filename)) : filename + end + end +end diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb new file mode 100644 index 00000000000..25ee07cf940 --- /dev/null +++ b/lib/gitlab/file_type_detection.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# File helpers methods. +# It needs the method filename to be defined. +module Gitlab + module FileTypeDetection + IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze + # We recommend using the .mp4 format over .mov. Videos in .mov format can + # still be used but you really need to make sure they are served with the + # proper MIME type video/mp4 and not video/quicktime or your videos won't play + # on IE >= 9. + # http://archive.sublimevideo.info/20150912/docs.sublimevideo.net/troubleshooting.html + VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze + # These extension types can contain dangerous code and should only be embedded inline with + # proper filtering. They should always be tagged as "Content-Disposition: attachment", not "inline". + DANGEROUS_EXT = %w[svg].freeze + + def image? + extension_match?(IMAGE_EXT) + end + + def video? + extension_match?(VIDEO_EXT) + end + + def image_or_video? + image? || video? + end + + def dangerous? + extension_match?(DANGEROUS_EXT) + end + + private + + def extension_match?(extensions) + return false unless filename + + extension = File.extname(filename).delete('.') + extensions.include?(extension.downcase) + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index a91de278cf3..98ea5b309a1 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -79,6 +79,7 @@ module Gitlab ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end + # rubocop: disable CodeReuse/ActiveRecord def user_info(person_id) user_hash = user_map[person_id.to_s] @@ -95,7 +96,9 @@ module Gitlab { name: user_name, gitlab_id: gitlab_id } end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def import_cases return unless @cases @@ -141,6 +144,7 @@ module Gitlab import_issue_comments(issue, comments) end end + # rubocop: enable CodeReuse/ActiveRecord def opened_content(comments) while comment = comments.shift diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 5b264868af0..74cdabfed9d 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -53,9 +53,6 @@ module Gitlab # Already a commit? return commit_id if commit_id.is_a?(Gitlab::Git::Commit) - # A rugged reference? - commit_id = Gitlab::Git::Ref.dereference_object(commit_id) - # Some weird thing? return nil unless commit_id.is_a?(String) @@ -127,8 +124,6 @@ module Gitlab # :topo, or any combination of them (in an array). Commit ordering types # are documented here: # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) repo.wrapped_gitaly_errors do Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) @@ -328,7 +323,6 @@ module Gitlab entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) return unless entry - # To be compatible with the rugged format entry = entry.to_h entry.delete(:data) entry[:name] = File.basename(path) @@ -346,8 +340,8 @@ module Gitlab subject: message_split[0] ? message_split[0].chomp.b : "", body: raw_commit.message.b, parent_ids: raw_commit.parent_ids, - author: gitaly_commit_author_from_rugged(raw_commit.author), - committer: gitaly_commit_author_from_rugged(raw_commit.committer) + author: gitaly_commit_author_from_raw(raw_commit.author), + committer: gitaly_commit_author_from_raw(raw_commit.committer) ) end @@ -381,7 +375,7 @@ module Gitlab SERIALIZE_KEYS end - def gitaly_commit_author_from_rugged(author_or_committer) + def gitaly_commit_author_from_raw(author_or_committer) Gitaly::CommitAuthor.new( name: author_or_committer[:name].b, email: author_or_committer[:email].b, diff --git a/lib/gitlab/git/committer_with_hooks.rb b/lib/gitlab/git/committer_with_hooks.rb deleted file mode 100644 index 4198be7c9c9..00000000000 --- a/lib/gitlab/git/committer_with_hooks.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module Git - class CommitterWithHooks < Gollum::Committer - attr_reader :gl_wiki - - def initialize(gl_wiki, options = {}) - @gl_wiki = gl_wiki - super(gl_wiki.gollum_wiki, options) - end - - def commit - # TODO: Remove after 10.8 - return super unless allowed_to_run_hooks? - - result = Gitlab::Git::OperationService.new(git_user, gl_wiki.repository).with_branch( - @wiki.ref, - start_branch_name: @wiki.ref - ) do |start_commit| - super(false) - end - - result[:newrev] - rescue Gitlab::Git::PreReceiveError => e - message = "Custom Hook failed: #{e.message}" - raise Gitlab::Git::Wiki::OperationError, message - end - - private - - # TODO: Remove after 10.8 - def allowed_to_run_hooks? - @options[:user_id] != 0 && @options[:username].present? - end - - def git_user - @git_user ||= Gitlab::Git::User.new(@options[:username], - @options[:name], - @options[:email], - gitlab_id) - end - - def gitlab_id - Gitlab::GlId.gl_id_from_id_value(@options[:user_id]) - end - end - end -end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 61ce10ca131..0d96211f4d4 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -1,6 +1,3 @@ -# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between. - -# Gitlab::Git::Diff is a wrapper around native Rugged::Diff object module Gitlab module Git class Diff @@ -22,13 +19,17 @@ module Gitlab alias_method :expanded?, :expanded - SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze + # The default maximum content size to display a diff patch. + # + # If this value ever changes, make sure to create a migration to update + # current records, and default of `ApplicationSettings#diff_max_patch_bytes`. + DEFAULT_MAX_PATCH_BYTES = 100.kilobytes - # The maximum size of a diff to display. - SIZE_LIMIT = 100.kilobytes + # This is a limitation applied on the source (Gitaly), therefore we don't allow + # persisting limits over that. + MAX_PATCH_BYTES_UPPER_BOUND = 500.kilobytes - # The maximum size before a diff is collapsed. - COLLAPSE_LIMIT = 10.kilobytes + SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze class << self def between(repo, head, base, options = {}, *paths) @@ -52,20 +53,31 @@ module Gitlab repo.diff(common_commit, head, actual_options, *paths) end - # Return a copy of the +options+ hash containing only keys that can be - # passed to Rugged. Allowed options are: + # Return a copy of the +options+ hash containing only recognized keys. + # Allowed options are: # # :ignore_whitespace_change :: # If true, changes in amount of whitespace will be ignored. # - # :disable_pathspec_match :: - # If true, the given +*paths+ will be applied as exact matches, - # instead of as fnmatch patterns. + # :max_files :: + # Limit how many files will patches be allowed for before collapsing + # + # :max_lines :: + # Limit how many patch lines (across all files) will be allowed for + # before collapsing + # + # :limits :: + # A hash with additional limits to check before collapsing patches. + # Allowed keys are: `max_bytes`, `safe_max_files`, `safe_max_lines` + # and `safe_max_bytes` # + # :expanded :: + # If true, patch raw data will not be included in the diff after + # `max_files`, `max_lines` or any of the limits in `limits` are + # exceeded def filter_diff_options(options, default_options = {}) - allowed_options = [:ignore_whitespace_change, - :disable_pathspec_match, :paths, - :max_files, :max_lines, :limits, :expanded] + allowed_options = [:ignore_whitespace_change, :max_files, :max_lines, + :limits, :expanded] if default_options actual_defaults = default_options.dup @@ -93,10 +105,30 @@ module Gitlab # # "Binary files a/file/path and b/file/path differ\n" # This is used when we detect that a diff is binary - # using CharlockHolmes when Rugged treats it as text. + # using CharlockHolmes. def binary_message(old_path, new_path) "Binary files #{old_path} and #{new_path} differ\n" end + + # Returns the limit of bytes a single diff file can reach before it + # appears as 'collapsed' for end-users. + # By convention, it's 10% of the persisted `diff_max_patch_bytes`. + # + # Example: If we have 100k for the `diff_max_patch_bytes`, it will be 10k by + # default. + # + # Patches surpassing this limit should still be persisted in the database. + def patch_safe_limit_bytes + patch_hard_limit_bytes / 10 + end + + # Returns the limit for a single diff file (patch). + # + # Patches surpassing this limit shouldn't be persisted in the database + # and will be presented as 'too large' for end-users. + def patch_hard_limit_bytes + Gitlab::CurrentSettings.diff_max_patch_bytes + end end def initialize(raw_diff, expanded: true) @@ -106,8 +138,6 @@ module Gitlab when Hash init_from_hash(raw_diff) prune_diff_if_eligible - when Rugged::Patch, Rugged::Diff::Delta - init_from_rugged(raw_diff) when Gitlab::GitalyClient::Diff init_from_gitaly(raw_diff) prune_diff_if_eligible @@ -144,7 +174,7 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= SIZE_LIMIT + @too_large = @diff.bytesize >= self.class.patch_hard_limit_bytes else @too_large end @@ -162,7 +192,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT + @collapsed = !expanded && @diff.bytesize >= self.class.patch_safe_limit_bytes end def collapse! @@ -184,31 +214,6 @@ module Gitlab private - def init_from_rugged(rugged) - if rugged.is_a?(Rugged::Patch) - init_from_rugged_patch(rugged) - d = rugged.delta - else - d = rugged - end - - @new_path = encode!(d.new_file[:path]) - @old_path = encode!(d.old_file[:path]) - @a_mode = d.old_file[:mode].to_s(8) - @b_mode = d.new_file[:mode].to_s(8) - @new_file = d.added? - @renamed_file = d.renamed? - @deleted_file = d.deleted? - end - - def init_from_rugged_patch(patch) - # Don't bother initializing diffs that are too large. If a diff is - # binary we're not going to display anything so we skip the size check. - return if !patch.delta.binary? && prune_large_patch(patch) - - @diff = encode!(strip_diff_headers(patch.to_s)) - end - def init_from_hash(hash) raw_diff = hash.symbolize_keys @@ -238,47 +243,6 @@ module Gitlab collapse! end end - - # If the patch surpasses any of the diff limits it calls the appropiate - # prune method and returns true. Otherwise returns false. - def prune_large_patch(patch) - size = 0 - - patch.each_hunk do |hunk| - hunk.each_line do |line| - size += line.content.bytesize - - if size >= SIZE_LIMIT - too_large! - return true # rubocop:disable Cop/AvoidReturnFromBlocks - end - end - end - - if !expanded && size >= COLLAPSE_LIMIT - collapse! - return true - end - - false - end - - # Strip out the information at the beginning of the patch's text to match - # Grit's output - def strip_diff_headers(diff_text) - # Delete everything up to the first line that starts with '---' or - # 'Binary' - diff_text.sub!(/\A.*?^(---|Binary)/m, '\1') - - if diff_text.start_with?('---', 'Binary') - diff_text - else - # If the diff_text did not contain a line starting with '---' or - # 'Binary', return the empty string. No idea why; we are just - # preserving behavior from before the refactor. - '' - end - end end end end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 219c69893ad..47ebca7c4a2 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -11,7 +11,7 @@ module Gitlab delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits - def self.collection_limits(options = {}) + def self.limits(options = {}) limits = {} limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) @@ -19,13 +19,14 @@ module Gitlab limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file + limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes OpenStruct.new(limits) end def initialize(iterator, options = {}) @iterator = iterator - @limits = self.class.collection_limits(options) + @limits = self.class.limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb new file mode 100644 index 00000000000..998c41497a2 --- /dev/null +++ b/lib/gitlab/git/diff_stats_collection.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class DiffStatsCollection + include Gitlab::Utils::StrongMemoize + include Enumerable + + def initialize(diff_stats) + @collection = diff_stats + end + + def each(&block) + @collection.each(&block) + end + + def find_by_path(path) + indexed_by_path[path] + end + + def paths + @collection.map(&:path) + end + + private + + def indexed_by_path + strong_memoize(:indexed_by_path) do + index_by { |stats| stats.path } + end + end + end + end +end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb deleted file mode 100644 index 5ff15a787f0..00000000000 --- a/lib/gitlab/git/gitlab_projects.rb +++ /dev/null @@ -1,253 +0,0 @@ -module Gitlab - module Git - class GitlabProjects - include Gitlab::Git::Popen - include Gitlab::Utils::StrongMemoize - - # Name of shard where repositories are stored. - # Example: nfs-file06 - attr_reader :shard_name - - # Relative path is a directory name for repository with .git at the end. - # Example: gitlab-org/gitlab-test.git - attr_reader :repository_relative_path - - # This is the path at which the gitlab-shell hooks directory can be found. - # It's essential for integration between git and GitLab proper. All new - # repositories should have their hooks directory symlinked here. - attr_reader :global_hooks_path - - attr_reader :logger - - def initialize(shard_name, repository_relative_path, global_hooks_path:, logger:) - @shard_name = shard_name - @repository_relative_path = repository_relative_path - - @logger = logger - @global_hooks_path = global_hooks_path - @output = StringIO.new - end - - def output - io = @output.dup - io.rewind - io.read - end - - # Absolute path to the repository. - # Example: /home/git/repositorities/gitlab-org/gitlab-test.git - # Probably will be removed when we fully migrate to Gitaly, part of - # https://gitlab.com/gitlab-org/gitaly/issues/1124. - def repository_absolute_path - strong_memoize(:repository_absolute_path) do - File.join(shard_path, repository_relative_path) - end - end - - def shard_path - strong_memoize(:shard_path) do - Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path - end - end - - # Import project via git clone --bare - # URL must be publicly cloneable - def import_project(source, timeout) - git_import_repository(source, timeout) - end - - def fork_repository(new_shard_name, new_repository_relative_path) - git_fork_repository(new_shard_name, new_repository_relative_path) - end - - def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) - logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = fetch_remote_command(name, tags, prune, force) - - setup_ssh_auth(ssh_key, known_hosts) do |env| - run_with_timeout(cmd, timeout, repository_absolute_path, env).tap do |success| - unless success - logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." - end - end - end - end - - def push_branches(remote_name, timeout, force, branch_names) - logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push) - cmd << '--force' if force - cmd += %W(-- #{remote_name}).concat(branch_names) - - success = run_with_timeout(cmd, timeout, repository_absolute_path) - - unless success - logger.error("Pushing branches to remote #{remote_name} failed.") - end - - success - end - - def delete_remote_branches(remote_name, branch_names) - branches = branch_names.map { |branch_name| ":#{branch_name}" } - - logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push -- #{remote_name}).concat(branches) - - success = run(cmd, repository_absolute_path) - - unless success - logger.error("Pushing deleted branches to remote #{remote_name} failed.") - end - - success - end - - protected - - def run(*args) - output, exitstatus = popen(*args) - @output << output - - exitstatus&.zero? - end - - def run_with_timeout(*args) - output, exitstatus = popen_with_timeout(*args) - @output << output - - exitstatus&.zero? - rescue Timeout::Error - @output.puts('Timed out') - - false - end - - def mask_password_in_url(url) - result = URI(url) - result.password = "*****" unless result.password.nil? - result.user = "*****" unless result.user.nil? # it's needed for oauth access_token - result - rescue - url - end - - def remove_origin_in_repo - cmd = %W(#{Gitlab.config.git.bin_path} remote rm origin) - run(cmd, repository_absolute_path) - end - - # Builds a small shell script that can be used to execute SSH with a set of - # custom options. - # - # Options are expanded as `'-oKey="Value"'`, so SSH will correctly interpret - # paths with spaces in them. We trust the user not to embed single or double - # quotes in the key or value. - def custom_ssh_script(options = {}) - args = options.map { |k, v| %Q{'-o#{k}="#{v}"'} }.join(' ') - - [ - "#!/bin/sh", - "exec ssh #{args} \"$@\"" - ].join("\n") - end - - # Known hosts data and private keys can be passed to gitlab-shell in the - # environment. If present, this method puts them into temporary files, writes - # a script that can substitute as `ssh`, setting the options to respect those - # files, and yields: { "GIT_SSH" => "/tmp/myScript" } - def setup_ssh_auth(key, known_hosts) - options = {} - - if key - key_file = Tempfile.new('gitlab-shell-key-file') - key_file.chmod(0o400) - key_file.write(key) - key_file.close - - options['IdentityFile'] = key_file.path - options['IdentitiesOnly'] = 'yes' - end - - if known_hosts - known_hosts_file = Tempfile.new('gitlab-shell-known-hosts') - known_hosts_file.chmod(0o400) - known_hosts_file.write(known_hosts) - known_hosts_file.close - - options['StrictHostKeyChecking'] = 'yes' - options['UserKnownHostsFile'] = known_hosts_file.path - end - - return yield({}) if options.empty? - - script = Tempfile.new('gitlab-shell-ssh-wrapper') - script.chmod(0o755) - script.write(custom_ssh_script(options)) - script.close - - yield('GIT_SSH' => script.path) - ensure - key_file&.close! - known_hosts_file&.close! - script&.close! - end - - private - - def fetch_remote_command(name, tags, prune, force) - %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet).tap do |cmd| - cmd << '--prune' if prune - cmd << '--force' if force - cmd << (tags ? '--tags' : '--no-tags') - end - end - - def git_import_repository(source, timeout) - # Skip import if repo already exists - return false if File.exist?(repository_absolute_path) - - masked_source = mask_password_in_url(source) - - logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare -- #{source} #{repository_absolute_path}) - - success = run_with_timeout(cmd, timeout, nil) - - unless success - logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") - FileUtils.rm_rf(repository_absolute_path) - return false - end - - Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) - - # The project was imported successfully. - # Remove the origin URL since it may contain password. - remove_origin_in_repo - - true - end - - def git_fork_repository(new_shard_name, new_repository_relative_path) - from_path = repository_absolute_path - new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path - to_path = File.join(new_shard_path, new_repository_relative_path) - - # The repository cannot already exist - if File.exist?(to_path) - logger.error "fork-repository failed: destination repository <#{to_path}> already exists." - return false - end - - # Ensure the namepsace / hashed storage directory exists - FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) - - logger.info "Forking repository from <#{from_path}> to <#{to_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare --no-local -- #{from_path} #{to_path}) - - run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) - end - end - end -end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb deleted file mode 100644 index 94ff5b4980a..00000000000 --- a/lib/gitlab/git/hook.rb +++ /dev/null @@ -1,108 +0,0 @@ -# Gitaly note: JV: looks like this is only used by Gitlab::Git::HooksService in -# app/services. We shouldn't bother migrating this until we know how -# Gitlab::Git::HooksService will be migrated. - -module Gitlab - module Git - class Hook - GL_PROTOCOL = 'web'.freeze - attr_reader :name, :path, :repository - - def initialize(name, repository) - @name = name - @repository = repository - @path = File.join(repo_path, 'hooks', name) - end - - def repo_path - repository.path - end - - def exists? - File.exist?(path) - end - - def trigger(gl_id, gl_username, oldrev, newrev, ref) - return [true, nil] unless exists? - - Bundler.with_clean_env do - case name - when "pre-receive", "post-receive" - call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) - when "update" - call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - end - end - end - - private - - def call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) - changes = [oldrev, newrev, ref].join(" ") - - exit_status = false - exit_message = nil - - vars = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username, - 'PWD' => repo_path, - 'GL_PROTOCOL' => GL_PROTOCOL, - 'GL_REPOSITORY' => repository.gl_repository - } - - options = { - chdir: repo_path - } - - Open3.popen3(vars, path, options) do |stdin, stdout, stderr, wait_thr| - exit_status = true - stdin.sync = true - - # in git, pre- and post- receive hooks may just exit without - # reading stdin. We catch the exception to avoid a broken pipe - # warning - begin - # inject all the changes as stdin to the hook - changes.lines do |line| - stdin.puts line - end - rescue Errno::EPIPE - end - - stdin.close - - unless wait_thr.value == 0 - exit_status = false - exit_message = retrieve_error_message(stderr, stdout) - end - end - - [exit_status, exit_message] - end - - def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - env = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username, - 'PWD' => repo_path - } - - options = { - chdir: repo_path - } - - args = [ref, oldrev, newrev] - - stdout, stderr, status = Open3.capture3(env, path, *args, options) - [status.success?, stderr.presence || stdout] - end - - def retrieve_error_message(stderr, stdout) - err_message = stderr.read - err_message = err_message.blank? ? stdout.read : err_message - err_message - end - end - end -end diff --git a/lib/gitlab/git/hook_env.rb b/lib/gitlab/git/hook_env.rb index 455e8451c10..620568d8817 100644 --- a/lib/gitlab/git/hook_env.rb +++ b/lib/gitlab/git/hook_env.rb @@ -17,18 +17,18 @@ module Gitlab ].freeze def self.set(gl_repository, env) - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? raise "missing gl_repository" if gl_repository.blank? - RequestStore.store[:gitlab_git_env] ||= {} - RequestStore.store[:gitlab_git_env][gl_repository] = whitelist_git_env(env) + Gitlab::SafeRequestStore[:gitlab_git_env] ||= {} + Gitlab::SafeRequestStore[:gitlab_git_env][gl_repository] = whitelist_git_env(env) end def self.all(gl_repository) - return {} unless RequestStore.active? + return {} unless Gitlab::SafeRequestStore.active? - h = RequestStore.fetch(:gitlab_git_env) { {} } + h = Gitlab::SafeRequestStore.fetch(:gitlab_git_env) { {} } h.fetch(gl_repository, {}) end diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb deleted file mode 100644 index e67cacdb95a..00000000000 --- a/lib/gitlab/git/hooks_service.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Gitlab - module Git - class HooksService - attr_accessor :oldrev, :newrev, :ref - - def execute(pusher, repository, oldrev, newrev, ref) - @repository = repository - @gl_id = pusher.gl_id - @gl_username = pusher.username - @oldrev = oldrev - @newrev = newrev - @ref = ref - - %w(pre-receive update).each do |hook_name| - status, message = run_hook(hook_name) - - unless status - raise PreReceiveError, message - end - end - - yield(self).tap do - run_hook('post-receive') - end - end - - private - - def run_hook(name) - hook = Gitlab::Git::Hook.new(name, @repository) - hook.trigger(@gl_id, @gl_username, oldrev, newrev, ref) - end - end - end -end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index d94082a3e30..c2e4274e3ee 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,157 +1,7 @@ -# Gitaly note: JV: When the time comes I think we will want to copy this -# class into Gitaly. None of its methods look like they should be RPC's. -# The RPC's will be at a higher level. - module Gitlab module Git class Index IndexError = Class.new(StandardError) - - DEFAULT_MODE = 0o100644 - - ACTIONS = %w(create create_dir update move delete).freeze - ACTION_OPTIONS = %i(file_path previous_path content encoding).freeze - - attr_reader :repository, :raw_index - - def initialize(repository) - @repository = repository - @raw_index = repository.rugged.index - end - - delegate :read_tree, :get, to: :raw_index - - def apply(action, options) - validate_action!(action) - public_send(action, options.slice(*ACTION_OPTIONS)) # rubocop:disable GitlabSecurity/PublicSend - end - - def write_tree - raw_index.write_tree(repository.rugged) - end - - def dir_exists?(path) - raw_index.find { |entry| entry[:path].start_with?("#{path}/") } - end - - def create(options) - options = normalize_options(options) - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - add_blob(options) - end - - def create_dir(options) - options = normalize_options(options) - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - if dir_exists?(options[:file_path]) - raise IndexError, "A directory with this name already exists" - end - - options = options.dup - options[:file_path] += '/.gitkeep' - options[:content] = '' - - add_blob(options) - end - - def update(options) - options = normalize_options(options) - - file_entry = get(options[:file_path]) - unless file_entry - raise IndexError, "A file with this name doesn't exist" - end - - add_blob(options, mode: file_entry[:mode]) - end - - def move(options) - options = normalize_options(options) - - file_entry = get(options[:previous_path]) - unless file_entry - raise IndexError, "A file with this name doesn't exist" - end - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - raw_index.remove(options[:previous_path]) - - add_blob(options, mode: file_entry[:mode]) - end - - def delete(options) - options = normalize_options(options) - - unless get(options[:file_path]) - raise IndexError, "A file with this name doesn't exist" - end - - raw_index.remove(options[:file_path]) - end - - private - - def normalize_options(options) - options = options.dup - options[:file_path] = normalize_path(options[:file_path]) if options[:file_path] - options[:previous_path] = normalize_path(options[:previous_path]) if options[:previous_path] - options - end - - def normalize_path(path) - unless path - raise IndexError, "You must provide a file path" - end - - pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) - - pathname.each_filename do |segment| - if segment == '..' - raise IndexError, 'Path cannot include directory traversal' - end - end - - pathname.to_s - end - - def add_blob(options, mode: nil) - content = options[:content] - unless content - raise IndexError, "You must provide content" - end - - content = Base64.decode64(content) if options[:encoding] == 'base64' - - detect = CharlockHolmes::EncodingDetector.new.detect(content) - unless detect && detect[:type] == :binary - # When writing to the repo directly as we are doing here, - # the `core.autocrlf` config isn't taken into account. - content.gsub!("\r\n", "\n") if repository.autocrlf - end - - oid = repository.rugged.write(content, :blob) - - raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) - rescue Rugged::IndexError => e - raise IndexError, e.message - end - - def validate_action!(action) - unless ACTIONS.include?(action.to_s) - raise ArgumentError, "Unknown action '#{action}'" - end - end end end end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index 57d748343be..0584629ac84 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -1,8 +1,6 @@ module Gitlab module Git class OperationService - include Gitlab::Git::Popen - BranchUpdate = Struct.new(:newrev, :repo_created, :branch_created) do alias_method :repo_created?, :repo_created alias_method :branch_created?, :branch_created @@ -17,177 +15,6 @@ module Gitlab ) end end - - attr_reader :user, :repository - - def initialize(user, new_repository) - if user - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - @user = user - end - - # Refactoring aid - Gitlab::Git.check_namespace!(new_repository) - - @repository = new_repository - end - - def add_branch(branch_name, newrev) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - oldrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) - end - - def rm_branch(branch) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch.name - oldrev = branch.target - newrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) - end - - def add_tag(tag_name, newrev, options = {}) - ref = Gitlab::Git::TAG_REF_PREFIX + tag_name - oldrev = Gitlab::Git::BLANK_SHA - - with_hooks(ref, newrev, oldrev) do |service| - # We want to pass the OID of the tag object to the hooks. For an - # annotated tag we don't know that OID until after the tag object - # (raw_tag) is created in the repository. That is why we have to - # update the value after creating the tag object. Only the - # "post-receive" hook will receive the correct value in this case. - raw_tag = repository.rugged.tags.create(tag_name, newrev, options) - service.newrev = raw_tag.target_id - end - end - - def rm_tag(tag) - ref = Gitlab::Git::TAG_REF_PREFIX + tag.name - oldrev = tag.target - newrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) do - repository.rugged.tags.delete(tag_name) - end - end - - # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist, - # it would be created from `start_branch_name`. - # If `start_repository` is passed, and the branch doesn't exist, - # it would try to find the commits from it instead of current repository. - def with_branch( - branch_name, - start_branch_name: nil, - start_repository: repository, - &block) - - Gitlab::Git.check_namespace!(start_repository) - start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - - start_branch_name = nil if start_repository.empty? - - if start_branch_name && !start_repository.branch_exists?(start_branch_name) - raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}" - end - - update_branch_with_hooks(branch_name) do - repository.with_repo_branch_commit( - start_repository, - start_branch_name || branch_name, - &block) - end - end - - def update_branch(branch_name, newrev, oldrev) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - update_ref_in_hooks(ref, newrev, oldrev) - end - - private - - # Returns [newrev, should_run_after_create, should_run_after_create_branch] - def update_branch_with_hooks(branch_name) - update_autocrlf_option - - was_empty = repository.empty? - - # Make commit - newrev = yield - - unless newrev - raise Gitlab::Git::CommitError.new('Failed to create commit') - end - - branch = repository.find_branch(branch_name) - oldrev = find_oldrev_from_branch(newrev, branch) - - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - update_ref_in_hooks(ref, newrev, oldrev) - - BranchUpdate.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) - end - - def find_oldrev_from_branch(newrev, branch) - return Gitlab::Git::BLANK_SHA unless branch - - oldrev = branch.target - - merge_base = repository.merge_base(newrev, branch.target) - raise Gitlab::Git::Repository::InvalidRef unless merge_base - - if oldrev == merge_base - oldrev - else - raise Gitlab::Git::CommitError.new('Branch diverged') - end - end - - def update_ref_in_hooks(ref, newrev, oldrev) - with_hooks(ref, newrev, oldrev) do - update_ref(ref, newrev, oldrev) - end - end - - def with_hooks(ref, newrev, oldrev) - Gitlab::Git::HooksService.new.execute( - user, - repository, - oldrev, - newrev, - ref) do |service| - - yield(service) - end - end - - # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites. - def update_ref(ref, newrev, oldrev) - # We use 'git update-ref' because libgit2/rugged currently does not - # offer 'compare and swap' ref updates. Without compare-and-swap we can - # (and have!) accidentally reset the ref to an earlier state, clobbering - # commits. See also https://github.com/libgit2/libgit2/issues/1534. - command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - - output, status = popen( - command, - repository.path) do |stdin| - stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") - end - - unless status.zero? - Gitlab::GitLogger.error("'git update-ref' in #{repository.path}: #{output}") - raise Gitlab::Git::CommitError.new( - "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ - " Please refresh and try again.") - end - end - - def update_autocrlf_option - if repository.autocrlf != :input - repository.autocrlf = :input - end - end end end end diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb deleted file mode 100644 index 7426688fc55..00000000000 --- a/lib/gitlab/git/popen.rb +++ /dev/null @@ -1,112 +0,0 @@ -# Gitaly note: JV: no RPC's here. - -require 'open3' - -module Gitlab - module Git - module Popen - FAST_GIT_PROCESS_TIMEOUT = 15.seconds - - def popen(cmd, path, vars = {}, lazy_block: nil) - unless cmd.is_a?(Array) - raise "System commands must be given as an array of strings" - end - - path ||= Dir.pwd - vars['PWD'] = path - options = { chdir: path } - - cmd_output = "" - cmd_status = 0 - Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| - stdout.set_encoding(Encoding::ASCII_8BIT) - - # stderr and stdout pipes can block if stderr/stdout aren't drained: https://bugs.ruby-lang.org/issues/9082 - # Mimic what Ruby does with capture3: https://github.com/ruby/ruby/blob/1ec544695fa02d714180ef9c34e755027b6a2103/lib/open3.rb#L257-L273 - err_reader = Thread.new { stderr.read } - - yield(stdin) if block_given? - stdin.close - - if lazy_block - cmd_output = lazy_block.call(stdout.lazy) - cmd_status = 0 - break - else - cmd_output << stdout.read - end - - cmd_output << err_reader.value - cmd_status = wait_thr.value.exitstatus - end - - [cmd_output, cmd_status] - end - - def popen_with_timeout(cmd, timeout, path, vars = {}) - unless cmd.is_a?(Array) - raise "System commands must be given as an array of strings" - end - - path ||= Dir.pwd - vars['PWD'] = path - - unless File.directory?(path) - FileUtils.mkdir_p(path) - end - - rout, wout = IO.pipe - rerr, werr = IO.pipe - - pid = Process.spawn(vars, *cmd, out: wout, err: werr, chdir: path, pgroup: true) - # stderr and stdout pipes can block if stderr/stdout aren't drained: https://bugs.ruby-lang.org/issues/9082 - # Mimic what Ruby does with capture3: https://github.com/ruby/ruby/blob/1ec544695fa02d714180ef9c34e755027b6a2103/lib/open3.rb#L257-L273 - out_reader = Thread.new { rout.read } - err_reader = Thread.new { rerr.read } - - begin - # close write ends so we could read them - wout.close - werr.close - - status = process_wait_with_timeout(pid, timeout) - - cmd_output = out_reader.value - cmd_output << err_reader.value # Copying the behaviour of `popen` which merges stderr into output - - [cmd_output, status.exitstatus] - rescue Timeout::Error => e - kill_process_group_for_pid(pid) - - raise e - ensure - wout.close unless wout.closed? - werr.close unless werr.closed? - - rout.close - rerr.close - end - end - - def process_wait_with_timeout(pid, timeout) - deadline = timeout.seconds.from_now - wait_time = 0.01 - - while deadline > Time.now - sleep(wait_time) - _, status = Process.wait2(pid, Process::WNOHANG) - - return status unless status.nil? - end - - raise Timeout::Error, "Timeout waiting for process ##{pid}" - end - - def kill_process_group_for_pid(pid) - Process.kill("KILL", -pid) - Process.wait(pid) - rescue Errno::ESRCH - end - end - end -end diff --git a/lib/gitlab/git/push.rb b/lib/gitlab/git/push.rb new file mode 100644 index 00000000000..b6577ba17f1 --- /dev/null +++ b/lib/gitlab/git/push.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class Push + include Gitlab::Utils::StrongMemoize + + attr_reader :ref, :oldrev, :newrev + + def initialize(project, oldrev, newrev, ref) + @project = project + @oldrev = oldrev.presence || Gitlab::Git::BLANK_SHA + @newrev = newrev.presence || Gitlab::Git::BLANK_SHA + @ref = ref + end + + def branch_name + strong_memoize(:branch_name) do + Gitlab::Git.branch_name(@ref) + end + end + + def branch_added? + Gitlab::Git.blank_ref?(@oldrev) + end + + def branch_removed? + Gitlab::Git.blank_ref?(@newrev) + end + + def branch_updated? + branch_push? && !branch_added? && !branch_removed? + end + + def force_push? + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + end + + def branch_push? + strong_memoize(:branch_push) do + Gitlab::Git.branch_ref?(@ref) + end + end + + def modified_paths + unless branch_updated? + raise ArgumentError, 'Unable to calculate modified paths!' + end + + strong_memoize(:modified_paths) do + @project.repository.diff_stats(@oldrev, @newrev).paths + end + end + end + end +end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index fa71a4e7ea7..31a280155bd 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged). - module Gitlab module Git class Ref @@ -26,13 +24,6 @@ module Gitlab str.gsub(%r{\Arefs/heads/}, '') end - # Gitaly: this method will probably be migrated indirectly via its call sites. - def self.dereference_object(object) - object = object.target while object.is_a?(Rugged::Tag::Annotation) - - object - end - def initialize(repository, name, target, dereferenced_target) @name = Gitlab::Git.ref_name(name) @dereferenced_target = dereferenced_target diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 9521a2d63a0..7732049b69b 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -6,18 +6,9 @@ module Gitlab module Git class Repository include Gitlab::Git::RepositoryMirroring - include Gitlab::Git::Popen include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize - ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY - GIT_ALTERNATE_OBJECT_DIRECTORIES - ].freeze - ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY_RELATIVE - GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE - ].freeze SEARCH_CONTEXT_LINES = 3 REV_LIST_COMMIT_LIMIT = 2_000 # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698 @@ -73,7 +64,7 @@ module Gitlab # Relative path of repo attr_reader :relative_path - attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path + attr_reader :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -82,13 +73,6 @@ module Gitlab @relative_path = relative_path @gl_repository = gl_repository - @gitlab_projects = Gitlab::Git::GitlabProjects.new( - storage, - relative_path, - global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, - logger: Rails.logger - ) - @name = @relative_path.split("/").last end @@ -112,19 +96,6 @@ module Gitlab raise Gitlab::Git::CommandError.new(e.message) end - # This method will be removed when Gitaly reaches v1.1. - def rugged - circuit_breaker.perform do - Rugged::Repository.new(path, alternates: alternate_object_directories) - end - rescue Rugged::RepositoryError, Rugged::OSError - raise NoRepository.new('no repository for such path') - end - - def cleanup - @rugged&.close - end - def circuit_breaker @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end @@ -148,10 +119,6 @@ module Gitlab end end - def reload_rugged - @rugged = nil - end - # Directly find a branch with a simple name (e.g. master) # def find_branch(name) @@ -250,15 +217,6 @@ module Gitlab end end - # Returns an Array of all ref names, except when it's matching pattern - # - # regexp - The pattern for ref names we don't want - def all_ref_names_except(prefixes) - rugged.references.reject do |ref| - prefixes.any? { |p| ref.name.start_with?(p) } - end.map(&:name) - end - def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) @@ -331,7 +289,7 @@ module Gitlab (size.to_f / 1024).round(2) end - # Use the Rugged Walker API to build an array of commits. + # Build an array of commits. # # Usage. # repo.log( @@ -463,6 +421,16 @@ module Gitlab Gitlab::Git::DiffCollection.new(iterator, options) end + def diff_stats(left_id, right_id) + stats = wrapped_gitaly_errors do + gitaly_commit_client.diff_stats(left_id, right_id) + end + + Gitlab::Git::DiffStatsCollection.new(stats) + rescue CommandError, TypeError + Gitlab::Git::DiffStatsCollection.new([]) + end + # Returns a RefName for a given SHA def ref_name_for_sha(ref_path, sha) raise ArgumentError, "sha can't be empty" unless sha.present? @@ -591,19 +559,6 @@ module Gitlab end end - def check_revert_content(target_commit, source_sha) - args = [target_commit.sha, source_sha] - args << { mainline: 1 } if target_commit.merge_commit? - - revert_index = rugged.revert_commit(*args) - return false if revert_index.conflicts? - - tree_id = revert_index.write_tree(rugged) - return false unless diff_exists?(source_sha, tree_id) - - tree_id - end - def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) args = { user: user, @@ -619,14 +574,6 @@ module Gitlab end end - def diff_exists?(sha1, sha2) - rugged.diff(sha1, sha2).size > 0 - end - - def user_to_committer(user) - Gitlab::Git.committer_hash(email: user.email, name: user.name) - end - # Delete the specified branch from the repository def delete_branch(branch_name) wrapped_gitaly_errors do @@ -666,18 +613,12 @@ module Gitlab end end - AUTOCRLF_VALUES = { - "true" => true, - "false" => false, - "input" => :input - }.freeze + def find_remote_root_ref(remote_name) + return unless remote_name.present? - def autocrlf - AUTOCRLF_VALUES[rugged.config['core.autocrlf']] - end - - def autocrlf=(value) - rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value] + wrapped_gitaly_errors do + gitaly_remote_client.find_remote_root_ref(remote_name) + end end # Returns result like "git ls-files" , recursive and full file path @@ -738,48 +679,6 @@ module Gitlab end end - def with_repo_branch_commit(start_repository, start_branch_name) - Gitlab::Git.check_namespace!(start_repository) - start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - - return yield nil if start_repository.empty? - - if start_repository.same_repository?(self) - yield commit(start_branch_name) - else - start_commit_id = start_repository.commit_id(start_branch_name) - - return yield nil unless start_commit_id - - if branch_commit = commit(start_commit_id) - yield branch_commit - else - with_repo_tmp_commit( - start_repository, start_branch_name, start_commit_id) do |tmp_commit| - yield tmp_commit - end - end - end - end - - def with_repo_tmp_commit(start_repository, start_branch_name, sha) - source_ref = start_branch_name - - unless Gitlab::Git.branch_ref?(source_ref) - source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}" - end - - tmp_ref = fetch_ref( - start_repository, - source_ref: source_ref, - target_ref: "refs/tmp/#{SecureRandom.hex}" - ) - - yield commit(sha) - ensure - delete_refs(tmp_ref) if tmp_ref - end - def fetch_source_branch!(source_repository, source_branch, local_ref) wrapped_gitaly_errors do gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) @@ -809,21 +708,6 @@ module Gitlab end end - # This method, fetch_ref, is used from within - # Gitlab::Git::OperationService. OperationService will eventually only - # exist in gitaly-ruby. When we delete OperationService from gitlab-ce - # we can also remove fetch_ref. - def fetch_ref(source_repository, source_ref:, target_ref:) - Gitlab::Git.check_namespace!(source_repository) - source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) - - args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref}) - message, status = run_git(args, env: source_repository.fetch_env) - raise Gitlab::Git::CommandError, message if status != 0 - - target_ref - end - # Refactoring aid; allows us to copy code from app/models/repository.rb def commit(ref = 'HEAD') Gitlab::Git::Commit.find(self, ref) @@ -891,24 +775,6 @@ module Gitlab end end - def push_remote_branches(remote_name, branch_names, forced: true) - success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - def bundle_to_disk(save_path) wrapped_gitaly_errors do gitaly_repository_client.create_bundle(save_path) @@ -1056,9 +922,10 @@ module Gitlab end end - def shell_blame(sha, path) - output, _status = run_git(%W(blame -p #{sha} -- #{path})) - output + def list_last_commits_for_tree(sha, path, offset: 0, limit: 25) + wrapped_gitaly_errors do + gitaly_commit_client.list_last_commits_for_tree(sha, path, offset: offset, limit: limit) + end end def last_commit_for_path(sha, path) @@ -1067,26 +934,6 @@ module Gitlab end end - def rev_list(including: [], excluding: [], options: [], objects: false, &block) - args = ['rev-list'] - - args.push(*rev_list_param(including)) - - exclude_param = *rev_list_param(excluding) - if exclude_param.any? - args.push('--not') - args.push(*exclude_param) - end - - args.push('--objects') if objects - - if options.any? - args.push(*options) - end - - run_git!(args, lazy_block: block) - end - def checksum # The exists? RPC is much cheaper, so we perform this request first raise NoRepository, "Repository does not exists" unless exists? @@ -1104,44 +951,6 @@ module Gitlab end end - def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - cmd = [Gitlab.config.git.bin_path, *args] - cmd.unshift("nice") if nice - - object_directories = alternate_object_directories - if object_directories.any? - env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR) - end - - circuit_breaker.perform do - popen(cmd, chdir, env, lazy_block: lazy_block, &block) - end - end - - def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block) - - raise GitError, output unless status.zero? - - output - end - - def run_git_with_timeout(args, timeout, env: {}) - circuit_breaker.perform do - popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) - end - end - - def git_env_for_user(user) - { - 'GIT_COMMITTER_NAME' => user.name, - 'GIT_COMMITTER_EMAIL' => user.email, - 'GL_ID' => Gitlab::GlId.gl_id(user), - 'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL, - 'GL_REPOSITORY' => gl_repository - } - end - def gitaly_merged_branch_names(branch_names, root_sha) qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } @@ -1184,31 +993,6 @@ module Gitlab found_module && found_module['url'] end - def alternate_object_directories - relative_object_directories.map { |d| File.join(path, d) } - end - - def relative_object_directories - Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact - end - - def sort_branches(branches, sort_by) - case sort_by - when 'name' - branches.sort_by(&:name) - when 'updated_desc' - branches.sort do |a, b| - b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date - end - when 'updated_asc' - branches.sort do |a, b| - a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date - end - else - branches - end - end - # Returns true if the given ref name exists # # Ref names must start with `refs/`. @@ -1223,14 +1007,6 @@ module Gitlab def gitaly_delete_refs(*ref_names) gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any? end - - def gitlab_projects_error - raise CommandError, @gitlab_projects.output - end - - def rev_list_param(spec) - spec == :all ? ['--all'] : spec - end end end end diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 62427ac9cc4..fcee9ae566c 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -11,7 +11,7 @@ module Gitlab to: :failure_info def self.for_storage(storage) - cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do + cached_circuitbreakers = Gitlab::SafeRequestStore.fetch(:circuitbreaker_cache) do Hash.new do |hash, storage_name| hash[storage_name] = build(storage_name) end diff --git a/lib/gitlab/git/storage/failure_info.rb b/lib/gitlab/git/storage/failure_info.rb index 387279c110d..1d28a850049 100644 --- a/lib/gitlab/git/storage/failure_info.rb +++ b/lib/gitlab/git/storage/failure_info.rb @@ -10,7 +10,7 @@ module Gitlab redis.del(*all_storage_keys) unless all_storage_keys.empty? end - RequestStore.delete(:circuitbreaker_cache) + Gitlab::SafeRequestStore.delete(:circuitbreaker_cache) end def self.load(cache_key) diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb index 90bbe85fd37..8e14acb4ccb 100644 --- a/lib/gitlab/git/storage/health.rb +++ b/lib/gitlab/git/storage/health.rb @@ -81,9 +81,11 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def total_failures @total_failures ||= failing_info.sum { |info_for_host| info_for_host[:failure_count] } end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index cb851b76a23..e0867aeb5a7 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -50,51 +50,6 @@ module Gitlab entry[:oid] end end - - def tree_entries_from_rugged(repository, sha, path, recursive) - current_path_entries = get_tree_entries_from_rugged(repository, sha, path) - ordered_entries = [] - - current_path_entries.each do |entry| - ordered_entries << entry - - if recursive && entry.dir? - ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) - end - end - - ordered_entries - end - - def get_tree_entries_from_rugged(repository, sha, path) - commit = repository.lookup(sha) - root_tree = commit.tree - - tree = if path - id = find_id_by_path(repository, root_tree.oid, path) - if id - repository.lookup(id) - else - [] - end - else - root_tree - end - - tree.map do |entry| - new( - id: entry[:oid], - root_id: root_tree.oid, - name: entry[:name], - type: entry[:type], - mode: entry[:filemode].to_s(8), - path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha - ) - end - rescue Rugged::ReferenceError - [] - end end def initialize(options) diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index e573cd0e143..338e1a30c45 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -4,7 +4,7 @@ module Gitlab attr_reader :username, :name, :email, :gl_id def self.from_gitlab(gitlab_user) - new(gitlab_user.username, gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) + new(gitlab_user.username, gitlab_user.name, gitlab_user.commit_email, Gitlab::GlId.gl_id(gitlab_user)) end def self.from_gitaly(gitaly_user) diff --git a/lib/gitlab/git/version.rb b/lib/gitlab/git/version.rb index 1e14e8b652a..4bd91898457 100644 --- a/lib/gitlab/git/version.rb +++ b/lib/gitlab/git/version.rb @@ -1,8 +1,6 @@ module Gitlab module Git module Version - extend Gitlab::Git::Popen - def self.git_version Gitlab::VersionInfo.parse(Gitaly::Server.all.first.git_binary_version) end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 9d992be66eb..072019dfb0a 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -1,9 +1,16 @@ +# We only need Gollum::Page so let's not load all of gollum-lib. +require 'gollum-lib/pagination' +require 'gollum-lib/wiki' +require 'gollum-lib/page' + module Gitlab module Git class Wiki DuplicatePageError = Class.new(StandardError) OperationError = Class.new(StandardError) + DEFAULT_PAGINATION = Kaminari.config.default_per_page + CommitDetails = Struct.new(:user_id, :username, :name, :email, :message) do def to_h { user_id: user_id, username: username, name: name, email: email, message: message } @@ -74,7 +81,7 @@ module Gitlab # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 # per page, but also fetches 20 if `limit` or `per_page` < 20. # Slicing returns an array with the expected number of items. - slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page + slice_bound = options[:limit] || options[:per_page] || DEFAULT_PAGINATION versions[0..slice_bound] end @@ -104,30 +111,6 @@ module Gitlab private - def new_page(gollum_page) - Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id)) - end - - def new_version(gollum_page, commit_id) - Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format) - end - - def version(commit_id) - commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) } - - if RequestStore.active? - RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call } - else - commit_find_proc.call - end - end - - def assert_type!(object, klass) - unless object.is_a?(klass) - raise ArgumentError, "expected a #{klass}, got #{object.inspect}" - end - end - def gitaly_wiki_client @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) end @@ -163,20 +146,6 @@ module Gitlab Gitlab::Git::WikiPage.new(wiki_page, version) end end - - def committer_with_hooks(commit_details) - Gitlab::Git::CommitterWithHooks.new(self, commit_details.to_h) - end - - def with_committer_with_hooks(commit_details, &block) - committer = committer_with_hooks(commit_details) - - yield committer - - committer.commit - - nil - end end end end diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb index 84335aca4bc..64313bb04e8 100644 --- a/lib/gitlab/git/wiki_file.rb +++ b/lib/gitlab/git/wiki_file.rb @@ -3,17 +3,12 @@ module Gitlab class WikiFile attr_reader :mime_type, :raw_data, :name, :path - # This class is meant to be serializable so that it can be constructed - # by Gitaly and sent over the network to GitLab. - # - # Because Gollum::File is not serializable we must get all the data from - # 'gollum_file' during initialization, and NOT store it in an instance - # variable. - def initialize(gollum_file) - @mime_type = gollum_file.mime_type - @raw_data = gollum_file.raw_data - @name = gollum_file.name - @path = gollum_file.path + # This class wraps Gitlab::GitalyClient::WikiFile + def initialize(gitaly_file) + @mime_type = gitaly_file.mime_type + @raw_data = gitaly_file.raw_data + @name = gitaly_file.name + @path = gitaly_file.path end end end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb index 669ae11a423..c4087c9ebdc 100644 --- a/lib/gitlab/git/wiki_page.rb +++ b/lib/gitlab/git/wiki_page.rb @@ -3,25 +3,15 @@ module Gitlab class WikiPage attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data - # This class is meant to be serializable so that it can be constructed - # by Gitaly and sent over the network to GitLab. - # - # Because Gollum::Page is not serializable we must get all the data from - # 'gollum_page' during initialization, and NOT store it in an instance - # variable. - # - # Note that 'version' is a WikiPageVersion instance which it itself - # serializable. That means it's OK to store 'version' in an instance - # variable. - def initialize(gollum_page, version) - @url_path = gollum_page.url_path - @title = gollum_page.title - @format = gollum_page.format - @path = gollum_page.path - @raw_data = gollum_page.raw_data - @name = gollum_page.name - @historical = gollum_page.historical? - @formatted_data = gollum_page.formatted_data if gollum_page.is_a?(Gollum::Page) + # This class abstracts away Gitlab::GitalyClient::WikiPage + def initialize(gitaly_page, version) + @url_path = gitaly_page.url_path + @title = gitaly_page.title + @format = gitaly_page.format + @path = gitaly_page.path + @raw_data = gitaly_page.raw_data + @name = gitaly_page.name + @historical = gitaly_page.historical? @version = version end diff --git a/lib/gitlab/git/wiki_page_version.rb b/lib/gitlab/git/wiki_page_version.rb index 55f1afedcab..d5e7e70fd31 100644 --- a/lib/gitlab/git/wiki_page_version.rb +++ b/lib/gitlab/git/wiki_page_version.rb @@ -3,11 +3,6 @@ module Gitlab class WikiPageVersion attr_reader :commit, :format - # This class is meant to be serializable so that it can be constructed - # by Gitaly and sent over the network to GitLab. - # - # Both 'commit' (a Gitlab::Git::Commit) and 'format' (a string) are - # serializable. def initialize(commit, format) @commit = commit @format = format diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 35808149b90..240a0d7d1b8 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -24,8 +24,8 @@ module Gitlab cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." }.freeze - DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze - PUSH_COMMANDS = %w{ git-receive-pack }.freeze + DOWNLOAD_COMMANDS = %w{git-upload-pack git-upload-archive}.freeze + PUSH_COMMANDS = %w{git-receive-pack}.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes @@ -50,6 +50,10 @@ module Gitlab check_authentication_abilities!(cmd) check_command_disabled!(cmd) check_command_existence!(cmd) + + custom_action = check_custom_action(cmd) + return custom_action if custom_action + check_db_accessibility!(cmd) ensure_project_on_push!(cmd, changes) @@ -65,7 +69,7 @@ module Gitlab check_push_access! end - true + ::Gitlab::GitAccessResult::Success.new end def guest_can_download_code? @@ -92,6 +96,10 @@ module Gitlab private + def check_custom_action(cmd) + nil + end + def check_valid_actor! return unless actor.is_a?(Key) diff --git a/lib/gitlab/git_access_result/custom_action.rb b/lib/gitlab/git_access_result/custom_action.rb new file mode 100644 index 00000000000..a05a4baed82 --- /dev/null +++ b/lib/gitlab/git_access_result/custom_action.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GitAccessResult + class CustomAction + attr_reader :payload, :message + + # Example of payload: + # + # { + # 'action' => 'geo_proxy_to_primary', + # 'data' => { + # 'api_endpoints' => %w{geo/proxy_git_push_ssh/info_refs geo/proxy_git_push_ssh/push}, + # 'gl_username' => user.username, + # 'primary_repo' => geo_primary_http_url_to_repo(project_or_wiki) + # } + # } + # + def initialize(payload, message) + @payload = payload + @message = message + end + end + end +end diff --git a/lib/gitlab/git_access_result/success.rb b/lib/gitlab/git_access_result/success.rb new file mode 100644 index 00000000000..7bb9f24cb0e --- /dev/null +++ b/lib/gitlab/git_access_result/success.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module GitAccessResult + class Success + end + end +end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c27972a84a4..4ec87f6a3e7 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -174,10 +174,29 @@ module Gitlab end private_class_method :current_transaction_labels + # For some time related tasks we can't rely on `Time.now` since it will be + # affected by Timecop in some tests, and the clock of some gitaly-related + # components (grpc's c-core and gitaly server) use system time instead of + # timecop's time, so tests will fail. + # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will circumvent + # timecop. + def self.real_time + Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + end + private_class_method :real_time + + def self.authorization_token(storage) + token = token(storage).to_s + issued_at = real_time.to_i.to_s + hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, token, issued_at) + + "v2.#{hmac}.#{issued_at}" + end + private_class_method :authorization_token + def self.request_kwargs(storage, timeout, remote_storage: nil) - encoded_token = Base64.strict_encode64(token(storage).to_s) metadata = { - 'authorization' => "Bearer #{encoded_token}", + 'authorization' => "Bearer #{authorization_token(storage)}", 'client_name' => CLIENT_NAME } @@ -195,12 +214,7 @@ module Gitlab return result unless timeout > 0 - # Do not use `Time.now` for deadline calculation, since it - # will be affected by Timecop in some tests, but grpc's c-core - # uses system time instead of timecop's time, so tests will fail - # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will - # circumvent timecop - deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout + deadline = real_time + timeout result[:deadline] = deadline result @@ -302,7 +316,7 @@ module Gitlab # Ensures that Gitaly is not being abuse through n+1 misuse etc def self.enforce_gitaly_request_limits(call_site) # Only count limits in request-response environments (not sidekiq for example) - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? # This is this actual number of times this call was made. Used for information purposes only actual_call_count = increment_call_count("gitaly_#{call_site}_actual") @@ -326,7 +340,7 @@ module Gitlab end def self.allow_n_plus_1_calls - return yield unless RequestStore.active? + return yield unless Gitlab::SafeRequestStore.active? begin increment_call_count(:gitaly_call_count_exception_block_depth) @@ -337,25 +351,25 @@ module Gitlab end def self.get_call_count(key) - RequestStore.store[key] || 0 + Gitlab::SafeRequestStore[key] || 0 end private_class_method :get_call_count def self.increment_call_count(key) - RequestStore.store[key] ||= 0 - RequestStore.store[key] += 1 + Gitlab::SafeRequestStore[key] ||= 0 + Gitlab::SafeRequestStore[key] += 1 end private_class_method :increment_call_count def self.decrement_call_count(key) - RequestStore.store[key] -= 1 + Gitlab::SafeRequestStore[key] -= 1 end private_class_method :decrement_call_count # Returns an estimate of the number of Gitaly calls made for this # request def self.get_request_count - return 0 unless RequestStore.active? + return 0 unless Gitlab::SafeRequestStore.active? gitaly_migrate_count = get_call_count("gitaly_migrate_actual") gitaly_call_count = get_call_count("gitaly_call_actual") @@ -372,28 +386,28 @@ module Gitlab end def self.reset_counts - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? %w[migrate call].each do |call_site| - RequestStore.store["gitaly_#{call_site}_actual"] = 0 - RequestStore.store["gitaly_#{call_site}_permitted"] = 0 + Gitlab::SafeRequestStore["gitaly_#{call_site}_actual"] = 0 + Gitlab::SafeRequestStore["gitaly_#{call_site}_permitted"] = 0 end end def self.add_call_details(details) id = details.delete(:id) - return unless id && RequestStore.active? && RequestStore.store[:peek_enabled] + return unless id && Gitlab::SafeRequestStore[:peek_enabled] - RequestStore.store['gitaly_call_details'] ||= {} - RequestStore.store['gitaly_call_details'][id] ||= {} - RequestStore.store['gitaly_call_details'][id].merge!(details) + Gitlab::SafeRequestStore['gitaly_call_details'] ||= {} + Gitlab::SafeRequestStore['gitaly_call_details'][id] ||= {} + Gitlab::SafeRequestStore['gitaly_call_details'][id].merge!(details) end def self.list_call_details - return {} unless RequestStore.active? && RequestStore.store[:peek_enabled] + return {} unless Gitlab::SafeRequestStore[:peek_enabled] - RequestStore.store['gitaly_call_details'] || {} + Gitlab::SafeRequestStore['gitaly_call_details'] || {} end def self.expected_server_version @@ -431,22 +445,22 @@ module Gitlab # Count a stack. Used for n+1 detection def self.count_stack - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? stack_string = Gitlab::Profiler.clean_backtrace(caller).drop(1).join("\n") - RequestStore.store[:stack_counter] ||= Hash.new + Gitlab::SafeRequestStore[:stack_counter] ||= Hash.new - count = RequestStore.store[:stack_counter][stack_string] || 0 - RequestStore.store[:stack_counter][stack_string] = count + 1 + count = Gitlab::SafeRequestStore[:stack_counter][stack_string] || 0 + Gitlab::SafeRequestStore[:stack_counter][stack_string] = count + 1 end private_class_method :count_stack # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection def self.max_call_count - return 0 unless RequestStore.active? + return 0 unless Gitlab::SafeRequestStore.active? - stack_counter = RequestStore.store[:stack_counter] + stack_counter = Gitlab::SafeRequestStore[:stack_counter] return 0 unless stack_counter stack_counter.values.max @@ -455,9 +469,9 @@ module Gitlab # Returns the stacks that calls Gitaly the most times. Used for n+1 detection def self.max_stacks - return nil unless RequestStore.active? + return nil unless Gitlab::SafeRequestStore.active? - stack_counter = RequestStore.store[:stack_counter] + stack_counter = Gitlab::SafeRequestStore[:stack_counter] return nil unless stack_counter max = max_call_count diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 6a97cd8ed17..085b2a127a5 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -148,6 +148,24 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end + def list_last_commits_for_tree(revision, path, offset: 0, limit: 25) + request = Gitaly::ListLastCommitsForTreeRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision), + path: encode_binary(path.to_s), + offset: offset, + limit: limit + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout) + + response.each_with_object({}) do |gitaly_response, hsh| + gitaly_response.commits.each do |commit_for_tree| + hsh[commit_for_tree.path] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit) + end + end + end + def last_commit_for_path(revision, path) request = Gitaly::LastCommitForPathRequest.new( repository: @gitaly_repo, @@ -172,6 +190,17 @@ module Gitlab consume_commits_response(response) end + def diff_stats(left_commit_sha, right_commit_sha) + request = Gitaly::DiffStatsRequest.new( + repository: @gitaly_repo, + left_commit_id: left_commit_sha, + right_commit_id: right_commit_sha + ) + + response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) + response.flat_map(&:stats) + end + def find_all_commits(opts = {}) request = Gitaly::FindAllCommitsRequest.new( repository: @gitaly_repo, @@ -229,27 +258,29 @@ module Gitlab end def find_commit(revision) - if RequestStore.active? - # We don't use RequeStstore.fetch(key) { ... } directly because `revision` - # can be a branch name, so we can't use it as a key as it could point - # to another commit later on (happens a lot in tests). + if Gitlab::SafeRequestStore.active? + # We don't use Gitlab::SafeRequestStore.fetch(key) { ... } directly + # because `revision` can be a branch name, so we can't use it as a key + # as it could point to another commit later on (happens a lot in + # tests). key = { storage: @gitaly_repo.storage_name, relative_path: @gitaly_repo.relative_path, commit_id: revision } - return RequestStore[key] if RequestStore.exist?(key) + return Gitlab::SafeRequestStore[key] if Gitlab::SafeRequestStore.exist?(key) commit = call_find_commit(revision) return unless commit key[:commit_id] = commit.id - RequestStore[key] = commit + Gitlab::SafeRequestStore[key] = commit else call_find_commit(revision) end end + # rubocop: disable CodeReuse/ActiveRecord def patch(revision) request = Gitaly::CommitPatchRequest.new( repository: @gitaly_repo, @@ -259,6 +290,7 @@ module Gitlab response.sum(&:data) end + # rubocop: enable CodeReuse/ActiveRecord def commit_stats(revision) request = Gitaly::CommitStatsRequest.new( @@ -369,7 +401,7 @@ module Gitlab request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) request_params[:enforce_limits] = options.fetch(:limits, true) request_params[:collapse_diffs] = !options.fetch(:expanded, true) - request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) + request_params.merge!(Gitlab::Git::DiffCollection.limits(options).to_h) request = Gitaly::CommitDiffRequest.new(request_params) response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 54c78fdb680..0f148614b20 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -333,7 +333,8 @@ module Gitlab action: action[:action].upcase.to_sym, file_path: encode_binary(action[:file_path]), previous_path: encode_binary(action[:previous_path]), - base64_content: action[:encoding] == 'base64' + base64_content: action[:encoding] == 'base64', + execute_filemode: !!action[:execute_filemode] ) rescue RangeError raise ArgumentError, "Unknown action '#{action[:action]}'" diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 1381e033d4b..4661448621b 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class RemoteService + include Gitlab::EncodingHelper + MAX_MSG_SIZE = 128.kilobytes.freeze def self.exists?(remote_url) @@ -52,6 +54,18 @@ module Gitlab response.result end + def find_remote_root_ref(remote_name) + request = Gitaly::FindRemoteRootRefRequest.new( + repository: @gitaly_repo, + remote: remote_name + ) + + response = GitalyClient.call(@storage, :remote_service, + :find_remote_root_ref, request) + + encode_utf8(response.ref) + end + def update_remote_mirror(ref_name, only_branches_matching) req_enum = Enumerator.new do |y| y.yield Gitaly::UpdateRemoteMirrorRequest.new( diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb index eb0e910665b..3a26dd58ff4 100644 --- a/lib/gitlab/gitaly_client/storage_service.rb +++ b/lib/gitlab/gitaly_client/storage_service.rb @@ -5,6 +5,14 @@ module Gitlab @storage = storage end + # Returns all directories in the git storage directory, lexically ordered + def list_directories(depth: 1) + request = Gitaly::ListDirectoriesRequest.new(storage_name: @storage, depth: depth) + + GitalyClient.call(@storage, :storage_service, :list_directories, request) + .flat_map(&:paths) + end + # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation. def delete_all_repositories request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage) diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 8e530de174d..26d1f53f26c 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -13,7 +13,7 @@ module Gitlab Storage is invalid because it has no `path` key. For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example. - If you're using the Gitlab Development Kit, you can update your configuration running `gdk reconfigure`. + If you're using the GitLab Development Kit, you can update your configuration running `gdk reconfigure`. MSG # This class will give easily recognizable NoMethodErrors diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 75be7d1f5a0..7c2c228ad01 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -110,7 +110,7 @@ module Gitlab repository: @gitaly_repo, page_path: encode_binary(page_path), page: options[:page] || 1, - per_page: options[:per_page] || Gollum::Page.per_page + per_page: options[:per_page] || Gitlab::Git::Wiki::DEFAULT_PAGINATION ) stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb index a73033d35ba..80246fa1b77 100644 --- a/lib/gitlab/github_import/importer/labels_importer.rb +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -10,11 +10,13 @@ module Gitlab # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. + # rubocop: disable CodeReuse/ActiveRecord def initialize(project, client) @project = project @client = client @existing_labels = project.labels.pluck(:title).to_set end + # rubocop: enable CodeReuse/ActiveRecord def execute bulk_insert(Label, build_labels) diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index 94eb9136b9a..8d54b27374c 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -10,11 +10,13 @@ module Gitlab # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` + # rubocop: disable CodeReuse/ActiveRecord def initialize(project, client) @project = project @client = client @existing_milestones = project.milestones.pluck(:iid).to_set end + # rubocop: enable CodeReuse/ActiveRecord def execute # We insert records in bulk, by-passing any standard model callbacks. diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 100f459fdcc..0e7c9ee0d00 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -10,11 +10,13 @@ module Gitlab # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` + # rubocop: disable CodeReuse/ActiveRecord def initialize(project, client) @project = project @client = client @existing_tags = project.releases.pluck(:tag).to_set end + # rubocop: enable CodeReuse/ActiveRecord def execute bulk_insert(Release, build_releases) diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 01168abde6c..374dc9d3c00 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -14,11 +14,13 @@ module Gitlab end # Returns true if we should import the wiki for the project. + # rubocop: disable CodeReuse/ActiveRecord def import_wiki? client.repository(project.import_source)&.has_wiki && !project.wiki_repository_exists? && Gitlab::GitalyClient::RemoteService.exists?(wiki_url) end + # rubocop: enable CodeReuse/ActiveRecord # Imports the repository data. # diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb index 9be071141db..d2479a8f565 100644 --- a/lib/gitlab/github_import/label_finder.rb +++ b/lib/gitlab/github_import/label_finder.rb @@ -18,6 +18,7 @@ module Gitlab Caching.read_integer(cache_key_for(name)) end + # rubocop: disable CodeReuse/ActiveRecord def build_cache mapping = @project .labels @@ -28,6 +29,7 @@ module Gitlab Caching.write_multiple(mapping) end + # rubocop: enable CodeReuse/ActiveRecord def cache_key_for(name) CACHE_KEY % { project: project.id, name: name } diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb index 208d15dc144..5625730e796 100644 --- a/lib/gitlab/github_import/milestone_finder.rb +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -21,6 +21,7 @@ module Gitlab Caching.read_integer(cache_key_for(issuable.milestone_number)) end + # rubocop: disable CodeReuse/ActiveRecord def build_cache mapping = @project .milestones @@ -31,6 +32,7 @@ module Gitlab Caching.write_multiple(mapping) end + # rubocop: enable CodeReuse/ActiveRecord def cache_key_for(iid) CACHE_KEY % { project: project.id, iid: iid } diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index be1259662a7..30283f147ef 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -136,13 +136,17 @@ module Gitlab Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id) end + # rubocop: disable CodeReuse/ActiveRecord def query_id_for_github_id(id) User.for_github_id(id).pluck(:id).first end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def query_id_for_github_email(email) User.by_any_email(email).pluck(:id).first end + # rubocop: enable CodeReuse/ActiveRecord # Reads an ID from the cache. # diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 195672f5a12..047487f1d24 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -52,10 +52,12 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def gitlab_user_id(project, gitlab_id) user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s) (user && user.id) || project.creator_id end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index 07c0abcce23..b54e45de4fe 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -4,6 +4,7 @@ module Gitlab "#{is_wiki ? 'wiki' : 'project'}-#{project.id}" end + # rubocop: disable CodeReuse/ActiveRecord def self.parse(gl_repository) match_data = /\A(project|wiki)-([1-9][0-9]*)\z/.match(gl_repository) unless match_data @@ -16,5 +17,6 @@ module Gitlab [project, wiki] end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 5070f4e3cfe..94c15739231 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -78,6 +78,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def import_issues return unless repo.issues @@ -123,6 +124,7 @@ module Gitlab import_issue_comments(issue, comments) end end + # rubocop: enable CodeReuse/ActiveRecord def import_issue_labels(raw_issue) labels = [] diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 2716834f566..2bc081a6181 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -26,6 +26,7 @@ module Gitlab !!(signature_text && signed_text) end + # rubocop: disable CodeReuse/ActiveRecord def signature return unless has_signature? @@ -36,6 +37,7 @@ module Gitlab @signature = create_cached_signature! end + # rubocop: enable CodeReuse/ActiveRecord def update_signature!(cached_signature) using_keychain do |gpg_key| @@ -113,9 +115,11 @@ module Gitlab gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} end + # rubocop: disable CodeReuse/ActiveRecord def find_gpg_key(keyid) GpgKey.find_by(primary_keyid: keyid) || GpgKeySubkey.find_by(keyid: keyid) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index 1991911ef6a..6972bd685f7 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -5,6 +5,7 @@ module Gitlab @gpg_key = gpg_key end + # rubocop: disable CodeReuse/ActiveRecord def run GpgSignature .select(:id, :commit_sha, :project_id) @@ -12,6 +13,7 @@ module Gitlab .where(gpg_key_primary_keyid: @gpg_key.keyids) .find_each { |sig| sig.gpg_commit&.update_signature!(sig) } end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/grape_logging/loggers/perf_logger.rb b/lib/gitlab/grape_logging/loggers/perf_logger.rb new file mode 100644 index 00000000000..e3b9c59bd6e --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/perf_logger.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# This module adds additional performance metrics to the grape logger +module Gitlab + module GrapeLogging + module Loggers + class PerfLogger < ::GrapeLogging::Loggers::Base + def parameters(_, _) + { gitaly_calls: Gitlab::GitalyClient.get_request_count } + end + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/route_logger.rb b/lib/gitlab/grape_logging/loggers/route_logger.rb new file mode 100644 index 00000000000..f3146b4dfd9 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/route_logger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# This grape_logging module (https://github.com/aserafin/grape_logging) makes it +# possible to log the details of the action +module Gitlab + module GrapeLogging + module Loggers + class RouteLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + endpoint = request.env[Grape::Env::API_ENDPOINT] + route = endpoint&.route&.pattern&.origin + + return {} unless route + + { route: route } + rescue + # endpoint.route calls env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] + # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response + # so we're rescuing exceptions and bailing out + {} + end + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset_connection.rb b/lib/gitlab/graphql/connections/keyset_connection.rb index abee2afe144..3c0d7e9784a 100644 --- a/lib/gitlab/graphql/connections/keyset_connection.rb +++ b/lib/gitlab/graphql/connections/keyset_connection.rb @@ -6,6 +6,7 @@ module Gitlab encode(node[order_field].to_s) end + # rubocop: disable CodeReuse/ActiveRecord def sliced_nodes @sliced_nodes ||= begin @@ -17,7 +18,9 @@ module Gitlab sliced end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def paged_nodes if first && last raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") @@ -29,6 +32,7 @@ module Gitlab sliced_nodes.limit(limit_value) end end + # rubocop: enable CodeReuse/ActiveRecord private diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb index 42ded7c286f..8fbfa1a86bf 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/group_hierarchy.rb @@ -19,9 +19,11 @@ module Gitlab # Returns the set of descendants of a given relation, but excluding the given # relation + # rubocop: disable CodeReuse/ActiveRecord def descendants base_and_descendants.where.not(id: descendants_base.select(:id)) end + # rubocop: enable CodeReuse/ActiveRecord # Returns the set of ancestors of a given relation, but excluding the given # relation @@ -29,9 +31,11 @@ module Gitlab # Passing an `upto` will stop the recursion once the specified parent_id is # reached. So all ancestors *lower* than the specified ancestor will be # included. + # rubocop: disable CodeReuse/ActiveRecord def ancestors(upto: nil) base_and_ancestors(upto: upto).where.not(id: ancestors_base.select(:id)) end + # rubocop: enable CodeReuse/ActiveRecord # Returns a relation that includes the ancestors_base set of groups # and all their ancestors (recursively). @@ -75,6 +79,7 @@ module Gitlab # Rails thinking it's selecting data the usual way. # # If nested groups are not supported, ancestors_base is returned. + # rubocop: disable CodeReuse/ActiveRecord def all_groups return ancestors_base unless Group.supports_nested_groups? @@ -84,20 +89,22 @@ module Gitlab ancestors_table = ancestors.alias_to(groups_table) descendants_table = descendants.alias_to(groups_table) - union = SQL::Union.new([model.unscoped.from(ancestors_table), - model.unscoped.from(descendants_table)]) - relation = model .unscoped .with .recursive(ancestors.to_arel, descendants.to_arel) - .from("(#{union.to_sql}) #{model.table_name}") + .from_union([ + model.unscoped.from(ancestors_table), + model.unscoped.from(descendants_table) + ]) read_only(relation) end + # rubocop: enable CodeReuse/ActiveRecord private + # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors_cte(stop_id = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) @@ -113,7 +120,9 @@ module Gitlab cte << parent_query cte end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def base_and_descendants_cte cte = SQL::RecursiveCTE.new(:base_and_descendants) @@ -127,6 +136,7 @@ module Gitlab cte end + # rubocop: enable CodeReuse/ActiveRecord def groups_table model.arel_table diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index d11fcc6a3e3..4edc251facb 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -22,6 +22,7 @@ module Gitlab # # @param [Object] start first project id for the range # @param [Object] finish last project id for the range + # rubocop: disable CodeReuse/ActiveRecord def bulk_migrate(start, finish) projects = build_relation(start, finish) @@ -29,6 +30,7 @@ module Gitlab migrate(project) end end + # rubocop: enable CodeReuse/ActiveRecord # Flag a project to be migrated # @@ -43,6 +45,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def build_relation(start, finish) relation = Project table = Project.arel_table @@ -52,6 +55,7 @@ module Gitlab relation end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 303b05e6a9a..22edd5f999d 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -21,6 +21,7 @@ module Gitlab !range_from.nil? && range_from == range_to end + # rubocop: disable CodeReuse/ActiveRecord def self.project_id_batches(&block) Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches ids = relation.pluck(:id) @@ -28,20 +29,25 @@ module Gitlab yield ids.min, ids.max end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def self.legacy_attachments_relation Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) JOIN projects ON (uploads.model_type='Project' AND uploads.model_id=projects.id) SQL end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def self.hashed_attachments_relation Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments]) JOIN projects ON (uploads.model_type='Project' AND uploads.model_id=projects.id) SQL end + # rubocop: enable CodeReuse/ActiveRecord def self.relation_summary(relation_name, relation) relation_count = relation.count @@ -62,6 +68,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def self.listing(relation_name, relation) relation_count = relation_summary(relation_name, relation) return unless relation_count > 0 @@ -78,6 +85,7 @@ module Gitlab break if index + 1 >= limit end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb index 0eb9b77634a..2f6c4db12bb 100644 --- a/lib/gitlab/health_checks/redis/cache_check.rb +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -19,11 +19,13 @@ module Gitlab result == 'PONG' end + # rubocop: disable CodeReuse/ActiveRecord def check catch_timeout 10.seconds do Gitlab::Redis::Cache.with(&:ping) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb index f322fe831b8..63d2882c5b2 100644 --- a/lib/gitlab/health_checks/redis/queues_check.rb +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -19,11 +19,13 @@ module Gitlab result == 'PONG' end + # rubocop: disable CodeReuse/ActiveRecord def check catch_timeout 10.seconds do Gitlab::Redis::Queues.with(&:ping) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb index 07e6f707998..f1ea1ffe1be 100644 --- a/lib/gitlab/health_checks/redis/shared_state_check.rb +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -19,11 +19,13 @@ module Gitlab result == 'PONG' end + # rubocop: disable CodeReuse/ActiveRecord def check catch_timeout 10.seconds do Gitlab::Redis::SharedState.with(&:ping) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 5408a1a6838..0b6cc893db1 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,5 +1,8 @@ module Gitlab class Highlight + TIMEOUT_BACKGROUND = 30.seconds + TIMEOUT_FOREGROUND = 3.seconds + def self.highlight(blob_name, blob_content, repository: nil, plain: false) new(blob_name, blob_content, repository: repository) .highlight(blob_content, continue: false, plain: plain) @@ -51,11 +54,20 @@ module Gitlab end def highlight_rich(text, continue: true) - @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe + tag = lexer.tag + tokens = lexer.lex(text, continue: continue) + Timeout.timeout(timeout_time) { @formatter.format(tokens, tag: tag).html_safe } + rescue Timeout::Error => e + Gitlab::Sentry.track_exception(e) + highlight_plain(text) rescue highlight_plain(text) end + def timeout_time + Sidekiq.server? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND + end + def link_dependencies(text, highlighted_text) Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) end diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 3f3f10596c5..a8b93f1d4b2 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -28,6 +28,7 @@ module Gitlab end # Tries to identify a user based on a user identifier (e.g. "user-123"). + # rubocop: disable CodeReuse/ActiveRecord def identify_using_user(identifier) user_id = identifier.gsub("user-", "") @@ -35,6 +36,7 @@ module Gitlab User.find_by(id: user_id) end end + # rubocop: enable CodeReuse/ActiveRecord # Tries to identify a user based on an SSH key identifier (e.g. "key-123"). def identify_using_ssh_key(identifier) diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb index 80857061933..5b3f30d894a 100644 --- a/lib/gitlab/import/database_helpers.rb +++ b/lib/gitlab/import/database_helpers.rb @@ -8,6 +8,7 @@ module Gitlab # attributes - The attributes/columns to set. # relation - An ActiveRecord::Relation to use for finding the ID of the row # when using MySQL. + # rubocop: disable CodeReuse/ActiveRecord def insert_and_return_id(attributes, relation) # We use bulk_insert here so we can bypass any queries executed by # callbacks or validation rules, as doing this wouldn't scale when @@ -20,6 +21,7 @@ module Gitlab result.first || relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/import/logger.rb b/lib/gitlab/import/logger.rb new file mode 100644 index 00000000000..8414954d141 --- /dev/null +++ b/lib/gitlab/import/logger.rb @@ -0,0 +1,9 @@ +module Gitlab + module Import + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'importer' + end + end + end +end diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb index 8ba70700dc1..97dc1a987c4 100644 --- a/lib/gitlab/import/merge_request_helpers.rb +++ b/lib/gitlab/import/merge_request_helpers.rb @@ -5,6 +5,7 @@ module Gitlab module MergeRequestHelpers include DatabaseHelpers + # rubocop: disable CodeReuse/ActiveRecord def create_merge_request_without_hooks(project, attributes, iid) # This work must be wrapped in a transaction as otherwise we can leave # behind incomplete data in the event of an error. This can then lead @@ -39,7 +40,9 @@ module Gitlab # existing row. [project.merge_requests.find_by(iid: iid), true] end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def insert_or_replace_git_data(merge_request, source_branch_sha, target_branch_sha, already_exists = false) # These fields are set so we can create the correct merge request # diffs. @@ -65,6 +68,7 @@ module Gitlab diff.save diff.save_git_content end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index be3710c5b7f..53fe2f8e436 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -40,10 +40,6 @@ module Gitlab "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end - def object_storage? - Feature.enabled?(:import_export_object_storage) - end - def version VERSION end diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index 83134bb0769..7cbf653dd97 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -53,7 +53,7 @@ module Gitlab end def self.lock_file_path(project) - return unless project.export_path || object_storage? + return unless project.export_path || export_file_exists? lock_path = project.import_export_shared.archive_path @@ -83,8 +83,8 @@ module Gitlab errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } end - def object_storage? - project.export_project_object_exists? + def export_file_exists? + project.export_file_exists? end end end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index dce8f89c0ab..4f29bdcea2c 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -23,7 +23,7 @@ module Gitlab def strategy_execute handle_response_error(send_file) - project.remove_exported_project_file + project.remove_exports end def handle_response_error(response) @@ -40,15 +40,11 @@ module Gitlab def send_file Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend ensure - export_file.close if export_file && !object_storage? + export_file.close if export_file end def export_file - if object_storage? - project.import_export_upload.export_file.file.open - else - File.open(project.export_project_path) - end + project.export_file.open end def send_file_options @@ -63,11 +59,7 @@ module Gitlab end def export_size - if object_storage? - project.import_export_upload.export_file.file.size - else - File.size(project.export_project_path) - end + project.export_file.file.size end end end diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb index cfa595629f4..17796430811 100644 --- a/lib/gitlab/import_export/avatar_restorer.rb +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -19,7 +19,7 @@ module Gitlab private def avatar_export_file - @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first + @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].find { |f| File.file?(f) } end def avatar_export_path diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 31ef0490cb3..6ffebf83dd2 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class AvatarSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared @@ -14,19 +12,12 @@ module Gitlab Gitlab::ImportExport::UploadsManager.new( project: @project, shared: @shared, - relative_export_path: 'avatar', - from: avatar_path + relative_export_path: 'avatar' ).save rescue => e @shared.error(e) false end - - private - - def avatar_path - @project.avatar.path - end end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index f69f98a78a3..2bed470514b 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -19,6 +19,9 @@ project_tree: - milestone: - events: - :push_event_payload + - resource_label_events: + - label: + :priorities - :issue_assignees - snippets: - :award_emoji @@ -45,6 +48,9 @@ project_tree: - milestone: - events: - :push_event_payload + - resource_label_events: + - label: + :priorities - pipelines: - notes: - :author @@ -64,6 +70,7 @@ project_tree: - :create_access_levels - :project_feature - :custom_attributes + - :prometheus_metrics - :project_badges - :ci_cd_settings @@ -108,6 +115,9 @@ excluded_attributes: - :remote_mirror_available_overridden - :description_html - :repository_languages + prometheus_metrics: + - :common + - :identifier snippets: - :expired_at merge_request_diff: @@ -133,6 +143,16 @@ excluded_attributes: - :event_id project_badges: - :group_id + resource_label_events: + - :reference + - :reference_html + - :epic_id + hooks: + - :token + - :encrypted_token + - :encrypted_token_iv + - :encrypted_url + - :encrypted_url_iv methods: labels: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 4e179f63d8c..72d5b9b830c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -92,8 +92,6 @@ module Gitlab end def remove_import_file - return unless Gitlab::ImportExport.object_storage? - upload = @project.import_export_upload return unless upload&.import_file&.file diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index f4106e03a57..3d693d23c99 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -136,9 +136,18 @@ module Gitlab return if tree_hash[relation_key].blank? tree_array = [tree_hash[relation_key]].flatten + null_iid_pipelines = [] # Avoid keeping a possible heavy object in memory once we are done with it - while relation_item = tree_array.shift + while relation_item = (tree_array.shift || null_iid_pipelines.shift) + if nil_iid_pipeline?(relation_key, relation_item) && tree_array.any? + # Move pipelines with NULL IIDs to the end + # so they don't clash with existing IIDs. + null_iid_pipelines << relation_item + + next + end + # The transaction at this level is less speedy than one single transaction # But we can't have it in the upper level or GC won't get rid of the AR objects # after we save the batch. @@ -199,7 +208,11 @@ module Gitlab end def excluded_keys_for_relation(relation) - @reader.attributes_finder.find_excluded_keys(relation) + reader.attributes_finder.find_excluded_keys(relation) + end + + def nil_iid_pipeline?(relation_key, relation_item) + relation_key == 'pipelines' && relation_item['iid'].nil? end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 81807ed659c..2486b1e4921 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -86,7 +86,6 @@ module Gitlab case @relation_name when :merge_request_diff_files then setup_diff when :notes then setup_note - when 'Ci::Pipeline' then setup_pipeline end update_user_references @@ -94,6 +93,8 @@ module Gitlab update_group_references remove_duplicate_assignees + setup_pipeline if @relation_name == 'Ci::Pipeline' + reset_tokens! remove_encrypted_attributes! end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 3cd153a4fd2..59a74083395 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -18,7 +18,7 @@ module Gitlab Rails.logger.info("Saved project export #{archive_file}") - save_on_object_storage if use_object_storage? + save_upload else @shared.error(Gitlab::ImportExport::Error.new(error_message)) false @@ -27,10 +27,8 @@ module Gitlab @shared.error(e) false ensure - if use_object_storage? - remove_archive - remove_export_path - end + remove_archive + remove_export_path end private @@ -51,7 +49,7 @@ module Gitlab @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) end - def save_on_object_storage + def save_upload upload = ImportExportUpload.find_or_initialize_by(project: @project) File.open(archive_file) { |file| upload.export_file = file } @@ -59,12 +57,8 @@ module Gitlab upload.save! end - def use_object_storage? - Gitlab::ImportExport.object_storage? - end - def error_message - "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}" + "Unable to save #{archive_file} into #{@shared.export_path}." end end end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index e0d4235e65b..8511319cb1c 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -5,18 +5,13 @@ module Gitlab UPLOADS_BATCH_SIZE = 100 - def initialize(project:, shared:, relative_export_path: 'uploads', from: nil) + def initialize(project:, shared:, relative_export_path: 'uploads') @project = project @shared = shared @relative_export_path = relative_export_path - @from = from || default_uploads_path end def save - if File.file?(@from) && @relative_export_path == 'avatar' - copy_files(@from, File.join(uploads_export_path, @project.avatar.filename)) - end - copy_project_uploads true @@ -55,17 +50,11 @@ module Gitlab copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path)) else - next unless Gitlab::ImportExport.object_storage? - download_and_copy(uploader) end end end - def default_uploads_path - FileUploader.absolute_base_dir(@project) - end - def uploads_export_path @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path) end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index 25f85936227..b4313ff4cb4 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -2,30 +2,14 @@ module Gitlab module ImportExport class UploadsRestorer < UploadsSaver def restore - if Gitlab::ImportExport.object_storage? - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared - ).restore - elsif File.directory?(uploads_export_path) - copy_files(uploads_export_path, uploads_path) - - true - else - true # Proceed without uploads - end + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).restore rescue => e @shared.error(e) false end - - def uploads_path - FileUploader.absolute_base_dir(@project) - end - - def uploads_export_path - @uploads_export_path ||= File.join(@shared.export_path, 'uploads') - end end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index b3f17af5661..0275f686c5e 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class UploadsSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb index 505810964bc..b5657a36998 100644 --- a/lib/gitlab/issuables_count_for_state.rb +++ b/lib/gitlab/issuables_count_for_state.rb @@ -1,7 +1,7 @@ module Gitlab # Class for counting and caching the number of issuables per state. class IssuablesCountForState - # The name of the RequestStore cache key. + # The name of the Gitlab::SafeRequestStore cache key. CACHE_KEY = :issuables_count_for_state # The state values that can be safely casted to a Symbol. @@ -10,12 +10,7 @@ module Gitlab # finder - The finder class to use for retrieving the issuables. def initialize(finder) @finder = finder - @cache = - if RequestStore.active? - RequestStore[CACHE_KEY] ||= initialize_cache - else - initialize_cache - end + @cache = Gitlab::SafeRequestStore[CACHE_KEY] ||= initialize_cache end def for_state_or_opened(state = nil) diff --git a/lib/gitlab/kubernetes/cluster_role_binding.rb b/lib/gitlab/kubernetes/cluster_role_binding.rb new file mode 100644 index 00000000000..ebea8aff5be --- /dev/null +++ b/lib/gitlab/kubernetes/cluster_role_binding.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class ClusterRoleBinding + attr_reader :name, :cluster_role_name, :subjects + + def initialize(name, cluster_role_name, subjects) + @name = name + @cluster_role_name = cluster_role_name + @subjects = subjects + end + + def generate + ::Kubeclient::Resource.new.tap do |resource| + resource.metadata = metadata + resource.roleRef = role_ref + resource.subjects = subjects + end + end + + private + + def metadata + { name: name } + end + + def role_ref + { + apiGroup: 'rbac.authorization.k8s.io', + kind: 'ClusterRole', + name: cluster_role_name + } + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 530ccf88053..4a1bdf34c3e 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -3,6 +3,9 @@ module Gitlab module Helm HELM_VERSION = '2.7.2'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze + SERVICE_ACCOUNT = 'tiller'.freeze + CLUSTER_ROLE_BINDING = 'tiller-admin'.freeze + CLUSTER_ROLE = 'cluster-admin'.freeze end end end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index d65374cc23b..e21bc531444 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -9,7 +9,17 @@ module Gitlab def install(command) namespace.ensure_exists! + + create_service_account(command) + create_cluster_role_binding(command) create_config_map(command) + + kubeclient.create_pod(command.pod_resource) + end + + def update(command) + namespace.ensure_exists! + update_config_map(command) kubeclient.create_pod(command.pod_resource) end @@ -32,6 +42,12 @@ module Gitlab kubeclient.delete_pod(pod_name, namespace.name) end + def get_config_map(config_map_name) + namespace.ensure_exists! + + kubeclient.get_config_map(config_map_name, namespace.name) + end + private attr_reader :kubeclient, :namespace @@ -41,6 +57,56 @@ module Gitlab kubeclient.create_config_map(config_map_resource) end end + + def update_config_map(command) + command.config_map_resource.tap do |config_map_resource| + kubeclient.update_config_map(config_map_resource) + end + end + + def create_service_account(command) + command.service_account_resource.tap do |service_account_resource| + break unless service_account_resource + + if service_account_exists?(service_account_resource) + kubeclient.update_service_account(service_account_resource) + else + kubeclient.create_service_account(service_account_resource) + end + end + end + + def create_cluster_role_binding(command) + command.cluster_role_binding_resource.tap do |cluster_role_binding_resource| + break unless cluster_role_binding_resource + + if cluster_role_binding_exists?(cluster_role_binding_resource) + kubeclient.update_cluster_role_binding(cluster_role_binding_resource) + else + kubeclient.create_cluster_role_binding(cluster_role_binding_resource) + end + end + end + + def service_account_exists?(resource) + resource_exists? do + kubeclient.get_service_account(resource.metadata.name, resource.metadata.namespace) + end + end + + def cluster_role_binding_exists?(resource) + resource_exists? do + kubeclient.get_cluster_role_binding(resource.metadata.name) + end + end + + def resource_exists? + yield + rescue ::Kubeclient::HttpError => e + raise e unless e.error_code == 404 + + false + end end end end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb index afcfd109de0..6752f2cff43 100644 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -3,7 +3,9 @@ module Gitlab module Helm module BaseCommand def pod_resource - Gitlab::Kubernetes::Helm::Pod.new(self, namespace).generate + pod_service_account_name = rbac? ? service_account_name : nil + + Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate end def generate_script @@ -26,6 +28,14 @@ module Gitlab Gitlab::Kubernetes::ConfigMap.new(name, files).generate end + def service_account_resource + nil + end + + def cluster_role_binding_resource + nil + end + def file_names files.keys end @@ -34,6 +44,10 @@ module Gitlab raise "Not implemented" end + def rbac? + raise "Not implemented" + end + def files raise "Not implemented" end @@ -47,6 +61,10 @@ module Gitlab def namespace Gitlab::Kubernetes::Helm::NAMESPACE end + + def service_account_name + Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT + end end end end diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb index a4546509515..c7046a9ea75 100644 --- a/lib/gitlab/kubernetes/helm/init_command.rb +++ b/lib/gitlab/kubernetes/helm/init_command.rb @@ -6,9 +6,10 @@ module Gitlab attr_reader :name, :files - def initialize(name:, files:) + def initialize(name:, files:, rbac:) @name = name @files = files + @rbac = rbac end def generate_script @@ -17,15 +18,62 @@ module Gitlab ].join("\n") end + def rbac? + @rbac + end + + def service_account_resource + return unless rbac? + + Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate + end + + def cluster_role_binding_resource + return unless rbac? + + subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] + + Gitlab::Kubernetes::ClusterRoleBinding.new( + cluster_role_binding_name, + cluster_role_name, + subjects + ).generate + end + private def init_helm_command - tls_flags = "--tiller-tls" \ - " --tiller-tls-verify --tls-ca-cert #{files_dir}/ca.pem" \ - " --tiller-tls-cert #{files_dir}/cert.pem" \ - " --tiller-tls-key #{files_dir}/key.pem" + command = %w[helm init] + init_command_flags + + command.shelljoin + " >/dev/null\n" + end + + def init_command_flags + tls_flags + optional_service_account_flag + end + + def tls_flags + [ + '--tiller-tls', + '--tiller-tls-verify', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tiller-tls-cert', "#{files_dir}/cert.pem", + '--tiller-tls-key', "#{files_dir}/key.pem" + ] + end + + def optional_service_account_flag + return [] unless rbac? + + ['--service-account', service_account_name] + end + + def cluster_role_binding_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING + end - "helm init #{tls_flags} >/dev/null" + def cluster_role_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE end end end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 9672f80687e..1be7924d6ac 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -6,10 +6,11 @@ module Gitlab attr_reader :name, :files, :chart, :version, :repository - def initialize(name:, chart:, files:, version: nil, repository: nil) + def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil) @name = name @chart = chart @version = version + @rbac = rbac @files = files @repository = repository end @@ -22,6 +23,10 @@ module Gitlab ].compact.join("\n") end + def rbac? + @rbac + end + private def init_command @@ -29,28 +34,51 @@ module Gitlab end def repository_command - "helm repo add #{name} #{repository}" if repository + ['helm', 'repo', 'add', name, repository].shelljoin if repository end def script_command - init_flags = "--name #{name}#{optional_tls_flags}#{optional_version_flag}" \ - " --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE}" \ - " -f /data/helm/#{name}/config/values.yaml" + command = ['helm', 'install', chart] + install_command_flags + + command.shelljoin + " >/dev/null\n" + end + + def install_command_flags + name_flag = ['--name', name] + namespace_flag = ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] + value_flag = ['-f', "/data/helm/#{name}/config/values.yaml"] - "helm install #{chart} #{init_flags} >/dev/null\n" + name_flag + + optional_tls_flags + + optional_version_flag + + optional_rbac_create_flag + + namespace_flag + + value_flag + end + + def optional_rbac_create_flag + return [] unless rbac? + + # jupyterhub helm chart is using rbac.enabled + # https://github.com/jupyterhub/zero-to-jupyterhub-k8s/tree/master/jupyterhub + %w[--set rbac.create=true,rbac.enabled=true] end def optional_version_flag - " --version #{version}" if version + return [] unless version + + ['--version', version] end def optional_tls_flags - return unless files.key?(:'ca.pem') + return [] unless files.key?(:'ca.pem') - " --tls" \ - " --tls-ca-cert #{files_dir}/ca.pem" \ - " --tls-cert #{files_dir}/cert.pem" \ - " --tls-key #{files_dir}/key.pem" + [ + '--tls', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tls-cert', "#{files_dir}/cert.pem", + '--tls-key', "#{files_dir}/key.pem" + ] end end end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index 6e5d3388405..95192b11c0d 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -2,9 +2,10 @@ module Gitlab module Kubernetes module Helm class Pod - def initialize(command, namespace_name) + def initialize(command, namespace_name, service_account_name: nil) @command = command @namespace_name = namespace_name + @service_account_name = service_account_name end def generate @@ -12,13 +13,14 @@ module Gitlab spec[:volumes] = volumes_specification spec[:containers][0][:volumeMounts] = volume_mounts_specification + spec[:serviceAccountName] = service_account_name if service_account_name ::Kubeclient::Resource.new(metadata: metadata, spec: spec) end private - attr_reader :command, :namespace_name, :kubeclient, :config_map + attr_reader :command, :namespace_name, :service_account_name def container_specification { diff --git a/lib/gitlab/kubernetes/helm/upgrade_command.rb b/lib/gitlab/kubernetes/helm/upgrade_command.rb new file mode 100644 index 00000000000..74188046739 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/upgrade_command.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + class UpgradeCommand + include BaseCommand + + attr_reader :name, :chart, :version, :repository, :files + + def initialize(name, chart:, files:, rbac:, version: nil, repository: nil) + @name = name + @chart = chart + @rbac = rbac + @version = version + @files = files + @repository = repository + end + + def generate_script + super + [ + init_command, + repository_command, + script_command + ].compact.join("\n") + end + + def rbac? + @rbac + end + + def pod_name + "upgrade-#{name}" + end + + private + + def init_command + 'helm init --client-only >/dev/null' + end + + def repository_command + "helm repo add #{name} #{repository}" if repository + end + + def script_command + upgrade_flags = "#{optional_version_flag}#{optional_tls_flags}" \ + " --reset-values" \ + " --install" \ + " --namespace #{::Gitlab::Kubernetes::Helm::NAMESPACE}" \ + " -f /data/helm/#{name}/config/values.yaml" + + "helm upgrade #{name} #{chart}#{upgrade_flags} >/dev/null\n" + end + + def optional_version_flag + " --version #{version}" if version + end + + def optional_tls_flags + return unless files.key?(:'ca.pem') + + " --tls" \ + " --tls-ca-cert #{files_dir}/ca.pem" \ + " --tls-cert #{files_dir}/cert.pem" \ + " --tls-key #{files_dir}/key.pem" + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb new file mode 100644 index 00000000000..588238de608 --- /dev/null +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +require 'uri' + +module Gitlab + module Kubernetes + # Wrapper around Kubeclient::Client to dispatch + # the right message to the client that can respond to the message. + # We must have a kubeclient for each ApiGroup as there is no + # other way to use the Kubeclient gem. + # + # See https://github.com/abonas/kubeclient/issues/348. + class KubeClient + include Gitlab::Utils::StrongMemoize + + SUPPORTED_API_GROUPS = [ + 'api', + 'apis/rbac.authorization.k8s.io', + 'apis/extensions' + ].freeze + + # Core API methods delegates to the core api group client + delegate :get_pods, + :get_secrets, + :get_config_map, + :get_namespace, + :get_pod, + :get_secret, + :get_service, + :get_service_account, + :delete_pod, + :create_config_map, + :create_namespace, + :create_pod, + :create_secret, + :create_service_account, + :update_config_map, + :update_service_account, + to: :core_client + + # RBAC methods delegates to the apis/rbac.authorization.k8s.io api + # group client + delegate :create_cluster_role_binding, + :get_cluster_role_binding, + :update_cluster_role_binding, + to: :rbac_client + + # Deployments resource is currently on the apis/extensions api group + delegate :get_deployments, + to: :extensions_client + + # non-entity methods that can only work with the core client + # as it uses the pods/log resource + delegate :get_pod_log, + :watch_pod_log, + to: :core_client + + def initialize(api_prefix, api_groups = ['api'], api_version = 'v1', **kubeclient_options) + raise ArgumentError unless check_api_groups_supported?(api_groups) + + @api_prefix = api_prefix + @api_groups = api_groups + @api_version = api_version + @kubeclient_options = kubeclient_options + end + + def discover! + clients.each(&:discover) + end + + def clients + hashed_clients.values + end + + def core_client + hashed_clients['api'] + end + + def rbac_client + hashed_clients['apis/rbac.authorization.k8s.io'] + end + + def extensions_client + hashed_clients['apis/extensions'] + end + + def hashed_clients + strong_memoize(:hashed_clients) do + @api_groups.map do |api_group| + api_url = join_api_url(@api_prefix, api_group) + [api_group, ::Kubeclient::Client.new(api_url, @api_version, **@kubeclient_options)] + end.to_h + end + end + + private + + def check_api_groups_supported?(api_groups) + api_groups.all? {|api_group| SUPPORTED_API_GROUPS.include?(api_group) } + end + + def join_api_url(api_prefix, api_path) + url = URI.parse(api_prefix) + prefix = url.path.sub(%r{/+\z}, '') + + url.path = [prefix, api_path].join("/") + + url.to_s + end + end + end +end diff --git a/lib/gitlab/kubernetes/service_account.rb b/lib/gitlab/kubernetes/service_account.rb new file mode 100644 index 00000000000..d58fc1c3976 --- /dev/null +++ b/lib/gitlab/kubernetes/service_account.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class ServiceAccount + attr_reader :name, :namespace_name + + def initialize(name, namespace_name) + @name = name + @namespace_name = namespace_name + end + + def generate + ::Kubeclient::Resource.new(metadata: metadata) + end + + private + + def metadata + { + name: name, + namespace: namespace_name + } + end + end + end +end diff --git a/lib/gitlab/kubernetes/service_account_token.rb b/lib/gitlab/kubernetes/service_account_token.rb new file mode 100644 index 00000000000..2e912b26c09 --- /dev/null +++ b/lib/gitlab/kubernetes/service_account_token.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class ServiceAccountToken + attr_reader :name, :service_account_name, :namespace_name + + def initialize(name, service_account_name, namespace_name) + @name = name + @service_account_name = service_account_name + @namespace_name = namespace_name + end + + def generate + ::Kubeclient::Resource.new(metadata: metadata, type: service_acount_token_type) + end + + private + + # as per https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#to-create-additional-api-tokens + def service_acount_token_type + 'kubernetes.io/service-account-token' + end + + def metadata + { + name: name, + namespace: namespace_name, + annotations: { + "kubernetes.io/service-account.name": service_account_name + } + } + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb index 2f07fde406c..11d1300e51e 100644 --- a/lib/gitlab/legacy_github_import/base_formatter.rb +++ b/lib/gitlab/legacy_github_import/base_formatter.rb @@ -10,6 +10,7 @@ module Gitlab @formatter = Gitlab::ImportFormatter.new end + # rubocop: disable CodeReuse/ActiveRecord def create! association = project.public_send(project_association) # rubocop:disable GitlabSecurity/PublicSend @@ -17,6 +18,7 @@ module Gitlab record.attributes = attributes end end + # rubocop: enable CodeReuse/ActiveRecord def url raw_data.url || '' diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index b04d678cf98..c5bde681365 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -113,6 +113,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def import_issues fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| issues.each do |raw| @@ -133,6 +134,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| @@ -193,6 +195,7 @@ module Gitlab issuable.update_attribute(:label_ids, label_ids) end + # rubocop: disable CodeReuse/ActiveRecord def import_comments(issuable_type) resource_type = "#{issuable_type}_comments".to_sym @@ -213,7 +216,9 @@ module Gitlab create_comments(comments) end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def create_comments(comments) ActiveRecord::Base.no_touching do comments.each do |raw| @@ -238,6 +243,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def discard_inserted_comments(comments, last_note) last_note_attrs = nil diff --git a/lib/gitlab/legacy_github_import/issuable_formatter.rb b/lib/gitlab/legacy_github_import/issuable_formatter.rb index de55382d3ad..7db4a54267e 100644 --- a/lib/gitlab/legacy_github_import/issuable_formatter.rb +++ b/lib/gitlab/legacy_github_import/issuable_formatter.rb @@ -55,12 +55,14 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def milestone if raw_data.milestone.present? milestone = MilestoneFormatter.new(project, raw_data.milestone) project.milestones.find_by(milestone.find_condition) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/legacy_github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb index c3eed12e739..e9663650903 100644 --- a/lib/gitlab/legacy_github_import/label_formatter.rb +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -13,6 +13,7 @@ module Gitlab :labels end + # rubocop: disable CodeReuse/ActiveRecord def create! params = attributes.except(:project) service = ::Labels::FindOrCreateService.new(nil, project, params) @@ -22,6 +23,7 @@ module Gitlab label end + # rubocop: enable CodeReuse/ActiveRecord private diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index 6d8055622f1..3794380e2d0 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -29,6 +29,7 @@ module Gitlab .try(:id) end + # rubocop: disable CodeReuse/ActiveRecord def find_by_external_uid return nil unless id @@ -40,6 +41,7 @@ module Gitlab .first .try(:id) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index e58927a40b9..3d7c049c17f 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -30,7 +30,7 @@ module Gitlab end def self.build - RequestStore[self.cache_key] ||= new(self.full_log_path) + Gitlab::SafeRequestStore[self.cache_key] ||= new(self.full_log_path) end def self.full_log_path diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index c205f348023..04107296ae3 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -6,9 +6,15 @@ module Gitlab include Gitlab::Metrics::Methods attach_to :active_record + IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze + def sql(event) return unless current_transaction + payload = event.payload + + return if payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) + self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) current_transaction.increment(:sql_duration, event.duration, false) diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 3d588918adf..10cb1e7127e 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -83,7 +83,7 @@ module Gitlab def open_file(params, key) allowed_paths = [ - FileUploader.root, + ::FileUploader.root, Gitlab.config.uploads.storage_path, File.join(Rails.root, 'public/uploads/tmp') ] diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index fd5de73c526..fab7a43dd30 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -53,6 +53,7 @@ module Gitlab @first_collection_page_count = first_collection_page.total_pages end + # rubocop: disable CodeReuse/ActiveRecord def first_collection_last_page_size return @first_collection_last_page_size if defined?(@first_collection_last_page_size) @@ -60,5 +61,6 @@ module Gitlab .except(:select) .size end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/null_request_store.rb b/lib/gitlab/null_request_store.rb new file mode 100644 index 00000000000..8db331dcb9f --- /dev/null +++ b/lib/gitlab/null_request_store.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# Used by Gitlab::SafeRequestStore +module Gitlab + # The methods `begin!`, `clear!`, and `end!` are not defined because they + # should only be called directly on `RequestStore`. + class NullRequestStore + def store + {} + end + + def active? + end + + def read(key) + end + + def [](key) + end + + def write(key, value) + value + end + + def []=(key, value) + value + end + + def exist?(key) + false + end + + def fetch(key, &block) + yield + end + + def delete(key, &block) + yield(key) if block_given? + end + end +end diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index 22332474945..ca5d49eedb9 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -26,6 +26,7 @@ module Gitlab @filename = filename end + # rubocop: disable CodeReuse/ActiveRecord def rotate!(old_key:, new_key:) old_key ||= Gitlab::Application.secrets.otp_key_base @@ -47,7 +48,9 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def rollback! ActiveRecord::Base.transaction do CSV.foreach(filename, headers: HEADERS, return_headers: false) do |row| @@ -55,6 +58,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord private diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb new file mode 100644 index 00000000000..a9f6cfb19cb --- /dev/null +++ b/lib/gitlab/patch/prependable.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +# We're patching `ActiveSupport::Concern` in +# config/initializers/0_as_concern.rb +# +# We want to patch `ActiveSupport::Concern` for two reasons: +# 1. Allow defining class methods via: `class_methods` method +# 2. Allow `prepended do; end` work like `included do; end` +# If we don't need anything above, we don't need this patch nor the concern! + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Patch + module Prependable + class MultiplePrependedBlocks < StandardError + def initialize + super "Cannot define multiple 'prepended' blocks for a Concern" + end + end + + def prepend_features(base) + return false if prepended?(base) + + super + + if const_defined?(:ClassMethods) + klass_methods = const_get(:ClassMethods) + base.singleton_class.prepend klass_methods + base.instance_variable_set(:@_prepended_class_methods, klass_methods) + end + + if instance_variable_defined?(:@_prepended_block) + base.class_eval(&@_prepended_block) + end + + true + end + + def class_methods + super + + if instance_variable_defined?(:@_prepended_class_methods) + const_get(:ClassMethods).prepend @_prepended_class_methods + end + end + + def prepended(base = nil, &block) + if base.nil? + raise MultiplePrependedBlocks if + instance_variable_defined?(:@_prepended_block) + + @_prepended_block = block + else + super + end + end + + def prepended?(base) + index = base.ancestors.index(base) + + base.ancestors[0...index].index(self) + end + end + end +end diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 92a308a12dc..fda61000f6a 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -15,6 +15,7 @@ module Gitlab Gitlab::CurrentSettings.performance_bar_allowed_group_id end + # rubocop: disable CodeReuse/ActiveRecord def self.allowed_user_ids Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do group = Group.find_by_id(allowed_group_id) @@ -26,6 +27,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def self.expire_allowed_user_ids_cache Rails.cache.delete(ALLOWED_USER_IDS_KEY) diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index f2825db59ae..37ff32b1296 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -23,7 +23,7 @@ module Gitlab end subscribe('sql.active_record') do |_, start, finish, _, data| - if RequestStore.active? && RequestStore.store[:peek_enabled] + if Gitlab::SafeRequestStore.store[:peek_enabled] # data[:cached] is only available starting from Rails 5.1.0 # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113 # Before that, data[:name] was set to 'CACHE' diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index c5bb4648572..15391b1330b 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -34,6 +34,7 @@ module Gitlab # # - private_token: instead of providing a user instance, the token can be # given as a string. Takes precedence over the user option. + # rubocop: disable CodeReuse/ActiveRecord def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil) app = ActionDispatch::Integration::Session.new(Rails.application) verb = :get @@ -76,6 +77,7 @@ module Gitlab result end + # rubocop: enable CodeReuse/ActiveRecord def self.create_custom_logger(logger, private_token: nil) return unless logger @@ -135,6 +137,7 @@ module Gitlab result end + # rubocop: disable CodeReuse/ActiveRecord def self.log_load_times_by_model(logger) return unless logger.respond_to?(:load_times_by_model) @@ -146,6 +149,7 @@ module Gitlab logger.info("#{model} total (#{query_count}): #{time.round(2)}ms") end end + # rubocop: enable CodeReuse/ActiveRecord def self.print_by_total_time(result, options = {}) default_options = { sort_method: :total_time } diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb index e3da1634fa5..448c3f3a7d8 100644 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ b/lib/gitlab/project_authorizations/with_nested_groups.rb @@ -49,13 +49,11 @@ module Gitlab .where('p_ns.share_with_group_lock IS FALSE') ] - union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization .unscoped .with .recursive(cte.to_arel) - .select_from_union(union) + .select_from_union(relations) end private diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb index 7d0c00c7f36..ed2287dcc7e 100644 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ b/lib/gitlab/project_authorizations/without_nested_groups.rb @@ -24,11 +24,9 @@ module Gitlab user.groups.joins(:shared_projects).select_for_project_authorization ] - union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization .unscoped - .select_from_union(union) + .select_from_union(relations) end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 62f9e538c04..71e9cc7f238 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -29,6 +29,7 @@ module Gitlab @blobs_count ||= blobs.count end + # rubocop: disable CodeReuse/ActiveRecord def limited_notes_count return @limited_notes_count if defined?(@limited_notes_count) @@ -42,6 +43,7 @@ module Gitlab @limited_notes_count end + # rubocop: enable CodeReuse/ActiveRecord def wiki_blobs_count @wiki_blobs_count ||= wiki_blobs.count @@ -118,9 +120,11 @@ module Gitlab @notes ||= notes_finder(nil) end + # rubocop: disable CodeReuse/ActiveRecord def notes_finder(type) NotesFinder.new(project, @current_user, search: query, target_type: type).execute.user.order('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord def commits @commits ||= find_commits(query) diff --git a/lib/gitlab/project_service_logger.rb b/lib/gitlab/project_service_logger.rb new file mode 100644 index 00000000000..e84dca97962 --- /dev/null +++ b/lib/gitlab/project_service_logger.rb @@ -0,0 +1,7 @@ +module Gitlab + class ProjectServiceLogger < Gitlab::JsonLogger + def self.file_name_noext + 'integrations_json' + end + end +end diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index bb1172f82a1..a240d090074 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -5,7 +5,7 @@ module Gitlab MUTEX = Mutex.new extend self - def load_groups_from_yaml(file_name = 'additional_metrics.yml') + def load_groups_from_yaml(file_name) yaml_metrics_raw(file_name).map(&method(:group_from_entry)) end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index e91c6fb2e27..d696a8fc00c 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -4,10 +4,13 @@ module Gitlab include ActiveModel::Model attr_accessor :name, :priority, :metrics + validates :name, :priority, :metrics, presence: true def self.common_metrics - AdditionalMetricsParser.load_groups_from_yaml + ::PrometheusMetric.common.group_by(&:group_title).map do |name, metrics| + MetricGroup.new(name: name, priority: 0, metrics: metrics.map(&:to_query_metric)) + end end # EE only diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 8534afcc849..fa86d2dfd6c 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -4,6 +4,7 @@ module Gitlab class AdditionalMetricsDeploymentQuery < BaseQuery include QueryAdditionalMetrics + # rubocop: disable CodeReuse/ActiveRecord def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( @@ -17,6 +18,7 @@ module Gitlab ) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb index e3af217b202..09f8f1103d2 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -4,6 +4,7 @@ module Gitlab class AdditionalMetricsEnvironmentQuery < BaseQuery include QueryAdditionalMetrics + # rubocop: disable CodeReuse/ActiveRecord def query(environment_id) ::Environment.find_by(id: environment_id).try do |environment| query_metrics( @@ -13,6 +14,7 @@ module Gitlab ) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index c2626581897..3a609a795ba 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -2,6 +2,7 @@ module Gitlab module Prometheus module Queries class DeploymentQuery < BaseQuery + # rubocop: disable CodeReuse/ActiveRecord def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| environment_slug = deployment.environment.slug @@ -25,6 +26,7 @@ module Gitlab } end end + # rubocop: enable CodeReuse/ActiveRecord def self.transform_reactive_result(result) result[:metrics] = result.delete :data diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index b62910c8de6..4d8b136d7af 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -2,6 +2,7 @@ module Gitlab module Prometheus module Queries class EnvironmentQuery < BaseQuery + # rubocop: disable CodeReuse/ActiveRecord def query(environment_id) ::Environment.find_by(id: environment_id).try do |environment| environment_slug = environment.slug @@ -19,6 +20,7 @@ module Gitlab } end end + # rubocop: enable CodeReuse/ActiveRecord def self.transform_reactive_result(result) result[:metrics] = result.delete :data diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb index b1bf3ca4143..a03ce07b6a1 100644 --- a/lib/gitlab/repository_cache.rb +++ b/lib/gitlab/repository_cache.rb @@ -29,5 +29,21 @@ module Gitlab def read(key) backend.read(cache_key(key)) end + + def write(key, value) + backend.write(cache_key(key), value) + end + + def fetch_without_caching_false(key, &block) + value = read(key) + return value if value + + value = yield + + # Don't cache false values + write(key, value) if value + + value + end end end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 2ec871f0754..d95024fccf7 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -1,23 +1,80 @@ module Gitlab module RepositoryCacheAdapter extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize class_methods do - # Wraps around the given method and caches its output in Redis and an instance - # variable. + # Caches and strongly memoizes the method. # # This only works for methods that do not take any arguments. - def cache_method(name, fallback: nil, memoize_only: false) - original = :"_uncached_#{name}" + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + cache_method_output(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + end - alias_method(original, name) + # Caches truthy values from the method. All values are strongly memoized, + # and cached in RequestStore. + # + # Currently only used to cache `exists?` since stale false values are + # particularly troublesome. This can occur, for example, when an NFS mount + # is temporarily down. + # + # This only works for methods that do not take any arguments. + # + # name - The name of the method to be cached. + def cache_method_asymmetrically(name) + uncached_name = alias_uncached_method(name) define_method(name) do - cache_method_output(name, fallback: fallback, memoize_only: memoize_only) do - __send__(original) # rubocop:disable GitlabSecurity/PublicSend + cache_method_output_asymmetrically(name) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend end end end + + # Strongly memoizes the method. + # + # This only works for methods that do not take any arguments. + # + # name - The name of the method to be memoized. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. The fallback value + # is not memoized. + def memoize_method(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + memoize_method_output(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + + # Prepends "_uncached_" to the target method name + # + # Returns the uncached method name + def alias_uncached_method(name) + uncached_name = :"_uncached_#{name}" + + alias_method(uncached_name, name) + + uncached_name + end + end + + # RequestStore-backed RepositoryCache to be used. Should be overridden by + # the including class + def request_store_cache + raise NotImplementedError end # RepositoryCache to be used. Should be overridden by the including class @@ -30,65 +87,93 @@ module Gitlab raise NotImplementedError end - # Caches the supplied block both in a cache and in an instance variable. + # Caches and strongly memoizes the supplied block. # - # The cache key and instance variable are named the same way as the value of - # the `key` argument. + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_output(name, fallback: nil, &block) + memoize_method_output(name, fallback: fallback) do + cache.fetch(name, &block) + end + end + + # Caches truthy values from the supplied block. All values are strongly + # memoized, and cached in RequestStore. # - # This method will return `nil` if the corresponding instance variable is also - # set to `nil`. This ensures we don't keep yielding the block when it returns - # `nil`. + # Currently only used to cache `exists?` since stale false values are + # particularly troublesome. This can occur, for example, when an NFS mount + # is temporarily down. # - # key - The name of the key to cache the data in. - # fallback - A value to fall back to in the event of a Git error. - def cache_method_output(key, fallback: nil, memoize_only: false, &block) - ivar = cache_instance_variable_name(key) - - if instance_variable_defined?(ivar) - instance_variable_get(ivar) - else - # If the repository doesn't exist and a fallback was specified we return - # that value inmediately. This saves us Rugged/gRPC invocations. - return fallback unless fallback.nil? || cache.repository.exists? - - begin - value = - if memoize_only - yield - else - cache.fetch(key, &block) - end - - instance_variable_set(ivar, value) - rescue Gitlab::Git::Repository::NoRepository - # Even if the above `#exists?` check passes these errors might still - # occur (for example because of a non-existing HEAD). We want to - # gracefully handle this and not cache anything - fallback + # name - The name of the method to be cached. + def cache_method_output_asymmetrically(name, &block) + memoize_method_output(name) do + request_store_cache.fetch(name) do + cache.fetch_without_caching_false(name, &block) end end end + # Strongly memoizes the supplied block. + # + # name - The name of the method to be memoized. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. The fallback value is + # not memoized. + def memoize_method_output(name, fallback: nil, &block) + no_repository_fallback(name, fallback: fallback) do + strong_memoize(memoizable_name(name), &block) + end + end + + # Returns the fallback value if the repository does not exist + def no_repository_fallback(name, fallback: nil, &block) + # Avoid unnecessary gRPC invocations + return fallback if fallback && fallback_early?(name) + + yield + rescue Gitlab::Git::Repository::NoRepository + # Even if the `#exists?` check in `fallback_early?` passes, these errors + # might still occur (for example because of a non-existing HEAD). We + # want to gracefully handle this and not memoize anything. + fallback + end + # Expires the caches of a specific set of methods def expire_method_caches(methods) - methods.each do |key| - unless cached_methods.include?(key.to_sym) - Rails.logger.error "Requested to expire non-existent method '#{key}' for Repository" + methods.each do |name| + unless cached_methods.include?(name.to_sym) + Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" next end - cache.expire(key) + cache.expire(name) - ivar = cache_instance_variable_name(key) - - remove_instance_variable(ivar) if instance_variable_defined?(ivar) + clear_memoization(memoizable_name(name)) end + + expire_request_store_method_caches(methods) end private - def cache_instance_variable_name(key) - :"@#{key.to_s.tr('?!', '')}" + def memoizable_name(name) + "#{name.to_s.tr('?!', '')}" + end + + def expire_request_store_method_caches(methods) + methods.each do |name| + request_store_cache.expire(name) + end + end + + # All cached repository methods depend on the existence of a Git repository, + # so if the repository doesn't exist, we already know not to call it. + def fallback_early?(method_name) + # Avoid infinite loop + return false if method_name == :exists? + + !exists? end end end diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index fef536ecb0b..8562d4515aa 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -2,7 +2,7 @@ module Gitlab class RequestContext class << self def client_ip - RequestStore[:client_ip] + Gitlab::SafeRequestStore[:client_ip] end end @@ -13,7 +13,7 @@ module Gitlab def call(env) req = Rack::Request.new(env) - RequestStore[:client_ip] = req.ip + Gitlab::SafeRequestStore[:client_ip] = req.ip @app.call(env) end diff --git a/lib/gitlab/safe_request_store.rb b/lib/gitlab/safe_request_store.rb new file mode 100644 index 00000000000..4e82353adb6 --- /dev/null +++ b/lib/gitlab/safe_request_store.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module SafeRequestStore + NULL_STORE = Gitlab::NullRequestStore.new + + class << self + # These methods should always run directly against RequestStore + delegate :clear!, :begin!, :end!, :active?, to: :RequestStore + + # These methods will run against NullRequestStore if RequestStore is disabled + delegate :read, :[], :write, :[]=, :exist?, :fetch, :delete, to: :store + end + + def self.store + if RequestStore.active? + RequestStore + else + NULL_STORE + end + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 1e45d074e0a..d1cf8e10228 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -62,10 +62,13 @@ module Gitlab without_count ? collection.without_count : collection end + # rubocop: disable CodeReuse/ActiveRecord def limited_projects_count @limited_projects_count ||= projects.limit(count_limit).count end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def limited_issues_count return @limited_issues_count if @limited_issues_count @@ -77,14 +80,19 @@ module Gitlab sum = issues(public_only: true).limit(count_limit).count @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def limited_merge_requests_count @limited_merge_requests_count ||= merge_requests.limit(count_limit).count end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def limited_milestones_count @limited_milestones_count ||= milestones.limit(count_limit).count end + # rubocop: enable CodeReuse/ActiveRecord def single_commit_result? false @@ -100,6 +108,7 @@ module Gitlab limit_projects.search(query) end + # rubocop: disable CodeReuse/ActiveRecord def issues(finder_params = {}) issues = IssuesFinder.new(current_user, finder_params).execute unless default_project_filter @@ -115,13 +124,17 @@ module Gitlab issues.reorder('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def milestones milestones = Milestone.where(project_id: project_ids_relation) milestones = milestones.search(query) milestones.reorder('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def merge_requests merge_requests = MergeRequestsFinder.new(current_user).execute unless default_project_filter @@ -137,13 +150,16 @@ module Gitlab merge_requests.reorder('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord def default_scope 'projects' end + # rubocop: disable CodeReuse/ActiveRecord def project_ids_relation limit_projects.select(:id).reorder(nil) end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index a17cd27e82d..89d2028d7b0 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -225,6 +225,7 @@ module Gitlab # Ex. # remove_keys_not_found_in_db # + # rubocop: disable CodeReuse/ActiveRecord def remove_keys_not_found_in_db return unless self.authorized_keys_enabled? @@ -243,6 +244,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord # Iterate over all ssh key IDs from gitlab shell, in batches # @@ -326,9 +328,11 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # + # rubocop: disable CodeReuse/ActiveRecord def exists?(storage, dir_name) Gitlab::GitalyClient::NamespaceService.new(storage).exists?(dir_name) end + # rubocop: enable CodeReuse/ActiveRecord protected @@ -368,15 +372,6 @@ module Gitlab private - def gitlab_projects(shard_name, disk_path) - Gitlab::Git::GitlabProjects.new( - shard_name, - disk_path, - global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, - logger: Rails.logger - ) - end - def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb deleted file mode 100644 index 5512afa45a8..00000000000 --- a/lib/gitlab/sidekiq_throttler.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - class SidekiqThrottler - class << self - def execute! - if Gitlab::CurrentSettings.sidekiq_throttling_enabled? - require 'sidekiq-limit_fetch' - - Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue| - Sidekiq::Queue[queue].limit = queue_limit - end - end - end - - private - - def queue_limit - @queue_limit ||= - begin - factor = Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_factor - (factor * Sidekiq.options[:concurrency]).ceil - end - end - end - end -end diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb index 466554e398c..0c76378d51c 100644 --- a/lib/gitlab/slash_commands/base_command.rb +++ b/lib/gitlab/slash_commands/base_command.rb @@ -40,9 +40,11 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def find_by_iid(iid) collection.find_by(iid: iid) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/slash_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb index 93e00ab75a1..b308fd9637f 100644 --- a/lib/gitlab/slash_commands/deploy.rb +++ b/lib/gitlab/slash_commands/deploy.rb @@ -36,6 +36,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def find_action(from, to) environment = project.environments.find_by(name: from) return unless environment @@ -50,6 +51,7 @@ module Gitlab actions.first end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/slash_commands/issue_search.rb b/lib/gitlab/slash_commands/issue_search.rb index acba84b54b4..ee78f0f832e 100644 --- a/lib/gitlab/slash_commands/issue_search.rb +++ b/lib/gitlab/slash_commands/issue_search.rb @@ -9,6 +9,7 @@ module Gitlab "issue search <your query>" end + # rubocop: disable CodeReuse/ActiveRecord def execute(match) issues = collection.search(match[:query]).limit(QUERY_LIMIT) @@ -18,6 +19,7 @@ module Gitlab Presenters::Access.new(issues).not_found end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 4f86b3e8f73..95e37dfbdab 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -30,13 +30,17 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def snippet_titles limit_snippets.search(query).order('updated_at DESC').includes(:author) end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def snippet_blobs limit_snippets.search_code(query).order('updated_at DESC').includes(:author) end + # rubocop: enable CodeReuse/ActiveRecord def default_scope 'snippet_blobs' diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb index b19aa6dea35..1c87c56c45e 100644 --- a/lib/gitlab/string_regex_marker.rb +++ b/lib/gitlab/string_regex_marker.rb @@ -1,5 +1,6 @@ module Gitlab class StringRegexMarker < StringRangeMarker + # rubocop: disable CodeReuse/ActiveRecord def mark(regex, group: 0, &block) ranges = [] @@ -11,5 +12,6 @@ module Gitlab super(ranges, &block) end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 7393574ac13..4456217017f 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -1,21 +1,32 @@ module Gitlab module Template class BaseTemplate - def initialize(path, project = nil) + attr_reader :category + + def initialize(path, project = nil, category: nil) @path = path + @category = category @finder = self.class.finder(project) end def name File.basename(@path, self.class.extension) end + alias_method :key, :name def content @finder.read(@path) end + # Present for compatibility with license templates, which can replace text + # like `[fullname]` with a user-specified string. This is a no-op for + # other templates + def resolve!(_placeholders = {}) + self + end + def to_json - { name: name, content: content } + { key: key, name: name, content: content } end def <=>(other) @@ -62,7 +73,7 @@ module Gitlab directory = category_directory(category) files = finder(project).list_files_for(directory) - files.map { |f| new(f, project) }.sort + files.map { |f| new(f, project, category: category) }.sort end def category_directory(category) diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 831da45191f..b08d9a99e99 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -1,4 +1,4 @@ -# Searches and reads file present on Gitlab installation directory +# Searches and reads file present on GitLab installation directory module Gitlab module Template module Finders diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index 29bc2393ff9..9140ace879f 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -1,4 +1,4 @@ -# Searches and reads files present on each Gitlab project repository +# Searches and reads files present on each GitLab project repository module Gitlab module Template module Finders diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index fd040148a1e..deae53cc61b 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -20,7 +20,7 @@ module Gitlab end def base_dir - Rails.root.join('vendor/gitlab-ci-yml') + Rails.root.join('lib/gitlab/ci/templates') end def finder(project = nil) diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb index f24a01e6cf5..fc498dde723 100644 --- a/lib/gitlab/template_helper.rb +++ b/lib/gitlab/template_helper.rb @@ -1,24 +1,9 @@ module Gitlab module TemplateHelper - include Gitlab::Utils::StrongMemoize - def prepare_template_environment(file) return unless file - if Gitlab::ImportExport.object_storage? - params[:import_export_upload] = ImportExportUpload.new(import_file: file) - else - FileUtils.mkdir_p(File.dirname(import_upload_path)) - FileUtils.copy_entry(file.path, import_upload_path) - - params[:import_source] = import_upload_path - end - end - - def import_upload_path - strong_memoize(:import_upload_path) do - Gitlab::ImportExport.import_upload_path(filename: tmp_filename) - end + params[:import_export_upload] = ImportExportUpload.new(import_file: file) end def tmp_filename diff --git a/lib/gitlab/temporarily_allow.rb b/lib/gitlab/temporarily_allow.rb index 880e55f71df..8d7dacc6c54 100644 --- a/lib/gitlab/temporarily_allow.rb +++ b/lib/gitlab/temporarily_allow.rb @@ -10,7 +10,7 @@ module Gitlab end def temporarily_allowed?(key) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? temporarily_allow_request_store[key] > 0 else TEMPORARILY_ALLOW_MUTEX.synchronize do @@ -26,11 +26,11 @@ module Gitlab end def temporarily_allow_request_store - RequestStore[:temporarily_allow] ||= Hash.new(0) + Gitlab::SafeRequestStore[:temporarily_allow] ||= Hash.new(0) end def temporarily_allow_add(key, value) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? temporarily_allow_request_store[key] += value else TEMPORARILY_ALLOW_MUTEX.synchronize do diff --git a/lib/gitlab/testing/request_inspector_middleware.rb b/lib/gitlab/testing/request_inspector_middleware.rb index e387667480d..c251e78f5c5 100644 --- a/lib/gitlab/testing/request_inspector_middleware.rb +++ b/lib/gitlab/testing/request_inspector_middleware.rb @@ -35,11 +35,15 @@ module Gitlab request_headers = env_http_headers(env) status, headers, body = @app.call(env) + full_body = '' + body.each { |b| full_body << b } + request = OpenStruct.new( url: url, status_code: status, request_headers: request_headers, - response_headers: headers + response_headers: headers, + body: full_body ) log_request request diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb new file mode 100644 index 00000000000..c2955cd374c --- /dev/null +++ b/lib/gitlab/tree_summary.rb @@ -0,0 +1,119 @@ +module Gitlab + class TreeSummary + include ::Gitlab::Utils::StrongMemoize + + attr_reader :commit, :project, :path, :offset, :limit + + attr_reader :resolved_commits + private :resolved_commits + + def initialize(commit, project, params = {}) + @commit = commit + @project = project + + @path = params.fetch(:path, nil).presence + @offset = params.fetch(:offset, 0).to_i + @limit = (params.fetch(:limit, 25) || 25).to_i + + # Ensure that if multiple tree entries share the same last commit, they share + # a ::Commit instance. This prevents us from rendering the same commit title + # multiple times + @resolved_commits = {} + end + + # Creates a summary of the tree entries for a commit, within the window of + # entries defined by the offset and limit parameters. This consists of two + # return values: + # + # - An Array of Hashes containing the following keys: + # - file_name: The full path of the tree entry + # - type: One of :blob, :tree, or :submodule + # - commit: The last ::Commit to touch this entry in the tree + # - commit_path: URI of the commit in the web interface + # - An Array of the unique ::Commit objects in the first value + def summarize + summary = contents + .map { |content| build_entry(content) } + .tap { |summary| fill_last_commits!(summary) } + + [summary, commits] + end + + # Does the tree contain more entries after the given offset + limit? + def more? + all_contents[next_offset].present? + end + + # The offset of the next batch of tree entries. If more? returns false, this + # batch will be empty + def next_offset + [all_contents.size + 1, offset + limit].min + end + + private + + def contents + all_contents[offset, limit] + end + + def commits + resolved_commits.values + end + + def repository + project.repository + end + + def entry_path(entry) + File.join(*[path, entry[:file_name]].compact) + end + + def build_entry(entry) + { file_name: entry.name, type: entry.type } + end + + def fill_last_commits!(entries) + # Ensure the path is in "path/" format + ensured_path = + if path + File.join(*[path, ""]) + end + + commits_hsh = repository.list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit) + + entries.each do |entry| + path_key = entry_path(entry) + commit = cache_commit(commits_hsh[path_key]) + + if commit + entry[:commit] = commit + entry[:commit_path] = commit_path(commit) + end + end + end + + def cache_commit(commit) + return nil unless commit.present? + + resolved_commits[commit.id] ||= commit + end + + def commit_path(commit) + Gitlab::Routing.url_helpers.project_commit_path(project, commit) + end + + def all_contents + strong_memoize(:all_contents) do + [ + *tree.trees, + *tree.blobs, + *tree.submodules + ] + end + end + + def tree + strong_memoize(:tree) { repository.tree(commit.id, path) } + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 308a95d2f09..29672d68cad 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -3,7 +3,7 @@ module Gitlab ALLOWED_SCHEMES = %w[http https ssh git].freeze def self.sanitize(content) - regexp = URI::Parser.new.make_regexp(ALLOWED_SCHEMES) + regexp = URI::DEFAULT_PARSER.make_regexp(ALLOWED_SCHEMES) content.gsub(regexp) { |url| new(url).masked_url } rescue Addressable::URI::InvalidURIError diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 7797bd5fab2..5097c3253c9 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -10,6 +10,7 @@ module Gitlab .merge(features_usage_data) .merge(components_usage_data) .merge(cycle_analytics_usage_data) + .merge(usage_counters) end def to_json(force_refresh: false) @@ -22,7 +23,7 @@ module Gitlab hostname: Gitlab.config.gitlab.host, version: Gitlab::VERSION, installation_type: Gitlab::INSTALLATION_TYPE, - active_user_count: User.active.count, + active_user_count: count(User.active), recorded_at: Time.now, edition: 'CE' } @@ -31,57 +32,59 @@ module Gitlab end # rubocop:disable Metrics/AbcSize + # rubocop: disable CodeReuse/ActiveRecord def system_usage_data { counts: { - assignee_lists: List.assignee.count, - boards: Board.count, - ci_builds: ::Ci::Build.count, - ci_internal_pipelines: ::Ci::Pipeline.internal.count, - ci_external_pipelines: ::Ci::Pipeline.external.count, - ci_pipeline_config_auto_devops: ::Ci::Pipeline.auto_devops_source.count, - ci_pipeline_config_repository: ::Ci::Pipeline.repository_source.count, - ci_runners: ::Ci::Runner.count, - ci_triggers: ::Ci::Trigger.count, - ci_pipeline_schedules: ::Ci::PipelineSchedule.count, - auto_devops_enabled: ::ProjectAutoDevops.enabled.count, - auto_devops_disabled: ::ProjectAutoDevops.disabled.count, - deploy_keys: DeployKey.count, - deployments: Deployment.count, - environments: ::Environment.count, - clusters: ::Clusters::Cluster.count, - clusters_enabled: ::Clusters::Cluster.enabled.count, - clusters_disabled: ::Clusters::Cluster.disabled.count, - clusters_platforms_gke: ::Clusters::Cluster.gcp_installed.enabled.count, - clusters_platforms_user: ::Clusters::Cluster.user_provided.enabled.count, - clusters_applications_helm: ::Clusters::Applications::Helm.installed.count, - clusters_applications_ingress: ::Clusters::Applications::Ingress.installed.count, - clusters_applications_prometheus: ::Clusters::Applications::Prometheus.installed.count, - clusters_applications_runner: ::Clusters::Applications::Runner.installed.count, - in_review_folder: ::Environment.in_review_folder.count, - groups: Group.count, - issues: Issue.count, - keys: Key.count, - label_lists: List.label.count, - labels: Label.count, - lfs_objects: LfsObject.count, - merge_requests: MergeRequest.count, - milestone_lists: List.milestone.count, - milestones: Milestone.count, - notes: Note.count, - pages_domains: PagesDomain.count, - projects: Project.count, - projects_imported_from_github: Project.where(import_type: 'github').count, - protected_branches: ProtectedBranch.count, - releases: Release.count, - remote_mirrors: RemoteMirror.count, - snippets: Snippet.count, - todos: Todo.count, - uploads: Upload.count, - web_hooks: WebHook.count + assignee_lists: count(List.assignee), + boards: count(Board), + ci_builds: count(::Ci::Build), + ci_internal_pipelines: count(::Ci::Pipeline.internal), + ci_external_pipelines: count(::Ci::Pipeline.external), + ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source), + ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source), + ci_runners: count(::Ci::Runner), + ci_triggers: count(::Ci::Trigger), + ci_pipeline_schedules: count(::Ci::PipelineSchedule), + auto_devops_enabled: count(::ProjectAutoDevops.enabled), + auto_devops_disabled: count(::ProjectAutoDevops.disabled), + deploy_keys: count(DeployKey), + deployments: count(Deployment), + environments: count(::Environment), + clusters: count(::Clusters::Cluster), + clusters_enabled: count(::Clusters::Cluster.enabled), + clusters_disabled: count(::Clusters::Cluster.disabled), + clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), + clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), + clusters_applications_helm: count(::Clusters::Applications::Helm.installed), + clusters_applications_ingress: count(::Clusters::Applications::Ingress.installed), + clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.installed), + clusters_applications_runner: count(::Clusters::Applications::Runner.installed), + in_review_folder: count(::Environment.in_review_folder), + groups: count(Group), + issues: count(Issue), + keys: count(Key), + label_lists: count(List.label), + labels: count(Label), + lfs_objects: count(LfsObject), + merge_requests: count(MergeRequest), + milestone_lists: count(List.milestone), + milestones: count(Milestone), + notes: count(Note), + pages_domains: count(PagesDomain), + projects: count(Project), + projects_imported_from_github: count(Project.where(import_type: 'github')), + protected_branches: count(ProtectedBranch), + releases: count(Release), + remote_mirrors: count(RemoteMirror), + snippets: count(Snippet), + todos: count(Todo), + uploads: count(Upload), + web_hooks: count(WebHook) }.merge(services_usage) } end + # rubocop: enable CodeReuse/ActiveRecord def cycle_analytics_usage_data Gitlab::CycleAnalytics::UsageData.new.to_json @@ -104,6 +107,12 @@ module Gitlab } end + def usage_counters + { + web_ide_commits: Gitlab::WebIdeCommitsCounter.total_count + } + end + def components_usage_data { gitlab_pages: { enabled: Gitlab.config.pages.enabled, version: Gitlab::Pages::VERSION }, @@ -112,6 +121,7 @@ module Gitlab } end + # rubocop: disable CodeReuse/ActiveRecord def services_usage types = { JiraService: :projects_jira_active, @@ -120,9 +130,16 @@ module Gitlab PrometheusService: :projects_prometheus_active } - results = Service.unscoped.where(type: types.keys, active: true).group(:type).count - results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value } + results = count(Service.unscoped.where(type: types.keys, active: true).group(:type), fallback: Hash.new(-1)) + types.each_with_object({}) { |(klass, key), response| response[key] = results[klass.to_s] || 0 } + end + + def count(relation, fallback: -1) + relation.count + rescue ActiveRecord::StatementInvalid + fallback end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb new file mode 100644 index 00000000000..bd0d24e4369 --- /dev/null +++ b/lib/gitlab/user_extractor.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +# This class extracts all users found in a piece of text by the username or the +# email adress + +module Gitlab + class UserExtractor + # Not using `Devise.email_regexp` to filter out any chars that an email + # does not end with and not pinning the email to a start of end of a string. + EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/ + USERNAME_REGEXP = User.reference_pattern + + def initialize(text) + @text = text + end + + # rubocop: disable CodeReuse/ActiveRecord + def users + return User.none unless @text.present? + + @users ||= User.from_union(union_relations) + end + # rubocop: enable CodeReuse/ActiveRecord + + def usernames + matches[:usernames] + end + + def emails + matches[:emails] + end + + def references + @references ||= matches.values.flatten + end + + def matches + @matches ||= { + emails: @text.scan(EMAIL_REGEXP).flatten.uniq, + usernames: @text.scan(USERNAME_REGEXP).flatten.uniq + } + end + + private + + def union_relations + relations = [] + + relations << User.by_any_email(emails) if emails.any? + relations << User.by_username(usernames) if usernames.any? + + relations + end + end +end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 7b2a62fed48..d00921e6cdc 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -89,15 +89,19 @@ module Gitlab def included(base = nil) super - queue_verification(base) + queue_verification(base) if base end - alias_method :prepended, :included + def prepended(base = nil) + super + + queue_verification(base) if base + end - def extended(mod) + def extended(mod = nil) super - queue_verification(mod.singleton_class) + queue_verification(mod.singleton_class) if mod end def queue_verification(base) diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb index 73fc43cb590..201fcc7de7f 100644 --- a/lib/gitlab/verify/uploads.rb +++ b/lib/gitlab/verify/uploads.rb @@ -11,9 +11,11 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def all_relation Upload.all.preload(:model) end + # rubocop: enable CodeReuse/ActiveRecord def local?(upload) upload.local? diff --git a/lib/gitlab/web_ide_commits_counter.rb b/lib/gitlab/web_ide_commits_counter.rb new file mode 100644 index 00000000000..1cd9b5295b9 --- /dev/null +++ b/lib/gitlab/web_ide_commits_counter.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module WebIdeCommitsCounter + WEB_IDE_COMMITS_KEY = "WEB_IDE_COMMITS_COUNT".freeze + + class << self + def increment + Gitlab::Redis::SharedState.with { |redis| redis.incr(WEB_IDE_COMMITS_KEY) } + end + + def total_count + Gitlab::Redis::SharedState.with { |redis| redis.get(WEB_IDE_COMMITS_KEY).to_i } + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index a9629a92a50..30a8c3578d8 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -22,18 +22,27 @@ module Gitlab project = repository.project - { + attrs = { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), GL_USERNAME: user&.username, ShowAllRefs: show_all_refs, Repository: repository.gitaly_repository.to_h, RepoPath: 'ignored but not allowed to be empty in gitlab-workhorse', + GitConfigOptions: [], GitalyServer: { address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) } } + + # Custom option for git-receive-pack command + receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i + if receive_max_input_size > 0 + attrs[:GitConfigOptions] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" + end + + attrs end def send_git_blob(repository, blob) diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 36859b4d025..77b6610286f 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -50,7 +50,7 @@ module GoogleApi service.get_zone_cluster(project_id, zone, cluster_id, options: user_agent_header) end - def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:) + def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:, legacy_abac:) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token @@ -63,7 +63,7 @@ module GoogleApi "machine_type": machine_type }, "legacy_abac": { - "enabled": true + "enabled": legacy_abac } } } diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb index 61a69e7ffe4..97f56e10ccf 100644 --- a/lib/object_storage/direct_upload.rb +++ b/lib/object_storage/direct_upload.rb @@ -41,7 +41,9 @@ module ObjectStorage GetURL: get_url, StoreURL: store_url, DeleteURL: delete_url, - MultipartUpload: multipart_upload_hash + MultipartUpload: multipart_upload_hash, + CustomPutHeaders: true, + PutHeaders: upload_options }.compact end @@ -87,7 +89,7 @@ module ObjectStorage method: 'PUT', bucket_name: bucket_name, object_name: object_name, - query: { uploadId: upload_id, partNumber: part_number }, + query: { 'uploadId' => upload_id, 'partNumber' => part_number }, headers: upload_options }, expire_at) end @@ -98,7 +100,7 @@ module ObjectStorage method: 'POST', bucket_name: bucket_name, object_name: object_name, - query: { uploadId: upload_id }, + query: { 'uploadId' => upload_id }, headers: { 'Content-Type' => 'application/xml' } }, expire_at) end @@ -109,7 +111,7 @@ module ObjectStorage method: 'DELETE', bucket_name: bucket_name, object_name: object_name, - query: { uploadId: upload_id } + query: { 'uploadId' => upload_id } }, expire_at) end @@ -156,7 +158,7 @@ module ObjectStorage end def upload_options - { 'Content-Type' => 'application/octet-stream' } + {} end def connection diff --git a/lib/quality/helm_client.rb b/lib/quality/helm_client.rb new file mode 100644 index 00000000000..49d953da681 --- /dev/null +++ b/lib/quality/helm_client.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'time' +require_relative '../gitlab/popen' unless defined?(Gitlab::Popen) + +module Quality + class HelmClient + attr_reader :namespace + + Release = Struct.new(:name, :revision, :last_update, :status, :chart, :namespace) do + def revision + @revision ||= self[:revision].to_i + end + + def last_update + @last_update ||= Time.parse(self[:last_update]) + end + end + + def initialize(namespace: ENV['KUBE_NAMESPACE']) + @namespace = namespace + end + + def releases(args: []) + command = ['list', %(--namespace "#{namespace}"), *args] + + run_command(command) + .stdout + .lines + .select { |line| line.include?(namespace) } + .map { |line| Release.new(*line.split(/\t/).map(&:strip)) } + end + + def delete(release_name:) + run_command(['delete', '--purge', release_name]) + end + + private + + def run_command(command) + final_command = ['helm', *command].join(' ') + puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output + + Gitlab::Popen.popen_with_detail([final_command]) + end + end +end diff --git a/lib/quality/kubernetes_client.rb b/lib/quality/kubernetes_client.rb new file mode 100644 index 00000000000..e366a688e3e --- /dev/null +++ b/lib/quality/kubernetes_client.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require_relative '../gitlab/popen' unless defined?(Gitlab::Popen) + +module Quality + class KubernetesClient + attr_reader :namespace + + def initialize(namespace: ENV['KUBE_NAMESPACE']) + @namespace = namespace + end + + def cleanup(release_name:) + command = ['kubectl'] + command << %(-n "#{namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1) + command << '|' << %(grep "#{release_name}") + command << '|' << "awk '{print $1}'" + command << '|' << %(xargs kubectl -n "#{namespace}" delete) + command << '||' << 'true' + + run_command(command) + end + + private + + def run_command(command) + puts "Running command: `#{command.join(' ')}`" # rubocop:disable Rails/Output + + Gitlab::Popen.popen_with_detail(command) + end + end +end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 72eb8adcce2..fc984d737d5 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -17,7 +17,7 @@ ## See installation.md#using-https for additional HTTPS configuration details. upstream gitlab-workhorse { - # Gitlab socket file, + # GitLab socket file, # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -112,7 +112,7 @@ server { error_page 502 /502.html; error_page 503 /503.html; location ~ ^/(404|422|500|502|503)\.html$ { - # Location to the Gitlab's public directory, + # Location to the GitLab's public directory, # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public. root /home/git/gitlab/public; internal; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 2e3799d5e1b..ba01e250bbb 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -21,7 +21,7 @@ ## See installation.md#using-https for additional HTTPS configuration details. upstream gitlab-workhorse { - # Gitlab socket file, + # GitLab socket file, # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -162,7 +162,7 @@ server { error_page 502 /502.html; error_page 503 /503.html; location ~ ^/(404|422|500|502|503)\.html$ { - # Location to the Gitlab's public directory, + # Location to the GitLab's public directory, # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public root /home/git/gitlab/public; internal; diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb index e55bea86d3f..3550c5796b0 100644 --- a/lib/system_check/incoming_email/imap_authentication_check.rb +++ b/lib/system_check/incoming_email/imap_authentication_check.rb @@ -7,7 +7,7 @@ module SystemCheck if config try_connect_imap else - @error = "#{mail_room_config_path} does not have mailboxes setup" + @error = "#{mail_room_config_path} does not have mailboxes set up" false end end diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake deleted file mode 100644 index 4bec013a141..00000000000 --- a/lib/tasks/flay.rake +++ /dev/null @@ -1,9 +0,0 @@ -desc 'Code duplication analyze via flay' -task :flay do - output = `bundle exec flay --mass 35 app/ lib/gitlab/ ee/ 2> #{File::NULL}` - - if output.include?("Similar code found") || output.include?("IDENTICAL code found") - puts output - exit 1 - end -end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index c6942d22926..560a52053d8 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -86,7 +86,7 @@ namespace :gemojione do SPRITESHEET_WIDTH = 860 SPRITESHEET_HEIGHT = 840 - # Setup a map to rename image files + # Set up a map to rename image files emoji_unicode_string_to_name_map = {} Gitlab::Emoji.emojis.each do |name, emoji_hash| # Ignore aliases diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index bfca4bfb3f7..e7634d2ed4f 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -15,7 +15,7 @@ namespace :gitlab do build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE) build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE) - logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage") + logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage") rescue => e logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index c8a8863443e..e8ae5dfa540 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -1,40 +1,29 @@ -# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/954 -# +# frozen_string_literal: true +require 'set' + namespace :gitlab do namespace :cleanup do - HASHED_REPOSITORY_NAME = '@hashed'.freeze - desc "GitLab | Cleanup | Clean namespaces" task dirs: :gitlab_environment do - warn_user_is_not_gitlab + namespaces = Set.new(Namespace.pluck(:path)) + namespaces << Storage::HashedProject::ROOT_PATH_PREFIX - namespaces = Namespace.pluck(:path) - namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored - Gitlab.config.repositories.storages.each do |name, repository_storage| - git_base_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path } - all_dirs = Dir.glob(git_base_path + '/*') + Gitaly::Server.all.each do |server| + all_dirs = Gitlab::GitalyClient::StorageService + .new(server.storage) + .list_directories(depth: 0) + .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) } - puts git_base_path.color(:yellow) puts "Looking for directories to remove... " - - all_dirs.reject! do |dir| - # skip if git repo - dir =~ /.git$/ - end - - all_dirs.reject! do |dir| - dir_name = File.basename dir - - # skip if namespace present - namespaces.include?(dir_name) - end - all_dirs.each do |dir_path| if remove? - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".color(:red) - else - puts "Cannot remove #{dir_path}".color(:red) + begin + Gitlab::GitalyClient::NamespaceService.new(server.storage) + .remove(dir_path) + + puts "Removed...#{dir_path}" + rescue StandardError => e + puts "Cannot remove #{dir_path}: #{e.message}".color(:red) end else puts "Can be removed: #{dir_path}".color(:red) @@ -49,29 +38,29 @@ namespace :gitlab do desc "GitLab | Cleanup | Clean repositories" task repos: :gitlab_environment do - warn_user_is_not_gitlab - move_suffix = "+orphaned+#{Time.now.to_i}" - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_root = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path } - - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| - find.each_line do |path| - path.chomp! - repo_with_namespace = path - .sub(repo_root, '') - .sub(%r{^/*}, '') - .chomp('.git') - .chomp('.wiki') - - # TODO ignoring hashed repositories for now. But revisit to fully support - # possible orphaned hashed repos - next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace) - - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - File.rename(path, new_path) + + Gitaly::Server.all.each do |server| + Gitlab::GitalyClient::StorageService + .new(server.storage) + .list_directories + .each do |path| + repo_with_namespace = path.chomp('.git').chomp('.wiki') + + # TODO ignoring hashed repositories for now. But revisit to fully support + # possible orphaned hashed repos + next if repo_with_namespace.start_with?(Storage::HashedProject::ROOT_PATH_PREFIX) + next if Project.find_by_full_path(repo_with_namespace) + + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + + begin + Gitlab::GitalyClient::NamespaceService + .new(server.storage) + .rename(path, new_path) + rescue StandardError => e + puts "Error occured while moving the repository: #{e.message}".color(:red) end end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 69166851816..74cd70c6e9f 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -51,6 +51,8 @@ namespace :gitlab do if ActiveRecord::Base.connection.tables.count > 1 Rake::Task['db:migrate'].invoke else + # Add post-migrate paths to ensure we mark all migrations as up + Gitlab::Database.add_post_migrate_path_to_rails(force: true) Rake::Task['db:schema:load'].invoke Rake::Task['db:seed_fu'].invoke end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 4fcbbbf8c9d..0ebc6f00793 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -92,9 +92,11 @@ namespace :gitlab do def setup warn_user_is_not_gitlab + ensure_write_to_authorized_keys_is_enabled + unless ENV['force'] == 'yes' - puts "This will rebuild an authorized_keys file." - puts "You will lose any data stored in authorized_keys file." + puts "This task will now rebuild the authorized_keys file." + puts "You will lose any data stored in the authorized_keys file." ask_to_continue puts "" end @@ -118,4 +120,44 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end + + def ensure_write_to_authorized_keys_is_enabled + return if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled + + puts authorized_keys_is_disabled_warning + + unless ENV['force'] == 'yes' + puts 'Do you want to permanently enable the "Write to authorized_keys file" setting now?' + ask_to_continue + end + + puts 'Enabling the "Write to authorized_keys file" setting...' + Gitlab::CurrentSettings.current_application_settings.update!(authorized_keys_enabled: true) + + puts 'Successfully enabled "Write to authorized_keys file"!' + puts '' + end + + def authorized_keys_is_disabled_warning + <<-MSG.strip_heredoc + WARNING + + The "Write to authorized_keys file" setting is disabled, which prevents + the file from being rebuilt! + + It should be enabled for most GitLab installations. Large installations + may wish to disable it as part of speeding up SSH operations. + + See https://docs.gitlab.com/ee/administration/operations/fast_ssh_key_lookup.html + + If you did not intentionally disable this option in Admin Area > Settings, + then you may have been affected by the 9.3.0 bug in which the new setting + was disabled by default. + + https://gitlab.com/gitlab-org/gitlab-ee/issues/2738 + + It was reverted in 9.3.1 and fixed in 9.3.3, however, if Settings were + saved while the setting was unchecked, then it is still disabled. + MSG + end end diff --git a/lib/tasks/gitlab/site_statistics.rake b/lib/tasks/gitlab/site_statistics.rake index 7d24ec72a9d..d97f11b2ed5 100644 --- a/lib/tasks/gitlab/site_statistics.rake +++ b/lib/tasks/gitlab/site_statistics.rake @@ -10,14 +10,6 @@ namespace :gitlab do SiteStatistic.update_all('repositories_count = (SELECT COUNT(*) FROM projects)') end puts 'OK!'.color(:green) - - print '* Wikis... ' - SiteStatistic.transaction do - # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 - ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? - SiteStatistic.update_all('wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)') - end - puts 'OK!'.color(:green) puts end end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index a25f7ce59c7..abe10f5580e 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -6,6 +6,8 @@ namespace :gitlab do desc "GitLab | Update project templates" task :update_project_templates do + include Gitlab::ImportExport::CommandLineUtil + if Rails.env.production? puts "This rake task is not meant fo production instances".red exit(1) @@ -52,7 +54,7 @@ namespace :gitlab do end Projects::ImportExport::ExportService.new(project, admin).execute - FileUtils.cp(project.export_project_path, template.archive_path) + download_or_copy_upload(project.export_file, template.archive_path) Projects::DestroyService.new(admin, project).execute puts "Exported #{template.name}".green end @@ -98,10 +100,6 @@ namespace :gitlab do /(\.{1,2}|LICENSE|Global|\.gitignore)\z/ ), Template.new( - "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", - /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/ - ), - Template.new( "https://gitlab.com/gitlab-org/Dockerfile.git", /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/ ) diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index f548a266b99..1c93609a006 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -1,6 +1,30 @@ namespace :gitlab do namespace :uploads do - desc 'GitLab | Uploads | Migrate the uploaded files to object storage' + namespace :migrate do + desc "GitLab | Uploads | Migrate all uploaded files to object storage" + task all: :environment do + categories = [%w(AvatarUploader Project :avatar), + %w(AvatarUploader Group :avatar), + %w(AvatarUploader User :avatar), + %w(AttachmentUploader Note :attachment), + %w(AttachmentUploader Appearance :logo), + %w(AttachmentUploader Appearance :header_logo), + %w(FaviconUploader Appearance :favicon), + %w(FileUploader Project), + %w(PersonalFileUploader Snippet), + %w(NamespaceFileUploader Snippet), + %w(FileUploader MergeRequest)] + + categories.each do |args| + Rake::Task["gitlab:uploads:migrate"].invoke(*args) + Rake::Task["gitlab:uploads:migrate"].reenable + end + end + end + + # The following is the actual rake task that migrates uploads of specified + # category to object storage + desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage' task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args| batch_size = ENV.fetch('BATCH', 200).to_i @to_store = ObjectStorage::Store::REMOTE diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 006fcdd31a4..5d673a1a285 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -34,7 +34,6 @@ unless Rails.env.production? config_lint lint:haml scss_lint - flay gettext:lint gettext:updated_check lint:static_verification diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake index 9b05876034c..c77fa49d586 100644 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ b/lib/tasks/migrate/add_limits_mysql.rake @@ -3,6 +3,7 @@ require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql') require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql') require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql') +require Rails.root.join('db/migrate/prometheus_metrics_limits_to_mysql') desc "GitLab | Add limits to strings in mysql database" task add_limits_mysql: :environment do @@ -12,4 +13,5 @@ task add_limits_mysql: :environment do MergeRequestDiffFileLimitsToMysql.new.up LimitsCiBuildTraceChunksRawDataForMysql.new.up IncreaseMysqlTextLimitForGpgKeys.new.up + PrometheusMetricsLimitsToMysql.new.up end |