diff options
| author | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 20:06:12 +0100 |
|---|---|---|
| committer | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 20:06:12 +0100 |
| commit | 5a69b51bc870f5b42ee3406ba77de02f44ef8d32 (patch) | |
| tree | c2a6e5b2c171826236b5d0f5e1ed8d02bd1554d2 /lib | |
| parent | b1f8d8a1739ff48412c8205f0007a2af8399d097 (diff) | |
| parent | b39d0c318921bae2e3a11df9ee6828291dad9864 (diff) | |
| download | gitlab-ce-5a69b51bc870f5b42ee3406ba77de02f44ef8d32.tar.gz | |
Merge commit 'b39d0c318921bae2e3a11df9ee6828291dad9864' into object-storage-ee-to-ce-backport
Diffstat (limited to 'lib')
255 files changed, 4794 insertions, 1943 deletions
diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb new file mode 100644 index 00000000000..4750a2c373a --- /dev/null +++ b/lib/after_commit_queue.rb @@ -0,0 +1,30 @@ +module AfterCommitQueue + extend ActiveSupport::Concern + + included do + after_commit :_run_after_commit_queue + after_rollback :_clear_after_commit_queue + end + + def run_after_commit(method = nil, &block) + _after_commit_queue << proc { self.send(method) } if method # rubocop:disable GitlabSecurity/PublicSend + _after_commit_queue << block if block + true + end + + protected + + def _run_after_commit_queue + while action = _after_commit_queue.pop + self.instance_eval(&action) + end + end + + def _after_commit_queue + @after_commit_queue ||= [] + end + + def _clear_after_commit_queue + _after_commit_queue.clear + end +end diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index cdacf9839e5..374b611f55e 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: { id: %r{[^/]+} } do + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "Gets a list of access requests for a #{source_type}." do detail 'This feature was introduced in GitLab 8.11.' success Entities::AccessRequester @@ -67,10 +67,12 @@ module API end delete ":id/access_requests/:user_id" do source = find_source(source_type, params[:id]) + member = source.requesters.find_by!(user_id: params[:user_id]) - status 204 - ::Members::DestroyService.new(source, current_user, params) - .execute(:requesters) + destroy_conditionally!(member) do + ::Members::DestroyService.new(source, current_user, params) + .execute(:requesters) + end end end end diff --git a/lib/api/api.rb b/lib/api/api.rb index 94df543853b..ee4e1688e12 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -2,6 +2,17 @@ module API class API < Grape::API include APIGuard + LOG_FILENAME = Rails.root.join("log", "api_json.log") + + use GrapeLogging::Middleware::RequestLogger, + logger: Logger.new(LOG_FILENAME), + formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, + include: [ + GrapeLogging::Loggers::Response.new, + GrapeLogging::Loggers::FilterParameters.new, + GrapeLogging::Loggers::ClientEnv.new + ] + allow_access_with_scope :api prefix :api @@ -108,6 +119,7 @@ module API mount ::API::Internal mount ::API::Issues mount ::API::Jobs + mount ::API::JobArtifacts mount ::API::Keys mount ::API::Labels mount ::API::Lint @@ -143,6 +155,7 @@ module API mount ::API::Variables mount ::API::GroupVariables mount ::API::Version + mount ::API::Wikis route :any, '*path' do error!('404 Not Found', 404) diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 0d2d71e336a..c4c0fdda665 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -122,7 +122,7 @@ module API error_classes = [MissingTokenError, TokenNotFoundError, ExpiredError, RevokedError, InsufficientScopeError] - base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler + base.__send__(:rescue_from, *error_classes, oauth2_bearer_token_error_handler) # rubocop:disable GitlabSecurity/PublicSend end def oauth2_bearer_token_error_handler diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 5a028fc9d0b..c3d93996816 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do AWARDABLES.each do |awardable_params| awardable_string = awardable_params[:type].pluralize awardable_id_string = "#{awardable_params[:type]}_#{awardable_params[:find_by]}" @@ -88,8 +88,7 @@ module API unauthorized! unless award.user == current_user || current_user.admin? - status 204 - award.destroy + destroy_conditionally!(award) end end end diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 5a2d7a681e3..366b0dc9a6f 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all project boards' do detail 'This feature was introduced in 8.13' success Entities::Board @@ -122,13 +122,13 @@ module API end delete "/lists/:list_id" do authorize!(:admin_list, user_project) - list = board_lists.find(params[:list_id]) - service = ::Boards::Lists::DestroyService.new(user_project, current_user) - - unless service.execute(list) - render_api_error!({ error: 'List could not be deleted!' }, 400) + destroy_conditionally!(list) do |list| + service = ::Boards::Lists::DestroyService.new(user_project, current_user) + unless service.execute(list) + render_api_error!({ error: 'List could not be deleted!' }, 400) + end end end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index d3dbf941298..642c1140fcc 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -24,17 +24,22 @@ module API present paginate(branches), with: Entities::RepoBranch, project: user_project end - desc 'Get a single branch' do - success Entities::RepoBranch - end - params do - requires :branch, type: String, desc: 'The name of the branch' - end - get ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do - branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch") unless branch + resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do + desc 'Get a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, desc: 'The name of the branch' + end + head do + user_project.repository.branch_exists?(params[:branch]) ? status(204) : status(404) + end + get do + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::RepoBranch, project: user_project + end end # Note: This API will be deprecated in favor of the protected branches API. @@ -125,11 +130,18 @@ module API delete ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_push_project - result = DeleteBranchService.new(user_project, current_user) - .execute(params[:branch]) + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch + + commit = user_project.repository.commit(branch.dereferenced_target) + + destroy_conditionally!(commit, last_updated: commit.authored_date) do + result = DeleteBranchService.new(user_project, current_user) + .execute(params[:branch]) - if result[:status] != :success - render_api_error!(result[:message], result[:return_code]) + if result[:status] != :success + render_api_error!(result[:message], result[:return_code]) + end end end diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 9980aec4752..d7138b2f2fe 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -20,7 +20,7 @@ module API use :pagination end get do - messages = BroadcastMessage.all + messages = BroadcastMessage.all.order_id_desc present paginate(messages), with: Entities::BroadcastMessage end @@ -91,8 +91,7 @@ module API delete ':id' do message = find_message - status 204 - message.destroy + destroy_conditionally!(message) end end end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 485b680cd5f..829eef18795 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -5,7 +5,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do include PaginationParams before { authenticate! } @@ -74,7 +74,8 @@ module API source: :external, sha: commit.sha, ref: ref, - user: current_user) + user: current_user, + protected: @project.protected_for?(ref)) end status = GenericCommitStatus.running_or_pending.find_or_initialize_by( @@ -82,7 +83,8 @@ module API pipeline: pipeline, name: name, ref: ref, - user: current_user + user: current_user, + protected: @project.protected_for?(ref) ) optional_attributes = @@ -101,7 +103,7 @@ module API when 'success' status.success! when 'failed' - status.drop! + status.drop!(:api_failure) when 'canceled' status.cancel! else diff --git a/lib/api/commits.rb b/lib/api/commits.rb index ea78737288a..4b8d248f5f7 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -104,7 +104,7 @@ module API not_found! 'Commit' unless commit - commit.raw_diffs.to_a + present commit.raw_diffs.to_a, with: Entities::RepoDiff end desc "Get a commit's comments" do diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 42e7c1486b0..281269b1190 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -17,7 +17,7 @@ module API params do requires :id, type: String, desc: 'The ID of the project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do before { authorize_admin_project } desc "Get a specific project's deploy keys" do @@ -125,8 +125,7 @@ module API key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) not_found!('Deploy Key') unless key - status 204 - key.destroy + destroy_conditionally!(key) end end end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 46b936897f6..1efee9a1324 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -8,7 +8,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all deployments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Deployment diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 6ba4005dd0b..52c49e5caa9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1,11 +1,21 @@ module API module Entities + class WikiPageBasic < Grape::Entity + expose :format + expose :slug + expose :title + end + + class WikiPage < WikiPageBasic + expose :content + end + class UserSafe < Grape::Entity - expose :name, :username + expose :id, :name, :username end class UserBasic < UserSafe - expose :id, :state + expose :state expose :avatar_url do |user, options| user.avatar_url(only_path: false) end @@ -35,7 +45,7 @@ module API expose :confirmed_at expose :last_activity_on expose :email - expose :color_scheme_id, :projects_limit, :current_sign_in_at + expose :theme_id, :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity expose :can_create_group?, as: :can_create_group expose :can_create_project?, as: :can_create_project @@ -83,7 +93,7 @@ module API expose :created_at, :last_activity_at end - class Project < BasicProjectDetails + class Project < BasicProjectDetails include ::API::Helpers::RelatedResourcesHelpers expose :_links do @@ -119,6 +129,7 @@ module API expose :archived?, as: :archived expose :visibility expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } + expose :resolve_outdated_diff_discussions expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible @@ -132,7 +143,7 @@ module API expose :lfs_enabled?, as: :lfs_enabled expose :creator_id expose :namespace, using: 'API::Entities::Namespace' - expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } + expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } expose :avatar_url do |user, options| @@ -290,10 +301,11 @@ module API end class RepoDiff < Grape::Entity - expose :old_path, :new_path, :a_mode, :b_mode, :diff + expose :old_path, :new_path, :a_mode, :b_mode expose :new_file?, as: :new_file expose :renamed_file?, as: :renamed_file expose :deleted_file?, as: :deleted_file + expose :json_safe_diff, as: :diff end class ProtectedRefAccess < Grape::Entity @@ -320,7 +332,10 @@ module API end class IssueBasic < ProjectEntity - expose :label_names, as: :labels + expose :labels do |issue, options| + # Avoids an N+1 query since labels are preloaded + issue.labels.map(&:title).sort + end expose :milestone, using: Entities::Milestone expose :assignees, :author, using: Entities::UserBasic @@ -329,13 +344,32 @@ module API end expose :user_notes_count - expose :upvotes, :downvotes + expose :upvotes do |issue, options| + if options[:issuable_metadata] + # Avoids an N+1 query when metadata is included + options[:issuable_metadata][issue.id].upvotes + else + issue.upvotes + end + end + expose :downvotes do |issue, options| + if options[:issuable_metadata] + # Avoids an N+1 query when metadata is included + options[:issuable_metadata][issue.id].downvotes + else + issue.downvotes + end + end expose :due_date expose :confidential expose :web_url do |issue, options| Gitlab::UrlBuilder.build(issue) end + + expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |issue| + issue + end end class Issue < IssueBasic @@ -365,10 +399,22 @@ module API end class IssuableTimeStats < Grape::Entity + format_with(:time_tracking_formatter) do |time_spent| + Gitlab::TimeTrackingFormatter.output(time_spent) + end + expose :time_estimate expose :total_time_spent expose :human_time_estimate - expose :human_total_time_spent + + with_options(format_with: :time_tracking_formatter) do + expose :total_time_spent, as: :human_total_time_spent + end + + def total_time_spent + # Avoids an N+1 query since timelogs are preloaded + object.timelogs.map(&:time_spent).sum + end end class ExternalIssue < Grape::Entity @@ -418,6 +464,10 @@ module API expose :web_url do |merge_request, options| Gitlab::UrlBuilder.build(merge_request) end + + expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |merge_request| + merge_request + end end class MergeRequest < MergeRequestBasic @@ -453,6 +503,10 @@ module API expose :user, using: Entities::UserPublic end + class GPGKey < Grape::Entity + expose :id, :key, :created_at + end + class Note < Grape::Entity # Only Issue and MergeRequest have iid NOTEABLE_TYPES_WITH_IID = %w(Issue MergeRequest).freeze @@ -497,14 +551,24 @@ module API expose :author, using: Entities::UserBasic end + class PushEventPayload < Grape::Entity + expose :commit_count, :action, :ref_type, :commit_from, :commit_to + expose :ref, :commit_title + end + class Event < Grape::Entity - expose :title, :project_id, :action_name + expose :project_id, :action_name expose :target_id, :target_iid, :target_type, :author_id - expose :data, :target_title + expose :target_title expose :created_at expose :note, using: Entities::Note, if: ->(event, options) { event.note? } expose :author, using: Entities::UserBasic, if: ->(event, options) { event.author } + expose :push_event_payload, + as: :push_data, + using: PushEventPayload, + if: -> (event, _) { event.push? } + expose :author_username do |event, options| event.author&.username end @@ -531,8 +595,9 @@ module API target_url = "namespace_project_#{target_type}_url" target_anchor = "note_#{todo.note_id}" if todo.note_id? - Gitlab::Routing.url_helpers.public_send(target_url, - todo.project.namespace, todo.project, todo.target, anchor: target_anchor) + Gitlab::Routing + .url_helpers + .public_send(target_url, todo.project.namespace, todo.project, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend end expose :body @@ -726,6 +791,7 @@ module API expose :tag_list expose :run_untagged expose :locked + expose :access_level expose :version, :revision, :platform, :architecture expose :contacted_at expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.is_shared? } @@ -769,7 +835,7 @@ module API class Variable < Grape::Entity expose :key, :value - expose :protected?, as: :protected + expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) } end class Pipeline < PipelineBasic @@ -790,6 +856,7 @@ module API class PipelineScheduleDetails < PipelineSchedule expose :last_pipeline, using: Entities::PipelineBasic + expose :variables, using: Entities::Variable end class EnvironmentBasic < Grape::Entity diff --git a/lib/api/environments.rb b/lib/api/environments.rb index c774a5c6685..5c63ec028d9 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all environments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Environment @@ -79,8 +79,7 @@ module API environment = user_project.environments.find(params[:environment_id]) - status 204 - environment.destroy + destroy_conditionally!(environment) end desc 'Stops an existing environment' do diff --git a/lib/api/events.rb b/lib/api/events.rb index dabdf579119..b0713ff1d54 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -67,7 +67,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "List a Project's visible events" do success Entities::Event end diff --git a/lib/api/files.rb b/lib/api/files.rb index 450334fee84..1598d3c00b8 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,5 +1,10 @@ module API class Files < Grape::API + FILE_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX) + + # Prevents returning plain/text responses for files with .txt extension + after_validation { content_type "application/json" } + helpers do def commit_params(attrs) { @@ -55,13 +60,13 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do desc 'Get raw file contents from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag commit' end - get ":id/repository/files/:file_path/raw" do + get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! send_git_blob @repo, @blob @@ -72,7 +77,7 @@ module API requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag or commit' end - get ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! { @@ -92,7 +97,7 @@ module API params do use :extended_file_params end - post ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + post ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project file_params = declared_params(include_missing: false) @@ -110,7 +115,7 @@ module API params do use :extended_file_params end - put ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + put ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project file_params = declared_params(include_missing: false) @@ -134,7 +139,7 @@ module API params do use :simple_file_params end - delete ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + delete ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project file_params = declared_params(include_missing: false) diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index b85eb59dc0a..93fa0b95857 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of group milestones' do success Entities::Milestone end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index f64da4ab77b..92800ce6450 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -9,7 +9,7 @@ module API requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get group-level variables' do success Entities::Variable end @@ -88,8 +88,7 @@ module API variable = user_group.variables.find_by(key: params[:key]) not_found!('GroupVariable') unless variable - status 204 - variable.destroy + destroy_conditionally!(variable) end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 49c3b2278c7..e817dcbbc4b 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -2,12 +2,16 @@ module API class Groups < Grape::API include PaginationParams - before { authenticate! } + before { authenticate_non_get! } helpers do params :optional_params_ce do optional :description, type: String, desc: 'The description of the group' - optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group' + optional :visibility, type: String, + values: Gitlab::VisibilityLevel.string_values, + default: Gitlab::VisibilityLevel.string_level( + Gitlab::CurrentSettings.current_application_settings.default_group_visibility), + desc: 'The visibility of the group' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' @@ -47,16 +51,8 @@ module API use :pagination end get do - groups = if params[:owned] - current_user.owned_groups - elsif current_user.admin - Group.all - elsif params[:all_available] - GroupsFinder.new(current_user).execute - else - current_user.groups - end - + find_params = { all_available: params[:all_available], owned: params[:owned] } + groups = GroupsFinder.new(current_user, find_params).execute groups = groups.search(params[:search]) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.reorder(params[:order_by] => params[:sort]) @@ -78,7 +74,12 @@ module API use :optional_params end post do - authorize! :create_group + parent_group = find_group!(params[:parent_id]) if params[:parent_id].present? + if parent_group + authorize! :create_subgroup, parent_group + else + authorize! :create_group + end group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute @@ -93,7 +94,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Update a group. Available only for users who can administrate groups.' do success Entities::Group end @@ -126,8 +127,9 @@ module API group = find_group!(params[:id]) authorize! :admin_group, group - status 204 - ::Groups::DestroyService.new(group, current_user).execute + destroy_conditionally!(group) do |group| + ::Groups::DestroyService.new(group, current_user).execute + end end desc 'Get a list of projects in this group.' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 4056d06bcc7..6a0be74623e 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -11,6 +11,27 @@ module API declared(params, options).to_h.symbolize_keys end + def check_unmodified_since!(last_modified) + if_unmodified_since = Time.parse(headers['If-Unmodified-Since']) rescue nil + + if if_unmodified_since && last_modified && last_modified > if_unmodified_since + render_api_error!('412 Precondition Failed', 412) + end + end + + def destroy_conditionally!(resource, last_updated: nil) + last_updated ||= resource.updated_at + + check_unmodified_since!(last_updated) + + status 204 + if block_given? + yield resource + else + resource.destroy + end + end + def current_user return @current_user if defined?(@current_user) @@ -35,6 +56,12 @@ module API @project ||= find_project!(params[:id]) end + def wiki_page + page = user_project.wiki.find_page(params[:slug]) + + page || not_found!('Wiki Page') + end + def available_labels @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute end @@ -66,7 +93,7 @@ module API end def find_group(id) - if id =~ /^\d+$/ + if id.to_s =~ /^\d+$/ Group.find_by(id: id) else Group.find_by_full_path(id) @@ -107,6 +134,10 @@ module API merge_request end + def find_build!(id) + user_project.builds.find(id.to_i) + end + def authenticate! unauthorized! unless current_user && can?(initial_current_user, :access_api) end @@ -139,6 +170,14 @@ module API authorize! :admin_project, user_project end + def authorize_read_builds! + authorize! :read_build, user_project + end + + def authorize_update_builds! + authorize! :update_build, user_project + end + def require_gitlab_workhorse! unless env['HTTP_GITLAB_WORKHORSE'].present? forbidden!('Request should be executed via GitLab Workhorse') @@ -189,7 +228,7 @@ module API def bad_request!(attribute) message = ["400 (Bad request)"] - message << "\"" + attribute.to_s + "\" not given" + message << "\"" + attribute.to_s + "\" not given" if attribute render_api_error!(message.join(' '), 400) end @@ -257,7 +296,15 @@ module API message << " " << trace.join("\n ") API.logger.add Logger::FATAL, message - rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) + + response_message = + if Rails.env.test? + message + else + '500 Internal Server Error' + end + + rack_response({ 'message' => response_message }.to_json, 500) end # project helpers @@ -282,7 +329,7 @@ module API def uploaded_file(field, uploads_path) if params[field] - bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) + bad_request!("#{field} is not a file") unless params[field][:filename] return params[field] end @@ -403,6 +450,10 @@ module API header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format)) end + def send_artifacts_entry(build, entry) + header(*Gitlab::Workhorse.send_artifacts_entry(build, entry)) + end + # The Grape Error Middleware only has access to env but no params. We workaround this by # defining a method that returns the right value. def define_params_for_grape_middleware diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index ecb79317093..4c0db4d42b1 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -42,6 +42,19 @@ module API ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) end + def merge_request_urls + ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) + end + + def redis_ping + result = Gitlab::Redis::SharedState.with { |redis| redis.ping } + + result == 'PONG' + rescue => e + Rails.logger.warn("GitLab: An unexpected error occurred in pinging to Redis: #{e}") + false + end + private def set_project diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index d9cae1501f8..a50ea0b52aa 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -1,8 +1,10 @@ +# rubocop:disable GitlabSecurity/PublicSend + module API module Helpers module MembersHelpers def find_source(source_type, id) - public_send("find_#{source_type}!", id) + public_send("find_#{source_type}!", id) # rubocop:disable GitlabSecurity/PublicSend end def authorize_admin_source!(source_type, source) diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 0764b58fb4c..95108292aac 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -11,7 +11,7 @@ module API def add_pagination_headers(paginated_data) header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', paginated_data.total_pages.to_s + header 'X-Total-Pages', total_pages(paginated_data).to_s header 'X-Per-Page', paginated_data.limit_value.to_s header 'X-Page', paginated_data.current_page.to_s header 'X-Next-Page', paginated_data.next_page.to_s @@ -26,20 +26,25 @@ module API links = [] - request_params[:page] = paginated_data.current_page - 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? + request_params[:page] = paginated_data.prev_page + links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") if request_params[:page] - request_params[:page] = paginated_data.current_page + 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? + request_params[:page] = paginated_data.next_page + links << %(<#{request_url}?#{request_params.to_query}>; rel="next") if request_params[:page] request_params[:page] = 1 links << %(<#{request_url}?#{request_params.to_query}>; rel="first") - request_params[:page] = paginated_data.total_pages + request_params[:page] = total_pages(paginated_data) links << %(<#{request_url}?#{request_params.to_query}>; rel="last") links.join(', ') end + + def total_pages(paginated_data) + # Ensure there is in total at least 1 page + [paginated_data.total_pages, 1].max + end end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index f8645e364ce..282af32ca94 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,6 +1,8 @@ module API module Helpers module Runner + include Gitlab::CurrentSettings + JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :token UPDATE_RUNNER_EVERY = 10 * 60 diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 8b007869dc3..c0fef56378f 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -68,7 +68,7 @@ module API end get "/merge_request_urls" do - ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) + merge_request_urls end # @@ -88,7 +88,8 @@ module API { api_version: API.version, gitlab_version: Gitlab::VERSION, - gitlab_rev: Gitlab::REVISION + gitlab_rev: Gitlab::REVISION, + redis: redis_ping } end @@ -142,6 +143,14 @@ module API { success: true, recovery_codes: codes } end + post '/pre_receive' do + status 200 + + reference_counter_increased = Gitlab::ReferenceCounter.new(params[:gl_repository]).increase + + { reference_counter_increased: reference_counter_increased } + end + post "/notify_post_receive" do status 200 @@ -155,6 +164,21 @@ module API # render_api_error!(e, 500) # end end + + post '/post_receive' do + status 200 + + PostReceive.perform_async(params[:gl_repository], params[:identifier], + params[:changes]) + broadcast_message = BroadcastMessage.current&.last&.message + reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease + + { + merge_request_urls: merge_request_urls, + broadcast_message: broadcast_message, + reference_counter_decreased: reference_counter_decreased + } + end end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4cec1145f3a..1729df2aad0 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -4,6 +4,8 @@ module API before { authenticate! } + helpers ::Gitlab::IssuableMetadata + helpers do def find_issues(args = {}) args = params.merge(args) @@ -13,6 +15,7 @@ module API args[:label_name] = args.delete(:labels) issues = IssuesFinder.new(current_user, args).execute + .preload(:assignees, :labels, :notes, :timelogs) issues.reorder(args[:order_by] => args[:sort]) end @@ -33,6 +36,7 @@ module API optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' + optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' use :pagination end @@ -65,14 +69,20 @@ module API get do issues = find_issues - present paginate(issues), with: Entities::IssueBasic, current_user: current_user + options = { + with: Entities::IssueBasic, + current_user: current_user, + issuable_metadata: issuable_meta_data(issues, 'Issue') + } + + present paginate(issues), options end end params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of group issues' do success Entities::IssueBasic end @@ -86,14 +96,20 @@ module API issues = find_issues(group_id: group.id) - present paginate(issues), with: Entities::IssueBasic, current_user: current_user + options = { + with: Entities::IssueBasic, + current_user: current_user, + issuable_metadata: issuable_meta_data(issues, 'Issue') + } + + present paginate(issues), options end end params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do include TimeTrackingEndpoints desc 'Get a list of project issues' do @@ -109,7 +125,14 @@ module API issues = find_issues(project_id: project.id) - present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project + options = { + with: Entities::IssueBasic, + current_user: current_user, + project: user_project, + issuable_metadata: issuable_meta_data(issues, 'Issue') + } + + present paginate(issues), options end desc 'Get a single project issue' do @@ -230,8 +253,8 @@ module API not_found!('Issue') unless issue authorize!(:destroy_issue, issue) - status 204 - issue.destroy + + destroy_conditionally!(issue) end desc 'List merge requests closing issue' do diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb new file mode 100644 index 00000000000..2a8fa7659bf --- /dev/null +++ b/lib/api/job_artifacts.rb @@ -0,0 +1,80 @@ +module API + class JobArtifacts < Grape::API + before { authenticate_non_get! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Download the artifacts file from a job' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the job' + end + get ':id/jobs/artifacts/:ref_name/download', + requirements: { ref_name: /.+/ } do + authorize_read_builds! + + builds = user_project.latest_successful_builds_for(params[:ref_name]) + latest_build = builds.find_by!(name: params[:job]) + + present_artifacts!(latest_build.artifacts_file) + end + + desc 'Download the artifacts file from a job' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + get ':id/jobs/:job_id/artifacts' do + authorize_read_builds! + + build = find_build!(params[:job_id]) + + present_artifacts!(build.artifacts_file) + end + + desc 'Download a specific file from artifacts archive' do + detail 'This feature was introduced in GitLab 10.0' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + requires :artifact_path, type: String, desc: 'Artifact path' + end + get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do + authorize_read_builds! + + build = find_build!(params[:job_id]) + not_found! unless build.artifacts? + + path = Gitlab::Ci::Build::Artifacts::Path + .new(params[:artifact_path]) + bad_request! unless path.valid? + + send_artifacts_entry(build, path) + end + + desc 'Keep the artifacts to prevent them from being deleted' do + success Entities::Job + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + post ':id/jobs/:job_id/artifacts/keep' do + authorize_update_builds! + + build = find_build!(params[:job_id]) + authorize!(:update_build, build) + return not_found!(build) unless build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: Entities::Job + end + end + end +end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 8a67de10bca..3c1c412ba42 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do params :optional_scope do optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', @@ -16,9 +16,9 @@ module API case scope when String [scope] - when Hashie::Mash + when ::Hash scope.values - when Hashie::Array + when ::Array scope else ['unknown'] @@ -66,42 +66,11 @@ module API get ':id/jobs/:job_id' do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) present build, with: Entities::Job end - desc 'Download the artifacts file from a job' do - detail 'This feature was introduced in GitLab 8.5' - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - get ':id/jobs/:job_id/artifacts' do - authorize_read_builds! - - build = get_build!(params[:job_id]) - - present_artifacts!(build.artifacts_file) - end - - desc 'Download the artifacts file from a job' do - detail 'This feature was introduced in GitLab 8.10' - end - params do - requires :ref_name, type: String, desc: 'The ref from repository' - requires :job, type: String, desc: 'The name for the job' - end - get ':id/jobs/artifacts/:ref_name/download', - requirements: { ref_name: /.+/ } do - authorize_read_builds! - - builds = user_project.latest_successful_builds_for(params[:ref_name]) - latest_build = builds.find_by!(name: params[:job]) - - present_artifacts!(latest_build.artifacts_file) - end - # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. @@ -112,7 +81,7 @@ module API get ':id/jobs/:job_id/trace' do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" content_type 'text/plain' @@ -131,7 +100,7 @@ module API post ':id/jobs/:job_id/cancel' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) build.cancel @@ -148,7 +117,7 @@ module API post ':id/jobs/:job_id/retry' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) return forbidden!('Job is not retryable') unless build.retryable? @@ -166,7 +135,7 @@ module API post ':id/jobs/:job_id/erase' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) return forbidden!('Job is not erasable!') unless build.erasable? @@ -174,25 +143,6 @@ module API present build, with: Entities::Job end - desc 'Keep the artifacts to prevent them from being deleted' do - success Entities::Job - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - post ':id/jobs/:job_id/artifacts/keep' do - authorize_update_builds! - - build = get_build!(params[:job_id]) - authorize!(:update_build, build) - return not_found!(build) unless build.artifacts? - - build.keep_artifacts! - - status 200 - present build, with: Entities::Job - end - desc 'Trigger a manual job' do success Entities::Job detail 'This feature was added in GitLab 8.11' @@ -203,7 +153,7 @@ module API post ":id/jobs/:job_id/play" do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) bad_request!("Unplayable Job") unless build.playable? @@ -216,14 +166,6 @@ module API end helpers do - def find_build(id) - user_project.builds.find_by(id: id.to_i) - end - - def get_build!(id) - find_build(id) || not_found! - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? @@ -234,14 +176,6 @@ module API builds.where(status: available_statuses && scope) end - - def authorize_read_builds! - authorize! :read_build, user_project - end - - def authorize_update_builds! - authorize! :update_build, user_project - end end end end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 4520c98d951..e41a1720ac1 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all labels of the project' do success Entities::Label end @@ -56,8 +56,7 @@ module API label = user_project.labels.find_by(title: params[:name]) not_found!('Label') unless label - status 204 - label.destroy + destroy_conditionally!(label) end desc 'Update an existing label. At least one optional parameter is required.' do diff --git a/lib/api/members.rb b/lib/api/members.rb index bb970b7cd54..22e4bdead41 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: { id: %r{[^/]+} } do + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Gets a list of group or project members viewable by the authenticated user.' do success Entities::Member end @@ -93,11 +93,11 @@ module API end delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) - # Ensure that memeber exists - source.members.find_by!(user_id: params[:user_id]) + member = source.members.find_by!(user_id: params[:user_id]) - status 204 - ::Members::DestroyService.new(source, current_user, declared_params).execute + destroy_conditionally!(member) do + ::Members::DestroyService.new(source, current_user, declared_params).execute + end end end end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 4b79eac2b8b..95ef8f42954 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -8,7 +8,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of merge request diff versions' do detail 'This feature was introduced in GitLab 8.12.' success Entities::MergeRequestDiff @@ -21,7 +21,7 @@ module API get ":id/merge_requests/:merge_request_iid/versions" do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present paginate(merge_request.merge_request_diffs), with: Entities::MergeRequestDiff + present paginate(merge_request.merge_request_diffs.order_id_desc), with: Entities::MergeRequestDiff end desc 'Get a single merge request diff version' do diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 8810d4e441d..56d72d511da 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -21,7 +21,7 @@ module API return merge_requests if args[:view] == 'simple' merge_requests - .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels) + .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels, :timelogs) end params :merge_requests_params do @@ -40,6 +40,7 @@ module API optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' + optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' use :pagination end end @@ -72,7 +73,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do include TimeTrackingEndpoints helpers do @@ -164,8 +165,8 @@ module API merge_request = find_project_merge_request(params[:merge_request_iid]) authorize!(:destroy_merge_request, merge_request) - status 204 - merge_request.destroy + + destroy_conditionally!(merge_request) end params do diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb index ef09d9505d2..c570eace862 100644 --- a/lib/api/milestone_responses.rb +++ b/lib/api/milestone_responses.rb @@ -28,7 +28,7 @@ module API end def list_milestones_for(parent) - milestones = parent.milestones + milestones = parent.milestones.order_id_desc milestones = Milestone.filter_by_state(milestones, params[:state]) milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present? milestones = filter_by_search(milestones, params[:search]) if params[:search] diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 65ff89edf65..d6e7203adaf 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do NOTEABLE_TYPES.each do |noteable_type| noteables_str = noteable_type.to_s.underscore.pluralize @@ -129,17 +129,19 @@ module API end delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do note = user_project.notes.find(params[:note_id]) + authorize! :admin_note, note - status 204 - ::Notes::DestroyService.new(user_project, current_user).execute(note) + destroy_conditionally!(note) do |note| + ::Notes::DestroyService.new(user_project, current_user).execute(note) + end end end end helpers do def find_project_noteable(noteables_str, noteable_id) - public_send("find_project_#{noteables_str.singularize}", noteable_id) + public_send("find_project_#{noteables_str.singularize}", noteable_id) # rubocop:disable GitlabSecurity/PublicSend end def noteable_read_ability_name(noteable) diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index 5d113c94b22..bcc0833aa5c 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -54,7 +54,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: { id: %r{[^/]+} } do + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "Get #{source_type} level notification level settings, defaults to Global" do detail 'This feature was introduced in GitLab 8.12' success Entities::NotificationSetting diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index dbeaf9e17ef..37f32411296 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all pipeline schedules' do success Entities::PipelineSchedule end @@ -31,10 +31,6 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end get ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule - present pipeline_schedule, with: Entities::PipelineScheduleDetails end @@ -74,9 +70,6 @@ module API optional :active, type: Boolean, desc: 'The activation of pipeline schedule' end put ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.update(declared_params(include_missing: false)) @@ -93,9 +86,6 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do - authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.own!(current_user) @@ -112,22 +102,84 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end delete ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :read_pipeline_schedule, user_project + authorize! :admin_pipeline_schedule, pipeline_schedule + + destroy_conditionally!(pipeline_schedule) + end + + desc 'Create a new pipeline schedule variable' do + success Entities::Variable + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :key, type: String, desc: 'The key of the variable' + requires :value, type: String, desc: 'The value of the variable' + end + post ':id/pipeline_schedules/:pipeline_schedule_id/variables' do + authorize! :update_pipeline_schedule, pipeline_schedule - not_found!('PipelineSchedule') unless pipeline_schedule + variable_params = declared_params(include_missing: false) + variable = pipeline_schedule.variables.create(variable_params) + if variable.persisted? + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end + end + + desc 'Edit a pipeline schedule variable' do + success Entities::Variable + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :key, type: String, desc: 'The key of the variable' + optional :value, type: String, desc: 'The value of the variable' + end + put ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do + authorize! :update_pipeline_schedule, pipeline_schedule + + if pipeline_schedule_variable.update(declared_params(include_missing: false)) + present pipeline_schedule_variable, with: Entities::Variable + else + render_validation_error!(pipeline_schedule_variable) + end + end + + desc 'Delete a pipeline schedule variable' do + success Entities::Variable + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :key, type: String, desc: 'The key of the variable' + end + delete ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do authorize! :admin_pipeline_schedule, pipeline_schedule status :accepted - present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails + present pipeline_schedule_variable.destroy, with: Entities::Variable end end helpers do def pipeline_schedule @pipeline_schedule ||= - user_project.pipeline_schedules - .preload(:owner, :last_pipeline) - .find_by(id: params.delete(:pipeline_schedule_id)) + user_project + .pipeline_schedules + .preload(:owner, :last_pipeline) + .find_by(id: params.delete(:pipeline_schedule_id)).tap do |pipeline_schedule| + unless can?(current_user, :read_pipeline_schedule, pipeline_schedule) + not_found!('Pipeline Schedule') + end + end + end + + def pipeline_schedule_variable + @pipeline_schedule_variable ||= + pipeline_schedule.variables.find_by(key: params[:key]).tap do |pipeline_schedule_variable| + unless pipeline_schedule_variable + not_found!('Pipeline Schedule Variable') + end + end end end end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index e505cae3992..74b3376a1f3 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all Pipelines of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::PipelineBasic diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 649dd891f56..86066e2b58f 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -24,7 +24,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get project hooks' do success Entities::ProjectHook end @@ -96,8 +96,7 @@ module API delete ":id/hooks/:hook_id" do hook = user_project.hooks.find(params.delete(:hook_id)) - status 204 - hook.destroy + destroy_conditionally!(hook) end end end diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 451998c726a..0cb209a02d0 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of project milestones' do success Entities::Milestone end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index f3d905b0068..2ccda1c1aa1 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? @@ -116,8 +116,8 @@ module API not_found!('Snippet') unless snippet authorize! :admin_project_snippet, snippet - status 204 - snippet.destroy + + destroy_conditionally!(snippet) end desc 'Get a raw project snippet' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 89dda88d3f5..7dc19788462 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -1,7 +1,6 @@ require_dependency 'declarative_policy' module API - # Projects API class Projects < Grape::API include PaginationParams @@ -17,6 +16,7 @@ module API optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' @@ -96,7 +96,7 @@ module API end end - resource :users, requirements: { user_id: %r{[^/]+} } do + resource :users, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a user projects' do success Entities::BasicProjectDetails end @@ -184,7 +184,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a single project' do success Entities::ProjectWithAccess end @@ -237,6 +237,7 @@ module API at_least_one_of_ce = [ :jobs_enabled, + :resolve_outdated_diff_discussions, :container_registry_enabled, :default_branch, :description, @@ -334,7 +335,10 @@ module API desc 'Remove a project' delete ":id" do authorize! :remove_project, user_project - ::Projects::DestroyService.new(user_project, current_user, {}).async_execute + + destroy_conditionally!(user_project) do + ::Projects::DestroyService.new(user_project, current_user, {}).async_execute + end accepted! end @@ -351,6 +355,8 @@ module API if user_project.forked_from_project.nil? user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) + + ::Projects::ForksCountService.new(forked_from_project).refresh_cache else render_api_error!("Project already forked", 409) end @@ -361,8 +367,7 @@ module API authorize! :remove_fork_project, user_project if user_project.forked? - status 204 - user_project.forked_project_link.destroy + destroy_conditionally!(user_project.forked_project_link) else not_modified! end @@ -406,8 +411,7 @@ module API link = user_project.project_group_links.find_by(group_id: params[:group_id]) not_found!('Group Link') unless link - status 204 - link.destroy + destroy_conditionally!(link) end desc 'Upload a file' diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index d742f2e18d0..15fcb9e8e27 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -61,7 +61,7 @@ module API service_args = [user_project, current_user, protected_branch_params] protected_branch = ::ProtectedBranches::CreateService.new(*service_args).execute - + if protected_branch.persisted? present protected_branch, with: Entities::ProtectedBranch, project: user_project else @@ -76,9 +76,7 @@ module API delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do protected_branch = user_project.protected_branches.find_by!(name: params[:name]) - protected_branch.destroy - - status 204 + destroy_conditionally!(protected_branch) end end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 14d2bff9cb5..2255fb1b70d 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 88fc62d33df..a3987c560dd 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -45,8 +45,10 @@ module API end delete '/' do authenticate_runner! - status 204 - Ci::Runner.find_by_token(params[:token]).destroy + + runner = Ci::Runner.find_by_token(params[:token]) + + destroy_conditionally!(runner) end desc 'Validates authentication credentials' do @@ -78,7 +80,7 @@ module API no_content! unless current_runner.active? update_runner_info - if current_runner.is_runner_queue_value_latest?(params[:last_update]) + if current_runner.runner_queue_value_latest?(params[:last_update]) header 'X-GitLab-Last-Update', params[:last_update] Gitlab::Metrics.add_event(:build_not_found_cached) return no_content! @@ -112,6 +114,8 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :trace, type: String, desc: %q(Job's full trace) optional :state, type: String, desc: %q(Job's status: success, failed) + optional :failure_reason, type: String, values: CommitStatus.failure_reasons.keys, + desc: %q(Job's failure_reason) end put '/:id' do job = authenticate_job! @@ -125,7 +129,7 @@ module API when 'success' job.success when 'failed' - job.drop + job.drop(params[:failure_reason] || :unknown_failure) end end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 5bf5a18e42f..d3559ef71be 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -55,7 +55,9 @@ module API optional :tag_list, type: Array[String], desc: 'The list of tags for a runner' optional :run_untagged, type: Boolean, desc: 'Flag indicating the runner can execute untagged jobs' optional :locked, type: Boolean, desc: 'Flag indicating the runner is locked' - at_least_one_of :description, :active, :tag_list, :run_untagged, :locked + optional :access_level, type: String, values: Ci::Runner.access_levels.keys, + desc: 'The access_level of the runner' + at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level end put ':id' do runner = get_runner(params.delete(:id)) @@ -77,17 +79,17 @@ module API end delete ':id' do runner = get_runner(params[:id]) + authenticate_delete_runner!(runner) - status 204 - runner.destroy! + destroy_conditionally!(runner) end end params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do before { authorize_admin_project } desc 'Get runners available for project' do @@ -135,8 +137,7 @@ module API runner = runner_project.runner forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1 - status 204 - runner_project.destroy + destroy_conditionally!(runner_project) end end @@ -153,7 +154,7 @@ module API render_api_error!('Scope contains invalid value', 400) end - runners.send(scope) + runners.public_send(scope) # rubocop:disable GitlabSecurity/PublicSend end def get_runner(id) diff --git a/lib/api/services.rb b/lib/api/services.rb index 843c05ae32e..2cbd0517dc3 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -601,7 +601,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do before { authenticate! } before { authorize_admin_project } @@ -656,12 +656,14 @@ module API delete ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) - attrs = service_attributes(service).inject({}) do |hash, key| - hash.merge!(key => nil) - end + destroy_conditionally!(service) do + attrs = service_attributes(service).inject({}) do |hash, key| + hash.merge!(key => nil) + end - unless service.update_attributes(attrs.merge(active: false)) - render_api_error!('400 Bad Request', 400) + unless service.update_attributes(attrs.merge(active: false)) + render_api_error!('400 Bad Request', 400) + end end end @@ -689,7 +691,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "Trigger a slash command for #{service_slug}" do detail 'Added in GitLab 8.13' end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index d55a61fa638..851b226e9e5 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -29,6 +29,7 @@ module API desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' + optional :project_export_enabled, type: Boolean, desc: 'Enable project export' optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' @@ -121,6 +122,13 @@ module API optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' + ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| + optional :"#{type}_key_restriction", + type: Integer, + values: KeyRestrictionValidator.supported_key_restrictions(type), + desc: "Restrictions on the complexity of uploaded #{type.upcase} keys. A value of #{ApplicationSetting::FORBIDDEN_KEY_VALUE} disables all #{type.upcase} keys." + end + optional(*::ApplicationSettingsHelper.visible_attributes) at_least_one_of(*::ApplicationSettingsHelper.visible_attributes) end diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 35ece56c65c..00eb7c60f16 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -123,8 +123,7 @@ module API authorize! :destroy_personal_snippet, snippet - status 204 - snippet.destroy + destroy_conditionally!(snippet) end desc 'Get a raw snippet' do diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 91567909998..b3e1e23031a 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -12,7 +12,7 @@ module API requires :id, type: String, desc: 'The ID of a project' requires :subscribable_id, type: String, desc: 'The ID of a resource' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do subscribable_types.each do |type, finder| type_singularized = type.singularize entity_class = Entities.const_get(type_singularized.camelcase) diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index c0179037440..6b6a03e3300 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -66,8 +66,7 @@ module API hook = SystemHook.find_by(id: params[:id]) not_found!('System hook') unless hook - status 204 - hook.destroy + destroy_conditionally!(hook) end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 1333747cced..912415e3a7f 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -65,11 +65,18 @@ module API delete ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do authorize_push_project - result = ::Tags::DestroyService.new(user_project, current_user) - .execute(params[:tag_name]) + tag = user_project.repository.find_tag(params[:tag_name]) + not_found!('Tag') unless tag + + commit = user_project.repository.commit(tag.dereferenced_target) + + destroy_conditionally!(commit, last_updated: commit.authored_date) do + result = ::Tags::DestroyService.new(user_project, current_user) + .execute(params[:tag_name]) - if result[:status] != :success - render_api_error!(result[:message], result[:return_code]) + if result[:status] != :success + render_api_error!(result[:message], result[:return_code]) + end end end diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 0fc13b35d5b..f70bc0622b7 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -57,7 +57,7 @@ module API end get "templates/licenses" do options = { - featured: declared(params).popular.present? ? true : nil + featured: declared(params)[:popular].present? ? true : nil } licences = ::Kaminari.paginate_array(Licensee::License.all(options)) present paginate(licences), with: Entities::RepoLicense @@ -71,7 +71,7 @@ module API requires :name, type: String, desc: 'The name of the template' end get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ } do - not_found!('License') unless Licensee::License.find(declared(params).name) + not_found!('License') unless Licensee::License.find(declared(params)[:name]) template = parsed_license_template @@ -102,7 +102,7 @@ module API requires :name, type: String, desc: 'The name of the template' end get "templates/#{template_type}/:name" do - new_template = klass.find(declared(params).name) + new_template = klass.find(declared(params)[:name]) render_response(template_type, new_template) end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 55191169dd4..ffccfebe752 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do ISSUABLE_TYPES.each do |type, finder| type_id_str = "#{type.singularize}_iid".to_sym diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index edfdb63d183..dd6801664b1 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -5,7 +5,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Trigger a GitLab project pipeline' do success Entities::Pipeline end @@ -140,8 +140,7 @@ module API trigger = user_project.triggers.find(params.delete(:trigger_id)) return not_found!('Trigger') unless trigger - status 204 - trigger.destroy + destroy_conditionally!(trigger) end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index a590f2692a2..1825c90a23b 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -79,22 +79,17 @@ module API end desc 'Get a single user' do - success Entities::UserBasic + success Entities::User end params do requires :id, type: Integer, desc: 'The ID of the user' end get ":id" do user = User.find_by(id: params[:id]) - not_found!('User') unless user + not_found!('User') unless user && can?(current_user, :read_user, user) - if current_user && current_user.admin? - present user, with: Entities::UserPublic - elsif can?(current_user, :read_user, user) - present user, with: Entities::User - else - render_api_error!("User not found.", 404) - end + opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : {} + present user, opts end desc 'Create a user. Available only for admins.' do @@ -235,10 +230,89 @@ module API key = user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key + destroy_conditionally!(key) + end + + desc 'Add a GPG key to a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key, type: String, desc: 'The new GPG key' + end + post ':id/gpg_keys' do + authenticated_as_admin! + + user = User.find_by(id: params.delete(:id)) + not_found!('User') unless user + + key = user.gpg_keys.new(declared_params(include_missing: false)) + + if key.save + present key, with: Entities::GPGKey + else + render_validation_error!(key) + end + end + + desc 'Get the GPG keys of a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/gpg_keys' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + present paginate(user.gpg_keys), with: Entities::GPGKey + end + + desc 'Delete an existing GPG key from a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + delete ':id/gpg_keys/:key_id' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + key = user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + status 204 key.destroy end + desc 'Revokes an existing GPG key from a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + post ':id/gpg_keys/:key_id/revoke' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + key = user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + key.revoke + status :accepted + end + desc 'Add an email address to a specified user. Available only for admins.' do success Entities::Email end @@ -292,7 +366,11 @@ module API email = user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email - Emails::DestroyService.new(user, email: email.email).execute + destroy_conditionally!(email) do |email| + Emails::DestroyService.new(current_user, email: email.email).execute + end + + user.update_secondary_emails! end desc 'Delete a user. Available only for admins.' do @@ -304,11 +382,13 @@ module API end delete ":id" do authenticated_as_admin! + user = User.find_by(id: params[:id]) not_found!('User') unless user - status 204 - user.delete_async(deleted_by: current_user, params: params) + destroy_conditionally!(user) do + user.delete_async(deleted_by: current_user, params: params) + end end desc 'Block a user. Available only for admins.' @@ -408,8 +488,11 @@ module API requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token' end delete ':impersonation_token_id' do - status 204 - find_impersonation_token.revoke! + token = find_impersonation_token + + destroy_conditionally!(token) do + token.revoke! + end end end end @@ -486,6 +569,75 @@ module API key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key + destroy_conditionally!(key) + end + + desc "Get the currently authenticated user's GPG keys" do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + use :pagination + end + get 'gpg_keys' do + present paginate(current_user.gpg_keys), with: Entities::GPGKey + end + + desc 'Get a single GPG key owned by currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + get 'gpg_keys/:key_id' do + key = current_user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + present key, with: Entities::GPGKey + end + + desc 'Add a new GPG key to the currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :key, type: String, desc: 'The new GPG key' + end + post 'gpg_keys' do + key = current_user.gpg_keys.new(declared_params) + + if key.save + present key, with: Entities::GPGKey + else + render_validation_error!(key) + end + end + + desc 'Revoke a GPG key owned by currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + post 'gpg_keys/:key_id/revoke' do + key = current_user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + key.revoke + status :accepted + end + + desc 'Delete a GPG key from the currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :key_id, type: Integer, desc: 'The ID of the SSH key' + end + delete 'gpg_keys/:key_id' do + key = current_user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + status 204 key.destroy end @@ -538,8 +690,11 @@ module API email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email - status 204 - Emails::DestroyService.new(current_user, email: email.email).execute + destroy_conditionally!(email) do |email| + Emails::DestroyService.new(current_user, email: email.email).execute + end + + current_user.update_secondary_emails! end desc 'Get a list of user activities' diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 93ad9eb26b8..c189d486f50 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -16,7 +16,7 @@ module API coerce_with: ->(scope) { if scope.is_a?(String) [scope] - elsif scope.is_a?(Hashie::Mash) + elsif scope.is_a?(::Hash) scope.values else ['unknown'] diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 773f667abe0..c928ce5265b 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -25,14 +25,24 @@ module API expose(:downvote?) { |note| false } end + class PushEventPayload < Grape::Entity + expose :commit_count, :action, :ref_type, :commit_from, :commit_to + expose :ref, :commit_title + end + class Event < Grape::Entity - expose :title, :project_id, :action_name + expose :project_id, :action_name expose :target_id, :target_type, :author_id - expose :data, :target_title + expose :target_title expose :created_at expose :note, using: Entities::Note, if: ->(event, options) { event.note? } expose :author, using: ::API::Entities::UserBasic, if: ->(event, options) { event.author } + expose :push_event_payload, + as: :push_data, + using: PushEventPayload, + if: -> (event, _) { event.push? } + expose :author_username do |event, options| event.author&.username end @@ -54,6 +64,7 @@ module API expose :owner, using: ::API::Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace + expose :resolve_outdated_diff_discussions expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible @@ -68,7 +79,7 @@ module API expose :lfs_enabled?, as: :lfs_enabled expose :creator_id expose :namespace, using: 'API::Entities::Namespace' - expose :forked_from_project, using: ::API::Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } + expose :forked_from_project, using: ::API::Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :avatar_url do |user, options| user.avatar_url(only_path: false) end diff --git a/lib/api/v3/merge_request_diffs.rb b/lib/api/v3/merge_request_diffs.rb index 35f462e907b..22866fc2845 100644 --- a/lib/api/v3/merge_request_diffs.rb +++ b/lib/api/v3/merge_request_diffs.rb @@ -20,7 +20,7 @@ module API get ":id/merge_requests/:merge_request_id/versions" do merge_request = find_merge_request_with_access(params[:merge_request_id]) - present merge_request.merge_request_diffs, with: ::API::Entities::MergeRequestDiff + present merge_request.merge_request_diffs.order_id_desc, with: ::API::Entities::MergeRequestDiff end desc 'Get a single merge request diff version' do diff --git a/lib/api/v3/milestones.rb b/lib/api/v3/milestones.rb index 4c7061d4939..9be4cf9d22a 100644 --- a/lib/api/v3/milestones.rb +++ b/lib/api/v3/milestones.rb @@ -34,6 +34,7 @@ module API milestones = user_project.milestones milestones = filter_milestones_state(milestones, params[:state]) milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present? + milestones = milestones.order_id_desc present paginate(milestones), with: ::API::Entities::Milestone end diff --git a/lib/api/v3/notes.rb b/lib/api/v3/notes.rb index 23fe95e42e4..d49772b92f2 100644 --- a/lib/api/v3/notes.rb +++ b/lib/api/v3/notes.rb @@ -22,7 +22,7 @@ module API use :pagination end get ":id/#{noteables_str}/:noteable_id/notes" do - noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + noteable = user_project.public_send(noteables_str.to_sym).find(params[:noteable_id]) # rubocop:disable GitlabSecurity/PublicSend if can?(current_user, noteable_read_ability_name(noteable), noteable) # We exclude notes that are cross-references and that cannot be viewed @@ -50,7 +50,7 @@ module API requires :noteable_id, type: Integer, desc: 'The ID of the noteable' end get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + noteable = user_project.public_send(noteables_str.to_sym).find(params[:noteable_id]) # rubocop:disable GitlabSecurity/PublicSend note = noteable.notes.find(params[:note_id]) can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) @@ -76,7 +76,7 @@ module API noteable_id: params[:noteable_id] } - noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + noteable = user_project.public_send(noteables_str.to_sym).find(params[:noteable_id]) # rubocop:disable GitlabSecurity/PublicSend if can?(current_user, noteable_read_ability_name(noteable), noteable) if params[:created_at] && (current_user.admin? || user_project.owner == current_user) diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index eb090453b48..7c260b8d910 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -18,6 +18,7 @@ module API optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' optional :public, type: Boolean, desc: 'Create a public project. The same as visibility_level = 20.' @@ -119,7 +120,7 @@ module API get do authenticate! - present_projects current_user.authorized_projects, + present_projects current_user.authorized_projects.order_id_desc, with: ::API::V3::Entities::ProjectWithAccess end @@ -296,9 +297,9 @@ module API use :optional_params at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, - :shared_runners_enabled, :container_registry_enabled, - :lfs_enabled, :public, :visibility_level, :public_builds, - :request_access_enabled, :only_allow_merge_if_build_succeeds, + :shared_runners_enabled, :resolve_outdated_diff_discussions, + :container_registry_enabled, :lfs_enabled, :public, :visibility_level, + :public_builds, :request_access_enabled, :only_allow_merge_if_build_succeeds, :only_allow_merge_if_all_discussions_are_resolved, :path, :default_branch end @@ -388,6 +389,8 @@ module API if user_project.forked_from_project.nil? user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) + + ::Projects::ForksCountService.new(forked_from_project).refresh_cache else render_api_error!("Project already forked", 409) end diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb index 4c577a8d2b7..2a2fb59045c 100644 --- a/lib/api/v3/templates.rb +++ b/lib/api/v3/templates.rb @@ -59,7 +59,7 @@ module API end get route do options = { - featured: declared(params).popular.present? ? true : nil + featured: declared(params)[:popular].present? ? true : nil } present Licensee::License.all(options), with: ::API::Entities::RepoLicense end @@ -76,7 +76,7 @@ module API requires :name, type: String, desc: 'The name of the template' end get route, requirements: { name: /[\w\.-]+/ } do - not_found!('License') unless Licensee::License.find(declared(params).name) + not_found!('License') unless Licensee::License.find(declared(params)[:name]) template = parsed_license_template @@ -111,7 +111,7 @@ module API requires :name, type: String, desc: 'The name of the template' end get route do - new_template = klass.find(declared(params).name) + new_template = klass.find(declared(params)[:name]) render_response(template_type, new_template) end diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index e9d4c35307b..534911fde5c 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -16,25 +16,31 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/builds", requirements: { ref: /.+/ } do - project = find_project(params[:id]) - trigger = Ci::Trigger.find_by_token(params[:token].to_s) - not_found! unless project && trigger - unauthorized! unless trigger.project == project - # validate variables - variables = params[:variables].to_h - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + params[:variables] = params[:variables].to_h + unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } render_api_error!('variables needs to be a map of key-valued strings', 400) end - # create request and trigger builds - result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref].to_s, variables) - pipeline = result.pipeline + project = find_project(params[:id]) + not_found! unless project + + result = Ci::PipelineTriggerService.new(project, nil, params).execute + not_found! unless result - if pipeline.persisted? - present result.trigger_request, with: ::API::V3::Entities::TriggerRequest + if result[:http_status] + render_api_error!(result[:message], result[:http_status]) else - render_validation_error!(pipeline) + pipeline = result[:pipeline] + + # We switched to Ci::PipelineVariable from Ci::TriggerRequest.variables. + # Ci::TriggerRequest doesn't save variables anymore. + # Here is copying Ci::PipelineVariable to Ci::TriggerRequest.variables for presenting the variables. + # The same endpoint in v4 API pressents Pipeline instead of TriggerRequest, so it doesn't need such a process. + trigger_request = pipeline.trigger_requests.last + trigger_request.variables = params[:variables] + + present trigger_request, with: ::API::V3::Entities::TriggerRequest end end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 7c0fdd3d1be..d08876ae1b9 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -9,7 +9,7 @@ module API requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get project variables' do success Entities::Variable end @@ -88,6 +88,7 @@ module API variable = user_project.variables.find_by(key: params[:key]) not_found!('Variable') unless variable + # Variables don't have any timestamp. Therfore, destroy unconditionally. status 204 variable.destroy end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb new file mode 100644 index 00000000000..b3fc4e876ad --- /dev/null +++ b/lib/api/wikis.rb @@ -0,0 +1,89 @@ +module API + class Wikis < Grape::API + helpers do + params :wiki_page_params do + requires :content, type: String, desc: 'Content of a wiki page' + requires :title, type: String, desc: 'Title of a wiki page' + optional :format, + type: String, + values: ProjectWiki::MARKUPS.values.map(&:to_s), + default: 'markdown', + desc: 'Format of a wiki page. Available formats are markdown, rdoc, and asciidoc' + end + end + + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Get a list of wiki pages' do + success Entities::WikiPageBasic + end + params do + optional :with_content, type: Boolean, default: false, desc: "Include pages' content" + end + get ':id/wikis' do + authorize! :read_wiki, user_project + + entity = params[:with_content] ? Entities::WikiPage : Entities::WikiPageBasic + present user_project.wiki.pages, with: entity + end + + desc 'Get a wiki page' do + success Entities::WikiPage + end + params do + requires :slug, type: String, desc: 'The slug of a wiki page' + end + get ':id/wikis/:slug' do + authorize! :read_wiki, user_project + + present wiki_page, with: Entities::WikiPage + end + + desc 'Create a wiki page' do + success Entities::WikiPage + end + params do + use :wiki_page_params + end + post ':id/wikis' do + authorize! :create_wiki, user_project + + page = WikiPages::CreateService.new(user_project, current_user, params).execute + + if page.valid? + present page, with: Entities::WikiPage + else + render_validation_error!(page) + end + end + + desc 'Update a wiki page' do + success Entities::WikiPage + end + params do + use :wiki_page_params + end + put ':id/wikis/:slug' do + authorize! :create_wiki, user_project + + page = WikiPages::UpdateService.new(user_project, current_user, params).execute(wiki_page) + + if page.valid? + present page, with: Entities::WikiPage + else + render_validation_error!(page) + end + end + + desc 'Delete a wiki page' + params do + requires :slug, type: String, desc: 'The slug of a wiki page' + end + delete ':id/wikis/:slug' do + authorize! :admin_wiki, user_project + + status 204 + WikiPages::DestroyService.new(user_project, current_user).execute(wiki_page) + end + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index ca6d6848d41..b9a573d3542 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -198,11 +198,11 @@ module Backup end def archives_to_backup - ARCHIVES_TO_BACKUP.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + ARCHIVES_TO_BACKUP.map { |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup - FOLDERS_TO_BACKUP.reject{ |name| skipped?(name) } + FOLDERS_TO_BACKUP.reject { |name| skipped?(name) } end def disabled_features diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 88821ae56e0..4e92be85110 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -75,7 +75,7 @@ module Backup path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) - project.ensure_storage_path_exist + project.ensure_storage_path_exists cmd = if File.exist?(path_to_project_bundle) %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) diff --git a/lib/banzai/commit_renderer.rb b/lib/banzai/commit_renderer.rb new file mode 100644 index 00000000000..f5ff95e3eb3 --- /dev/null +++ b/lib/banzai/commit_renderer.rb @@ -0,0 +1,11 @@ +module Banzai + module CommitRenderer + ATTRIBUTES = [:description, :title].freeze + + def self.render(commits, project, user = nil) + obj_renderer = ObjectRenderer.new(project, user) + + ATTRIBUTES.each { |attr| obj_renderer.render(commits, attr) } + end + end +end diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index 53a229256a5..ed01a72ff9f 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -95,10 +95,10 @@ module Banzai private def external_issues_cached(attribute) - return project.public_send(attribute) unless RequestStore.active? + return project.public_send(attribute) unless RequestStore.active? # rubocop:disable GitlabSecurity/PublicSend cached_attributes = RequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } - cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? + cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? # rubocop:disable GitlabSecurity/PublicSend cached_attributes[project.id][attribute] end end diff --git a/lib/banzai/filter/image_lazy_load_filter.rb b/lib/banzai/filter/image_lazy_load_filter.rb index 7a81d583b82..bcb4f332267 100644 --- a/lib/banzai/filter/image_lazy_load_filter.rb +++ b/lib/banzai/filter/image_lazy_load_filter.rb @@ -6,9 +6,9 @@ module Banzai doc.xpath('descendant-or-self::img').each do |img| img['class'] ||= '' << 'lazy' img['data-src'] = img['src'] - img['src'] = LazyImageTagHelper.placeholder_image + img['src'] = LazyImageTagHelper.placeholder_image end - + doc end end diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 8e7084f2543..47151626208 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -22,40 +22,94 @@ module Banzai result[:toc] = "" headers = Hash.new(0) + header_root = current_header = HeaderNode.new doc.css('h1, h2, h3, h4, h5, h6').each do |node| - text = node.text + if header_content = node.children.first + id = node + .text + .downcase + .gsub(PUNCTUATION_REGEXP, '') # remove punctuation + .tr(' ', '-') # replace spaces with dash + .squeeze('-') # replace multiple dashes with one - id = text.downcase - id.gsub!(PUNCTUATION_REGEXP, '') # remove punctuation - id.tr!(' ', '-') # replace spaces with dash - id.squeeze!('-') # replace multiple dashes with one + uniq = headers[id] > 0 ? "-#{headers[id]}" : '' + headers[id] += 1 + href = "#{id}#{uniq}" - uniq = (headers[id] > 0) ? "-#{headers[id]}" : '' - headers[id] += 1 + current_header = HeaderNode.new(node: node, href: href, previous_header: current_header) - if header_content = node.children.first - # namespace detection will be automatically handled via javascript (see issue #22781) - namespace = "user-content-" - href = "#{id}#{uniq}" - push_toc(href, text) - header_content.add_previous_sibling(anchor_tag("#{namespace}#{href}", href)) + header_content.add_previous_sibling(anchor_tag(href)) end end - result[:toc] = %Q{<ul class="section-nav">\n#{result[:toc]}</ul>} unless result[:toc].empty? + push_toc(header_root.children, root: true) doc end private - def anchor_tag(id, href) - %Q{<a id="#{id}" class="anchor" href="##{href}" aria-hidden="true"></a>} + def anchor_tag(href) + %Q{<a id="user-content-#{href}" class="anchor" href="##{href}" aria-hidden="true"></a>} end - def push_toc(href, text) - result[:toc] << %Q{<li><a href="##{href}">#{text}</a></li>\n} + def push_toc(children, root: false) + return if children.empty? + + klass = ' class="section-nav"' if root + + result[:toc] << "<ul#{klass}>" + children.each { |child| push_anchor(child) } + result[:toc] << '</ul>' + end + + def push_anchor(header_node) + result[:toc] << %Q{<li><a href="##{header_node.href}">#{header_node.text}</a>} + push_toc(header_node.children) + result[:toc] << '</li>' + end + + class HeaderNode + attr_reader :node, :href, :parent, :children + + def initialize(node: nil, href: nil, previous_header: nil) + @node = node + @href = href + @children = [] + + @parent = find_parent(previous_header) + @parent.children.push(self) if @parent + end + + def level + return 0 unless node + + @level ||= node.name[1].to_i + end + + def text + return '' unless node + + @text ||= node.text + end + + private + + def find_parent(previous_header) + return unless previous_header + + if level == previous_header.level + parent = previous_header.parent + elsif level > previous_header.level + parent = previous_header + else + parent = previous_header + parent = parent.parent while parent.level >= level + end + + parent + end end end end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 002a3341ccd..e40556e869c 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -37,8 +37,8 @@ module Banzai objects.each_with_index do |object, index| redacted_data = redacted[index] - object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) - object.user_visible_reference_count = redacted_data[:visible_reference_count] + object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) # rubocop:disable GitlabSecurity/PublicSend + object.user_visible_reference_count = redacted_data[:visible_reference_count] if object.respond_to?(:user_visible_reference_count) end end diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb index 321fd5bbe14..3ae3bed570d 100644 --- a/lib/banzai/pipeline/base_pipeline.rb +++ b/lib/banzai/pipeline/base_pipeline.rb @@ -18,7 +18,7 @@ module Banzai define_method(meth) do |text, context| context = transform_context(context) - html_pipeline.send(meth, text, context) + html_pipeline.__send__(meth, text, context) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index ad08c0905e2..ceca9296851 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -36,6 +36,10 @@ module Banzai # The context to use is managed by the object and cannot be changed. # Use #render, passing it the field text, if a custom rendering is needed. def self.render_field(object, field) + unless object.respond_to?(:cached_markdown_fields) + return cacheless_render_field(object, field) + end + object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) object.cached_html_for(field) @@ -43,7 +47,7 @@ module Banzai # Same as +render_field+, but without consulting or updating the cache field def self.cacheless_render_field(object, field, options = {}) - text = object.__send__(field) + text = object.__send__(field) # rubocop:disable GitlabSecurity/PublicSend context = object.banzai_render_context(field).merge(options) cacheless_render(text, context) @@ -156,7 +160,7 @@ module Banzai # method. def self.full_cache_multi_key(cache_key, pipeline_name) return unless cache_key - Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) + Rails.cache.__send__(:expanded_key, full_cache_key(cache_key, pipeline_name)) # rubocop:disable GitlabSecurity/PublicSend end # GitLab EE needs to disable updates on GET requests in Geo diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb index 3a9379ff680..a78495dbf5e 100644 --- a/lib/bitbucket/collection.rb +++ b/lib/bitbucket/collection.rb @@ -13,7 +13,7 @@ module Bitbucket def method_missing(method, *args) return super unless self.respond_to?(method) - self.send(method, *args) do |item| + self.__send__(method, *args) do |item| # rubocop:disable GitlabSecurity/PublicSend block_given? ? yield(item) : item end end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index 55402101e43..b9e9f9f7f4a 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -208,7 +208,7 @@ module Ci return unless command = stack.shift() if self.respond_to?("on_#{command}", true) - self.send("on_#{command}", stack) + self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend end evaluate_command_stack(stack) @@ -254,7 +254,7 @@ module Ci def state state = STATE_PARAMS.inject({}) do |h, param| - h[param] = send(param) + h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend h end Base64.urlsafe_encode64(state.to_json) @@ -266,7 +266,7 @@ module Ci return if state[:offset].to_i > stream.size STATE_PARAMS.each do |param| - send("#{param}=".to_sym, state[param]) + send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb deleted file mode 100644 index 24bb3649a76..00000000000 --- a/lib/ci/api/api.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Ci - module API - class API < Grape::API - include ::API::APIGuard - version 'v1', using: :path - - rescue_from ActiveRecord::RecordNotFound do - rack_response({ 'message' => '404 Not found' }.to_json, 404) - end - - # Retain 405 error rather than a 500 error for Grape 0.15.0+. - # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes - rescue_from Grape::Exceptions::MethodNotAllowed do |e| - error! e.message, e.status, e.headers - end - - rescue_from Grape::Exceptions::Base do |e| - error! e.message, e.status, e.headers - end - - rescue_from :all do |exception| - handle_api_exception(exception) - end - - content_type :txt, 'text/plain' - content_type :json, 'application/json' - format :json - - helpers ::SentryHelper - helpers ::Ci::API::Helpers - helpers ::API::Helpers - helpers Gitlab::CurrentSettings - - mount ::Ci::API::Builds - mount ::Ci::API::Runners - mount ::Ci::API::Triggers - end - end -end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb deleted file mode 100644 index 812ba8ed603..00000000000 --- a/lib/ci/api/builds.rb +++ /dev/null @@ -1,219 +0,0 @@ -module Ci - module API - # Builds API - class Builds < Grape::API - resource :builds do - # Runs oldest pending build by runner - Runners only - # - # Parameters: - # token (required) - The uniq token of runner - # - # Example Request: - # POST /builds/register - post "register" do - authenticate_runner! - required_attributes! [:token] - not_found! unless current_runner.active? - update_runner_info - - if current_runner.is_runner_queue_value_latest?(params[:last_update]) - header 'X-GitLab-Last-Update', params[:last_update] - Gitlab::Metrics.add_event(:build_not_found_cached) - return build_not_found! - end - - new_update = current_runner.ensure_runner_queue_value - - result = Ci::RegisterJobService.new(current_runner).execute - - if result.valid? - if result.build - Gitlab::Metrics.add_event(:build_found, - project: result.build.project.full_path) - - present result.build, with: Entities::BuildDetails - else - Gitlab::Metrics.add_event(:build_not_found) - - header 'X-GitLab-Last-Update', new_update - - build_not_found! - end - else - # We received build that is invalid due to concurrency conflict - Gitlab::Metrics.add_event(:build_invalid) - conflict! - end - end - - # Update an existing build - Runners only - # - # Parameters: - # id (required) - The ID of a project - # state (optional) - The state of a build - # trace (optional) - The trace of a build - # Example Request: - # PUT /builds/:id - put ":id" do - authenticate_runner! - build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) - validate_build!(build) - - update_runner_info - - build.trace.set(params[:trace]) if params[:trace] - - Gitlab::Metrics.add_event(:update_build, - project: build.project.full_path) - - case params[:state].to_s - when 'success' - build.success - when 'failed' - build.drop - end - end - - # Send incremental log update - Runners only - # - # Parameters: - # id (required) - The ID of a build - # Body: - # content of logs to append - # Headers: - # Content-Range (required) - range of content that was sent - # BUILD-TOKEN (required) - The build authorization token - # Example Request: - # PATCH /builds/:id/trace.txt - patch ":id/trace.txt" do - build = authenticate_build! - - error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') - content_range = request.headers['Content-Range'] - content_range = content_range.split('-') - - stream_size = build.trace.append(request.body.read, content_range[0].to_i) - if stream_size < 0 - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) - end - - status 202 - header 'Build-Status', build.status - header 'Range', "0-#{stream_size}" - end - - # Authorize artifacts uploading for build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # filesize (optional) - the size of uploaded file - # Example Request: - # POST /builds/:id/artifacts/authorize - post ":id/artifacts/authorize" do - require_gitlab_workhorse! - Gitlab::Workhorse.verify_api_request!(headers) - not_allowed! unless Gitlab.config.artifacts.enabled - build = authenticate_build! - forbidden!('build is not running') unless build.running? - - if params[:filesize] - file_size = params[:filesize].to_i - file_to_large! unless file_size < max_artifacts_size - end - - status 200 - content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - Gitlab::Workhorse.artifact_upload_ok - end - - # Upload artifacts to build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # file (required) - Artifacts file - # expire_in (optional) - Specify when artifacts should expire (ex. 7d) - # Parameters (accelerated by GitLab Workhorse): - # file.path - path to locally stored body (generated by Workhorse) - # file.name - real filename as send in Content-Disposition - # file.type - real content type as send in Content-Type - # metadata.path - path to locally stored body (generated by Workhorse) - # metadata.name - filename (generated by Workhorse) - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Body: - # The file content - # - # Example Request: - # POST /builds/:id/artifacts - post ":id/artifacts" do - require_gitlab_workhorse! - not_allowed! unless Gitlab.config.artifacts.enabled - build = authenticate_build! - forbidden!('Build is not running!') unless build.running? - - artifacts_upload_path = ArtifactUploader.artifacts_upload_path - artifacts = uploaded_file(:file, artifacts_upload_path) - metadata = uploaded_file(:metadata, artifacts_upload_path) - - bad_request!('Missing artifacts file!') unless artifacts - file_to_large! unless artifacts.size < max_artifacts_size - - build.artifacts_file = artifacts - build.artifacts_metadata = metadata - build.artifacts_expire_in = - params['expire_in'] || - Gitlab::CurrentSettings.current_application_settings - .default_artifacts_expire_in - - if build.save - present(build, with: Entities::BuildDetails) - else - render_validation_error!(build) - end - end - - # Download the artifacts file from build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Example Request: - # GET /builds/:id/artifacts - get ":id/artifacts" do - build = authenticate_build! - artifacts_file = build.artifacts_file - - unless artifacts_file.exists? - not_found! - end - - unless artifacts_file.file_storage? - return redirect(build.artifacts_file.url) - end - - present_file!(artifacts_file.path, artifacts_file.filename) - end - - # Remove the artifacts file from build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Example Request: - # DELETE /builds/:id/artifacts - delete ":id/artifacts" do - build = authenticate_build! - - status(200) - build.erase_artifacts! - end - end - end - end -end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb deleted file mode 100644 index 31f66dd5a58..00000000000 --- a/lib/ci/api/entities.rb +++ /dev/null @@ -1,93 +0,0 @@ -module Ci - module API - module Entities - class Commit < Grape::Entity - expose :id, :sha, :project_id, :created_at - expose :status, :finished_at, :duration - expose :git_commit_message, :git_author_name, :git_author_email - end - - class CommitWithBuilds < Commit - expose :builds - end - - class ArtifactFile < Grape::Entity - expose :filename, :size - end - - class BuildOptions < Grape::Entity - expose :image - expose :services - expose :artifacts - expose :cache - expose :dependencies - expose :after_script - end - - class Build < Grape::Entity - expose :id, :ref, :tag, :sha, :status - expose :name, :token, :stage - expose :project_id - expose :project_name - expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } - end - - class BuildCredentials < Grape::Entity - expose :type, :url, :username, :password - end - - class BuildDetails < Build - expose :commands - expose :repo_url - expose :before_sha - expose :allow_git_fetch - expose :token - expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } - - expose :options do |model| - # This part ensures that output of old API is still the same after adding support - # for extended docker configuration options, used by new API - # - # I'm leaving this here, not in the model, because it should be removed at the same time - # when old API will be removed (planned for August 2017). - model.options.dup.tap do |options| - options[:image] = options[:image][:name] if options[:image].is_a?(Hash) - options[:services]&.map! do |service| - if service.is_a?(Hash) - service[:name] - else - service - end - end - end - end - - expose :timeout do |model| - model.timeout - end - - expose :variables - expose :depends_on_builds, using: Build - - expose :credentials, using: BuildCredentials - end - - class Runner < Grape::Entity - expose :id, :token - end - - class RunnerProject < Grape::Entity - expose :id, :project_id, :runner_id - end - - class WebHook < Grape::Entity - expose :id, :project_id, :url - end - - class TriggerRequest < Grape::Entity - expose :id, :variables - expose :pipeline, using: Commit, as: :commit - end - end - end -end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb deleted file mode 100644 index a40b6ab6c9f..00000000000 --- a/lib/ci/api/helpers.rb +++ /dev/null @@ -1,89 +0,0 @@ -module Ci - module API - module Helpers - BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN".freeze - BUILD_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 10 * 60 - - def authenticate_runners! - forbidden! unless runner_registration_token_valid? - end - - def authenticate_runner! - forbidden! unless current_runner - end - - def authenticate_build! - build = Ci::Build.find_by_id(params[:id]) - - validate_build!(build) do - forbidden! unless build_token_valid?(build) - end - - build - end - - def validate_build!(build) - not_found! unless build - - yield if block_given? - - project = build.project - forbidden!('Project has been deleted!') if project.nil? || project.pending_delete? - forbidden!('Build has been erased!') if build.erased? - end - - def runner_registration_token_valid? - ActiveSupport::SecurityUtils.variable_size_secure_compare( - params[:token], - current_application_settings.runners_registration_token) - end - - def build_token_valid?(build) - token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s - - # We require to also check `runners_token` to maintain compatibility with old version of runners - token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) - end - - def update_runner_info - return unless update_runner? - - current_runner.contacted_at = Time.now - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - - def update_runner? - # Use a random threshold to prevent beating DB updates. - # It generates a distribution between [40m, 80m]. - # - contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - - current_runner.contacted_at.nil? || - (Time.now - current_runner.contacted_at) >= contacted_at_max_age - end - - def build_not_found! - if headers['User-Agent'].to_s =~ /gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? / - no_content! - else - not_found! - end - end - - def current_runner - @runner ||= Runner.find_by_token(params[:token].to_s) - end - - def get_runner_version_from_params - return unless params["info"].present? - attributes_for_keys(%w(name version revision platform architecture), params["info"]) - end - - def max_artifacts_size - current_application_settings.max_artifacts_size.megabytes.to_i - end - end - end -end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb deleted file mode 100644 index 45aa2adccf5..00000000000 --- a/lib/ci/api/runners.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Ci - module API - class Runners < Grape::API - resource :runners do - desc 'Delete a runner' - params do - requires :token, type: String, desc: 'The unique token of the runner' - end - delete "delete" do - authenticate_runner! - - status(200) - Ci::Runner.find_by_token(params[:token]).destroy - end - - desc 'Register a new runner' do - success Entities::Runner - end - params do - requires :token, type: String, desc: 'The unique token of the runner' - optional :description, type: String, desc: 'The description of the runner' - optional :tag_list, type: Array[String], desc: 'A list of tags the runner should run for' - optional :run_untagged, type: Boolean, desc: 'Flag if the runner should execute untagged jobs' - optional :locked, type: Boolean, desc: 'Lock this runner for this specific project' - end - post "register" do - runner_params = declared(params, include_missing: false).except(:token) - - runner = - if runner_registration_token_valid? - # Create shared runner. Requires admin access - Ci::Runner.create(runner_params.merge(is_shared: true)) - elsif project = Project.find_by(runners_token: params[:token]) - # Create a specific runner for project. - project.runners.create(runner_params) - end - - return forbidden! unless runner - - if runner.id - runner.update(get_runner_version_from_params) - present runner, with: Entities::Runner - else - not_found! - end - end - end - end - end -end diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb deleted file mode 100644 index 6225203f223..00000000000 --- a/lib/ci/api/triggers.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Ci - module API - class Triggers < Grape::API - resource :projects do - desc 'Trigger a GitLab CI project build' do - success Entities::TriggerRequest - end - params do - requires :id, type: Integer, desc: 'The ID of a CI project' - requires :ref, type: String, desc: "The name of project's branch or tag" - requires :token, type: String, desc: 'The unique token of the trigger' - optional :variables, type: Hash, desc: 'Optional build variables' - end - post ":id/refs/:ref/trigger" do - project = Project.find_by(ci_id: params[:id]) - trigger = Ci::Trigger.find_by_token(params[:token]) - not_found! unless project && trigger - unauthorized! unless trigger.project == project - - # Validate variables - variables = params[:variables].to_h - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } - render_api_error!('variables needs to be a map of key-valued strings', 400) - end - - # create request and trigger builds - result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref], variables) - pipeline = result.pipeline - - if pipeline.persisted? - present result.trigger_request, with: Entities::TriggerRequest - else - render_validation_error!(pipeline) - end - end - end - end - end -end diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 872e418c788..76a69bf8a83 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -47,7 +47,7 @@ module Ci def collect query = project.pipelines - .where("? > #{Ci::Pipeline.table_name}.created_at AND #{Ci::Pipeline.table_name}.created_at > ?", @to, @from) + .where("? > #{Ci::Pipeline.table_name}.created_at AND #{Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection totals_count = grouped_count(query) success_count = grouped_count(query.success) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 3a4911b23b0..62b44389b15 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -20,24 +20,6 @@ module Ci raise ValidationError, e.message end - def jobs_for_ref(ref, tag = false, source = nil) - @jobs.select do |_, job| - process?(job[:only], job[:except], ref, tag, source) - end - end - - def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).select do |_, job| - job[:stage] == stage - end - end - - def builds_for_ref(ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).map do |name, _| - build_attributes(name) - end - end - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| build_attributes(name) @@ -52,8 +34,7 @@ module Ci def stage_seeds(pipeline) seeds = @stages.uniq.map do |stage| - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) + builds = pipeline_stage_builds(stage, pipeline) Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? end @@ -101,6 +82,34 @@ module Ci private + def pipeline_stage_builds(stage, pipeline) + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, pipeline.source) + + builds.select do |build| + job = @jobs[build.fetch(:name).to_sym] + has_kubernetes = pipeline.has_kubernetes_active? + only_kubernetes = job.dig(:only, :kubernetes) + except_kubernetes = job.dig(:except, :kubernetes) + + [!only_kubernetes && !except_kubernetes, + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? + end + end + + def jobs_for_ref(ref, tag = false, source = nil) + @jobs.select do |_, job| + process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) + end + end + + def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_ref(ref, tag, source).select do |_, job| + job[:stage] == stage + end + end + def initial_parsing ## # Global config diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index 4c0aee6c48f..5bef29eb1da 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -6,6 +6,8 @@ class ProjectUrlConstrainer return false unless DynamicPathValidator.valid_project_path?(full_path) + # We intentionally allow SELECT(*) here so result of this query can be used + # as cache for further Project.find_by_full_path calls within request Project.find_by_full_path(full_path, follow_redirects: request.get?).present? end end diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb index df94cafb6a1..b028169f500 100644 --- a/lib/declarative_policy/base.rb +++ b/lib/declarative_policy/base.rb @@ -109,7 +109,7 @@ module DeclarativePolicy name = name.to_sym if delegation_block.nil? - delegation_block = proc { @subject.__send__(name) } + delegation_block = proc { @subject.__send__(name) } # rubocop:disable GitlabSecurity/PublicSend end own_delegations[name] = delegation_block @@ -221,7 +221,7 @@ module DeclarativePolicy end # computes the given ability and prints a helpful debugging output - # showing which + # showing which def debug(ability, *a) runner(ability).debug(*a) end diff --git a/lib/declarative_policy/dsl.rb b/lib/declarative_policy/dsl.rb index b26807a7622..6ba1e7a3c5c 100644 --- a/lib/declarative_policy/dsl.rb +++ b/lib/declarative_policy/dsl.rb @@ -93,7 +93,7 @@ module DeclarativePolicy def method_missing(m, *a, &b) return super unless @context_class.respond_to?(m) - @context_class.__send__(m, *a, &b) + @context_class.__send__(m, *a, &b) # rubocop:disable GitlabSecurity/PublicSend end def respond_to_missing?(m) diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb index 63f9f8d7a5a..f2bf3d0fb2b 100644 --- a/lib/email_template_interceptor.rb +++ b/lib/email_template_interceptor.rb @@ -1,6 +1,6 @@ # Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails class EmailTemplateInterceptor - include Gitlab::CurrentSettings + extend Gitlab::CurrentSettings def self.delivering_email(message) # Remove HTML part if HTML emails are disabled. diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index eb19ab45ac3..de391de9059 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -44,13 +44,13 @@ class FileSizeValidator < ActiveModel::EachValidator when Integer check_value when Symbol - record.send(check_value) + record.public_send(check_value) # rubocop:disable GitlabSecurity/PublicSend end value ||= [] if key == :maximum value_size = value.size - next if value_size.send(validity_check, check_value) + next if value_size.public_send(validity_check, check_value) # rubocop:disable GitlabSecurity/PublicSend errors_options = options.except(*RESERVED_OPTIONS) errors_options[:file_size] = help.number_to_human_size check_value diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb deleted file mode 100644 index 4e3c6d3c773..00000000000 --- a/lib/file_streamer.rb +++ /dev/null @@ -1,16 +0,0 @@ -class FileStreamer #:nodoc: - attr_reader :to_path - - def initialize(path) - @to_path = path - end - - # Stream the file's contents if Rack::Sendfile isn't present. - def each - File.open(to_path, 'rb') do |file| - while chunk = file.read(16384) - yield chunk - end - end - end -end diff --git a/lib/github/import.rb b/lib/github/import.rb index 4cc01593ef4..9354e142d3d 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -107,7 +107,7 @@ module Github # this means that repo has wiki enabled, but have no pages. So, # we can skip the import. if e.message !~ /repository not exported/ - errors(:wiki, wiki_url, e.message) + error(:wiki, wiki_url, e.message) end end @@ -226,49 +226,51 @@ module Github while url response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) - response.body.each do |raw| - representation = Github::Representation::Issue.new(raw, options) + response.body.each { |raw| populate_issue(raw) } - begin - # Every pull request is an issue, but not every issue - # is a pull request. For this reason, "shared" actions - # for both features, like manipulating assignees, labels - # and milestones, are provided within the Issues API. - if representation.pull_request? - next unless representation.has_labels? - - merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) - merge_request.update_attribute(:label_ids, label_ids(representation.labels)) - else - next if Issue.where(iid: representation.iid, project_id: project.id).exists? - - author_id = user_id(representation.author, project.creator_id) - issue = Issue.new - issue.iid = representation.iid - issue.project_id = project.id - issue.title = representation.title - issue.description = format_description(representation.description, representation.author) - issue.state = representation.state - issue.label_ids = label_ids(representation.labels) - issue.milestone_id = milestone_id(representation.milestone) - issue.author_id = author_id - issue.assignee_ids = [user_id(representation.assignee)] - issue.created_at = representation.created_at - issue.updated_at = representation.updated_at - issue.save!(validate: false) - - # Fetch comments - if representation.has_comments? - comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" - fetch_comments(issue, :comment, comments_url) - end - end - rescue => e - error(:issue, representation.url, e.message) + url = response.rels[:next] + end + end + + def populate_issue(raw) + representation = Github::Representation::Issue.new(raw, options) + + begin + # Every pull request is an issue, but not every issue + # is a pull request. For this reason, "shared" actions + # for both features, like manipulating assignees, labels + # and milestones, are provided within the Issues API. + if representation.pull_request? + return unless representation.has_labels? + + merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + else + return if Issue.where(iid: representation.iid, project_id: project.id).exists? + + author_id = user_id(representation.author, project.creator_id) + issue = Issue.new + issue.iid = representation.iid + issue.project_id = project.id + issue.title = representation.title + issue.description = format_description(representation.description, representation.author) + issue.state = representation.state + issue.label_ids = label_ids(representation.labels) + issue.milestone_id = milestone_id(representation.milestone) + issue.author_id = author_id + issue.assignee_ids = [user_id(representation.assignee)] + issue.created_at = representation.created_at + issue.updated_at = representation.updated_at + issue.save!(validate: false) + + # Fetch comments + if representation.has_comments? + comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" + fetch_comments(issue, :comment, comments_url) end end - - url = response.rels[:next] + rescue => e + error(:issue, representation.url, e.message) end end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb index c6fa928d565..823e8e9a9c4 100644 --- a/lib/github/representation/branch.rb +++ b/lib/github/representation/branch.rb @@ -41,7 +41,7 @@ module Github def remove!(name) repository.delete_branch(name) - rescue Rugged::ReferenceError => e + rescue Gitlab::Git::Repository::DeleteBranchError => e Rails.logger.error("#{self.class.name}: Could not remove branch #{name}: #{e}") end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 4714ab18cc1..b4012ebbb99 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -67,10 +67,14 @@ module Gitlab def protection_values protection_options.values end + + def human_access(access) + options_with_owner.key(access) + end end def human_access - Gitlab::Access.options_with_owner.key(access_field) + Gitlab::Access.human_access(access_field) end def owner? diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 3d41ac76406..cead1c7eacd 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -6,6 +6,8 @@ module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc + extend Gitlab::CurrentSettings + DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 7d3aa532750..3fd81759d25 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -19,6 +19,8 @@ module Gitlab OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze class << self + include Gitlab::CurrentSettings + def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -48,10 +50,6 @@ module Gitlab # Avoid resource intensive login checks if password is not provided return unless password.present? - # Nothing to do here if internal auth is disabled and LDAP is - # not configured - return unless current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? - Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) @@ -101,7 +99,7 @@ module Gitlab if Service.available_services_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included # in the Service.available_services_names whitelist. - service = project.public_send("#{underscored_service}_service") + service = project.public_send("#{underscored_service}_service") # rubocop:disable GitlabSecurity/PublicSend if service && service.activated? && service.valid_token?(password) Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities) @@ -149,7 +147,7 @@ module Gitlab def abilities_for_scope(scopes) scopes.map do |scope| - self.public_send(:"#{scope}_scope_authentication_abilities") + self.public_send(:"#{scope}_scope_authentication_abilities") # rubocop:disable GitlabSecurity/PublicSend end.flatten.uniq end diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb index 1089bc9f89e..e6173d45af3 100644 --- a/lib/gitlab/auth/ip_rate_limiter.rb +++ b/lib/gitlab/auth/ip_rate_limiter.rb @@ -11,11 +11,11 @@ module Gitlab def enabled? config.enabled end - + def reset! Rack::Attack::Allow2Ban.reset(ip, config) end - + def register_fail! # Allow2Ban.filter will return false if this IP has not failed too often yet @banned = Rack::Attack::Allow2Ban.filter(ip, config) do @@ -23,17 +23,17 @@ module Gitlab ip_can_be_banned? end end - + def banned? @banned end - + private - + def config Gitlab.config.rack_attack.git_basic_auth end - + def ip_can_be_banned? config.ip_whitelist.exclude?(ip) end diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index 0fbc6b70989..3fde1b09efb 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -81,10 +81,15 @@ module Gitlab relative_order: index ) - # Compatibility with old diffs created with Psych. diff_hash.tap do |hash| diff_text = hash[:diff] + hash[:too_large] = !!hash[:too_large] + + hash[:a_mode] ||= guess_mode(hash[:new_file], hash[:diff]) + hash[:b_mode] ||= guess_mode(hash[:deleted_file], hash[:diff]) + + # Compatibility with old diffs created with Psych. if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only? hash[:binary] = true hash[:diff] = [diff_text].pack('m0') @@ -95,6 +100,15 @@ module Gitlab [commit_rows, file_rows] end + # This doesn't have to be 100% accurate, because it's only used for + # display - it won't change file modes in the repository. Submodules are + # created as 600, regular files as 644. + def guess_mode(file_missing, diff) + return '0' if file_missing + + diff.include?('Subproject commit') ? '160000' : '100644' + end + # Unlike MergeRequestDiff#valid_raw_diff?, don't count Rugged objects as # valid, because we don't render them usefully anyway. def valid_raw_diffs?(diffs) diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb new file mode 100644 index 00000000000..432f7c3e706 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -0,0 +1,176 @@ +module Gitlab + module BackgroundMigration + # Class that migrates events for the new push event payloads setup. All + # events are copied to a shadow table, and push events will also have a row + # created in the push_event_payloads table. + class MigrateEventsToPushEventPayloads + class Event < ActiveRecord::Base + self.table_name = 'events' + + serialize :data + + BLANK_REF = ('0' * 40).freeze + TAG_REF_PREFIX = 'refs/tags/'.freeze + MAX_INDEX = 69 + PUSHED = 5 + + def push_event? + action == PUSHED && data.present? + end + + def commit_title + commit = commits.last + + return nil unless commit && commit[:message] + + index = commit[:message].index("\n") + message = index ? commit[:message][0..index] : commit[:message] + + message.strip.truncate(70) + end + + def commit_from_sha + if create? + nil + else + data[:before] + end + end + + def commit_to_sha + if remove? + nil + else + data[:after] + end + end + + def data + super || {} + end + + def commits + data[:commits] || [] + end + + def commit_count + data[:total_commits_count] || 0 + end + + def ref + data[:ref] + end + + def trimmed_ref_name + if ref_type == :tag + ref[10..-1] + else + ref[11..-1] + end + end + + def create? + data[:before] == BLANK_REF + end + + def remove? + data[:after] == BLANK_REF + end + + def push_action + if create? + :created + elsif remove? + :removed + else + :pushed + end + end + + def ref_type + if ref.start_with?(TAG_REF_PREFIX) + :tag + else + :branch + end + end + end + + class EventForMigration < ActiveRecord::Base + self.table_name = 'events_for_migration' + end + + class PushEventPayload < ActiveRecord::Base + self.table_name = 'push_event_payloads' + + enum action: { + created: 0, + removed: 1, + pushed: 2 + } + + enum ref_type: { + branch: 0, + tag: 1 + } + end + + # start_id - The start ID of the range of events to process + # end_id - The end ID of the range to process. + def perform(start_id, end_id) + return unless migrate? + + find_events(start_id, end_id).each { |event| process_event(event) } + end + + def process_event(event) + replicate_event(event) + create_push_event_payload(event) if event.push_event? + end + + def replicate_event(event) + new_attributes = event.attributes + .with_indifferent_access.except(:title, :data) + + EventForMigration.create!(new_attributes) + rescue ActiveRecord::InvalidForeignKey + # A foreign key error means the associated event was removed. In this + # case we'll just skip migrating the event. + end + + def create_push_event_payload(event) + commit_from = pack(event.commit_from_sha) + commit_to = pack(event.commit_to_sha) + + PushEventPayload.create!( + event_id: event.id, + commit_count: event.commit_count, + ref_type: event.ref_type, + action: event.push_action, + commit_from: commit_from, + commit_to: commit_to, + ref: event.trimmed_ref_name, + commit_title: event.commit_title + ) + rescue ActiveRecord::InvalidForeignKey + # A foreign key error means the associated event was removed. In this + # case we'll just skip migrating the event. + end + + def find_events(start_id, end_id) + Event + .where('NOT EXISTS (SELECT true FROM events_for_migration WHERE events_for_migration.id = events.id)') + .where(id: start_id..end_id) + end + + def migrate? + Event.table_exists? && PushEventPayload.table_exists? && + EventForMigration.table_exists? + end + + def pack(value) + value ? [value].pack('H*') : nil + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb new file mode 100644 index 00000000000..b1ff0900709 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -0,0 +1,77 @@ +module Gitlab + module BackgroundMigration + class MigrateStageStatus + STATUSES = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + + scope :latest, -> { where(retried: [false, nil]) } + scope :created, -> { where(status: 'created') } + scope :running, -> { where(status: 'running') } + scope :pending, -> { where(status: 'pending') } + scope :success, -> { where(status: 'success') } + scope :failed, -> { where(status: 'failed') } + scope :canceled, -> { where(status: 'canceled') } + scope :skipped, -> { where(status: 'skipped') } + scope :manual, -> { where(status: 'manual') } + + scope :failed_but_allowed, -> do + where(allow_failure: true, status: [:failed, :canceled]) + end + + scope :exclude_ignored, -> do + where("allow_failure = ? OR status IN (?)", + false, %w[created pending running success skipped]) + end + + def self.status_sql + scope_relevant = latest.exclude_ignored + scope_warnings = latest.failed_but_allowed + + builds = scope_relevant.select('count(*)').to_sql + created = scope_relevant.created.select('count(*)').to_sql + success = scope_relevant.success.select('count(*)').to_sql + manual = scope_relevant.manual.select('count(*)').to_sql + pending = scope_relevant.pending.select('count(*)').to_sql + running = scope_relevant.running.select('count(*)').to_sql + skipped = scope_relevant.skipped.select('count(*)').to_sql + canceled = scope_relevant.canceled.select('count(*)').to_sql + warnings = scope_warnings.select('count(*) > 0').to_sql + + <<-SQL.strip_heredoc + (CASE + WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} + WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} + WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} + WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} + WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} + WHEN (#{created}) > 0 THEN #{STATUSES[:running]} + ELSE #{STATUSES[:failed]} + END) + SQL + end + end + + def perform(start_id, stop_id) + status_sql = Build + .where('ci_builds.commit_id = ci_stages.pipeline_id') + .where('ci_builds.stage = ci_stages.name') + .status_sql + + sql = <<-SQL + UPDATE ci_stages SET status = (#{status_sql}) + WHERE ci_stages.status IS NULL + AND ci_stages.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + SQL + + ActiveRecord::Base.connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/move_personal_snippet_files.rb b/lib/gitlab/background_migration/move_personal_snippet_files.rb new file mode 100644 index 00000000000..07cec96bcc3 --- /dev/null +++ b/lib/gitlab/background_migration/move_personal_snippet_files.rb @@ -0,0 +1,79 @@ +module Gitlab + module BackgroundMigration + class MovePersonalSnippetFiles + delegate :select_all, :execute, :quote_string, to: :connection + + def perform(relative_source, relative_destination) + @source_relative_location = relative_source + @destination_relative_location = relative_destination + + move_personal_snippet_files + end + + def move_personal_snippet_files + query = "SELECT uploads.path, uploads.model_id FROM uploads "\ + "INNER JOIN snippets ON snippets.id = uploads.model_id WHERE uploader = 'PersonalFileUploader'" + select_all(query).each do |upload| + secret = upload['path'].split('/')[0] + file_name = upload['path'].split('/')[1] + + move_file(upload['model_id'], secret, file_name) + update_markdown(upload['model_id'], secret, file_name) + end + end + + def move_file(snippet_id, secret, file_name) + source_dir = File.join(base_directory, @source_relative_location, snippet_id.to_s, secret) + destination_dir = File.join(base_directory, @destination_relative_location, snippet_id.to_s, secret) + + source_file_path = File.join(source_dir, file_name) + destination_file_path = File.join(destination_dir, file_name) + + unless File.exist?(source_file_path) + say "Source file `#{source_file_path}` doesn't exist. Skipping." + return + end + + say "Moving file #{source_file_path} -> #{destination_file_path}" + + FileUtils.mkdir_p(destination_dir) + FileUtils.move(source_file_path, destination_file_path) + end + + def update_markdown(snippet_id, secret, file_name) + source_markdown_path = File.join(@source_relative_location, snippet_id.to_s, secret, file_name) + destination_markdown_path = File.join(@destination_relative_location, snippet_id.to_s, secret, file_name) + + source_markdown = "](#{source_markdown_path})" + destination_markdown = "](#{destination_markdown_path})" + quoted_source = quote_string(source_markdown) + quoted_destination = quote_string(destination_markdown) + + execute("UPDATE snippets "\ + "SET description = replace(snippets.description, '#{quoted_source}', '#{quoted_destination}'), description_html = NULL "\ + "WHERE id = #{snippet_id}") + + query = "SELECT id, note FROM notes WHERE noteable_id = #{snippet_id} "\ + "AND noteable_type = 'Snippet' AND note IS NOT NULL" + select_all(query).each do |note| + text = note['note'].gsub(source_markdown, destination_markdown) + quoted_text = quote_string(text) + + execute("UPDATE notes SET note = '#{quoted_text}', note_html = NULL WHERE id = #{note['id']}") + end + end + + def base_directory + File.join(Rails.root, 'public') + end + + def connection + ActiveRecord::Base.connection + end + + def say(message) + Rails.logger.debug(message) + end + end + end +end diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb new file mode 100644 index 00000000000..9323bfc7fb2 --- /dev/null +++ b/lib/gitlab/bare_repository_importer.rb @@ -0,0 +1,96 @@ +module Gitlab + class BareRepositoryImporter + NoAdminError = Class.new(StandardError) + + def self.execute + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + git_base_path = repository_storage['path'] + repos_to_import = Dir.glob(git_base_path + '/**/*.git') + + repos_to_import.each do |repo_path| + if repo_path.end_with?('.wiki.git') + log " * Skipping wiki repo" + next + end + + log "Processing #{repo_path}".color(:yellow) + + repo_relative_path = repo_path[repository_storage['path'].length..-1] + .sub(/^\//, '') # Remove leading `/` + .sub(/\.git$/, '') # Remove `.git` at the end + new(storage_name, repo_relative_path).create_project_if_needed + end + end + end + + attr_reader :storage_name, :full_path, :group_path, :project_path, :user + delegate :log, to: :class + + def initialize(storage_name, repo_path) + @storage_name = storage_name + @full_path = repo_path + + unless @user = User.admins.order_id_asc.first + raise NoAdminError.new('No admin user found to import repositories') + end + + @group_path, @project_path = File.split(repo_path) + @group_path = nil if @group_path == '.' + end + + def create_project_if_needed + if project = Project.find_by_full_path(full_path) + log " * #{project.name} (#{full_path}) exists" + return project + end + + create_project + end + + private + + def create_project + group = find_or_create_group + + project_params = { + name: project_path, + path: project_path, + repository_storage: storage_name, + namespace_id: group&.id + } + + project = Projects::CreateService.new(user, project_params).execute + + if project.persisted? + log " * Created #{project.name} (#{full_path})".color(:green) + ProjectCacheWorker.perform_async(project.id) + else + log " * Failed trying to create #{project.name} (#{full_path})".color(:red) + log " Errors: #{project.errors.messages}".color(:red) + end + + project + end + + def find_or_create_group + return nil unless group_path + + if namespace = Namespace.find_by_full_path(group_path) + log " * Namespace #{group_path} exists.".color(:green) + return namespace + end + + log " * Creating Group: #{group_path}" + Groups::NestedCreateService.new(user, group_path: group_path).execute + end + + # This is called from within a rake task only used by Admins, so allow writing + # to STDOUT + # + # rubocop:disable Rails/Output + def self.log(message) + puts message + end + # rubocop:enable Rails/Output + end +end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index f1a04affd38..754a45c3257 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -69,7 +69,7 @@ module Gitlab instance_variable_set(ivar_name, {}) end - key = __send__(cache_key_method_name, args) + key = __send__(cache_key_method_name, args) # rubocop:disable GitlabSecurity/PublicSend store.fetch(key) { store[key] = super(*args) } end diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index 1e73f89158d..dc5d285ea65 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -5,12 +5,19 @@ module Gitlab return false if project.empty_repo? # Created or deleted branch - if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) - false - else - Gitlab::Git::RevList.new( - path_to_repo: project.repository.path_to_repo, - oldrev: oldrev, newrev: newrev).missed_ref.present? + return false if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) + + GitalyClient.migrate(:force_push) do |is_enabled| + if is_enabled + !project + .repository + .gitaly_commit_client + .ancestor?(oldrev, newrev) + else + Gitlab::Git::RevList.new( + path_to_repo: project.repository.path_to_repo, + oldrev: oldrev, newrev: newrev).missed_ref.present? + end end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index a375ccbece0..a788fb3fcbc 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -60,7 +60,7 @@ module Gitlab begin path = read_string(gz).force_encoding('UTF-8') meta = read_string(gz).force_encoding('UTF-8') - + next unless path.valid_encoding? && meta.valid_encoding? next unless path =~ match_pattern next if path =~ INVALID_PATH_PATTERN diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 2e073334abc..22941d48edf 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -1,129 +1,129 @@ module Gitlab - module Ci::Build::Artifacts - class Metadata - ## - # Class that represents an entry (path and metadata) to a file or - # directory in GitLab CI Build Artifacts binary file / archive - # - # This is IO-operations safe class, that does similar job to - # Ruby's Pathname but without the risk of accessing filesystem. - # - # This class is working only with UTF-8 encoded paths. - # - class Entry - attr_reader :path, :entries - attr_accessor :name - - def initialize(path, entries) - @path = path.dup.force_encoding('UTF-8') - @entries = entries - - if path.include?("\0") - raise ArgumentError, 'Path contains zero byte character!' - end + module Ci + module Build + module Artifacts + class Metadata + ## + # Class that represents an entry (path and metadata) to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # This class is working only with UTF-8 encoded paths. + # + class Entry + attr_reader :entries + attr_accessor :name + + def initialize(path, entries) + @entries = entries + @path = Artifacts::Path.new(path) + end + + delegate :empty?, to: :children + + def directory? + blank_node? || @path.directory? + end + + def file? + !directory? + end + + def blob + return unless file? + + @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + self.class.new(@path.to_s.chomp(basename), @entries) + end + + def basename + (directory? && !blank_node?) ? name + '/' : name + end + + def name + @name || @path.name + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path.to_s)}[^/]+/?$} + @children = select_entries { |path| path =~ child_pattern } + end + + def directories(opts = {}) + return [] unless directory? + dirs = children.select(&:directory?) + return dirs unless has_parent? && opts[:parent] + + dotted_parent = parent + dotted_parent.name = '..' + dirs.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @entries[@path.to_s] || {} + end + + def nodes + @path.nodes + (file? ? 1 : 0) + end + + def blank_node? + @path.to_s.empty? # "" is considered to be './' + end + + def exists? + blank_node? || @entries.include?(@path.to_s) + end + + def total_size + descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}} + entries.sum do |path, entry| + (entry[:size] if path =~ descendant_pattern).to_i + end + end + + def path + @path.to_s + end + + def to_s + @path.to_s + end + + def ==(other) + path == other.path && @entries == other.entries + end + + def inspect + "#{self.class.name}: #{self}" + end - unless path.valid_encoding? - raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + private + + def select_entries + selected = @entries.select { |path, _metadata| yield path } + selected.map { |path, _metadata| self.class.new(path, @entries) } + end end end - - delegate :empty?, to: :children - - def directory? - blank_node? || @path.end_with?('/') - end - - def file? - !directory? - end - - def blob - return unless file? - - @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) - end - - def has_parent? - nodes > 0 - end - - def parent - return nil unless has_parent? - self.class.new(@path.chomp(basename), @entries) - end - - def basename - (directory? && !blank_node?) ? name + '/' : name - end - - def name - @name || @path.split('/').last.to_s - end - - def children - return [] unless directory? - return @children if @children - - child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} - @children = select_entries { |path| path =~ child_pattern } - end - - def directories(opts = {}) - return [] unless directory? - dirs = children.select(&:directory?) - return dirs unless has_parent? && opts[:parent] - - dotted_parent = parent - dotted_parent.name = '..' - dirs.prepend(dotted_parent) - end - - def files - return [] unless directory? - children.select(&:file?) - end - - def metadata - @entries[@path] || {} - end - - def nodes - @path.count('/') + (file? ? 1 : 0) - end - - def blank_node? - @path.empty? # "" is considered to be './' - end - - def exists? - blank_node? || @entries.include?(@path) - end - - def total_size - descendant_pattern = %r{^#{Regexp.escape(@path)}} - entries.sum do |path, entry| - (entry[:size] if path =~ descendant_pattern).to_i - end - end - - def to_s - @path - end - - def ==(other) - @path == other.path && @entries == other.entries - end - - def inspect - "#{self.class.name}: #{@path}" - end - - private - - def select_entries - selected = @entries.select { |path, _metadata| yield path } - selected.map { |path, _metadata| self.class.new(path, @entries) } - end end end end diff --git a/lib/gitlab/ci/build/artifacts/path.rb b/lib/gitlab/ci/build/artifacts/path.rb new file mode 100644 index 00000000000..9cd9b36c5f8 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/path.rb @@ -0,0 +1,51 @@ +module Gitlab + module Ci + module Build + module Artifacts + class Path + def initialize(path) + @path = path.dup.force_encoding('UTF-8') + end + + def valid? + nonzero? && utf8? + end + + def directory? + @path.end_with?('/') + end + + def name + @path.split('/').last.to_s + end + + def nodes + @path.count('/') + end + + def to_s + @path.tap do |path| + unless nonzero? + raise ArgumentError, 'Path contains zero byte character!' + end + + unless utf8? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end + end + end + + private + + def nonzero? + @path.exclude?("\0") + end + + def utf8? + @path.valid_encoding? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb index 1c8b55ee4c4..3e87a09704e 100644 --- a/lib/gitlab/ci/config/entry/attributable.rb +++ b/lib/gitlab/ci/config/entry/attributable.rb @@ -8,6 +8,10 @@ module Gitlab class_methods do def attributes(*attributes) attributes.flatten.each do |attribute| + if method_defined?(attribute) + raise ArgumentError, 'Method already defined!' + end + define_method(attribute) do return unless config.is_a?(Hash) diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index e05aca9881b..68b6742385a 100644 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -15,9 +15,10 @@ module Gitlab # module Configurable extend ActiveSupport::Concern - include Validatable included do + include Validatable + validations do validates :config, type: Hash end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 32f5c6ab142..91aac6df4b1 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -59,10 +59,10 @@ module Gitlab entry :services, Entry::Services, description: 'Services that will be used to execute this job.' - entry :only, Entry::Trigger, + entry :only, Entry::Policy, description: 'Refs policy this job will be executed for.' - entry :except, Entry::Trigger, + entry :except, Entry::Policy, description: 'Refs policy this job will be executed for.' entry :variables, Entry::Variables, diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb index a6a914d79c1..c868943c42e 100644 --- a/lib/gitlab/ci/config/entry/node.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -16,8 +16,9 @@ module Gitlab @metadata = metadata @entries = {} - @validator = self.class.validator.new(self) - @validator.validate(:new) + self.class.aspects.to_a.each do |aspect| + instance_exec(&aspect) + end end def [](key) @@ -47,7 +48,7 @@ module Gitlab end def errors - @validator.messages + descendants.flat_map(&:errors) + [] end def value @@ -70,6 +71,13 @@ module Gitlab true end + def location + name = @key.presence || self.class.name.to_s.demodulize + .underscore.humanize.downcase + + ancestors.map(&:key).append(name).compact.join(':') + end + def inspect val = leaf? ? config : descendants unspecified = specified? ? '' : '(unspecified) ' @@ -79,8 +87,8 @@ module Gitlab def self.default end - def self.validator - Validator + def self.aspects + @aspects ||= [] end end end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb new file mode 100644 index 00000000000..0027e9ec8c5 --- /dev/null +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -0,0 +1,53 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents an only/except trigger policy for the job. + # + class Policy < Simplifiable + strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } + strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) } + + class RefsPolicy < Entry::Node + include Entry::Validatable + + validations do + validates :config, array_of_strings_or_regexps: true + end + + def value + { refs: @config } + end + end + + class ComplexPolicy < Entry::Node + include Entry::Validatable + include Entry::Attributable + + attributes :refs, :kubernetes + + validations do + validates :config, presence: true + validates :config, allowed_keys: %i[refs kubernetes] + + with_options allow_nil: true do + validates :refs, array_of_strings_or_regexps: true + validates :kubernetes, allowed_values: %w[active] + end + end + end + + class UnknownStrategy < Entry::Node + def errors + ["#{location} has to be either an array of conditions or a hash"] + end + end + + def self.default + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb new file mode 100644 index 00000000000..12764629686 --- /dev/null +++ b/lib/gitlab/ci/config/entry/simplifiable.rb @@ -0,0 +1,43 @@ +module Gitlab + module Ci + class Config + module Entry + class Simplifiable < SimpleDelegator + EntryStrategy = Struct.new(:name, :condition) + + def initialize(config, **metadata) + unless self.class.const_defined?(:UnknownStrategy) + raise ArgumentError, 'UndefinedStrategy not available!' + end + + strategy = self.class.strategies.find do |variant| + variant.condition.call(config) + end + + entry = self.class.entry_class(strategy) + + super(entry.new(config, metadata)) + end + + def self.strategy(name, **opts) + EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy| + strategies.append(strategy) + end + end + + def self.strategies + @strategies ||= [] + end + + def self.entry_class(strategy) + if strategy.present? + self.const_get(strategy.name) + else + self::UnknownStrategy + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb deleted file mode 100644 index 16b234e6c59..00000000000 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ /dev/null @@ -1,18 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - ## - # Entry that represents a trigger policy for the job. - # - class Trigger < Node - include Validatable - - validations do - validates :config, array_of_strings_or_regexps: true - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb index f7f1b111571..5ced778d311 100644 --- a/lib/gitlab/ci/config/entry/validatable.rb +++ b/lib/gitlab/ci/config/entry/validatable.rb @@ -5,6 +5,17 @@ module Gitlab module Validatable extend ActiveSupport::Concern + def self.included(node) + node.aspects.append -> do + @validator = self.class.validator.new(self) + @validator.validate(:new) + end + end + + def errors + @validator.messages + descendants.flat_map(&:errors) + end + class_methods do def validator @validator ||= Class.new(Entry::Validator).tap do |validator| diff --git a/lib/gitlab/ci/config/entry/validator.rb b/lib/gitlab/ci/config/entry/validator.rb index 55343005fe3..2df23a3edcd 100644 --- a/lib/gitlab/ci/config/entry/validator.rb +++ b/lib/gitlab/ci/config/entry/validator.rb @@ -8,7 +8,6 @@ module Gitlab def initialize(entry) super(entry) - @entry = entry end def messages @@ -20,21 +19,6 @@ module Gitlab def self.name 'Validator' end - - private - - def location - predecessors = ancestors.map(&:key).compact - predecessors.append(key_name).join(':') - end - - def key_name - if key.blank? - @entry.class.name.demodulize.underscore.humanize - else - key - end - end end end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index b2ca3c881e4..0159179f0a9 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -14,6 +14,14 @@ module Gitlab end end + class AllowedValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless options[:in].include?(value.to_s) + record.errors.add(attribute, "unknown value: #{value}") + end + end + end + class ArrayOfStringsValidator < ActiveModel::EachValidator include LegacyValidationHelpers diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index f81f9347b4d..e19aae35a81 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -28,7 +28,8 @@ module Gitlab attributes.merge(project: project, ref: pipeline.ref, tag: pipeline.tag, - trigger_request: trigger) + trigger_request: trigger, + protected: protected_ref?) end end @@ -43,6 +44,12 @@ module Gitlab end end end + + private + + def protected_ref? + @protected_ref ||= project.protected_for?(pipeline.ref) + end end end end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 1611eba31da..90f83e0f810 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -18,7 +18,7 @@ module Gitlab new(merge_request, project).tap do |file_collection| project .repository - .with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do + .with_repo_branch_commit(merge_request.target_project.repository.raw_repository, merge_request.target_branch) do yield file_collection end @@ -77,8 +77,8 @@ EOM def initialize(merge_request, project) @merge_request = merge_request - @our_commit = merge_request.source_branch_head.raw.raw_commit - @their_commit = merge_request.target_branch_head.raw.raw_commit + @our_commit = merge_request.source_branch_head.raw.rugged_commit + @their_commit = merge_request.target_branch_head.raw.rugged_commit @project = project end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7fa02f3d7b3..642f0944354 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,5 +1,7 @@ module Gitlab module CurrentSettings + extend self + def current_application_settings if RequestStore.active? RequestStore.fetch(:current_application_settings) { ensure_application_settings! } diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index b260822788d..2479b4a7706 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -54,7 +54,7 @@ module Gitlab end def serialize_commit(event, commit, query) - commit = Commit.new(Gitlab::Git::Commit.new(commit.to_hash), @project) + commit = Commit.from_hash(commit.to_hash, @project) AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 5c5f507d44d..4ab5b3455a5 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -3,6 +3,35 @@ module Gitlab module Push extend self + SAMPLE_DATA = + { + object_kind: "push", + event_name: "push", + before: "95790bf891e76fee5e1747ab589903a6a1f80f22", + after: "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + ref: "refs/heads/master", + checkout_sha: "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + message: "Hello World", + user_id: 4, + user_name: "John Smith", + user_email: "john@example.com", + user_avatar: "https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80", + project_id: 15, + commits: [ + { + id: "c5feabde2d8cd023215af4d2ceeb7a64839fc428", + message: "Add simple search to projects in public area", + timestamp: "2013-05-13T18:18:08+00:00", + url: "https://test.example.com/gitlab/gitlabhq/commit/c5feabde2d8cd023215af4d2ceeb7a64839fc428", + author: { + name: "Test User", + email: "test@example.com" + } + } + ], + total_commits_count: 1 + }.freeze + # Produce a hash of post-receive data # # data = { @@ -74,6 +103,10 @@ module Gitlab build(project, user, commits.last&.id, commits.first&.id, ref, commits) end + def sample_data + SAMPLE_DATA + end + private def checkout_sha(repository, newrev, ref) diff --git a/lib/gitlab/data_builder/repository.rb b/lib/gitlab/data_builder/repository.rb index b42dc052949..c9c13ec6487 100644 --- a/lib/gitlab/data_builder/repository.rb +++ b/lib/gitlab/data_builder/repository.rb @@ -3,6 +3,23 @@ module Gitlab module Repository extend self + SAMPLE_DATA = { + event_name: 'repository_update', + user_id: 10, + user_name: 'john.doe', + user_email: 'test@example.com', + user_avatar: 'http://example.com/avatar/user.png', + project_id: 40, + changes: [ + { + before: "8205ea8d81ce0c6b90fbe8280d118cc9fdad6130", + after: "4045ea7a3df38697b3730a20fb73c8bed8a3e69e", + ref: "refs/heads/master" + } + ], + "refs": ["refs/heads/master"] + }.freeze + # Produce a hash of post-receive data def update(project, user, changes, refs) { @@ -30,6 +47,10 @@ module Gitlab ref: ref } end + + def sample_data + SAMPLE_DATA + end end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d7dab584a44..a6ec75da385 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -9,6 +9,14 @@ module Gitlab ActiveRecord::Base.configurations[Rails.env] end + def self.username + config['username'] || ENV['USER'] + end + + def self.database_name + config['database'] + end + def self.adapter_name config['adapter'] end @@ -25,6 +33,10 @@ module Gitlab database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end + def self.join_lateral_supported? + postgresql? && version.to_f >= 9.3 + end + def self.nulls_last_order(field, direction = 'ASC') order = "#{field} #{direction}" diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb new file mode 100644 index 00000000000..aee3981e79a --- /dev/null +++ b/lib/gitlab/database/grant.rb @@ -0,0 +1,34 @@ +module Gitlab + module Database + # Model that can be used for querying permissions of a SQL user. + class Grant < ActiveRecord::Base + self.table_name = + if Database.postgresql? + 'information_schema.role_table_grants' + else + 'mysql.user' + end + + def self.scope_to_current_user + if Database.postgresql? + where('grantee = user') + else + where("CONCAT(User, '@', Host) = current_user()") + end + end + + # Returns true if the current user can create and execute triggers on the + # given table. + def self.create_and_execute_trigger?(table) + priv = + if Database.postgresql? + where(privilege_type: 'TRIGGER', table_name: table) + else + where(Trigger_priv: 'Y') + end + + priv.scope_to_current_user.any? + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 69ca9aa596b..fb14798efe6 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -358,6 +358,8 @@ module Gitlab raise 'rename_column_concurrently can not be run inside a transaction' end + check_trigger_permissions!(table) + old_col = column_for(table, old) new_type = type || old_col.type @@ -430,6 +432,8 @@ module Gitlab def cleanup_concurrent_column_rename(table, old, new) trigger_name = rename_trigger_name(table, old, new) + check_trigger_permissions!(table) + if Database.postgresql? remove_rename_triggers_for_postgresql(table, trigger_name) else @@ -485,14 +489,14 @@ module Gitlab # Removes the triggers used for renaming a PostgreSQL column concurrently. def remove_rename_triggers_for_postgresql(table, trigger) - execute("DROP TRIGGER #{trigger} ON #{table}") - execute("DROP FUNCTION #{trigger}()") + execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}") + execute("DROP FUNCTION IF EXISTS #{trigger}()") end # Removes the triggers used for renaming a MySQL column concurrently. def remove_rename_triggers_for_mysql(trigger) - execute("DROP TRIGGER #{trigger}_insert") - execute("DROP TRIGGER #{trigger}_update") + execute("DROP TRIGGER IF EXISTS #{trigger}_insert") + execute("DROP TRIGGER IF EXISTS #{trigger}_update") end # Returns the (base) name to use for triggers when renaming columns. @@ -606,6 +610,49 @@ module Gitlab Arel::Nodes::SqlLiteral.new(replace.to_sql) end end + + def remove_foreign_key_without_error(*args) + remove_foreign_key(*args) + rescue ArgumentError + end + + def sidekiq_queue_migrate(queue_from, to:) + while sidekiq_queue_length(queue_from) > 0 + Sidekiq.redis do |conn| + conn.rpoplpush "queue:#{queue_from}", "queue:#{to}" + end + end + end + + def sidekiq_queue_length(queue_name) + Sidekiq.redis do |conn| + conn.llen("queue:#{queue_name}") + end + end + + def check_trigger_permissions!(table) + unless Grant.create_and_execute_trigger?(table) + dbname = Database.database_name + user = Database.username + + raise <<-EOF +Your database user is not allowed to create, drop, or execute triggers on the +table #{table}. + +If you are using PostgreSQL you can solve this by logging in to the GitLab +database (#{dbname}) using a super user and running: + + ALTER #{user} WITH SUPERUSER + +For MySQL you instead need to run: + + GRANT ALL PRIVILEGES ON *.* TO #{user}@'%' + +Both queries will grant the user super user permissions, ensuring you don't run +into similar problems in the future (e.g. when new tables are created). + EOF + end + end end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 6d7de52cb80..1dabd4ebdd0 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -186,7 +186,10 @@ module Gitlab end def content_changed? - old_blob && new_blob && old_blob.id != new_blob.id + return blobs_changed? if diff_refs + return false if new_file? || deleted_file? || renamed_file? + + text? && diff_lines.any? end def different_type? @@ -225,6 +228,10 @@ module Gitlab private + def blobs_changed? + old_blob && new_blob && old_blob.id != new_blob.id + end + def simple_viewer_class return DiffViewer::NotDiffable unless diffable? @@ -250,6 +257,8 @@ module Gitlab DiffViewer::Renamed elsif mode_changed? DiffViewer::ModeChanged + else + DiffViewer::NoPreview end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 2d89ccfc354..0603141e441 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -21,7 +21,7 @@ module Gitlab def to_hash hash = {} - serialize_keys.each { |key| hash[key] = send(key) } + serialize_keys.each { |key| hash[key] = send(key) } # rubocop:disable GitlabSecurity/PublicSend hash end diff --git a/lib/gitlab/diff/line_mapper.rb b/lib/gitlab/diff/line_mapper.rb index 576a761423e..cf71d47df8e 100644 --- a/lib/gitlab/diff/line_mapper.rb +++ b/lib/gitlab/diff/line_mapper.rb @@ -38,7 +38,7 @@ module Gitlab # - The first diff line with a higher line number, if it falls between diff contexts # - The last known diff line, if it falls after the last diff context diff_line = diff_lines.find do |diff_line| - diff_from_line = diff_line.send(from) + diff_from_line = diff_line.public_send(from) # rubocop:disable GitlabSecurity/PublicSend diff_from_line && diff_from_line >= from_line end diff_line ||= diff_lines.last @@ -47,8 +47,8 @@ module Gitlab # mapped line number is the same as the specified line number. return from_line unless diff_line - diff_from_line = diff_line.send(from) - diff_to_line = diff_line.send(to) + diff_from_line = diff_line.public_send(from) # rubocop:disable GitlabSecurity/PublicSend + diff_to_line = diff_line.public_send(to) # rubocop:disable GitlabSecurity/PublicSend # If the line was removed, there is no mapped line number. return unless diff_to_line diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 72d7d4f84d1..abd401224d8 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -98,10 +98,11 @@ module Gitlab if status.zero? @ee_branch_found = ee_branch_prefix - else - _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}]) + return end + _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}]) + if status.zero? @ee_branch_found = ee_branch_suffix else diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 781f9c56a42..7b3483a7f96 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -11,7 +11,7 @@ module Gitlab # obscure encoding with low confidence. # There is a lot more info with this merge request: # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 - ENCODING_CONFIDENCE_THRESHOLD = 40 + ENCODING_CONFIDENCE_THRESHOLD = 50 def encode!(message) return nil unless message.respond_to? :force_encoding @@ -22,10 +22,10 @@ module Gitlab # return message if message type is binary detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if detect && detect[:type] == :binary + return message.force_encoding("BINARY") if detect_binary?(message, detect) - # force detected encoding if we have sufficient confidence. if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + # force detected encoding if we have sufficient confidence. message.force_encoding(detect[:encoding]) end @@ -36,6 +36,19 @@ module Gitlab "--broken encoding: #{encoding}" end + def detect_binary?(data, detect = nil) + detect ||= CharlockHolmes::EncodingDetector.detect(data) + detect && detect[:type] == :binary && detect[:confidence] == 100 + end + + def detect_libgit2_binary?(data) + # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks + # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), + # which is what we use below to keep a consistent behavior. + detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) + detect && detect[:type] == :binary + end + def encode_utf8(message) detect = CharlockHolmes::EncodingDetector.detect(message) if detect && detect[:encoding] diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 3784f6c4947..3f7b42456af 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -25,6 +25,12 @@ module Gitlab end EOS + def self.get_uuid(key) + Gitlab::Redis::SharedState.with do |redis| + redis.get(redis_shared_state_key(key)) || false + end + end + def self.cancel(key, uuid) Gitlab::Redis::SharedState.with do |redis| redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) @@ -35,10 +41,10 @@ module Gitlab "gitlab:exclusive_lease:#{key}" end - def initialize(key, timeout:) + def initialize(key, uuid: nil, timeout:) @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout - @uuid = SecureRandom.uuid + @uuid = uuid || SecureRandom.uuid end # Try to obtain the lease. Return lease UUID on success, diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 093d9ed8092..10ffc345bd5 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -6,27 +6,48 @@ module Gitlab attr_reader :project, :ref + delegate :repository, to: :project + def initialize(project, ref) @project = project @ref = ref end def find(query) - blobs = project.repository.search_files_by_content(query, ref).first(BATCH_SIZE) - found_file_names = Set.new + by_content = find_by_content(query) - results = blobs.map do |blob| - blob = Gitlab::ProjectSearchResults.parse_search_result(blob) - found_file_names << blob.filename + already_found = Set.new(by_content.map(&:filename)) + by_filename = find_by_filename(query, except: already_found) - [blob.filename, blob] - end + (by_content + by_filename) + .sort_by(&:filename) + .map { |blob| [blob.filename, blob] } + end - project.repository.search_files_by_name(query, ref).first(BATCH_SIZE).each do |filename| - results << [filename, OpenStruct.new(ref: ref)] unless found_file_names.include?(filename) - end + private - results.sort_by(&:first) + def find_by_content(query) + results = repository.search_files_by_content(query, ref).first(BATCH_SIZE) + results.map { |result| Gitlab::ProjectSearchResults.parse_search_result(result) } + end + + def find_by_filename(query, except: []) + filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE) + filenames.delete_if { |filename| except.include?(filename) } unless except.empty? + + blob_refs = filenames.map { |filename| [ref, filename] } + blobs = Gitlab::Git::Blob.batch(repository, blob_refs, blob_size_limit: 1024) + + blobs.map do |blob| + Gitlab::SearchResults::FoundBlob.new( + id: blob.id, + filename: blob.path, + basename: File.basename(blob.path), + ref: ref, + startline: 1, + data: blob.data + ) + end end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index b6449f27034..8c9acbc9fbe 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -5,6 +5,7 @@ module Gitlab BRANCH_REF_PREFIX = "refs/heads/".freeze CommandError = Class.new(StandardError) + CommitError = Class.new(StandardError) class << self include Gitlab::EncodingHelper diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 8dbe25e55f6..31effdba292 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -16,7 +16,7 @@ module Gitlab def each @blames.each do |blame| yield( - Gitlab::Git::Commit.new(blame.commit), + Gitlab::Git::Commit.new(@repo, blame.commit), blame.line ) end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 77b81d2d437..8d96826f6ee 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -42,19 +42,11 @@ module Gitlab end end - def binary?(data) - # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks - # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), - # which is what we use below to keep a consistent behavior. - detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) - detect && detect[:type] == :binary - end - # Returns an array of Blob instances, specified in blob_references as # [[commit_sha, path], [commit_sha, path], ...]. If blob_size_limit < 0 then the # full blob contents are returned. If blob_size_limit >= 0 then each blob will # contain no more than limit bytes in its data attribute. - # + # # Keep in mind that this method may allocate a lot of memory. It is up # to the caller to limit the number of blobs and blob_size_limit. # @@ -65,6 +57,10 @@ module Gitlab end end + def binary?(data) + EncodingHelper.detect_libgit2_binary?(data) + end + private # Recursive search of blob id by path @@ -173,7 +169,7 @@ module Gitlab def initialize(options) %w(id name path size data mode commit_id binary).each do |key| - self.send("#{key}=", options[key.to_sym]) + self.__send__("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end @loaded_all_data = false diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 600d886e818..5ee6669050c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -14,7 +14,7 @@ module Gitlab attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator - delegate :tree, to: :raw_commit + delegate :tree, to: :rugged_commit def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) @@ -50,19 +50,29 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321 def find(repo, commit_id = "HEAD") + # Already a commit? return commit_id if commit_id.is_a?(Gitlab::Git::Commit) - return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) - obj = if commit_id.is_a?(String) - repo.rev_parse_target(commit_id) - else - Gitlab::Git::Ref.dereference_object(commit_id) - end + # A rugged reference? + commit_id = Gitlab::Git::Ref.dereference_object(commit_id) + return decorate(repo, commit_id) if commit_id.is_a?(Rugged::Commit) - return nil unless obj.is_a?(Rugged::Commit) + # Some weird thing? + return nil unless commit_id.is_a?(String) - decorate(obj) - rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, Gitlab::Git::Repository::NoRepository + commit = repo.gitaly_migrate(:find_commit) do |is_enabled| + if is_enabled + repo.gitaly_commit_client.find_commit(commit_id) + else + obj = repo.rev_parse_target(commit_id) + + obj.is_a?(Rugged::Commit) ? obj : nil + end + end + + decorate(repo, commit) if commit + rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, + Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository nil end @@ -102,7 +112,7 @@ module Gitlab if is_enabled repo.gitaly_commit_client.between(base, head) else - repo.rugged_commits_between(base, head).map { |c| decorate(c) } + repo.rugged_commits_between(base, head).map { |c| decorate(repo, c) } end end rescue Rugged::ReferenceError @@ -169,7 +179,7 @@ module Gitlab offset = actual_options[:skip] limit = actual_options[:max_count] walker.each(offset: offset, limit: limit) do |commit| - commits.push(decorate(commit)) + commits.push(decorate(repo, commit)) end walker.reset @@ -183,27 +193,8 @@ module Gitlab Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end - def decorate(commit, ref = nil) - Gitlab::Git::Commit.new(commit, ref) - end - - # Returns a diff object for the changes introduced by +rugged_commit+. - # If +rugged_commit+ doesn't have a parent, then the diff is between - # this commit and an empty repo. See Repository#diff for the keys - # allowed in the +options+ hash. - def diff_from_parent(rugged_commit, options = {}) - options ||= {} - break_rewrites = options[:break_rewrites] - actual_options = Gitlab::Git::Diff.filter_diff_options(options) - - diff = if rugged_commit.parents.empty? - rugged_commit.diff(actual_options.merge(reverse: true)) - else - rugged_commit.parents[0].diff(rugged_commit, actual_options) - end - - diff.find_similar!(break_rewrites: break_rewrites) - diff + def decorate(repository, commit, ref = nil) + Gitlab::Git::Commit.new(repository, commit, ref) end # Returns the `Rugged` sorting type constant for one or more given @@ -219,9 +210,19 @@ module Gitlab @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE) end + + def shas_with_signatures(repository, shas) + shas.select do |sha| + begin + Rugged::Commit.extract_signature(repository.rugged, sha) + rescue Rugged::OdbError + false + end + end + end end - def initialize(raw_commit, head = nil) + def initialize(repository, raw_commit, head = nil) raise "Nil as raw commit passed" unless raw_commit case raw_commit @@ -229,12 +230,13 @@ module Gitlab init_from_hash(raw_commit) when Rugged::Commit init_from_rugged(raw_commit) - when Gitlab::GitalyClient::Commit + when Gitaly::GitCommit init_from_gitaly(raw_commit) else raise "Invalid raw commit type: #{raw_commit.class}" end + @repository = repository @head = head end @@ -269,19 +271,56 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 def to_diff - diff_from_parent.patch + Gitlab::GitalyClient.migrate(:commit_patch) do |is_enabled| + if is_enabled + @repository.gitaly_commit_client.patch(id) + else + rugged_diff_from_parent.patch + end + end end # Returns a diff object for the changes from this commit's first parent. # If there is no parent, then the diff is between this commit and an - # empty repo. See Repository#diff for keys allowed in the +options+ + # empty repo. See Repository#diff for keys allowed in the +options+ # hash. def diff_from_parent(options = {}) - Commit.diff_from_parent(raw_commit, options) + Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| + if is_enabled + @repository.gitaly_commit_client.diff_from_parent(self, options) + else + rugged_diff_from_parent(options) + end + end + end + + def rugged_diff_from_parent(options = {}) + options ||= {} + break_rewrites = options[:break_rewrites] + actual_options = Gitlab::Git::Diff.filter_diff_options(options) + + diff = if rugged_commit.parents.empty? + rugged_commit.diff(actual_options.merge(reverse: true)) + else + rugged_commit.parents[0].diff(rugged_commit, actual_options) + end + + diff.find_similar!(break_rewrites: break_rewrites) + diff end def deltas - @deltas ||= diff_from_parent.each_delta.map { |d| Gitlab::Git::Diff.new(d) } + @deltas ||= begin + deltas = Gitlab::GitalyClient.migrate(:commit_deltas) do |is_enabled| + if is_enabled + @repository.gitaly_commit_client.commit_deltas(self) + else + rugged_diff_from_parent.each_delta + end + end + + deltas.map { |delta| Gitlab::Git::Diff.new(delta) } + end end def has_zero_stats? @@ -296,7 +335,7 @@ module Gitlab def to_hash serialize_keys.map.with_object({}) do |key, hash| - hash[key] = send(key) + hash[key] = send(key) # rubocop:disable GitlabSecurity/PublicSend end end @@ -309,23 +348,7 @@ module Gitlab end def parents - case raw_commit - when Rugged::Commit - raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } - when Gitlab::GitalyClient::Commit - parent_ids.map { |oid| self.class.find(raw_commit.repository, oid) }.compact - else - raise NotImplementedError, "commit source doesn't support #parents" - end - end - - # Get the gpg signature of this commit. - # - # Ex. - # commit.signature(repo) - # - def signature(repo) - Rugged::Commit.extract_signature(repo.rugged, sha) + parent_ids.map { |oid| self.class.find(@repository, oid) }.compact end def stats @@ -334,7 +357,7 @@ module Gitlab def to_patch(options = {}) begin - raw_commit.to_mbox(options) + rugged_commit.to_mbox(options) rescue Rugged::InvalidError => ex if ex.message =~ /commit \w+ is a merge commit/i 'Patch format is not currently supported for merge commits.' @@ -382,13 +405,21 @@ module Gitlab encode! @committer_email end + def rugged_commit + @rugged_commit ||= if raw_commit.is_a?(Rugged::Commit) + raw_commit + else + @repository.rev_parse_target(id) + end + end + private def init_from_hash(hash) raw_commit = hash.symbolize_keys serialize_keys.each do |key| - send("#{key}=", raw_commit[key]) + send("#{key}=", raw_commit[key]) # rubocop:disable GitlabSecurity/PublicSend end end @@ -415,10 +446,10 @@ module Gitlab # subject from the message to make it clearer when there's one # available but not the other. @message = (commit.body.presence || commit.subject).dup - @authored_date = Time.at(commit.author.date.seconds) + @authored_date = Time.at(commit.author.date.seconds).utc @author_name = commit.author.name.dup @author_email = commit.author.email.dup - @committed_date = Time.at(commit.committer.date.seconds) + @committed_date = Time.at(commit.committer.date.seconds).utc @committer_name = commit.committer.name.dup @committer_email = commit.committer.email.dup @parent_ids = commit.parent_ids diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 57c29ad112c..00acb4763e9 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -16,7 +16,7 @@ module Gitlab @deletions = 0 @total = 0 - diff = commit.diff_from_parent + diff = commit.rugged_diff_from_parent diff.each_patch do |p| # TODO: Use the new Rugged convenience methods when they're released diff --git a/lib/gitlab/git/committer.rb b/lib/gitlab/git/committer.rb new file mode 100644 index 00000000000..1f4bcf7a3a0 --- /dev/null +++ b/lib/gitlab/git/committer.rb @@ -0,0 +1,21 @@ +module Gitlab + module Git + class Committer + attr_reader :name, :email, :gl_id + + def self.from_user(user) + new(user.name, user.email, Gitlab::GlId.gl_id(user)) + end + + def initialize(name, email, gl_id) + @name = name + @email = email + @gl_id = gl_id + end + + def ==(other) + [name, email, gl_id] == [other.name, other.email, other.gl_id] + end + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 9e00abefd02..a23c8cf0dd1 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -116,6 +116,15 @@ module Gitlab filtered_opts end + + # Return a binary diff message like: + # + # "Binary files a/file/path and b/file/path differ\n" + # This is used when we detect that a diff is binary + # using CharlockHolmes when Rugged treats it as text. + def binary_message(old_path, new_path) + "Binary files #{old_path} and #{new_path} differ\n" + end end def initialize(raw_diff, expanded: true) @@ -143,7 +152,7 @@ module Gitlab hash = {} SERIALIZE_KEYS.each do |key| - hash[key] = send(key) + hash[key] = send(key) # rubocop:disable GitlabSecurity/PublicSend end hash @@ -190,6 +199,13 @@ module Gitlab @collapsed = true end + def json_safe_diff + return @diff unless detect_binary?(@diff) + + # the diff is binary, let's make a message for it + Diff.binary_message(@old_path, @new_path) + end + private def init_from_rugged(rugged) @@ -221,7 +237,7 @@ module Gitlab raw_diff = hash.symbolize_keys SERIALIZE_KEYS.each do |key| - send(:"#{key}=", raw_diff[key.to_sym]) + send(:"#{key}=", raw_diff[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 87ed9c3ea26..6a601561c2a 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -28,7 +28,6 @@ module Gitlab @limits = self.class.collection_limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) - @from_gitaly = options.fetch(:from_gitaly, false) @line_count = 0 @byte_count = 0 @@ -44,7 +43,7 @@ module Gitlab return if @iterator.nil? Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| - if is_enabled && @from_gitaly + if is_enabled && @iterator.is_a?(Gitlab::GitalyClient::DiffStitcher) each_gitaly_patch(&block) else each_rugged_patch(&block) diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 8f0c377ef4f..cc35d77c6e4 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -1,20 +1,23 @@ -# Gitaly note: JV: looks like this is only used by GitHooksService in +# Gitaly note: JV: looks like this is only used by Gitlab::Git::HooksService in # app/services. We shouldn't bother migrating this until we know how -# GitHooksService will be migrated. +# Gitlab::Git::HooksService will be migrated. module Gitlab module Git class Hook GL_PROTOCOL = 'web'.freeze - attr_reader :name, :repo_path, :path + attr_reader :name, :path, :repository - def initialize(name, project) + def initialize(name, repository) @name = name - @project = project - @repo_path = project.repository.path + @repository = repository @path = File.join(repo_path.strip, 'hooks', name) end + def repo_path + repository.path + end + def exists? File.exist?(path) end @@ -44,7 +47,7 @@ module Gitlab 'GL_ID' => gl_id, 'PWD' => repo_path, 'GL_PROTOCOL' => GL_PROTOCOL, - 'GL_REPOSITORY' => Gitlab::GlRepository.gl_repository(@project, false) + 'GL_REPOSITORY' => repository.gl_repository } options = { diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb new file mode 100644 index 00000000000..ea8a87a1290 --- /dev/null +++ b/lib/gitlab/git/hooks_service.rb @@ -0,0 +1,36 @@ +module Gitlab + module Git + class HooksService + PreReceiveError = Class.new(StandardError) + + attr_accessor :oldrev, :newrev, :ref + + def execute(committer, repository, oldrev, newrev, ref) + @repository = repository + @gl_id = committer.gl_id + @oldrev = oldrev + @newrev = newrev + @ref = ref + + %w(pre-receive update).each do |hook_name| + status, message = run_hook(hook_name) + + unless status + raise PreReceiveError, message + end + end + + yield(self).tap do + run_hook('post-receive') + end + end + + private + + def run_hook(name) + hook = Gitlab::Git::Hook.new(name, @repository) + hook.trigger(@gl_id, oldrev, newrev, ref) + end + end + end +end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb new file mode 100644 index 00000000000..9e6fca8c80c --- /dev/null +++ b/lib/gitlab/git/operation_service.rb @@ -0,0 +1,168 @@ +module Gitlab + module Git + class OperationService + attr_reader :committer, :repository + + def initialize(committer, new_repository) + committer = Gitlab::Git::Committer.from_user(committer) if committer.is_a?(User) + @committer = committer + + # Refactoring aid + unless new_repository.is_a?(Gitlab::Git::Repository) + raise "expected a Gitlab::Git::Repository, got #{new_repository}" + end + + @repository = new_repository + end + + def add_branch(branch_name, newrev) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + oldrev = Gitlab::Git::BLANK_SHA + + update_ref_in_hooks(ref, newrev, oldrev) + end + + def rm_branch(branch) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch.name + oldrev = branch.target + newrev = Gitlab::Git::BLANK_SHA + + update_ref_in_hooks(ref, newrev, oldrev) + end + + def add_tag(tag_name, newrev, options = {}) + ref = Gitlab::Git::TAG_REF_PREFIX + tag_name + oldrev = Gitlab::Git::BLANK_SHA + + with_hooks(ref, newrev, oldrev) do |service| + # We want to pass the OID of the tag object to the hooks. For an + # annotated tag we don't know that OID until after the tag object + # (raw_tag) is created in the repository. That is why we have to + # update the value after creating the tag object. Only the + # "post-receive" hook will receive the correct value in this case. + raw_tag = repository.rugged.tags.create(tag_name, newrev, options) + service.newrev = raw_tag.target_id + end + end + + def rm_tag(tag) + ref = Gitlab::Git::TAG_REF_PREFIX + tag.name + oldrev = tag.target + newrev = Gitlab::Git::BLANK_SHA + + update_ref_in_hooks(ref, newrev, oldrev) do + repository.rugged.tags.delete(tag_name) + end + end + + # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist, + # it would be created from `start_branch_name`. + # If `start_project` is passed, and the branch doesn't exist, + # it would try to find the commits from it instead of current repository. + def with_branch( + branch_name, + start_branch_name: nil, + start_repository: repository, + &block) + + # Refactoring aid + unless start_repository.is_a?(Gitlab::Git::Repository) + raise "expected a Gitlab::Git::Repository, got #{start_repository}" + end + + start_branch_name = nil if start_repository.empty_repo? + + if start_branch_name && !start_repository.branch_exists?(start_branch_name) + raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.full_path}" + end + + update_branch_with_hooks(branch_name) do + repository.with_repo_branch_commit( + start_repository, + start_branch_name || branch_name, + &block) + end + end + + private + + # Returns [newrev, should_run_after_create, should_run_after_create_branch] + def update_branch_with_hooks(branch_name) + update_autocrlf_option + + was_empty = repository.empty? + + # Make commit + newrev = yield + + unless newrev + raise Gitlab::Git::CommitError.new('Failed to create commit') + end + + branch = repository.find_branch(branch_name) + oldrev = find_oldrev_from_branch(newrev, branch) + + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + update_ref_in_hooks(ref, newrev, oldrev) + + [newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)] + end + + def find_oldrev_from_branch(newrev, branch) + return Gitlab::Git::BLANK_SHA unless branch + + oldrev = branch.target + + if oldrev == repository.rugged.merge_base(newrev, branch.target) + oldrev + else + raise Gitlab::Git::CommitError.new('Branch diverged') + end + end + + def update_ref_in_hooks(ref, newrev, oldrev) + with_hooks(ref, newrev, oldrev) do + update_ref(ref, newrev, oldrev) + end + end + + def with_hooks(ref, newrev, oldrev) + Gitlab::Git::HooksService.new.execute( + committer, + repository, + oldrev, + newrev, + ref) do |service| + + yield(service) + end + end + + # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites. + def update_ref(ref, newrev, oldrev) + # We use 'git update-ref' because libgit2/rugged currently does not + # offer 'compare and swap' ref updates. Without compare-and-swap we can + # (and have!) accidentally reset the ref to an earlier state, clobbering + # commits. See also https://github.com/libgit2/libgit2/issues/1534. + command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] + _, status = Gitlab::Popen.popen( + command, + repository.path) do |stdin| + stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") + end + + unless status.zero? + raise Gitlab::Git::CommitError.new( + "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ + " Please refresh and try again.") + end + end + + def update_autocrlf_option + if repository.autocrlf != :input + repository.autocrlf = :input + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f246393cfbc..efa13590a2c 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -17,6 +17,30 @@ module Gitlab NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) InvalidRef = Class.new(StandardError) + GitError = Class.new(StandardError) + DeleteBranchError = Class.new(StandardError) + + class << self + # Unlike `new`, `create` takes the storage path, not the storage name + def create(storage_path, name, bare: true, symlink_hooks_to: nil) + repo_path = File.join(storage_path, name) + repo_path += '.git' unless repo_path.end_with?('.git') + + FileUtils.mkdir_p(repo_path, mode: 0770) + + # Equivalent to `git --git-path=#{repo_path} init [--bare]` + repo = Rugged::Repository.init_at(repo_path, bare) + repo.close + + if symlink_hooks_to.present? + hooks_path = File.join(repo_path, 'hooks') + FileUtils.rm_rf(hooks_path) + FileUtils.ln_s(symlink_hooks_to, hooks_path) + end + + true + end + end # Full path to repo attr_reader :path @@ -24,16 +48,20 @@ module Gitlab # Directory name of repo attr_reader :name + # Relative path of repo + attr_reader :relative_path + # Rugged repo object attr_reader :rugged - attr_reader :storage + attr_reader :storage, :gl_repository, :relative_path # 'path' must be the path to a _bare_ git repository, e.g. # /path/to/my-repo.git - def initialize(storage, relative_path) + def initialize(storage, relative_path, gl_repository) @storage = storage @relative_path = relative_path + @gl_repository = gl_repository storage_path = Gitlab.config.repositories.storages[@storage]['path'] @path = File.join(storage_path, @relative_path) @@ -42,11 +70,14 @@ module Gitlab end delegate :empty?, - :bare?, to: :rugged delegate :exists?, to: :gitaly_repository_client + def ==(other) + path == other.path + end + # Default branch in the repository def root_ref @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| @@ -105,12 +136,18 @@ module Gitlab # be stale/invalid when packed-refs is changed. # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 def find_branch(name, force_reload = false) - reload_rugged if force_reload + gitaly_migrate(:find_branch) do |is_enabled| + if is_enabled + gitaly_ref_client.find_branch(name) + else + reload_rugged if force_reload - rugged_ref = rugged.branches[name] - if rugged_ref - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rugged_ref = rugged.branches[name] + if rugged_ref + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + end + end end end @@ -130,7 +167,7 @@ module Gitlab if is_enabled gitaly_ref_client.count_branch_names else - rugged.branches.count do |ref| + rugged.branches.each(:local).count do |ref| begin ref.name && ref.target # ensures the branch is valid @@ -178,25 +215,43 @@ module Gitlab end end + # Returns true if the given ref name exists + # + # Ref names must start with `refs/`. + def ref_exists?(ref_name) + gitaly_migrate(:ref_exists) do |is_enabled| + if is_enabled + gitaly_ref_exists?(ref_name) + else + rugged_ref_exists?(ref_name) + end + end + end + # Returns true if the given tag exists # # name - The name of the tag as a String. def tag_exists?(name) - !!rugged.tags[name] + gitaly_migrate(:ref_exists_tags) do |is_enabled| + if is_enabled + gitaly_ref_exists?("refs/tags/#{name}") + else + rugged_tag_exists?(name) + end + end end # Returns true if the given branch exists # # name - The name of the branch as a String. def branch_exists?(name) - rugged.branches.exists?(name) - - # If the branch name is invalid (e.g. ".foo") Rugged will raise an error. - # Whatever code calls this method shouldn't have to deal with that so - # instead we just return `false` (which is true since a branch doesn't - # exist when it has an invalid name). - rescue Rugged::ReferenceError - false + gitaly_migrate(:ref_exists_branches) do |is_enabled| + if is_enabled + gitaly_ref_exists?("refs/heads/#{name}") + else + rugged_branch_exists?(name) + end + end end # Returns an Array of branch and tag names @@ -204,8 +259,17 @@ module Gitlab branch_names + tag_names end - def has_commits? - !empty? + def delete_all_refs_except(prefixes) + delete_refs(*all_ref_names_except(prefixes)) + end + + # Returns an Array of all ref names, except when it's matching pattern + # + # regexp - The pattern for ref names we don't want + def all_ref_names_except(prefixes) + rugged.references.reject do |ref| + prefixes.any? { |p| ref.name.start_with?(p) } + end.map(&:name) end # Discovers the default branch based on the repository's available branches @@ -321,7 +385,24 @@ module Gitlab options[:limit] ||= 0 options[:offset] ||= 0 - raw_log(options).map { |c| Commit.decorate(c) } + raw_log(options).map { |c| Commit.decorate(self, c) } + end + + # Used in gitaly-ruby + def raw_log(options) + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end end def count_commits(options) @@ -384,8 +465,8 @@ module Gitlab end # Returns true is +from+ is direct ancestor to +to+, otherwise false - def is_ancestor?(from, to) - gitaly_commit_client.is_ancestor(from, to) + def ancestor?(from, to) + gitaly_commit_client.ancestor?(from, to) end # Return an array of Diff objects that represent the diff @@ -529,9 +610,77 @@ module Gitlab # TODO: implement this method end + def add_branch(branch_name, committer:, target:) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + OperationService.new(committer, self).add_branch(branch_name, target_object.oid) + find_branch(branch_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + end + + def add_tag(tag_name, committer:, target:, message: nil) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + committer = Committer.from_user(committer) if committer.is_a?(User) + + options = nil # Use nil, not the empty hash. Rugged cares about this. + if message + options = { + message: message, + tagger: Gitlab::Git.committer_hash(email: committer.email, name: committer.name) + } + end + + OperationService.new(committer, self).add_tag(tag_name, target_object.oid, options) + + find_tag(tag_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + end + + def rm_branch(branch_name, committer:) + OperationService.new(committer, self).rm_branch(find_branch(branch_name)) + end + + def rm_tag(tag_name, committer:) + OperationService.new(committer, self).rm_tag(find_tag(tag_name)) + end + + def find_tag(name) + tags.find { |tag| tag.name == name } + end + # Delete the specified branch from the repository def delete_branch(branch_name) - rugged.branches.delete(branch_name) + gitaly_migrate(:delete_branch) do |is_enabled| + if is_enabled + gitaly_ref_client.delete_branch(branch_name) + else + rugged.branches.delete(branch_name) + end + end + rescue Rugged::ReferenceError, CommandError => e + raise DeleteBranchError, e + end + + def delete_refs(*ref_names) + instructions = ref_names.map do |ref| + "delete #{ref}\x00\x00" + end + + command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] + message, status = Gitlab::Popen.popen( + command, + path) do |stdin| + stdin.write(instructions.join) + end + + unless status.zero? + raise GitError.new("Could not delete refs #{ref_names}: #{message}") + end end # Create a new branch named **ref+ based on **stat_point+, HEAD by default @@ -540,46 +689,35 @@ module Gitlab # create_branch("feature") # create_branch("other-feature", "master") def create_branch(ref, start_point = "HEAD") - rugged_ref = rugged.branches.create(ref, start_point) - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError => e - raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ - raise InvalidRef.new("Invalid reference #{start_point}") - end - - # Return an array of this repository's remote names - def remote_names - rugged.remotes.each_name.to_a + gitaly_migrate(:create_branch) do |is_enabled| + if is_enabled + gitaly_ref_client.create_branch(ref, start_point) + else + rugged_create_branch(ref, start_point) + end + end end # Delete the specified remote from this repository. def remote_delete(remote_name) rugged.remotes.delete(remote_name) + nil end - # Add a new remote to this repository. Returns a Rugged::Remote object + # Add a new remote to this repository. def remote_add(remote_name, url) rugged.remotes.create(remote_name, url) + nil end # Update the specified remote using the values in the +options+ hash # # Example # repo.update_remote("origin", url: "path/to/repo") - def remote_update(remote_name, options = {}) + def remote_update(remote_name, url:) # TODO: Implement other remote options - rugged.remotes.set_url(remote_name, options[:url]) if options[:url] - end - - # Fetch the specified remote - def fetch(remote_name) - rugged.remotes[remote_name].fetch - end - - # Push +*refspecs+ to the remote identified by +remote_name+. - def push(remote_name, *refspecs) - rugged.remotes[remote_name].push(refspecs) + rugged.remotes.set_url(remote_name, url) + nil end AUTOCRLF_VALUES = { @@ -603,60 +741,26 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) - actual_ref = ref || root_ref - - begin - sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree) - cmd += %w(-r) - cmd += %w(--full-tree) - cmd += %w(--full-name) - cmd += %W(-- #{actual_ref}) - - raw_output = IO.popen(cmd, &:read).split("\n").map do |f| - stuff, path = f.split("\t") - _mode, type, _sha = stuff.split(" ") - path if type == "blob" - # Contain only blob type + gitaly_migrate(:ls_files) do |is_enabled| + if is_enabled + gitaly_ls_files(ref) + else + git_ls_files(ref) + end end - - raw_output.compact end # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 def copy_gitattributes(ref) - begin - commit = lookup(ref) - rescue Rugged::ReferenceError - raise InvalidRef.new("Ref #{ref} is invalid") - end - - # Create the paths - info_dir_path = File.join(path, 'info') - info_attributes_path = File.join(info_dir_path, 'attributes') - - begin - # Retrieve the contents of the blob - gitattributes_content = blob_content(commit, '.gitattributes') - rescue InvalidBlobName - # No .gitattributes found. Should now remove any info/attributes and return - File.delete(info_attributes_path) if File.exist?(info_attributes_path) - return - end - - # Create the info directory if needed - Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path) - - # Write the contents of the .gitattributes file to info/attributes - # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8 - File.open(info_attributes_path, "wb") do |file| - file.write(gitattributes_content) + Gitlab::GitalyClient.migrate(:apply_gitattributes) do |is_enabled| + if is_enabled + gitaly_copy_gitattributes(ref) + else + rugged_copy_gitattributes(ref) + end end + rescue GRPC::InvalidArgument + raise InvalidRef end # Returns the Git attributes for the given file path. @@ -693,6 +797,106 @@ module Gitlab end end + def with_repo_branch_commit(start_repository, start_branch_name) + raise "expected Gitlab::Git::Repository, got #{start_repository}" unless start_repository.is_a?(Gitlab::Git::Repository) + + return yield nil if start_repository.empty_repo? + + if start_repository == self + yield commit(start_branch_name) + else + sha = start_repository.commit(start_branch_name).sha + + if branch_commit = commit(sha) + yield branch_commit + else + with_repo_tmp_commit( + start_repository, start_branch_name, sha) do |tmp_commit| + yield tmp_commit + end + end + end + end + + def with_repo_tmp_commit(start_repository, start_branch_name, sha) + tmp_ref = fetch_ref( + start_repository.path, + "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + "refs/tmp/#{SecureRandom.hex}/head" + ) + + yield commit(sha) + ensure + delete_refs(tmp_ref) if tmp_ref + end + + def fetch_source_branch(source_repository, source_branch, local_ref) + with_repo_branch_commit(source_repository, source_branch) do |commit| + if commit + write_ref(local_ref, commit.sha) + else + raise Rugged::ReferenceError, 'source repository is empty' + end + end + end + + def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) + with_repo_branch_commit(source_repository, source_branch_name) do |commit| + break unless commit + + Gitlab::Git::Compare.new( + self, + target_branch_name, + commit.sha, + straight: straight + ) + end + end + + def write_ref(ref_path, sha) + rugged.references.create(ref_path, sha, force: true) + end + + def fetch_ref(source_path, source_ref, target_ref) + args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) + message, status = run_git(args) + + # Make sure ref was created, and raise Rugged::ReferenceError when not + raise Rugged::ReferenceError, message if status != 0 + + target_ref + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb + def run_git(args) + circuit_breaker.perform do + popen([Gitlab.config.git.bin_path, *args], path) + end + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb + def commit(ref = 'HEAD') + Gitlab::Git::Commit.find(self, ref) + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb + def empty_repo? + !exists? || !has_visible_content? + end + + # + # Git repository can contains some hidden refs like: + # /refs/notes/* + # /refs/git-as-svn/* + # /refs/pulls/* + # This refs by default not visible in project page and not cloned to client side. + # + # This method return true if repository contains some content visible in project page. + # + def has_visible_content? + branch_count > 0 + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path) end @@ -733,22 +937,6 @@ module Gitlab sort_branches(branches, sort_by) end - def raw_log(options) - actual_ref = options[:ref] || root_ref - begin - sha = sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end - end - def log_using_shell?(options) options[:path].present? || options[:disable_walk] || @@ -826,6 +1014,8 @@ module Gitlab return unless commit_object && commit_object.type == :COMMIT gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) + return unless gitmodules + found_module = GitmodulesParser.new(gitmodules.data).parse[path] found_module && found_module['url'] @@ -973,6 +1163,117 @@ module Gitlab raw_output.to_i end + + def gitaly_ls_files(ref) + gitaly_commit_client.ls_files(ref) + end + + def git_ls_files(ref) + actual_ref = ref || root_ref + + begin + sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree) + cmd += %w(-r) + cmd += %w(--full-tree) + cmd += %w(--full-name) + cmd += %W(-- #{actual_ref}) + + raw_output = IO.popen(cmd, &:read).split("\n").map do |f| + stuff, path = f.split("\t") + _mode, type, _sha = stuff.split(" ") + path if type == "blob" + # Contain only blob type + end + + raw_output.compact + end + + # Returns true if the given ref name exists + # + # Ref names must start with `refs/`. + def rugged_ref_exists?(ref_name) + raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/') + rugged.references.exist?(ref_name) + rescue Rugged::ReferenceError + false + end + + # Returns true if the given ref name exists + # + # Ref names must start with `refs/`. + def gitaly_ref_exists?(ref_name) + gitaly_ref_client.ref_exists?(ref_name) + end + + # Returns true if the given tag exists + # + # name - The name of the tag as a String. + def rugged_tag_exists?(name) + !!rugged.tags[name] + end + + # Returns true if the given branch exists + # + # name - The name of the branch as a String. + def rugged_branch_exists?(name) + rugged.branches.exists?(name) + + # If the branch name is invalid (e.g. ".foo") Rugged will raise an error. + # Whatever code calls this method shouldn't have to deal with that so + # instead we just return `false` (which is true since a branch doesn't + # exist when it has an invalid name). + rescue Rugged::ReferenceError + false + end + + def rugged_create_branch(ref, start_point) + rugged_ref = rugged.branches.create(ref, start_point) + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError => e + raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Invalid reference #{start_point}") + end + + def gitaly_copy_gitattributes(revision) + gitaly_repository_client.apply_gitattributes(revision) + end + + def rugged_copy_gitattributes(ref) + begin + commit = lookup(ref) + rescue Rugged::ReferenceError + raise InvalidRef.new("Ref #{ref} is invalid") + end + + # Create the paths + info_dir_path = File.join(path, 'info') + info_attributes_path = File.join(info_dir_path, 'attributes') + + begin + # Retrieve the contents of the blob + gitattributes_content = blob_content(commit, '.gitattributes') + rescue InvalidBlobName + # No .gitattributes found. Should now remove any info/attributes and return + File.delete(info_attributes_path) if File.exist?(info_attributes_path) + return + end + + # Create the info directory if needed + Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path) + + # Write the contents of the .gitattributes file to info/attributes + # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8 + File.open(info_attributes_path, "wb") do |file| + file.write(gitattributes_content) + end + end end end end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 8e959c57c7c..5cf336af3c6 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -5,7 +5,7 @@ module Gitlab class Tree include Gitlab::EncodingHelper - attr_accessor :id, :root_id, :name, :path, :type, + attr_accessor :id, :root_id, :name, :path, :flat_path, :type, :mode, :commit_id, :submodule_url class << self @@ -19,8 +19,7 @@ module Gitlab Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled| if is_enabled - client = Gitlab::GitalyClient::CommitService.new(repository) - client.tree_entries(repository, sha, path) + repository.gitaly_commit_client.tree_entries(repository, sha, path) else tree_entries_from_rugged(repository, sha, path) end @@ -88,8 +87,8 @@ module Gitlab end def initialize(options) - %w(id root_id name path type mode commit_id).each do |key| - self.send("#{key}=", options[key.to_sym]) + %w(id root_id name path flat_path type mode commit_id).each do |key| + self.send("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end end @@ -101,6 +100,10 @@ module Gitlab encode! @path end + def flat_path + encode! @flat_path + end + def dir? type == :tree end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 0b62911958d..62d1ecae676 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -4,6 +4,7 @@ module Gitlab class GitAccess UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) + ProjectMovedError = Class.new(NotFoundError) ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', @@ -34,6 +35,7 @@ module Gitlab def check(cmd, changes) check_protocol! + check_valid_actor! check_active_user! check_project_accessibility! check_project_moved! @@ -69,6 +71,14 @@ module Gitlab private + def check_valid_actor! + return unless actor.is_a?(Key) + + unless actor.valid? + raise UnauthorizedError, "Your SSH key #{actor.errors[:key].first}." + end + end + def check_protocol! unless protocol_allowed? raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" @@ -90,18 +100,18 @@ module Gitlab end def check_project_moved! - if redirected_path - url = protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo - message = <<-MESSAGE.strip_heredoc - Project '#{redirected_path}' was moved to '#{project.full_path}'. + return unless redirected_path - Please update your Git remote and try again: + url = protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + message = <<-MESSAGE.strip_heredoc + Project '#{redirected_path}' was moved to '#{project.full_path}'. - git remote set-url origin #{url} - MESSAGE + Please update your Git remote and try again: - raise NotFoundError, message - end + git remote set-url origin #{url} + MESSAGE + + raise ProjectMovedError, message end def check_command_disabled!(cmd) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c90ef282fdd..a3dc2cd0b60 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -55,7 +55,7 @@ module Gitlab def self.call(storage, service, rpc, request) metadata = request_metadata(storage) metadata = yield(metadata) if block_given? - stub(service, storage).send(rpc, request, metadata) + stub(service, storage).__send__(rpc, request, metadata) # rubocop:disable GitlabSecurity/PublicSend end def self.request_metadata(storage) @@ -70,21 +70,41 @@ module Gitlab params['gitaly_token'].presence || Gitlab.config.gitaly['token'] end - def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN) + # Evaluates whether a feature toggle is on or off + def self.feature_enabled?(feature_name, status: MigrationStatus::OPT_IN) + # Disabled features are always off! return false if status == MigrationStatus::DISABLED - feature = Feature.get("gitaly_#{feature}") + feature = Feature.get("gitaly_#{feature_name}") - # If the feature hasn't been set, turn it on if it's opt-out - return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature) + # If the feature has been set, always evaluate + if Feature.persisted?(feature) + if feature.percentage_of_time_value > 0 + # Probabilistically enable this feature + return Random.rand() * 100 < feature.percentage_of_time_value + end + + return feature.enabled? + end - if feature.percentage_of_time_value > 0 - # Probabilistically enable this feature - return Random.rand() * 100 < feature.percentage_of_time_value + # If the feature has not been set, the default depends + # on it's status + case status + when MigrationStatus::OPT_OUT + true + when MigrationStatus::OPT_IN + opt_into_all_features? + else + false end + end - feature.enabled? + # opt_into_all_features? returns true when the current environment + # is one in which we opt into features automatically + def self.opt_into_all_features? + Rails.env.development? || ENV["GITALY_FEATURE_DEFAULT_ON"] == "1" end + private_class_method :opt_into_all_features? def self.migrate(feature, status: MigrationStatus::OPT_IN) is_enabled = feature_enabled?(feature, status: status) @@ -100,5 +120,9 @@ module Gitlab path = Rails.root.join(SERVER_VERSION_FILE) path.read.chomp end + + def self.encode(s) + s.dup.force_encoding(Encoding::ASCII_8BIT) + end end end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 7ea8e8d0857..a250eb75bd4 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -13,10 +13,17 @@ module Gitlab ) response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request) - blob = response.first - return unless blob.oid.present? + data = '' + blob = nil + response.each do |msg| + if blob.nil? + blob = msg + end - data = response.reduce(blob.data.dup) { |memo, msg| memo << msg.data.dup } + data << msg.data + end + + return nil if blob.oid.blank? Gitlab::Git::Blob.new( id: blob.oid, diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb deleted file mode 100644 index 61fe462d762..00000000000 --- a/lib/gitlab/gitaly_client/commit.rb +++ /dev/null @@ -1,14 +0,0 @@ -module Gitlab - module GitalyClient - class Commit - attr_reader :repository, :gitaly_commit - - delegate :id, :subject, :body, :author, :committer, :parent_ids, to: :gitaly_commit - - def initialize(repository, gitaly_commit) - @repository = repository - @gitaly_commit = gitaly_commit - end - end - end -end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 3f577ac8530..0825a3a7694 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -10,7 +10,19 @@ module Gitlab @repository = repository end - def is_ancestor(ancestor_id, child_id) + def ls_files(revision) + request = Gitaly::ListFilesRequest.new( + repository: @gitaly_repo, + revision: GitalyClient.encode(revision) + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request) + response.flat_map do |msg| + msg.paths.map { |d| d.dup.force_encoding(Encoding::UTF_8) } + end + end + + def ancestor?(ancestor_id, child_id) request = Gitaly::CommitIsAncestorRequest.new( repository: @gitaly_repo, ancestor_id: ancestor_id, @@ -29,56 +41,61 @@ module Gitlab request = Gitaly::CommitDiffRequest.new(request_params) response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) - Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options.merge(from_gitaly: true)) + GitalyClient::DiffStitcher.new(response) end def commit_deltas(commit) request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit)) response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request) - response.flat_map do |msg| - msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } - end + + response.flat_map { |msg| msg.deltas } end def tree_entry(ref, path, limit = nil) request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, revision: ref, - path: path.dup.force_encoding(Encoding::ASCII_8BIT), + path: GitalyClient.encode(path), limit: limit.to_i ) response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request) - entry = response.first - return unless entry.oid.present? - if entry.type == :BLOB - rest_of_data = response.reduce("") { |memo, msg| memo << msg.data } - entry.data += rest_of_data + entry = nil + data = '' + response.each do |msg| + if entry.nil? + entry = msg + + break unless entry.type == :BLOB + end + + data << msg.data end + entry.data = data - entry + entry unless entry.oid.blank? end def tree_entries(repository, revision, path) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, - revision: revision, - path: path.presence || '.' + revision: GitalyClient.encode(revision), + path: path.present? ? GitalyClient.encode(path) : '.' ) response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request) response.flat_map do |message| message.entries.map do |gitaly_tree_entry| - entry_path = gitaly_tree_entry.path.dup Gitlab::Git::Tree.new( id: gitaly_tree_entry.oid, root_id: gitaly_tree_entry.root_oid, type: gitaly_tree_entry.type.downcase, mode: gitaly_tree_entry.mode.to_s(8), - name: File.basename(entry_path), - path: entry_path, + name: File.basename(gitaly_tree_entry.path), + path: GitalyClient.encode(gitaly_tree_entry.path), + flat_path: GitalyClient.encode(gitaly_tree_entry.flat_path), commit_id: gitaly_tree_entry.commit_oid ) end @@ -100,15 +117,14 @@ module Gitlab def last_commit_for_path(revision, path) request = Gitaly::LastCommitForPathRequest.new( repository: @gitaly_repo, - revision: revision.force_encoding(Encoding::ASCII_8BIT), - path: path.to_s.force_encoding(Encoding::ASCII_8BIT) + revision: GitalyClient.encode(revision), + path: GitalyClient.encode(path.to_s) ) gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit return unless gitaly_commit - commit = GitalyClient::Commit.new(@repository, gitaly_commit) - Gitlab::Git::Commit.new(commit) + Gitlab::Git::Commit.new(@repository, gitaly_commit) end def between(from, to) @@ -159,18 +175,39 @@ module Gitlab def raw_blame(revision, path) request = Gitaly::RawBlameRequest.new( repository: @gitaly_repo, - revision: revision, - path: path + revision: GitalyClient.encode(revision), + path: GitalyClient.encode(path) ) response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request) response.reduce("") { |memo, msg| memo << msg.data } end + def find_commit(revision) + request = Gitaly::FindCommitRequest.new( + repository: @gitaly_repo, + revision: GitalyClient.encode(revision) + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request) + + response.commit + end + + def patch(revision) + request = Gitaly::CommitPatchRequest.new( + repository: @gitaly_repo, + revision: GitalyClient.encode(revision) + ) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request) + + response.sum(&:data) + end + private def commit_diff_request_params(commit, options = {}) - parent_id = commit.parents[0]&.id || EMPTY_TREE_ID + parent_id = commit.parent_ids.first || EMPTY_TREE_ID { repository: @gitaly_repo, @@ -183,8 +220,7 @@ module Gitlab def consume_commits_response(response) response.flat_map do |message| message.commits.map do |gitaly_commit| - commit = GitalyClient::Commit.new(@repository, gitaly_commit) - Gitlab::Git::Commit.new(commit) + Gitlab::Git::Commit.new(@repository, gitaly_commit) end end end diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb index d459c9a88fb..54df6304865 100644 --- a/lib/gitlab/gitaly_client/diff.rb +++ b/lib/gitlab/gitaly_client/diff.rb @@ -7,13 +7,13 @@ module Gitlab def initialize(params) params.each do |key, val| - public_send(:"#{key}=", val) + public_send(:"#{key}=", val) # rubocop:disable GitlabSecurity/PublicSend end end def ==(other) FIELDS.all? do |field| - public_send(field) == other.public_send(field) + public_send(field) == other.public_send(field) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index b0f7548b7dc..8ef873d5848 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -16,8 +16,7 @@ module Gitlab response.flat_map do |message| message.branches.map do |branch| - gitaly_commit = GitalyClient::Commit.new(@repository, branch.target) - target_commit = Gitlab::Git::Commit.decorate(gitaly_commit) + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target) Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) end end @@ -71,6 +70,62 @@ module Gitlab consume_tags_response(response) end + def ref_exists?(ref_name) + request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: GitalyClient.encode(ref_name)) + response = GitalyClient.call(@storage, :ref_service, :ref_exists, request) + response.value + rescue GRPC::InvalidArgument => e + raise ArgumentError, e.message + end + + def find_branch(branch_name) + request = Gitaly::FindBranchRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(branch_name) + ) + + response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request) + branch = response.branch + return unless branch + + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, encode!(branch.name.dup), branch.target_commit.id, target_commit) + end + + def create_branch(ref, start_point) + request = Gitaly::CreateBranchRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(ref), + start_point: GitalyClient.encode(start_point) + ) + + response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request) + + case response.status + when :OK + branch = response.branch + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target_commit.id, target_commit) + when :ERR_INVALID + invalid_ref!("Invalid ref name") + when :ERR_EXISTS + invalid_ref!("Branch #{ref} already exists") + when :ERR_INVALID_START_POINT + invalid_ref!("Invalid reference #{start_point}") + else + raise "Unknown response status: #{response.status}" + end + end + + def delete_branch(branch_name) + request = Gitaly::DeleteBranchRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(branch_name) + ) + + GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) + end + private def consume_refs_response(response) @@ -102,8 +157,7 @@ module Gitlab response.flat_map do |message| message.tags.map do |gitaly_tag| if gitaly_tag.target_commit.present? - commit = GitalyClient::Commit.new(@repository, gitaly_tag.target_commit) - gitaly_commit = Gitlab::Git::Commit.new(commit) + gitaly_commit = Gitlab::Git::Commit.decorate(@repository, gitaly_tag.target_commit) end Gitlab::Git::Tag.new( @@ -141,7 +195,11 @@ module Gitlab committer_email: response.commit_committer.email.dup } - Gitlab::Git::Commit.decorate(hash) + Gitlab::Git::Commit.decorate(@repository, hash) + end + + def invalid_ref!(message) + raise Gitlab::Git::Repository::InvalidRef.new(message) end end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 79ce784f2f2..177a1284f38 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -10,7 +10,7 @@ module Gitlab def exists? request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :exists, request).exists + GitalyClient.call(@storage, :repository_service, :repository_exists, request).exists end def garbage_collect(create_bitmap) @@ -32,6 +32,27 @@ module Gitlab request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) GitalyClient.call(@storage, :repository_service, :repository_size, request).size end + + def apply_gitattributes(revision) + request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: revision) + GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) + end + + def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false) + request = Gitaly::FetchRemoteRequest.new(repository: @gitaly_repo, remote: remote, force: forced, no_tags: no_tags) + + if ssh_auth&.ssh_import? + if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? + request.ssh_key = ssh_auth.ssh_private_key + end + + if ssh_auth.ssh_known_hosts.present? + request.known_hosts = ssh_auth.ssh_known_hosts + end + end + + GitalyClient.call(@storage, :repository_service, :fetch_remote, request) + end end end end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index f5a4c5493ef..8fc937496af 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -5,7 +5,9 @@ module Gitlab def repository(repository_storage, relative_path) Gitaly::Repository.new( storage_name: repository_storage, - relative_path: relative_path + relative_path: relative_path, + git_object_directory: Gitlab::Git::Env['GIT_OBJECT_DIRECTORY'].to_s, + git_alternate_object_directories: Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES']) ) end end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 8c80791e7c9..f330041cc00 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -11,7 +11,9 @@ module Gitlab end def create! - project.public_send(project_association).find_or_create_by!(find_condition) do |record| + association = project.public_send(project_association) # rubocop:disable GitlabSecurity/PublicSend + + association.find_or_create_by!(find_condition) do |record| record.attributes = attributes end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 7dbeec5b010..0550f9695bd 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -120,7 +120,7 @@ module Gitlab def request(method, *args, &block) sleep rate_limit_sleep_time if rate_limit_exceed? - data = api.send(method, *args) + data = api.__send__(method, *args) # rubocop:disable GitlabSecurity/PublicSend return data unless data.is_a?(Array) last_response = api.last_response diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 266b1a6fece..b8c07460ebb 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -166,7 +166,7 @@ module Gitlab def remove_branch(name) project.repository.delete_branch(name) - rescue Rugged::ReferenceError + rescue Gitlab::Git::Repository::DeleteBranchFailed errors << { type: :remove_branch, name: name } end @@ -289,7 +289,7 @@ module Gitlab opts.last[:page] = current_page(resource_type) - client.public_send(resource_type, *opts) do |resources| + client.public_send(resource_type, *opts) do |resources| # rubocop:disable GitlabSecurity/PublicSend yield resources increment_page(resource_type) end diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 86fb6c51765..f1007daab5d 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -71,7 +71,7 @@ module Gitlab end def config - Gitlab.config.omniauth.providers.find{|provider| provider.name == "gitlab"} + Gitlab.config.omniauth.providers.find {|provider| provider.name == "gitlab"} end def gitlab_options diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2d1ae6a5925..9bcc579278f 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,6 +3,7 @@ module Gitlab module GonHelper include WebpackHelper + include Gitlab::CurrentSettings def add_gon_variables gon.api_version = 'v4' diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index e1d1724295a..025f826e65f 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -2,6 +2,8 @@ module Gitlab module Gpg extend self + MUTEX = Mutex.new + module CurrentKeyChain extend self @@ -37,26 +39,42 @@ module Gitlab fingerprints = CurrentKeyChain.fingerprints_from_key(key) GPGME::Key.find(:public, fingerprints).flat_map do |raw_key| - raw_key.uids.map { |uid| { name: uid.name, email: uid.email } } + raw_key.uids.map { |uid| { name: uid.name, email: uid.email.downcase } } end end end - def using_tmp_keychain - Dir.mktmpdir do |dir| - @original_dirs ||= [GPGME::Engine.dirinfo('homedir')] - @original_dirs.push(dir) - - GPGME::Engine.home_dir = dir - - return_value = yield + # Allows thread safe switching of temporary keychain files + # + # 1. The current thread may use nesting of temporary keychain + # 2. Another thread needs to wait for the lock to be released + def using_tmp_keychain(&block) + if MUTEX.locked? && MUTEX.owned? + optimistic_using_tmp_keychain(&block) + else + MUTEX.synchronize do + optimistic_using_tmp_keychain(&block) + end + end + end - @original_dirs.pop + # 1. Returns the custom home directory if one has been set by calling + # `GPGME::Engine.home_dir=` + # 2. Returns the default home directory otherwise + def current_home_dir + GPGME::Engine.info.first.home_dir || GPGME::Engine.dirinfo('homedir') + end - GPGME::Engine.home_dir = @original_dirs[-1] + private - return_value + def optimistic_using_tmp_keychain + previous_dir = current_home_dir + Dir.mktmpdir do |dir| + GPGME::Engine.home_dir = dir + yield end + ensure + GPGME::Engine.home_dir = previous_dir end end end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 55428b85207..86bd9f5b125 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -1,12 +1,15 @@ module Gitlab module Gpg class Commit - attr_reader :commit - def initialize(commit) @commit = commit - @signature_text, @signed_text = commit.raw.signature(commit.project.repository) + @signature_text, @signed_text = + begin + Rugged::Commit.extract_signature(@commit.project.repository.rugged, @commit.sha) + rescue Rugged::OdbError + nil + end end def has_signature? @@ -16,18 +19,20 @@ module Gitlab def signature return unless has_signature? - cached_signature = GpgSignature.find_by(commit_sha: commit.sha) - return cached_signature if cached_signature.present? + return @signature if @signature - using_keychain do |gpg_key| - create_cached_signature!(gpg_key) - end + cached_signature = GpgSignature.find_by(commit_sha: @commit.sha) + return @signature = cached_signature if cached_signature.present? + + @signature = create_cached_signature! end def update_signature!(cached_signature) using_keychain do |gpg_key| cached_signature.update_attributes!(attributes(gpg_key)) end + + @signature = cached_signature end private @@ -55,26 +60,39 @@ module Gitlab end end - def create_cached_signature!(gpg_key) - GpgSignature.create!(attributes(gpg_key)) + def create_cached_signature! + using_keychain do |gpg_key| + GpgSignature.create!(attributes(gpg_key)) + end end def attributes(gpg_key) user_infos = user_infos(gpg_key) + verification_status = verification_status(gpg_key) { - commit_sha: commit.sha, - project: commit.project, + commit_sha: @commit.sha, + project: @commit.project, gpg_key: gpg_key, gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, gpg_key_user_name: user_infos[:name], gpg_key_user_email: user_infos[:email], - valid_signature: gpg_signature_valid_signature_value(gpg_key) + verification_status: verification_status } end - def gpg_signature_valid_signature_value(gpg_key) - !!(gpg_key && gpg_key.verified? && verified_signature.valid?) + def verification_status(gpg_key) + return :unknown_key unless gpg_key + return :unverified_key unless gpg_key.verified? + return :unverified unless verified_signature.valid? + + if gpg_key.verified_and_belongs_to_email?(@commit.committer_email) + :verified + elsif gpg_key.user.all_emails.include?(@commit.committer_email) + :same_user_different_email + else + :other_user + end end def user_infos(gpg_key) diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index 3bb491120ba..e085eab26c9 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -8,11 +8,9 @@ module Gitlab def run GpgSignature .select(:id, :commit_sha, :project_id) - .where('gpg_key_id IS NULL OR valid_signature = ?', false) + .where('gpg_key_id IS NULL OR verification_status <> ?', GpgSignature.verification_statuses[:verified]) .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) - .find_each do |gpg_signature| - Gitlab::Gpg::Commit.new(gpg_signature.commit).update_signature!(gpg_signature) - end + .find_each { |sig| sig.gpg_commit.update_signature!(sig) } end end end diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb new file mode 100644 index 00000000000..1e1fdabca93 --- /dev/null +++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb @@ -0,0 +1,19 @@ +module Gitlab + module GrapeLogging + module Formatters + class LogrageWithTimestamp + def call(severity, datetime, _, data) + time = data.delete :time + attributes = { + time: datetime.utc.iso8601(3), + severity: severity, + duration: time[:total], + db: time[:db], + view: time[:view] + }.merge(data) + ::Lograge.formatter.call(attributes) + "\n" + end + end + end + end +end diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index fd94984f8a2..e27e16ddaf6 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -10,7 +10,7 @@ module Gitlab 'db_ping' end - def is_successful?(result) + def successful?(result) result == '1' end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb index a28658d42d4..0eb9b77634a 100644 --- a/lib/gitlab/health_checks/redis/cache_check.rb +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -15,7 +15,7 @@ module Gitlab 'redis_cache_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb index f97d50d3947..f322fe831b8 100644 --- a/lib/gitlab/health_checks/redis/queues_check.rb +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -15,7 +15,7 @@ module Gitlab 'redis_queues_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index fe4e3c4a3ab..8ceb0a0aa46 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -11,7 +11,7 @@ module Gitlab 'redis_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb index e3244392902..07e6f707998 100644 --- a/lib/gitlab/health_checks/redis/shared_state_check.rb +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -15,7 +15,7 @@ module Gitlab 'redis_shared_state_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index f5026171ba4..96945ce5b20 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -5,7 +5,7 @@ module Gitlab def readiness check_result = check - if is_successful?(check_result) + if successful?(check_result) HealthChecks::Result.new(true) elsif check_result.is_a?(Timeout::Error) HealthChecks::Result.new(false, "#{human_name} check timed out") @@ -16,10 +16,10 @@ module Gitlab def metrics result, elapsed = with_timing(&method(:check)) - Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) [ metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), - metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), + metric("#{metric_prefix}_success", successful?(result) ? 1 : 0), metric("#{metric_prefix}_latency_seconds", elapsed) ] end @@ -30,7 +30,7 @@ module Gitlab raise NotImplementedError end - def is_successful?(result) + def successful?(result) raise NotImplementedError end diff --git a/lib/gitlab/i18n/metadata_entry.rb b/lib/gitlab/i18n/metadata_entry.rb new file mode 100644 index 00000000000..35d57459a3d --- /dev/null +++ b/lib/gitlab/i18n/metadata_entry.rb @@ -0,0 +1,27 @@ +module Gitlab + module I18n + class MetadataEntry + attr_reader :entry_data + + def initialize(entry_data) + @entry_data = entry_data + end + + def expected_plurals + return nil unless plural_information + + plural_information['nplurals'].to_i + end + + private + + def plural_information + return @plural_information if defined?(@plural_information) + + if plural_line = entry_data[:msgstr].detect { |metadata_line| metadata_line.starts_with?('Plural-Forms: ') } + @plural_information = Hash[plural_line.scan(/(\w+)=([^;\n]+)/)] + end + end + end + end +end diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb new file mode 100644 index 00000000000..7d3ff8c7f58 --- /dev/null +++ b/lib/gitlab/i18n/po_linter.rb @@ -0,0 +1,214 @@ +module Gitlab + module I18n + class PoLinter + attr_reader :po_path, :translation_entries, :metadata_entry, :locale + + VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze + + def initialize(po_path, locale = I18n.locale.to_s) + @po_path = po_path + @locale = locale + end + + def errors + @errors ||= validate_po + end + + def validate_po + if parse_error = parse_po + return 'PO-syntax errors' => [parse_error] + end + + validate_entries + end + + def parse_po + entries = SimplePoParser.parse(po_path) + + # The first entry is the metadata entry if there is one. + # This is an entry when empty `msgid` + if entries.first[:msgid].empty? + @metadata_entry = Gitlab::I18n::MetadataEntry.new(entries.shift) + else + return 'Missing metadata entry.' + end + + @translation_entries = entries.map do |entry_data| + Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_plurals) + end + + nil + rescue SimplePoParser::ParserError => e + @translation_entries = [] + e.message + end + + def validate_entries + errors = {} + + translation_entries.each do |entry| + errors_for_entry = validate_entry(entry) + errors[join_message(entry.msgid)] = errors_for_entry if errors_for_entry.any? + end + + errors + end + + def validate_entry(entry) + errors = [] + + validate_flags(errors, entry) + validate_variables(errors, entry) + validate_newlines(errors, entry) + validate_number_of_plurals(errors, entry) + validate_unescaped_chars(errors, entry) + + errors + end + + def validate_unescaped_chars(errors, entry) + if entry.msgid_contains_unescaped_chars? + errors << 'contains unescaped `%`, escape it using `%%`' + end + + if entry.plural_id_contains_unescaped_chars? + errors << 'plural id contains unescaped `%`, escape it using `%%`' + end + + if entry.translations_contain_unescaped_chars? + errors << 'translation contains unescaped `%`, escape it using `%%`' + end + end + + def validate_number_of_plurals(errors, entry) + return unless metadata_entry&.expected_plurals + return unless entry.translated? + + if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_plurals + errors << "should have #{metadata_entry.expected_plurals} "\ + "#{'translations'.pluralize(metadata_entry.expected_plurals)}" + end + end + + def validate_newlines(errors, entry) + if entry.msgid_contains_newlines? + errors << 'is defined over multiple lines, this breaks some tooling.' + end + + if entry.plural_id_contains_newlines? + errors << 'plural is defined over multiple lines, this breaks some tooling.' + end + + if entry.translations_contain_newlines? + errors << 'has translations defined over multiple lines, this breaks some tooling.' + end + end + + def validate_variables(errors, entry) + if entry.has_singular_translation? + validate_variables_in_message(errors, entry.msgid, entry.singular_translation) + end + + if entry.has_plural? + entry.plural_translations.each do |translation| + validate_variables_in_message(errors, entry.plural_id, translation) + end + end + end + + def validate_variables_in_message(errors, message_id, message_translation) + message_id = join_message(message_id) + required_variables = message_id.scan(VARIABLE_REGEX) + + validate_unnamed_variables(errors, required_variables) + validate_translation(errors, message_id, required_variables) + validate_variable_usage(errors, message_translation, required_variables) + end + + def validate_translation(errors, message_id, used_variables) + variables = fill_in_variables(used_variables) + + begin + Gitlab::I18n.with_locale(locale) do + translated = if message_id.include?('|') + FastGettext::Translation.s_(message_id) + else + FastGettext::Translation._(message_id) + end + + translated % variables + end + + # `sprintf` could raise an `ArgumentError` when invalid passing something + # other than a Hash when using named variables + # + # `sprintf` could raise `TypeError` when passing a wrong type when using + # unnamed variables + # + # FastGettext::Translation could raise `RuntimeError` (raised as a string), + # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` + # + # `FastGettext::Translation` could raise `ArgumentError` as subclassess + # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` + rescue ArgumentError, TypeError, RuntimeError => e + errors << "Failure translating to #{locale} with #{variables}: #{e.message}" + end + end + + def fill_in_variables(variables) + if variables.empty? + [] + elsif variables.any? { |variable| unnamed_variable?(variable) } + variables.map do |variable| + variable == '%d' ? Random.rand(1000) : Gitlab::Utils.random_string + end + else + variables.inject({}) do |hash, variable| + variable_name = variable[/\w+/] + hash[variable_name] = Gitlab::Utils.random_string + hash + end + end + end + + def validate_unnamed_variables(errors, variables) + if variables.size > 1 && variables.any? { |variable_name| unnamed_variable?(variable_name) } + errors << 'is combining multiple unnamed variables' + end + end + + def validate_variable_usage(errors, translation, required_variables) + translation = join_message(translation) + + # We don't need to validate when the message is empty. + # In this case we fall back to the default, which has all the the + # required variables. + return if translation.empty? + + found_variables = translation.scan(VARIABLE_REGEX) + + missing_variables = required_variables - found_variables + if missing_variables.any? + errors << "<#{translation}> is missing: [#{missing_variables.to_sentence}]" + end + + unknown_variables = found_variables - required_variables + if unknown_variables.any? + errors << "<#{translation}> is using unknown variables: [#{unknown_variables.to_sentence}]" + end + end + + def unnamed_variable?(variable_name) + !variable_name.start_with?('%{') + end + + def validate_flags(errors, entry) + errors << "is marked #{entry.flag}" if entry.flag + end + + def join_message(message) + Array(message).join + end + end + end +end diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb new file mode 100644 index 00000000000..e6c95afca7e --- /dev/null +++ b/lib/gitlab/i18n/translation_entry.rb @@ -0,0 +1,92 @@ +module Gitlab + module I18n + class TranslationEntry + PERCENT_REGEX = /(?:^|[^%])%(?!{\w*}|[a-z%])/.freeze + + attr_reader :nplurals, :entry_data + + def initialize(entry_data, nplurals) + @entry_data = entry_data + @nplurals = nplurals + end + + def msgid + entry_data[:msgid] + end + + def plural_id + entry_data[:msgid_plural] + end + + def has_plural? + plural_id.present? + end + + def singular_translation + all_translations.first if has_singular_translation? + end + + def all_translations + @all_translations ||= entry_data.fetch_values(*translation_keys) + .reject(&:empty?) + end + + def translated? + all_translations.any? + end + + def plural_translations + return [] unless has_plural? + return [] unless translated? + + @plural_translations ||= if has_singular_translation? + all_translations.drop(1) + else + all_translations + end + end + + def flag + entry_data[:flag] + end + + def has_singular_translation? + nplurals > 1 || !has_plural? + end + + def msgid_contains_newlines? + msgid.is_a?(Array) + end + + def plural_id_contains_newlines? + plural_id.is_a?(Array) + end + + def translations_contain_newlines? + all_translations.any? { |translation| translation.is_a?(Array) } + end + + def msgid_contains_unescaped_chars? + contains_unescaped_chars?(msgid) + end + + def plural_id_contains_unescaped_chars? + contains_unescaped_chars?(plural_id) + end + + def translations_contain_unescaped_chars? + all_translations.any? { |translation| contains_unescaped_chars?(translation) } + end + + def contains_unescaped_chars?(string) + string =~ PERCENT_REGEX + end + + private + + def translation_keys + @translation_keys ||= entry_data.keys.select { |key| key.to_s =~ /\Amsgstr(\[\d+\])?\z/ } + end + end + end +end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 3470a09eaf0..50ee879129c 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.8'.freeze + VERSION = '0.2.0'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index d230de781d5..56042ddecbf 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class AttributesFinder - def initialize(included_attributes:, excluded_attributes:, methods:) @included_attributes = included_attributes || {} @excluded_attributes = excluded_attributes || {} diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index ffd17118c91..989342389bc 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -47,12 +47,16 @@ module Gitlab end def remove_symlinks! - Dir["#{@shared.export_path}/**/*"].each do |path| + extracted_files.each do |path| FileUtils.rm(path) if File.lstat(path).symlink? end true end + + def extracted_files + Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ /.*\/\.{1,2}$/ } + end end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index c5c05bfe2fb..2171c6c7bbb 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -3,18 +3,22 @@ project_tree: - labels: :priorities - milestones: - - :events + - events: + - :push_event_payload - issues: - - :events + - events: + - :push_event_payload - :timelogs - notes: - :author - - :events + - events: + - :push_event_payload - label_links: - label: :priorities - milestone: - - :events + - events: + - :push_event_payload - snippets: - :award_emoji - notes: @@ -25,23 +29,28 @@ project_tree: - merge_requests: - notes: - :author - - :events + - events: + - :push_event_payload - merge_request_diff: - :merge_request_diff_commits - :merge_request_diff_files - - :events + - events: + - :push_event_payload - :timelogs - label_links: - label: :priorities - milestone: - - :events + - events: + - :push_event_payload - pipelines: - notes: - :author - - :events + - events: + - :push_event_payload - :stages - :statuses + - :auto_devops - :triggers - :pipeline_schedules - :services @@ -90,6 +99,7 @@ excluded_attributes: - :last_activity_at - :last_repository_updated_at - :last_repository_check_at + - :storage_version snippets: - :expired_at merge_request_diff: @@ -107,6 +117,9 @@ excluded_attributes: statuses: - :trace - :token + - :when + push_event_payload: + - :event_id methods: labels: @@ -123,5 +136,11 @@ methods: - :utf8_diff merge_requests: - :diff_head_sha + - :source_branch_sha + - :target_branch_sha project: - :description_html + events: + - :action + push_event_payload: + - :action diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index c20adc20bfd..81a213e8321 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -30,7 +30,7 @@ module Gitlab end def branch_exists?(branch_name) - @project.repository.branch_exists?(branch_name) + @project.repository.raw.branch_exists?(branch_name) end def fork_merge_request? diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 84ab1977dfa..3bc095a99a9 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,11 +1,16 @@ module Gitlab module ImportExport class ProjectTreeRestorer + # Relations which cannot have both group_id and project_id at the same time + RESTRICT_PROJECT_AND_GROUP = %i(milestones).freeze + def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @user = user @shared = shared @project = project + @project_id = project.id + @saved = true end def restore @@ -19,8 +24,10 @@ module Gitlab @project_members = @tree_hash.delete('project_members') - ActiveRecord::Base.no_touching do - create_relations + ActiveRecord::Base.uncached do + ActiveRecord::Base.no_touching do + create_relations + end end rescue => e @shared.error(e) @@ -45,21 +52,24 @@ module Gitlab # the configuration yaml file too. # Finally, it updates each attribute in the newly imported project. def create_relations - saved = [] default_relation_list.each do |relation| - next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? + if relation.is_a?(Hash) + create_sub_relations(relation, @tree_hash) + elsif @tree_hash[relation.to_s].present? + save_relation_hash(@tree_hash[relation.to_s], relation) + end + end - create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) + @saved + end - relation_key = relation.is_a?(Hash) ? relation.keys.first : relation - relation_hash_list = @tree_hash[relation_key.to_s] + def save_relation_hash(relation_hash_batch, relation_key) + relation_hash = create_relation(relation_key, relation_hash_batch) - next unless relation_hash_list + @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) - relation_hash = create_relation(relation_key, relation_hash_list) - saved << restored_project.append_or_update_attribute(relation_key, relation_hash) - end - saved.all? + # Restore the project again, extra query that skips holding the AR objects in memory + @restored_project = Project.find(@project_id) end def default_relation_list @@ -90,20 +100,42 @@ module Gitlab # issue, finds any subrelations such as notes, creates them and assign them back to the hash # # Recursively calls this method if the sub-relation is a hash containing more sub-relations - def create_sub_relations(relation, tree_hash) + def create_sub_relations(relation, tree_hash, save: true) relation_key = relation.keys.first.to_s return if tree_hash[relation_key].blank? - [tree_hash[relation_key]].flatten.each do |relation_item| - relation.values.flatten.each do |sub_relation| - # We just use author to get the user ID, do not attempt to create an instance. - next if sub_relation == :author + tree_array = [tree_hash[relation_key]].flatten + + # Avoid keeping a possible heavy object in memory once we are done with it + while relation_item = tree_array.shift + # The transaction at this level is less speedy than one single transaction + # But we can't have it in the upper level or GC won't get rid of the AR objects + # after we save the batch. + Project.transaction do + process_sub_relation(relation, relation_item) + + # For every subrelation that hangs from Project, save the associated records alltogether + # This effectively batches all records per subrelation item, only keeping those in memory + # We have to keep in mind that more batch granularity << Memory, but >> Slowness + if save + save_relation_hash([relation_item], relation_key) + tree_hash[relation_key].delete(relation_item) + end + end + end - create_sub_relations(sub_relation, relation_item) if sub_relation.is_a?(Hash) + tree_hash.delete(relation_key) if save + end - relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) - relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? - end + def process_sub_relation(relation, relation_item) + relation.values.flatten.each do |sub_relation| + # We just use author to get the user ID, do not attempt to create an instance. + next if sub_relation == :author + + create_sub_relations(sub_relation, relation_item, save: false) if sub_relation.is_a?(Hash) + + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) + relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? end end @@ -120,17 +152,25 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: parsed_relation_hash(relation_hash), + relation_hash: parsed_relation_hash(relation_hash, relation.to_sym), members_mapper: members_mapper, user: @user, - project: restored_project) + project: @restored_project) end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end - def parsed_relation_hash(relation_hash) - relation_hash.merge!('group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id) + def parsed_relation_hash(relation_hash, relation_type) + if RESTRICT_PROJECT_AND_GROUP.include?(relation_type) + params = {} + params['group_id'] = restored_project.group.try(:id) if relation_hash['group_id'] + params['project_id'] = restored_project.id if relation_hash['project_id'] + else + params = { 'group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id } + end + + relation_hash.merge(params) end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 20580459046..380b336395d 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -14,6 +14,7 @@ module Gitlab create_access_levels: 'ProtectedTag::CreateAccessLevel', labels: :project_labels, priorities: :label_priorities, + auto_devops: :project_auto_devops, label: :project_label }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze @@ -69,7 +70,6 @@ module Gitlab reset_tokens! remove_encrypted_attributes! - @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diff_commits if @relation_name == :merge_request_diff set_diff if @relation_name == :merge_request_diff_files end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 5d6de8bc475..9fd0b709ef2 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -16,7 +16,7 @@ module Gitlab error_out(error.message, caller[0].dup) @errors << error.message # Debug: - Rails.logger.error(error.backtrace) + Rails.logger.error(error.backtrace.join("\n")) end private diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb new file mode 100644 index 00000000000..505810964bc --- /dev/null +++ b/lib/gitlab/issuables_count_for_state.rb @@ -0,0 +1,50 @@ +module Gitlab + # Class for counting and caching the number of issuables per state. + class IssuablesCountForState + # The name of the RequestStore cache key. + CACHE_KEY = :issuables_count_for_state + + # The state values that can be safely casted to a Symbol. + STATES = %w[opened closed merged all].freeze + + # finder - The finder class to use for retrieving the issuables. + def initialize(finder) + @finder = finder + @cache = + if RequestStore.active? + RequestStore[CACHE_KEY] ||= initialize_cache + else + initialize_cache + end + end + + def for_state_or_opened(state = nil) + self[state || :opened] + end + + # Returns the count for the given state. + # + # state - The name of the state as either a String or a Symbol. + # + # Returns an Integer. + def [](state) + state = state.to_sym if cast_state_to_symbol?(state) + + cache_for_finder[state] || 0 + end + + private + + def cache_for_finder + @cache[@finder] + end + + def cast_state_to_symbol?(state) + state.is_a?(String) && STATES.include?(state) + end + + def initialize_cache + Hash.new { |hash, finder| hash[finder] = finder.count_by_state } + end + end +end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 208f0e1bbea..4d6bbda15f3 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -1,12 +1,31 @@ module Gitlab # JobWaiter can be used to wait for a number of Sidekiq jobs to complete. + # + # Its use requires the cooperation of the sidekiq jobs themselves. Set up the + # waiter, then start the jobs, passing them its `key`. Their `perform` methods + # should look like: + # + # def perform(args, notify_key) + # # do work + # ensure + # ::Gitlab::JobWaiter.notify(notify_key, jid) + # end + # + # The JobWaiter blocks popping items from a Redis array. All the sidekiq jobs + # push to that array when done. Once the waiter has popped `count` items, it + # knows all the jobs are done. class JobWaiter - # The sleep interval between checking keys, in seconds. - INTERVAL = 0.1 + def self.notify(key, jid) + Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } + end + + attr_reader :key, :jobs_remaining, :finished - # jobs - The job IDs to wait for. - def initialize(jobs) - @jobs = jobs + # jobs_remaining - the number of jobs left to wait for + def initialize(jobs_remaining) + @key = "gitlab:job_waiter:#{SecureRandom.uuid}" + @jobs_remaining = jobs_remaining + @finished = [] end # Waits for all the jobs to be completed. @@ -15,13 +34,33 @@ module Gitlab # ensures we don't indefinitely block a caller in case a job takes # long to process, or is never processed. def wait(timeout = 10) - start = Time.current + deadline = Time.now.utc + timeout + + Gitlab::Redis::SharedState.with do |redis| + # Fallback key expiry: allow a long grace period to reduce the chance of + # a job pushing to an expired key and recreating it + redis.expire(key, [timeout * 2, 10.minutes.to_i].max) + + while jobs_remaining > 0 + # Redis will not take fractional seconds. Prefer waiting too long over + # not waiting long enough + seconds_left = (deadline - Time.now.utc).ceil - while (Time.current - start) <= timeout - break if SidekiqStatus.all_completed?(@jobs) + # Redis interprets 0 as "wait forever", so skip the final `blpop` call + break if seconds_left <= 0 - sleep(INTERVAL) # to not overload Redis too much. + list, jid = redis.blpop(key, timeout: seconds_left) + break unless list && jid # timed out + + @finished << jid + @jobs_remaining -= 1 + end + + # All jobs have finished, so expire the key immediately + redis.expire(key, 0) if jobs_remaining == 0 end + + finished end end end diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb deleted file mode 100644 index d9a79f7c291..00000000000 --- a/lib/gitlab/key_fingerprint.rb +++ /dev/null @@ -1,48 +0,0 @@ -module Gitlab - class KeyFingerprint - attr_reader :key, :ssh_key - - # Unqualified MD5 fingerprint for compatibility - delegate :fingerprint, to: :ssh_key, allow_nil: true - - def initialize(key) - @key = key - - @ssh_key = - begin - Net::SSH::KeyFactory.load_data_public_key(key) - rescue Net::SSH::Exception, NotImplementedError - end - end - - def valid? - ssh_key.present? - end - - def type - return unless valid? - - parts = ssh_key.ssh_type.split('-') - parts.shift if parts[0] == 'ssh' - - parts[0].upcase - end - - def bits - return unless valid? - - case type - when 'RSA' - ssh_key.n.num_bits - when 'DSS', 'DSA' - ssh_key.p.num_bits - when 'ECDSA' - ssh_key.group.order.num_bits - when 'ED25519' - 256 - else - raise "Unsupported key type: #{type}" - end - end - end -end diff --git a/lib/gitlab/lazy.rb b/lib/gitlab/lazy.rb index 2a659ae4c74..99594577141 100644 --- a/lib/gitlab/lazy.rb +++ b/lib/gitlab/lazy.rb @@ -16,7 +16,7 @@ module Gitlab def method_missing(name, *args, &block) __evaluate__ - @result.__send__(name, *args, &block) + @result.__send__(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end def respond_to_missing?(name, include_private = false) diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 8867a91c244..cd7e4ca7b7e 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -73,7 +73,7 @@ module Gitlab private def user_options(field, value, limit) - options = { attributes: user_attributes } + options = { attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq } options[:size] = limit if limit if field.to_sym == :dn @@ -99,10 +99,6 @@ module Gitlab filter end end - - def user_attributes - %W(#{config.uid} cn dn) + config.attributes['username'] + config.attributes['email'] - end end end end diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index 95378e5a769..4fbc5fa5262 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -17,7 +17,7 @@ module Gitlab value = value.first if value break if value.present? end - + return super unless value Gitlab::Utils.force_utf8(value) diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 43eb73250b7..4d6f8ac79de 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -21,6 +21,15 @@ module Gitlab adapter.dn_matches_filter?(dn, AD_USER_DISABLED) end + def self.ldap_attributes(config) + [ + 'dn', # Used in `dn` + config.uid, # Used in `uid` + *config.attributes['name'], # Used in `name` + *config.attributes['email'] # Used in `email` + ] + end + def initialize(entry, provider) Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @@ -32,7 +41,7 @@ module Gitlab end def uid - entry.send(config.uid).first + entry.public_send(config.uid).first # rubocop:disable GitlabSecurity/PublicSend end def username @@ -65,7 +74,7 @@ module Gitlab return nil unless selected_attr - entry.public_send(selected_attr) + entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 39180dc17d9..3bf27b37ae6 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -36,7 +36,7 @@ module Gitlab end def find_by_email - ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_email? + ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_attribute?(:email) end def update_user_attributes @@ -60,7 +60,7 @@ module Gitlab ldap_config.block_auto_created_users end - def sync_email_from_provider? + def sync_profile_from_provider? true end diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index 59b21149a9a..6bffd410ed0 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -14,13 +14,9 @@ module Gitlab def self.read_latest path = Rails.root.join("log", file_name) - self.build unless File.exist?(path) - tail_output, _ = Gitlab::Popen.popen(%W(tail -n 2000 #{path})) - tail_output.split("\n") - end - def self.read_latest_for(filename) - path = Rails.root.join("log", filename) + return [] unless File.readable?(path) + tail_output, _ = Gitlab::Popen.popen(%W(tail -n 2000 #{path})) tail_output.split("\n") end diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb index 699d8b9fc07..306923902e0 100644 --- a/lib/gitlab/markdown/pipeline.rb +++ b/lib/gitlab/markdown/pipeline.rb @@ -23,7 +23,7 @@ module Gitlab define_method(meth) do |text, context| context = transform_context(context) - html_pipeline.send(meth, text, context) + html_pipeline.__send__(meth, text, context) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index d7c56463aac..7b06bb953aa 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -1,7 +1,7 @@ module Gitlab module Metrics module InfluxDb - extend Gitlab::CurrentSettings + include Gitlab::CurrentSettings extend self MUTEX = Mutex.new diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index 5d2d7d0026c..63c3372da51 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -8,7 +8,7 @@ module Gitlab def initialize(app) @app = app end - + def call(env) trans = Gitlab::Metrics.current_transaction proxy_start = env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence diff --git a/lib/gitlab/middleware/webpack_proxy.rb b/lib/gitlab/middleware/webpack_proxy.rb index 6105d165810..6aecf63231f 100644 --- a/lib/gitlab/middleware/webpack_proxy.rb +++ b/lib/gitlab/middleware/webpack_proxy.rb @@ -1,6 +1,7 @@ # This Rack middleware is intended to proxy the webpack assets directory to the # webpack-dev-server. It is only intended for use in development. +# :nocov: module Gitlab module Middleware class WebpackProxy < Rack::Proxy @@ -22,3 +23,4 @@ module Gitlab end end end +# :nocov: diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index 7d6911a1ab3..1f331b1e91d 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -32,8 +32,21 @@ module Gitlab @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) end - def has_email? - get_info(:email).present? + def location + location = get_info(:address) + if location.is_a?(Hash) + [location.locality.presence, location.country.presence].compact.join(', ') + else + location + end + end + + def has_attribute?(attribute) + if attribute == :location + get_info(:address).present? + else + get_info(attribute).present? + end end private diff --git a/lib/gitlab/o_auth/session.rb b/lib/gitlab/o_auth/session.rb index f33bfd0bd0e..30739f2a2c5 100644 --- a/lib/gitlab/o_auth/session.rb +++ b/lib/gitlab/o_auth/session.rb @@ -1,3 +1,4 @@ +# :nocov: module Gitlab module OAuth module Session @@ -15,3 +16,4 @@ module Gitlab end end end +# :nocov: diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index e8330917e91..7704bf715e4 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -12,7 +12,7 @@ module Gitlab def initialize(auth_hash) self.auth_hash = auth_hash - update_email + update_profile if sync_profile_from_provider? end def persisted? @@ -184,20 +184,30 @@ module Gitlab } end - def sync_email_from_provider? - auth_hash.provider.to_s == Gitlab.config.omniauth.sync_email_from_provider.to_s + def sync_profile_from_provider? + providers = Gitlab.config.omniauth.sync_profile_from_provider + + if providers.is_a?(Array) + providers.include?(auth_hash.provider) + else + providers + end end - def update_email - if auth_hash.has_email? && sync_email_from_provider? - if persisted? - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email - end + def update_profile + user_synced_attributes_metadata = gl_user.user_synced_attributes_metadata || gl_user.build_user_synced_attributes_metadata - gl_user.external_email = true - gl_user.email_provider = auth_hash.provider + UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| + if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) + gl_user[key] = auth_hash.public_send(key) # rubocop:disable GitlabSecurity/PublicSend + user_synced_attributes_metadata.set_attribute_synced(key, true) + else + user_synced_attributes_metadata.set_attribute_synced(key, false) + end end + + user_synced_attributes_metadata.provider = auth_hash.provider + gl_user.user_synced_attributes_metadata = user_synced_attributes_metadata end def log diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 894bd5efae5..7c02c9c5c48 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -26,6 +26,7 @@ module Gitlab apple-touch-icon.png assets autocomplete + boards ci dashboard deploy.html diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 56112ec2301..e73245b82c1 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -1,6 +1,6 @@ module Gitlab module PerformanceBar - include Gitlab::CurrentSettings + extend Gitlab::CurrentSettings ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze EXPIRY_TIME = 5.minutes diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb index f0c50584f07..4780675a492 100644 --- a/lib/gitlab/polling_interval.rb +++ b/lib/gitlab/polling_interval.rb @@ -1,6 +1,6 @@ module Gitlab class PollingInterval - include Gitlab::CurrentSettings + extend Gitlab::CurrentSettings HEADER_NAME = 'Poll-Interval'.freeze diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index cf461adf697..732fbf68dad 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -25,7 +25,9 @@ module Gitlab end TEMPLATES_TABLE = [ - ProjectTemplate.new('rails', 'Ruby on Rails') + ProjectTemplate.new('rails', 'Ruby on Rails'), + ProjectTemplate.new('spring', 'Spring'), + ProjectTemplate.new('express', 'NodeJS Express') ].freeze class << self diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index d4894c87f8d..4c3edccc71a 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -42,13 +42,13 @@ module Gitlab lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| client_series(*batched_series, start: timeframe_start, stop: timeframe_end) - .select(&method(:has_matching_label)) + .select(&method(:has_matching_label?)) .map { |series_info| [series_info['__name__'], true] } end lookup.to_h end - def has_matching_label(series_info) + def has_matching_label?(series_info) series_info.key?('environment') end diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb index 21aefc884be..09fa14764e6 100644 --- a/lib/gitlab/protocol_access.rb +++ b/lib/gitlab/protocol_access.rb @@ -1,5 +1,7 @@ module Gitlab module ProtocolAccess + extend Gitlab::CurrentSettings + def self.allowed?(protocol) if protocol == 'web' true diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index 4bc76ea033f..c463dd487a0 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -1,5 +1,7 @@ module Gitlab module Recaptcha + extend Gitlab::CurrentSettings + def self.load_configurations! if current_application_settings.recaptcha_enabled ::Recaptcha.configure do |config| diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index b0da516ff83..9bf019b72e6 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -7,9 +7,6 @@ module Gitlab CACHE_NAMESPACE = 'cache:gitlab'.freeze DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze - if defined?(::Rails) && ::Rails.root.present? - DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze - end class << self def default_url @@ -22,7 +19,7 @@ module Gitlab return file_name unless file_name.nil? # otherwise, if config files exists for this class, use it - file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + file_name = config_file_path('redis.cache.yml') return file_name if File.file?(file_name) # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index f9249d05565..e1695aafbeb 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -8,9 +8,6 @@ module Gitlab MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze - if defined?(::Rails) && ::Rails.root.present? - DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze - end class << self def default_url @@ -23,7 +20,7 @@ module Gitlab return file_name if file_name # otherwise, if config files exists for this class, use it - file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + file_name = config_file_path('redis.queues.yml') return file_name if File.file?(file_name) # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 395dcf082da..10bec7a90da 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -7,9 +7,6 @@ module Gitlab SESSION_NAMESPACE = 'session:gitlab'.freeze DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze - if defined?(::Rails) && ::Rails.root.present? - DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze - end class << self def default_url @@ -22,7 +19,7 @@ module Gitlab return file_name if file_name # otherwise, if config files exists for this class, use it - file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + file_name = config_file_path('redis.shared_state.yml') return file_name if File.file?(file_name) # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index c43b37dde74..8ad06480575 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -8,9 +8,6 @@ module Gitlab class Wrapper DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze - if defined?(::Rails) && ::Rails.root.present? - DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze - end class << self delegate :params, :url, to: :new @@ -49,13 +46,21 @@ module Gitlab DEFAULT_REDIS_URL end + # Return the absolute path to a Rails configuration file + # + # We use this instead of `Rails.root` because for certain tasks + # utilizing these classes, `Rails` might not be available. + def config_file_path(filename) + File.expand_path("../../../config/#{filename}", __dir__) + end + def config_file_name # if ENV set for wrapper class, use it even if it points to a file does not exist file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] return file_name unless file_name.nil? # otherwise, if config files exists for wrapper class, use it - file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + file_name = config_file_path('resque.yml') return file_name if File.file?(file_name) # nil will force use of DEFAULT_REDIS_URL when config file is absent diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb new file mode 100644 index 00000000000..bb26f1b610a --- /dev/null +++ b/lib/gitlab/reference_counter.rb @@ -0,0 +1,44 @@ +module Gitlab + class ReferenceCounter + REFERENCE_EXPIRE_TIME = 600 + + attr_reader :gl_repository, :key + + def initialize(gl_repository) + @gl_repository = gl_repository + @key = "git-receive-pack-reference-counter:#{gl_repository}" + end + + def value + Gitlab::Redis::SharedState.with { |redis| (redis.get(key) || 0).to_i } + end + + def increase + redis_cmd do |redis| + redis.incr(key) + redis.expire(key, REFERENCE_EXPIRE_TIME) + end + end + + def decrease + redis_cmd do |redis| + current_value = redis.decr(key) + if current_value < 0 + Rails.logger.warn("Reference counter for #{gl_repository} decreased" \ + " when its value was less than 1. Reseting the counter.") + redis.del(key) + end + end + end + + private + + def redis_cmd + Gitlab::Redis::SharedState.with { |redis| yield(redis) } + true + rescue => e + Rails.logger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") + false + end + end +end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index f5b757ace77..bc836dcc08d 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -45,7 +45,7 @@ module Gitlab end def all - REFERABLES.each { |referable| send(referable.to_s.pluralize) } + REFERABLES.each { |referable| send(referable.to_s.pluralize) } # rubocop:disable GitlabSecurity/PublicSend @references.values.flatten end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 1adc5ec952a..58f6245579a 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -53,7 +53,8 @@ module Gitlab end def kubernetes_namespace_regex_message - "can contain only letters, digits or '-', and cannot start or end with '-'" + "can contain only lowercase letters, digits, and '-'. " \ + "Must start with a letter, and cannot end with '-'" end def environment_slug_regex diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index 8a7cc690046..0f323a9e8b2 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -40,7 +40,7 @@ module Gitlab end def find_by_email - if auth_hash.has_email? + if auth_hash.has_attribute?(:email) user = ::User.find_by(email: auth_hash.email.downcase) user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user user diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 823f697f51c..f9ab9bd466f 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -1,3 +1,4 @@ +# :nocov: module DeliverNever def deliver_later self @@ -21,3 +22,4 @@ module Gitlab end end end +# :nocov: diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 2442c2ded3b..159d0e7952e 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -1,5 +1,7 @@ module Gitlab module Sentry + extend Gitlab::CurrentSettings + def self.enabled? Rails.env.production? && current_application_settings.sentry_enabled? end @@ -7,6 +9,8 @@ module Gitlab def self.context(current_user = nil) return unless self.enabled? + Raven.tags_context(locale: I18n.locale) + if current_user Raven.user_context( id: current_user.id, diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 0cb28732402..81ecdf43ef9 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -73,8 +73,10 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) - gitlab_shell_fast_execute([gitlab_shell_projects_path, - 'add-project', storage, "#{name}.git"]) + Gitlab::Git::Repository.create(storage, name, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) + rescue => err + Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}") + false end # Import repository @@ -96,33 +98,24 @@ module Gitlab # Fetch remote for repository # - # name - project path with namespace + # repository - an instance of Git::Repository # remote - remote name # forced - should we use --force flag? # no_tags - should we use --no-tags flag? # # Ex. - # fetch_remote("gitlab/gitlab-ci", "upstream") + # fetch_remote(my_repo, "upstream") # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 - def fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false) - args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] - args << '--force' if forced - args << '--no-tags' if no_tags - - vars = {} - - if ssh_auth&.ssh_import? - if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? - vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key - end - - if ssh_auth.ssh_known_hosts.present? - vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts + def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false) + gitaly_migrate(:fetch_remote) do |is_enabled| + if is_enabled + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) + else + storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] + local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) end end - - gitlab_shell_fast_execute_raise_error(args, vars) end # Move repository @@ -273,7 +266,11 @@ module Gitlab protected def gitlab_shell_path - Gitlab.config.gitlab_shell.path + File.expand_path(Gitlab.config.gitlab_shell.path) + end + + def gitlab_shell_hooks_path + File.expand_path(Gitlab.config.gitlab_shell.hooks_path) end def gitlab_shell_user_home @@ -296,6 +293,26 @@ module Gitlab private + def local_fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false) + args = [gitlab_shell_projects_path, 'fetch-remote', storage, name, remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] + args << '--force' if forced + args << '--no-tags' if no_tags + + vars = {} + + if ssh_auth&.ssh_import? + if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? + vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key + end + + if ssh_auth.ssh_known_hosts.present? + vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts + end + end + + gitlab_shell_fast_execute_raise_error(args, vars) + end + def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) @@ -319,5 +336,13 @@ module Gitlab # from wasting I/O by searching through GEM_PATH Bundler.with_original_env { Popen.popen(cmd, nil, vars) } end + + def gitaly_migrate(method, &block) + Gitlab::GitalyClient.migrate(method, &block) + rescue GRPC::NotFound, GRPC::BadStatus => e + # Old Popen code returns [Error, output] to the caller, so we + # need to do the same here... + raise Error, e + end end end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index ca8d3271541..a0a2769cf9e 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -90,9 +90,14 @@ module Gitlab # # Returns an array of completed JIDs def self.completed_jids(job_ids) - Sidekiq.redis do |redis| - job_ids.reject { |jid| redis.exists(key_for(jid)) } + statuses = job_status(job_ids) + + completed = [] + job_ids.zip(statuses).each do |job_id, status| + completed << job_id unless status end + + completed end def self.key_for(jid) diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb index d4d39a888e7..5512afa45a8 100644 --- a/lib/gitlab/sidekiq_throttler.rb +++ b/lib/gitlab/sidekiq_throttler.rb @@ -3,6 +3,8 @@ module Gitlab class << self def execute! if Gitlab::CurrentSettings.sidekiq_throttling_enabled? + require 'sidekiq-limit_fetch' + Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue| Sidekiq::Queue[queue].limit = queue_limit end diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index ea611a4d629..ab855319077 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -14,7 +14,7 @@ module Gitlab if text.start_with?('help') header_with_list("Available commands", full_commands(trigger)) else - header_with_list("Unknown command, these commands are available", full_commands(trigger)) + header_with_list("Unknown command, these commands are available", full_commands(trigger)) end end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb new file mode 100644 index 00000000000..7c2d1d8f887 --- /dev/null +++ b/lib/gitlab/sql/pattern.rb @@ -0,0 +1,46 @@ +module Gitlab + module SQL + module Pattern + extend ActiveSupport::Concern + + MIN_CHARS_FOR_PARTIAL_MATCHING = 3 + REGEX_QUOTED_WORD = /(?<=^| )"[^"]+"(?= |$)/ + + class_methods do + def to_pattern(query) + if partial_matching?(query) + "%#{sanitize_sql_like(query)}%" + else + sanitize_sql_like(query) + end + end + + def partial_matching?(query) + query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING + end + + def to_fuzzy_arel(column, query) + words = select_fuzzy_words(query) + + matches = words.map { |word| arel_table[column].matches(to_pattern(word)) } + + matches.reduce { |result, match| result.and(match) } + end + + def select_fuzzy_words(query) + quoted_words = query.scan(REGEX_QUOTED_WORD) + + query = quoted_words.reduce(query) { |q, quoted_word| q.sub(quoted_word, '') } + + words = query.split(/\s+/) + + quoted_words.map! { |quoted_word| quoted_word[1..-2] } + + words.concat(quoted_words) + + words.select { |word| partial_matching?(word) } + end + end + end + end +end diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb new file mode 100644 index 00000000000..89ca1298120 --- /dev/null +++ b/lib/gitlab/ssh_public_key.rb @@ -0,0 +1,71 @@ +module Gitlab + class SSHPublicKey + Technology = Struct.new(:name, :key_class, :supported_sizes) + + Technologies = [ + Technology.new(:rsa, OpenSSL::PKey::RSA, [1024, 2048, 3072, 4096]), + Technology.new(:dsa, OpenSSL::PKey::DSA, [1024, 2048, 3072]), + Technology.new(:ecdsa, OpenSSL::PKey::EC, [256, 384, 521]), + Technology.new(:ed25519, Net::SSH::Authentication::ED25519::PubKey, [256]) + ].freeze + + def self.technology(name) + Technologies.find { |tech| tech.name.to_s == name.to_s } + end + + def self.technology_for_key(key) + Technologies.find { |tech| key.is_a?(tech.key_class) } + end + + def self.supported_sizes(name) + technology(name)&.supported_sizes + end + + attr_reader :key_text, :key + + # Unqualified MD5 fingerprint for compatibility + delegate :fingerprint, to: :key, allow_nil: true + + def initialize(key_text) + @key_text = key_text + + @key = + begin + Net::SSH::KeyFactory.load_data_public_key(key_text) + rescue StandardError, NotImplementedError + end + end + + def valid? + key.present? + end + + def type + technology.name if valid? + end + + def bits + return unless valid? + + case type + when :rsa + key.n.num_bits + when :dsa + key.p.num_bits + when :ecdsa + key.group.order.num_bits + when :ed25519 + 256 + else + raise "Unsupported key type: #{type}" + end + end + + private + + def technology + @technology ||= + self.class.technology_for_key(key) || raise("Unsupported key type: #{key.class}") + end + end +end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index 94fba0a221a..11aeec1ebfa 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -1,21 +1,31 @@ module Gitlab class StringRangeMarker - attr_accessor :raw_line, :rich_line - - def initialize(raw_line, rich_line = raw_line) - @raw_line = raw_line - @rich_line = ERB::Util.html_escape(rich_line) + attr_accessor :raw_line, :rich_line, :html_escaped + + def initialize(raw_line, rich_line = nil) + @raw_line = raw_line.dup + if rich_line.nil? + @rich_line = raw_line.dup + @html_escaped = false + else + @rich_line = ERB::Util.html_escape(rich_line) + @html_escaped = true + end end def mark(marker_ranges) return rich_line unless marker_ranges - rich_marker_ranges = [] - marker_ranges.each do |range| - # Map the inline-diff range based on the raw line to character positions in the rich line - rich_positions = position_mapping[range].flatten - # Turn the array of character positions into ranges - rich_marker_ranges.concat(collapse_ranges(rich_positions)) + if html_escaped + rich_marker_ranges = [] + marker_ranges.each do |range| + # Map the inline-diff range based on the raw line to character positions in the rich line + rich_positions = position_mapping[range].flatten + # Turn the array of character positions into ranges + rich_marker_ranges.concat(collapse_ranges(rich_positions)) + end + else + rich_marker_ranges = marker_ranges end offset = 0 @@ -31,7 +41,7 @@ module Gitlab offset += text.length - original_text.length end - rich_line.html_safe + @html_escaped ? rich_line.html_safe : rich_line end private diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 7ebec8e2cff..7393574ac13 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -18,6 +18,10 @@ module Gitlab { name: name, content: content } end + def <=>(other) + name <=> other.name + end + class << self def all(project = nil) if categories.any? @@ -58,7 +62,7 @@ module Gitlab directory = category_directory(category) files = finder(project).list_files_for(directory) - files.map { |f| new(f, project) } + files.map { |f| new(f, project) }.sort end def category_directory(category) diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb new file mode 100644 index 00000000000..d43eff5ba4a --- /dev/null +++ b/lib/gitlab/themes.rb @@ -0,0 +1,84 @@ +module Gitlab + # Module containing GitLab's application theme definitions and helper methods + # for accessing them. + module Themes + extend self + + # Theme ID used when no `default_theme` configuration setting is provided. + APPLICATION_DEFAULT = 1 + + # Struct class representing a single Theme + Theme = Struct.new(:id, :name, :css_class) + + # All available Themes + THEMES = [ + Theme.new(1, 'Indigo', 'ui_indigo'), + Theme.new(2, 'Dark', 'ui_dark'), + Theme.new(3, 'Light', 'ui_light'), + Theme.new(4, 'Blue', 'ui_blue'), + Theme.new(5, 'Green', 'ui_green') + ].freeze + + # Convenience method to get a space-separated String of all the theme + # classes that might be applied to the `body` element + # + # Returns a String + def body_classes + THEMES.collect(&:css_class).uniq.join(' ') + end + + # Get a Theme by its ID + # + # If the ID is invalid, returns the default Theme. + # + # id - Integer ID + # + # Returns a Theme + def by_id(id) + THEMES.detect { |t| t.id == id } || default + end + + # Returns the number of defined Themes + def count + THEMES.size + end + + # Get the default Theme + # + # Returns a Theme + def default + by_id(default_id) + end + + # Iterate through each Theme + # + # Yields the Theme object + def each(&block) + THEMES.each(&block) + end + + # Get the Theme for the specified user, or the default + # + # user - User record + # + # Returns a Theme + def for_user(user) + if user + by_id(user.theme_id) + else + default + end + end + + private + + def default_id + @default_id ||= begin + id = Gitlab.config.gitlab.default_theme.to_i + theme_ids = THEMES.map(&:id) + + theme_ids.include?(id) ? id : APPLICATION_DEFAULT + end + end + end +end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 7e14a566696..fee1a127fd7 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -19,6 +19,8 @@ module Gitlab return false if internal?(uri) return true if blocked_port?(uri.port) + return true if blocked_user_or_hostname?(uri.user) + return true if blocked_user_or_hostname?(uri.hostname) server_ips = Resolv.getaddresses(uri.hostname) return true if (blocked_ips & server_ips).any? @@ -37,6 +39,12 @@ module Gitlab port < 1024 && !VALID_PORTS.include?(port) end + def blocked_user_or_hostname?(value) + return false if value.blank? + + value !~ /\A\p{Alnum}/ + end + def internal?(uri) internal_web?(uri) || internal_shell?(uri) end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index c81dc7e30d0..703adae12cb 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -9,7 +9,7 @@ module Gitlab end def self.valid?(url) - return false unless url + return false unless url.present? Addressable::URI.parse(url.strip) @@ -19,7 +19,12 @@ module Gitlab end def initialize(url, credentials: nil) - @url = Addressable::URI.parse(url.strip) + @url = Addressable::URI.parse(url.to_s.strip) + + %i[user password].each do |symbol| + credentials[symbol] = credentials[symbol].presence if credentials&.key?(symbol) + end + @credentials = credentials end @@ -29,13 +34,13 @@ module Gitlab def masked_url url = @url.dup - url.password = "*****" unless url.password.nil? - url.user = "*****" unless url.user.nil? + url.password = "*****" if url.password.present? + url.user = "*****" if url.user.present? url.to_s end def credentials - @credentials ||= { user: @url.user, password: @url.password } + @credentials ||= { user: @url.user.presence, password: @url.password.presence } end def full_url @@ -47,8 +52,10 @@ module Gitlab def generate_full_url return @url unless valid_credentials? @full_url = @url.dup - @full_url.user = credentials[:user] + @full_url.password = credentials[:password] + @full_url.user = credentials[:user] + @full_url end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 748e0a29184..3cf26625108 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -1,8 +1,8 @@ module Gitlab class UsageData - include Gitlab::CurrentSettings - class << self + include Gitlab::CurrentSettings + def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index fa182c4deda..abb3d3a02c3 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -14,6 +14,19 @@ module Gitlab str.force_encoding(Encoding::UTF_8) end + # A slugified version of the string, suitable for inclusion in URLs and + # domain names. Rules: + # + # * Lowercased + # * Anything not matching [a-z0-9-] is replaced with a - + # * Maximum length is 63 bytes + # * First/Last Character is not a hyphen + def slugify(str) + return str.downcase + .gsub(/[^a-z0-9]/, '-')[0..62] + .gsub(/(\A-+|-+\z)/, '') + end + def to_boolean(value) return value if [true, false].include?(value) return true if value =~ /^(true|t|yes|y|1|on)$/i @@ -29,5 +42,9 @@ module Gitlab 'No' end end + + def random_string + Random.rand(Float::MAX.to_i).to_s(36) + end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 3f25e463412..7a94af2f8f1 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -35,17 +35,13 @@ module Gitlab when 'git_receive_pack' Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) when 'git_upload_pack' - Gitlab::GitalyClient.feature_enabled?( - :post_upload_pack, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT - ) + true when 'info_refs' true else raise "Unsupported action: #{action}" end if feature_enabled - params[:GitalyAddress] = server[:address] # This field will be deprecated params[:GitalyServer] = server end @@ -125,10 +121,10 @@ module Gitlab ] end - def send_artifacts_entry(build, entry) + def send_artifacts_entry(build, path) params = { 'Archive' => build.artifacts_file.path, - 'Entry' => Base64.encode64(entry.path) + 'Entry' => Base64.encode64(path.to_s) } [ diff --git a/lib/rspec_flaky/example.rb b/lib/rspec_flaky/example.rb new file mode 100644 index 00000000000..b6e790cbbab --- /dev/null +++ b/lib/rspec_flaky/example.rb @@ -0,0 +1,46 @@ +module RspecFlaky + # This is a wrapper class for RSpec::Core::Example + class Example + delegate :status, :exception, to: :execution_result + + def initialize(rspec_example) + @rspec_example = rspec_example.try(:example) || rspec_example + end + + def uid + @uid ||= Digest::MD5.hexdigest("#{description}-#{file}") + end + + def example_id + rspec_example.id + end + + def file + metadata[:file_path] + end + + def line + metadata[:line_number] + end + + def description + metadata[:full_description] + end + + def attempts + rspec_example.try(:attempts) || 1 + end + + private + + attr_reader :rspec_example + + def metadata + rspec_example.metadata + end + + def execution_result + rspec_example.execution_result + end + end +end diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb new file mode 100644 index 00000000000..f81fb90e870 --- /dev/null +++ b/lib/rspec_flaky/flaky_example.rb @@ -0,0 +1,39 @@ +module RspecFlaky + # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file + class FlakyExample < OpenStruct + def initialize(example) + if example.respond_to?(:example_id) + super( + example_id: example.example_id, + file: example.file, + line: example.line, + description: example.description, + last_attempts_count: example.attempts, + flaky_reports: 1) + else + super + end + end + + def first_flaky_at + self[:first_flaky_at] || Time.now + end + + def last_flaky_at + Time.now + end + + def last_flaky_job + return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] + + "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" + end + + def to_h + super.merge( + first_flaky_at: first_flaky_at, + last_flaky_at: last_flaky_at, + last_flaky_job: last_flaky_job) + end + end +end diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb new file mode 100644 index 00000000000..ec2fbd9e36c --- /dev/null +++ b/lib/rspec_flaky/listener.rb @@ -0,0 +1,75 @@ +require 'json' + +module RspecFlaky + class Listener + attr_reader :all_flaky_examples, :new_flaky_examples + + def initialize + @new_flaky_examples = {} + @all_flaky_examples = init_all_flaky_examples + end + + def example_passed(notification) + current_example = RspecFlaky::Example.new(notification.example) + + return unless current_example.attempts > 1 + + flaky_example_hash = all_flaky_examples[current_example.uid] + + all_flaky_examples[current_example.uid] = + if flaky_example_hash + FlakyExample.new(flaky_example_hash).tap do |ex| + ex.last_attempts_count = current_example.attempts + ex.flaky_reports += 1 + end + else + FlakyExample.new(current_example).tap do |ex| + new_flaky_examples[current_example.uid] = ex + end + end + end + + def dump_summary(_) + write_report_file(all_flaky_examples, all_flaky_examples_report_path) + + if new_flaky_examples.any? + Rails.logger.warn "\nNew flaky examples detected:\n" + Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples)) + + write_report_file(new_flaky_examples, new_flaky_examples_report_path) + end + end + + def to_report(examples) + Hash[examples.map { |k, ex| [k, ex.to_h] }] + end + + private + + def init_all_flaky_examples + return {} unless File.exist?(all_flaky_examples_report_path) + + all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path)) + + Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }] + end + + def write_report_file(examples, file_path) + return unless ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + + report_path_dir = File.dirname(file_path) + FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) + File.write(file_path, JSON.pretty_generate(to_report(examples))) + end + + def all_flaky_examples_report_path + @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] || + Rails.root.join("rspec_flaky/all-report.json") + end + + def new_flaky_examples_report_path + @new_flaky_examples_report_path ||= ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || + Rails.root.join("rspec_flaky/new-report.json") + end + end +end diff --git a/lib/static_model.rb b/lib/static_model.rb index 185921d8fbe..60e2dd82e4e 100644 --- a/lib/static_model.rb +++ b/lib/static_model.rb @@ -18,7 +18,7 @@ module StaticModel # # Pass it along if we respond to it. def [](key) - send(key) if respond_to?(key) + send(key) if respond_to?(key) # rubocop:disable GitlabSecurity/PublicSend end def to_param diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index f25e66d54c8..54f51d9d633 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -25,6 +25,39 @@ map $http_upgrade $connection_upgrade_gitlab { '' close; } +## NGINX 'combined' log format with filtered query strings +log_format gitlab_access $remote_addr - $remote_user [$time_local] "$request_method $gitlab_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_filtered_http_referer" "$http_user_agent"; + +## Remove private_token from the request URI +# In: /foo?private_token=unfiltered&authenticity_token=unfiltered&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... +map $request_uri $gitlab_temp_request_uri_1 { + default $request_uri; + ~(?i)^(?<start>.*)(?<temp>[\?&]private[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; +} + +## Remove authenticity_token from the request URI +# In: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... +map $gitlab_temp_request_uri_1 $gitlab_temp_request_uri_2 { + default $gitlab_temp_request_uri_1; + ~(?i)^(?<start>.*)(?<temp>[\?&]authenticity[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; +} + +## Remove rss_token from the request URI +# In: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=[FILTERED]&... +map $gitlab_temp_request_uri_2 $gitlab_filtered_request_uri { + default $gitlab_temp_request_uri_2; + ~(?i)^(?<start>.*)(?<temp>[\?&]rss[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; +} + +## A version of the referer without the query string +map $http_referer $gitlab_filtered_http_referer { + default $http_referer; + ~^(?<temp>.*)\? $temp; +} + ## Normal HTTP host server { ## Either remove "default_server" from the listen line below, @@ -46,7 +79,7 @@ server { # set_real_ip_from YOUR_TRUSTED_ADDRESS; ## Replace this with something like 192.168.1.0/24 ## Individual nginx logs for this GitLab vhost - access_log /var/log/nginx/gitlab_access.log; + access_log /var/log/nginx/gitlab_access.log gitlab_access; error_log /var/log/nginx/gitlab_error.log; location / { diff --git a/lib/support/nginx/gitlab-pages b/lib/support/nginx/gitlab-pages index d9746c5c1aa..875c8bcbf3c 100644 --- a/lib/support/nginx/gitlab-pages +++ b/lib/support/nginx/gitlab-pages @@ -18,8 +18,11 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache off; + # The same address as passed to GitLab Pages: `-listen-proxy` - proxy_pass http://localhost:8090/; + proxy_pass http://localhost:8090/; } # Define custom error pages diff --git a/lib/support/nginx/gitlab-pages-ssl b/lib/support/nginx/gitlab-pages-ssl index a1ccf266835..62ed482e2bf 100644 --- a/lib/support/nginx/gitlab-pages-ssl +++ b/lib/support/nginx/gitlab-pages-ssl @@ -67,8 +67,11 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache off; + # The same address as passed to GitLab Pages: `-listen-proxy` - proxy_pass http://localhost:8090/; + proxy_pass http://localhost:8090/; } # Define custom error pages diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 2b40da18bab..ed8131ef24f 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -29,6 +29,41 @@ map $http_upgrade $connection_upgrade_gitlab_ssl { '' close; } + +## NGINX 'combined' log format with filtered query strings +log_format gitlab_ssl_access $remote_addr - $remote_user [$time_local] "$request_method $gitlab_ssl_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_ssl_filtered_http_referer" "$http_user_agent"; + +## Remove private_token from the request URI +# In: /foo?private_token=unfiltered&authenticity_token=unfiltered&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... +map $request_uri $gitlab_ssl_temp_request_uri_1 { + default $request_uri; + ~(?i)^(?<start>.*)(?<temp>[\?&]private[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; +} + +## Remove authenticity_token from the request URI +# In: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... +map $gitlab_ssl_temp_request_uri_1 $gitlab_ssl_temp_request_uri_2 { + default $gitlab_ssl_temp_request_uri_1; + ~(?i)^(?<start>.*)(?<temp>[\?&]authenticity[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; +} + +## Remove rss_token from the request URI +# In: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=[FILTERED]&... +map $gitlab_ssl_temp_request_uri_2 $gitlab_ssl_filtered_request_uri { + default $gitlab_ssl_temp_request_uri_2; + ~(?i)^(?<start>.*)(?<temp>[\?&]rss[\-_]token)=[^&]*(?<rest>.*)$ "$start$temp=[FILTERED]$rest"; +} + +## A version of the referer without the query string +map $http_referer $gitlab_ssl_filtered_http_referer { + default $http_referer; + ~^(?<temp>.*)\? $temp; +} + + ## Redirects all HTTP traffic to the HTTPS host server { ## Either remove "default_server" from the listen line below, @@ -40,7 +75,7 @@ server { server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com server_tokens off; ## Don't show the nginx version number, a security best practice return 301 https://$http_host$request_uri; - access_log /var/log/nginx/gitlab_access.log; + access_log /var/log/nginx/gitlab_access.log gitlab_ssl_access; error_log /var/log/nginx/gitlab_error.log; } @@ -93,7 +128,7 @@ server { # set_real_ip_from YOUR_TRUSTED_ADDRESS; ## Replace this with something like 192.168.1.0/24 ## Individual nginx logs for this GitLab vhost - access_log /var/log/nginx/gitlab_access.log; + access_log /var/log/nginx/gitlab_access.log gitlab_ssl_access; error_log /var/log/nginx/gitlab_error.log; location / { diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb index 198867f7ac6..d08a81639e3 100644 --- a/lib/system_check/app/git_config_check.rb +++ b/lib/system_check/app/git_config_check.rb @@ -20,7 +20,7 @@ module SystemCheck # Returns true if all subcommands were successful (according to their exit code) # Returns false if any or all subcommands failed. def repair! - return false unless is_gitlab_user? + return false unless gitlab_user? command_success = OPTIONS.map do |name, value| system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb new file mode 100644 index 00000000000..7b486d78cf0 --- /dev/null +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -0,0 +1,69 @@ +module SystemCheck + module App + class GitUserDefaultSSHConfigCheck < SystemCheck::BaseCheck + # These files are allowed in the .ssh directory. The `config` file is not + # whitelisted as it may change the SSH client's behaviour dramatically. + WHITELIST = %w[ + authorized_keys + authorized_keys2 + known_hosts + ].freeze + + set_name 'Git user has default SSH configuration?' + set_skip_reason 'skipped (git user is not present or configured)' + + def skip? + !home_dir || !File.directory?(home_dir) + end + + def check? + forbidden_files.empty? + end + + def show_error + backup_dir = "~/gitlab-check-backup-#{Time.now.to_i}" + + instructions = forbidden_files.map do |filename| + "sudo mv #{Shellwords.escape(filename)} #{backup_dir}" + end + + try_fixing_it("mkdir #{backup_dir}", *instructions) + for_more_information('doc/ssh/README.md in section "SSH on the GitLab server"') + fix_and_rerun + end + + private + + def git_user + Gitlab.config.gitlab.user + end + + def home_dir + return @home_dir if defined?(@home_dir) + + @home_dir = + begin + File.expand_path("~#{git_user}") + rescue ArgumentError + nil + end + end + + def ssh_dir + return nil unless home_dir + + File.join(home_dir, '.ssh') + end + + def forbidden_files + @forbidden_files ||= + begin + present = Dir[File.join(ssh_dir, '*')] + whitelisted = WHITELIST.map { |basename| File.join(ssh_dir, basename) } + + present - whitelisted + end + end + end + end +end diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb index 015c7ed1731..53a47eb0f42 100644 --- a/lib/system_check/app/init_script_up_to_date_check.rb +++ b/lib/system_check/app/init_script_up_to_date_check.rb @@ -7,26 +7,22 @@ module SystemCheck set_skip_reason 'skipped (omnibus-gitlab has no init script)' def skip? - omnibus_gitlab? - end + return true if omnibus_gitlab? - def multi_check - recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab') + unless init_file_exists? + self.skip_reason = "can't check because of previous errors" - unless File.exist?(SCRIPT_PATH) - $stdout.puts "can't check because of previous errors".color(:magenta) - return + true end + end + + def check? + recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab') recipe_content = File.read(recipe_path) script_content = File.read(SCRIPT_PATH) - if recipe_content == script_content - $stdout.puts 'yes'.color(:green) - else - $stdout.puts 'no'.color(:red) - show_error - end + recipe_content == script_content end def show_error @@ -38,6 +34,12 @@ module SystemCheck ) fix_and_rerun end + + private + + def init_file_exists? + File.exist?(SCRIPT_PATH) + end end end end diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index 5dcb3f0886b..0f5742dd67f 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -62,6 +62,25 @@ module SystemCheck call_or_return(@skip_reason) || 'skipped' end + # Define a reason why we skipped the SystemCheck (during runtime) + # + # This is used when you need dynamic evaluation like when you have + # multiple reasons why a check can fail + # + # @param [String] reason to be displayed + def skip_reason=(reason) + @skip_reason = reason + end + + # Skip reason defined during runtime + # + # This value have precedence over the one defined in the subclass + # + # @return [String] the reason + def skip_reason + @skip_reason + end + # Does the check support automatically repair routine? # # @return [Boolean] whether check implemented `#repair!` method or not @@ -73,7 +92,7 @@ module SystemCheck self.class.instance_methods(false).include?(:skip?) end - def is_multi_check? + def multi_check? self.class.instance_methods(false).include?(:multi_check) end diff --git a/lib/system_check/incoming_email/foreman_configured_check.rb b/lib/system_check/incoming_email/foreman_configured_check.rb new file mode 100644 index 00000000000..1db7bf2b782 --- /dev/null +++ b/lib/system_check/incoming_email/foreman_configured_check.rb @@ -0,0 +1,23 @@ +module SystemCheck + module IncomingEmail + class ForemanConfiguredCheck < SystemCheck::BaseCheck + set_name 'Foreman configured correctly?' + + def check? + path = Rails.root.join('Procfile') + + File.exist?(path) && File.read(path) =~ /^mail_room:/ + end + + def show_error + try_fixing_it( + 'Enable mail_room in your Procfile.' + ) + for_more_information( + 'doc/administration/reply_by_email.md' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb new file mode 100644 index 00000000000..dee108d987b --- /dev/null +++ b/lib/system_check/incoming_email/imap_authentication_check.rb @@ -0,0 +1,45 @@ +module SystemCheck + module IncomingEmail + class ImapAuthenticationCheck < SystemCheck::BaseCheck + set_name 'IMAP server credentials are correct?' + + def check? + if mailbox_config + begin + imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) + imap.starttls if config[:start_tls] + imap.login(config[:email], config[:password]) + connected = true + rescue + connected = false + end + end + + connected + end + + def show_error + try_fixing_it( + 'Check that the information in config/gitlab.yml is correct' + ) + for_more_information( + 'doc/administration/reply_by_email.md' + ) + fix_and_rerun + end + + private + + def mailbox_config + return @config if @config + + config_path = Rails.root.join('config', 'mail_room.yml').to_s + erb = ERB.new(File.read(config_path)) + erb.filename = config_path + config_file = YAML.load(erb.result) + + @config = config_file[:mailboxes]&.first + end + end + end +end diff --git a/lib/system_check/incoming_email/initd_configured_check.rb b/lib/system_check/incoming_email/initd_configured_check.rb new file mode 100644 index 00000000000..ea23b8ef49c --- /dev/null +++ b/lib/system_check/incoming_email/initd_configured_check.rb @@ -0,0 +1,32 @@ +module SystemCheck + module IncomingEmail + class InitdConfiguredCheck < SystemCheck::BaseCheck + set_name 'Init.d configured correctly?' + + def skip? + omnibus_gitlab? + end + + def check? + mail_room_configured? + end + + def show_error + try_fixing_it( + 'Enable mail_room in the init.d configuration.' + ) + for_more_information( + 'doc/administration/reply_by_email.md' + ) + fix_and_rerun + end + + private + + def mail_room_configured? + path = '/etc/default/gitlab' + File.exist?(path) && File.read(path).include?('mail_room_enabled=true') + end + end + end +end diff --git a/lib/system_check/incoming_email/mail_room_running_check.rb b/lib/system_check/incoming_email/mail_room_running_check.rb new file mode 100644 index 00000000000..c1807501829 --- /dev/null +++ b/lib/system_check/incoming_email/mail_room_running_check.rb @@ -0,0 +1,43 @@ +module SystemCheck + module IncomingEmail + class MailRoomRunningCheck < SystemCheck::BaseCheck + set_name 'MailRoom running?' + + def skip? + return true if omnibus_gitlab? + + unless mail_room_configured? + self.skip_reason = "can't check because of previous errors" + true + end + end + + def check? + mail_room_running? + end + + def show_error + try_fixing_it( + sudo_gitlab('RAILS_ENV=production bin/mail_room start') + ) + for_more_information( + see_installation_guide_section('Install Init Script'), + 'see log/mail_room.log for possible errors' + ) + fix_and_rerun + end + + private + + def mail_room_configured? + path = '/etc/default/gitlab' + File.exist?(path) && File.read(path).include?('mail_room_enabled=true') + end + + def mail_room_running? + ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) + ps_ux.include?("mail_room") + end + end + end +end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index e5986612908..00221f77cf4 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -23,7 +23,7 @@ module SystemCheck # # @param [BaseCheck] check class def <<(check) - raise ArgumentError unless check < BaseCheck + raise ArgumentError unless check.is_a?(Class) && check < BaseCheck @checks << check end @@ -48,12 +48,12 @@ module SystemCheck # When implements skip method, we run it first, and if true, skip the check if check.can_skip? && check.skip? - $stdout.puts check_klass.skip_reason.color(:magenta) + $stdout.puts check.skip_reason.try(:color, :magenta) || check_klass.skip_reason.color(:magenta) return end # When implements a multi check, we don't control the output - if check.is_multi_check? + if check.multi_check? check.multi_check return end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index b48e4dce445..35ba729c156 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -19,4 +19,46 @@ namespace :gettext do Rake::Task['gettext:pack'].invoke Rake::Task['gettext:po_to_json'].invoke end + + desc 'Lint all po files in `locale/' + task lint: :environment do + require 'simple_po_parser' + + FastGettext.silence_errors + files = Dir.glob(Rails.root.join('locale/*/gitlab.po')) + + linters = files.map do |file| + locale = File.basename(File.dirname(file)) + + Gitlab::I18n::PoLinter.new(file, locale) + end + + pot_file = Rails.root.join('locale/gitlab.pot') + linters.unshift(Gitlab::I18n::PoLinter.new(pot_file)) + + failed_linters = linters.select { |linter| linter.errors.any? } + + if failed_linters.empty? + puts 'All PO files are valid.' + else + failed_linters.each do |linter| + report_errors_for_file(linter.po_path, linter.errors) + end + + raise "Not all PO-files are valid: #{failed_linters.map(&:po_path).to_sentence}" + end + end + + def report_errors_for_file(file, errors_for_file) + puts "Errors in `#{file}`:" + + errors_for_file.each do |message_id, errors| + puts " #{message_id}" + errors.each do |error| + spaces = ' ' * 4 + error = error.lines.join("#{spaces}") + puts "#{spaces}#{error}" + end + end + end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index dbb3b827b9a..654f638c454 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -33,6 +33,7 @@ namespace :gitlab do SystemCheck::App::RedisVersionCheck, SystemCheck::App::RubyVersionCheck, SystemCheck::App::GitVersionCheck, + SystemCheck::App::GitUserDefaultSSHConfigCheck, SystemCheck::App::ActiveUsersCheck ] @@ -41,8 +42,6 @@ namespace :gitlab do end namespace :gitlab_shell do - include SystemCheck::Helpers - desc "GitLab | Check the configuration of GitLab Shell" task check: :environment do warn_user_is_not_gitlab @@ -249,8 +248,6 @@ namespace :gitlab do end namespace :sidekiq do - include SystemCheck::Helpers - desc "GitLab | Check the configuration of Sidekiq" task check: :environment do warn_user_is_not_gitlab @@ -309,143 +306,30 @@ namespace :gitlab do end namespace :incoming_email do - include SystemCheck::Helpers - desc "GitLab | Check the configuration of Reply by email" task check: :environment do warn_user_is_not_gitlab - start_checking "Reply by email" if Gitlab.config.incoming_email.enabled - check_imap_authentication + checks = [ + SystemCheck::IncomingEmail::ImapAuthenticationCheck + ] if Rails.env.production? - check_initd_configured_correctly - check_mail_room_running + checks << SystemCheck::IncomingEmail::InitdConfiguredCheck + checks << SystemCheck::IncomingEmail::MailRoomRunningCheck else - check_foreman_configured_correctly + checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck end - else - puts 'Reply by email is disabled in config/gitlab.yml' - end - - finished_checking "Reply by email" - end - - # Checks - ######################## - - def check_initd_configured_correctly - return if omnibus_gitlab? - - print "Init.d configured correctly? ... " - path = "/etc/default/gitlab" - - if File.exist?(path) && File.read(path).include?("mail_room_enabled=true") - puts "yes".color(:green) + SystemCheck.run('Reply by email', checks) else - puts "no".color(:red) - try_fixing_it( - "Enable mail_room in the init.d configuration." - ) - for_more_information( - "doc/administration/reply_by_email.md" - ) - fix_and_rerun - end - end - - def check_foreman_configured_correctly - print "Foreman configured correctly? ... " - - path = Rails.root.join("Procfile") - - if File.exist?(path) && File.read(path) =~ /^mail_room:/ - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Enable mail_room in your Procfile." - ) - for_more_information( - "doc/administration/reply_by_email.md" - ) - fix_and_rerun - end - end - - def check_mail_room_running - return if omnibus_gitlab? - - print "MailRoom running? ... " - - path = "/etc/default/gitlab" - - unless File.exist?(path) && File.read(path).include?("mail_room_enabled=true") - puts "can't check because of previous errors".color(:magenta) - return - end - - if mail_room_running? - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - sudo_gitlab("RAILS_ENV=production bin/mail_room start") - ) - for_more_information( - see_installation_guide_section("Install Init Script"), - "see log/mail_room.log for possible errors" - ) - fix_and_rerun - end - end - - def check_imap_authentication - print "IMAP server credentials are correct? ... " - - config_path = Rails.root.join('config', 'mail_room.yml').to_s - erb = ERB.new(File.read(config_path)) - erb.filename = config_path - config_file = YAML.load(erb.result) - - config = config_file[:mailboxes].first - - if config - begin - imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) - imap.starttls if config[:start_tls] - imap.login(config[:email], config[:password]) - connected = true - rescue - connected = false - end - end - - if connected - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Check that the information in config/gitlab.yml is correct" - ) - for_more_information( - "doc/administration/reply_by_email.md" - ) - fix_and_rerun + puts 'Reply by email is disabled in config/gitlab.yml' end end - - def mail_room_running? - ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) - ps_ux.include?("mail_room") - end end namespace :ldap do - include SystemCheck::Helpers - task :check, [:limit] => :environment do |_, args| # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. @@ -501,8 +385,6 @@ namespace :gitlab do end namespace :repo do - include SystemCheck::Helpers - desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do Gitlab.config.repositories.storages.each do |name, repository_storage| @@ -517,8 +399,6 @@ namespace :gitlab do end namespace :user do - include SystemCheck::Helpers - desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index f76bef5f4bf..8ae1b6a626a 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -111,7 +111,7 @@ namespace :gitlab do next unless id > max_iid project.deployments.find(id).create_ref - rugged.references.delete(ref) + project.repository.delete_refs(ref) end end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index aaf00bd703a..08677a98fc1 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -15,13 +15,17 @@ namespace :gitlab do checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) - command = status.zero? ? 'gmake' : 'make' + command = status.zero? ? ['gmake'] : ['make'] + + if Rails.env.test? + command += %W[BUNDLE_PATH=#{Bundler.bundle_path}] + end Dir.chdir(args.dir) do create_gitaly_configuration # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? - Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT] + [command]) } + Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + command) } end end end diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake index dd2d5861481..b0a24790c4a 100644 --- a/lib/tasks/gitlab/helpers.rake +++ b/lib/tasks/gitlab/helpers.rake @@ -4,5 +4,5 @@ require 'tasks/gitlab/task_helpers' StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] namespace :gitlab do - include Gitlab::TaskHelpers + extend SystemCheck::Helpers end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 48bd9139ce8..d227a0c8bdb 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -9,71 +9,16 @@ namespace :gitlab do # * The project owner will set to the first administator of the system # * Existing projects will be skipped # + # desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do - Gitlab.config.repositories.storages.each_value do |repository_storage| - git_base_path = repository_storage['path'] - repos_to_import = Dir.glob(git_base_path + '/**/*.git') - - repos_to_import.each do |repo_path| - # strip repo base path - repo_path[0..git_base_path.length] = '' - - path = repo_path.sub(/\.git$/, '') - group_name, name = File.split(path) - group_name = nil if group_name == '.' - - puts "Processing #{repo_path}".color(:yellow) - - if path.end_with?('.wiki') - puts " * Skipping wiki repo" - next - end - - project = Project.find_by_full_path(path) - - if project - puts " * #{project.name} (#{repo_path}) exists" - else - user = User.admins.reorder("id").first - - project_params = { - name: name, - path: name - } - - # find group namespace - if group_name - group = Namespace.find_by(path: group_name) - # create group namespace - unless group - group = Group.new(name: group_name) - group.path = group_name - group.owner = user - if group.save - puts " * Created Group #{group.name} (#{group.id})".color(:green) - else - puts " * Failed trying to create group #{group.name}".color(:red) - end - end - # set project group - project_params[:namespace_id] = group.id - end - - project = Projects::CreateService.new(user, project_params).execute + if Project.current_application_settings.hashed_storage_enabled + puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red) - if project.persisted? - puts " * Created #{project.name} (#{repo_path})".color(:green) - ProjectCacheWorker.perform_async(project.id) - else - puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) - puts " Errors: #{project.errors.messages}".color(:red) - end - end - end + exit 1 end - puts "Done!".color(:green) + Gitlab::BareRepositoryImporter.execute end end end diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index dd1825c8a9e..44074397c05 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -9,5 +9,16 @@ namespace :gitlab do task data: :environment do puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) end + + desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz' + task bump_test_version: :environment do + Dir.mktmpdir do |tmp_dir| + system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null") + File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') + system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null") + end + + puts "Updated to #{Gitlab::ImportExport.version}" + end end end diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index 28b2d86eed2..8a63f486fa3 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -5,6 +5,8 @@ module Gitlab TaskAbortedByUserError = Class.new(StandardError) module TaskHelpers + extend self + # Ask if the user wants to continue # # Returns "yes" the user chose to continue @@ -102,7 +104,7 @@ module Gitlab Gitlab.config.gitlab.user end - def is_gitlab_user? + def gitlab_user? return @is_gitlab_user unless @is_gitlab_user.nil? current_user = run_command(%w(whoami)).chomp @@ -112,7 +114,7 @@ module Gitlab def warn_user_is_not_gitlab return if @warned_user_not_gitlab - unless is_gitlab_user? + unless gitlab_user? current_user = run_command(%w(whoami)).chomp puts " Warning ".color(:black).background(:yellow) diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index a7e30423c7a..f44abc2b81b 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -21,13 +21,18 @@ namespace :gitlab do params = { import_url: template.clone_url, namespace_id: admin.namespace.id, - path: template.title, + path: template.name, skip_wiki: true } - puts "Creating project for #{template.name}" + puts "Creating project for #{template.title}" project = Projects::CreateService.new(admin, params).execute + unless project.persisted? + puts project.errors.messages + exit(1) + end + loop do if project.finished? puts "Import finished for #{template.name}" diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 96b8f59242c..4d485108cf6 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -72,23 +72,7 @@ class GithubImport return @current_user.namespace if names == @current_user.namespace_path return @current_user.namespace unless @current_user.can_create_group? - full_path_namespace = Namespace.find_by_full_path(names) - - return full_path_namespace if full_path_namespace - - names.split('/').inject(nil) do |parent, name| - begin - namespace = Group.create!(name: name, - path: name, - owner: @current_user, - parent: parent) - namespace.add_owner(@current_user) - - namespace - rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid - Namespace.where(parent: parent).find_by_path_or_name(name) - end - end + Groups::NestedCreateService.new(@current_user, group_path: names).execute end def full_path_namespace(names) @@ -96,7 +80,7 @@ class GithubImport end def visibility_level - @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility end end diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index 41dee5fdc06..4a3c40f88eb 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -27,7 +27,7 @@ class UploadedFile alias_method :local_path, :path def method_missing(method_name, *args, &block) #:nodoc: - @tempfile.__send__(method_name, *args, &block) + @tempfile.__send__(method_name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end def respond_to?(method_name, include_private = false) #:nodoc: |