diff options
| author | Lin Jen-Shin <godfat@godfat.org> | 2017-11-06 21:44:57 +0800 |
|---|---|---|
| committer | Lin Jen-Shin <godfat@godfat.org> | 2017-11-06 21:44:57 +0800 |
| commit | fc6aad0b4442c58fde1ac924cb2dd73823273537 (patch) | |
| tree | 3f4a46a5b649cf623ab5e8e42eaa2e06cb2b20cf /lib | |
| parent | 239332eed3fa870fd41be83864882c0f389840d8 (diff) | |
| parent | cfc932cad10b1d6c494222e9d91aa75583b56145 (diff) | |
| download | gitlab-ce-fc6aad0b4442c58fde1ac924cb2dd73823273537.tar.gz | |
Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules
* upstream/master: (1723 commits)
Resolve "Editor icons"
Refactor issuable destroy action
Ignore routes matching legacy_*_redirect in route specs
Gitlab::Git::RevList and LfsChanges use lazy popen
Gitlab::Git::Popen can lazily hand output to a block
Merge branch 'master-i18n' into 'master'
Remove unique validation from external_url in Environment
Expose `duration` in Job API entity
Add TimeCop freeze for DST and Regular time
Harcode project visibility
update a changelog
Put a condition to old migration that adds fast_forward column to MRs
Expose project visibility as CI variable
fix flaky tests by removing unneeded clicks and focus actions
fix flaky test in gfm_autocomplete_spec.rb
Use Gitlab::Git operations for repository mirroring
Encapsulate git operations for mirroring in Gitlab::Git
Create a Wiki Repository's raw_repository properly
Add `Gitlab::Git::Repository#fetch` command
Fix Gitlab::Metrics::System#real_time and #monotonic_time doc
...
Diffstat (limited to 'lib')
213 files changed, 5755 insertions, 1507 deletions
diff --git a/lib/additional_email_headers_interceptor.rb b/lib/additional_email_headers_interceptor.rb index 2358fa6bbfd..3cb1694b9f1 100644 --- a/lib/additional_email_headers_interceptor.rb +++ b/lib/additional_email_headers_interceptor.rb @@ -1,8 +1,6 @@ class AdditionalEmailHeadersInterceptor def self.delivering_email(message) - message.headers( - 'Auto-Submitted' => 'auto-generated', - 'X-Auto-Response-Suppress' => 'All' - ) + message.header['Auto-Submitted'] ||= 'auto-generated' + message.header['X-Auto-Response-Suppress'] ||= 'All' end end diff --git a/lib/api/api.rb b/lib/api/api.rb index 79e55a2f4f7..c37e596eb9d 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -4,6 +4,10 @@ module API LOG_FILENAME = Rails.root.join("log", "api_json.log") + NO_SLASH_URL_PART_REGEX = %r{[^/]+} + PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze + COMMIT_ENDPOINT_REQUIREMENTS = PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze + use GrapeLogging::Middleware::RequestLogger, logger: Logger.new(LOG_FILENAME), formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, @@ -96,9 +100,6 @@ module API helpers ::API::Helpers helpers ::API::Helpers::CommonHelpers - NO_SLASH_URL_PART_REGEX = %r{[^/]+} - PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze - # Keep in alphabetical order mount ::API::AccessRequests mount ::API::AwardEmoji @@ -130,6 +131,7 @@ module API mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings + mount ::API::PagesDomains mount ::API::Pipelines mount ::API::PipelineSchedules mount ::API::ProjectHooks @@ -140,7 +142,6 @@ module API mount ::API::Runner mount ::API::Runners mount ::API::Services - mount ::API::Session mount ::API::Settings mount ::API::SidekiqMetrics mount ::API::Snippets diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 9933439c43b..b9c7d443f6c 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -42,76 +42,101 @@ module API # Helper Methods for Grape Endpoint module HelperMethods - attr_reader :current_user + def find_current_user! + user = find_user_from_access_token || find_user_from_warden + return unless user - # Invokes the doorkeeper guard. - # - # If token is presented and valid, then it sets @current_user. - # - # If the token does not have sufficient scopes to cover the requred scopes, - # then it raises InsufficientScopeError. - # - # If the token is expired, then it raises ExpiredError. - # - # If the token is revoked, then it raises RevokedError. - # - # If the token is not found (nil), then it returns nil - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - # rubocop:disable Cop/ModuleWithInstanceVariables - def doorkeeper_guard(scopes: []) - access_token = find_access_token - return nil unless access_token + forbidden!('User is blocked') unless Gitlab::UserAccess.new(user).allowed? && user.can?(:access_api) + + user + end + + def access_token + return @access_token if defined?(@access_token) + + @access_token = find_oauth_access_token || find_personal_access_token + end + + def validate_access_token!(scopes: []) + return unless access_token case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) when AccessTokenValidationService::INSUFFICIENT_SCOPE raise InsufficientScopeError.new(scopes) - when AccessTokenValidationService::EXPIRED raise ExpiredError - when AccessTokenValidationService::REVOKED raise RevokedError - - when AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) end end - def find_user_by_private_token(scopes: []) - token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + private + + def find_user_from_access_token + return unless access_token - return nil unless token_string.present? + validate_access_token! - find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes) + access_token.user || raise(UnauthorizedError) end - private + # Check the Rails session for valid authentication details + def find_user_from_warden + warden.try(:authenticate) if verified_request? + end - def find_user_by_authentication_token(token_string) - User.find_by_authentication_token(token_string) + def warden + env['warden'] end - def find_user_by_personal_access_token(token_string, scopes) - access_token = PersonalAccessToken.active.find_by_token(token_string) - return unless access_token + # Check if the request is GET/HEAD, or if CSRF token is valid. + def verified_request? + Gitlab::RequestForgeryProtection.verified?(env) + end - if AccessTokenValidationService.new(access_token, request: request).include_any_scope?(scopes) - User.find(access_token.user_id) - end + def find_oauth_access_token + token = Doorkeeper::OAuth::Token.from_request(doorkeeper_request, *Doorkeeper.configuration.access_token_methods) + return unless token + + # Expiration, revocation and scopes are verified in `find_user_by_access_token` + access_token = OauthAccessToken.by_token(token) + raise UnauthorizedError unless access_token + + access_token.revoke_previous_refresh_token! + access_token end - def find_access_token - @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) + def find_personal_access_token + token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + return unless token.present? + + # Expiration, revocation and scopes are verified in `find_user_by_access_token` + access_token = PersonalAccessToken.find_by(token: token) + raise UnauthorizedError unless access_token + + access_token end def doorkeeper_request @doorkeeper_request ||= ActionDispatch::Request.new(env) end + + # An array of scopes that were registered (using `allow_access_with_scope`) + # for the current endpoint class. It also returns scopes registered on + # `API::API`, since these are meant to apply to all API routes. + def scopes_registered_for_endpoint + @scopes_registered_for_endpoint ||= + begin + endpoint_classes = [options[:for].presence, ::API::API].compact + endpoint_classes.reduce([]) do |memo, endpoint| + if endpoint.respond_to?(:allowed_scopes) + memo.concat(endpoint.allowed_scopes) + else + memo + end + end + end + end end module ClassMethods @@ -168,11 +193,12 @@ module API TokenNotFoundError = Class.new(StandardError) ExpiredError = Class.new(StandardError) RevokedError = Class.new(StandardError) + UnauthorizedError = Class.new(StandardError) class InsufficientScopeError < StandardError attr_reader :scopes def initialize(scopes) - @scopes = scopes + @scopes = scopes.map { |s| s.try(:name) || s } end end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 643c8e6fb8e..19152c9f395 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -8,12 +8,22 @@ module API before { authorize! :download_code, user_project } + helpers do + def find_branch!(branch_name) + begin + user_project.repository.find_branch(branch_name) || not_found!('Branch') + rescue Gitlab::Git::CommandError + render_api_error!('The branch refname is invalid', 400) + end + end + end + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository branches' do - success Entities::RepoBranch + success Entities::Branch end params do use :pagination @@ -23,13 +33,13 @@ module API # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442 Gitlab::GitalyClient.allow_n_plus_1_calls do - present paginate(branches), with: Entities::RepoBranch, project: user_project + present paginate(branches), with: Entities::Branch, project: user_project end end resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do desc 'Get a single branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -38,10 +48,9 @@ module API user_project.repository.branch_exists?(params[:branch]) ? status(204) : status(404) end get do - branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless branch + branch = find_branch!(params[:branch]) - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::Branch, project: user_project end end @@ -50,7 +59,7 @@ module API # in `gitlab-org/gitlab-ce!5081`. The API interface has not been changed (to maintain compatibility), # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`. desc 'Protect a single branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -60,8 +69,7 @@ module API put ':id/repository/branches/:branch/protect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project - branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless branch + branch = find_branch!(params[:branch]) protected_branch = user_project.protected_branches.find_by(name: branch.name) @@ -80,7 +88,7 @@ module API end if protected_branch.valid? - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::Branch, project: user_project else render_api_error!(protected_branch.errors.full_messages, 422) end @@ -88,7 +96,7 @@ module API # Note: This API will be deprecated in favor of the protected branches API. desc 'Unprotect a single branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -96,16 +104,15 @@ module API put ':id/repository/branches/:branch/unprotect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project - branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch") unless branch + branch = find_branch!(params[:branch]) protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch&.destroy - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::Branch, project: user_project end desc 'Create branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -119,7 +126,7 @@ module API if result[:status] == :success present result[:branch], - with: Entities::RepoBranch, + with: Entities::Branch, project: user_project else render_api_error!(result[:message], 400) @@ -133,8 +140,7 @@ module API delete ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_push_project - branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless branch + branch = find_branch!(params[:branch]) commit = user_project.repository.commit(branch.dereferenced_target) diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 4b8d248f5f7..2685dc27252 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -4,8 +4,6 @@ module API class Commits < Grape::API include PaginationParams - COMMIT_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: API::NO_SLASH_URL_PART_REGEX) - before { authorize! :download_code, user_project } params do @@ -13,7 +11,7 @@ module API end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository commits' do - success Entities::RepoCommit + success Entities::Commit end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -46,11 +44,11 @@ module API paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count) - present paginate(paginated_commits), with: Entities::RepoCommit + present paginate(paginated_commits), with: Entities::Commit end desc 'Commit multiple file changes as one commit' do - success Entities::RepoCommitDetail + success Entities::CommitDetail detail 'This feature was introduced in GitLab 8.13' end params do @@ -72,25 +70,25 @@ module API if result[:status] == :success commit_detail = user_project.repository.commit(result[:result]) - present commit_detail, with: Entities::RepoCommitDetail + present commit_detail, with: Entities::CommitDetail else render_api_error!(result[:message], 400) end end desc 'Get a specific commit of a project' do - success Entities::RepoCommitDetail + success Entities::CommitDetail failure [[404, 'Commit Not Found']] end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ':id/repository/commits/:sha', requirements: COMMIT_ENDPOINT_REQUIREMENTS do + get ':id/repository/commits/:sha', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - present commit, with: Entities::RepoCommitDetail + present commit, with: Entities::CommitDetail end desc 'Get the diff for a specific commit of a project' do @@ -99,12 +97,12 @@ module API params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ':id/repository/commits/:sha/diff', requirements: COMMIT_ENDPOINT_REQUIREMENTS do + get ':id/repository/commits/:sha/diff', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - present commit.raw_diffs.to_a, with: Entities::RepoDiff + present commit.raw_diffs.to_a, with: Entities::Diff end desc "Get a commit's comments" do @@ -115,7 +113,7 @@ module API use :pagination requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ':id/repository/commits/:sha/comments', requirements: COMMIT_ENDPOINT_REQUIREMENTS do + get ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -126,13 +124,13 @@ module API desc 'Cherry pick commit into a branch' do detail 'This feature was introduced in GitLab 8.15' - success Entities::RepoCommit + success Entities::Commit end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked' requires :branch, type: String, desc: 'The name of the branch' end - post ':id/repository/commits/:sha/cherry_pick', requirements: COMMIT_ENDPOINT_REQUIREMENTS do + post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project commit = user_project.commit(params[:sha]) @@ -151,7 +149,7 @@ module API if result[:status] == :success branch = user_project.repository.find_branch(params[:branch]) - present user_project.repository.commit(branch.dereferenced_target), with: Entities::RepoCommit + present user_project.repository.commit(branch.dereferenced_target), with: Entities::Commit else render_api_error!(result[:message], 400) end @@ -169,7 +167,7 @@ module API requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' end end - post ':id/repository/commits/:sha/comments', requirements: COMMIT_ENDPOINT_REQUIREMENTS do + post ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -186,7 +184,7 @@ module API lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] - break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end break if opts[:line_code] diff --git a/lib/api/custom_attributes_endpoints.rb b/lib/api/custom_attributes_endpoints.rb new file mode 100644 index 00000000000..5000aa0d9ac --- /dev/null +++ b/lib/api/custom_attributes_endpoints.rb @@ -0,0 +1,77 @@ +module API + module CustomAttributesEndpoints + extend ActiveSupport::Concern + + included do + attributable_class = name.demodulize.singularize + attributable_key = attributable_class.underscore + attributable_name = attributable_class.humanize(capitalize: false) + attributable_finder = "find_#{attributable_key}" + + helpers do + params :custom_attributes_key do + requires :key, type: String, desc: 'The key of the custom attribute' + end + end + + desc "Get all custom attributes on a #{attributable_name}" do + success Entities::CustomAttribute + end + get ':id/custom_attributes' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :read_custom_attribute + + present resource.custom_attributes, with: Entities::CustomAttribute + end + + desc "Get a custom attribute on a #{attributable_name}" do + success Entities::CustomAttribute + end + params do + use :custom_attributes_key + end + get ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :read_custom_attribute + + custom_attribute = resource.custom_attributes.find_by!(key: params[:key]) + + present custom_attribute, with: Entities::CustomAttribute + end + + desc "Set a custom attribute on a #{attributable_name}" + params do + use :custom_attributes_key + requires :value, type: String, desc: 'The value of the custom attribute' + end + put ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :update_custom_attribute + + custom_attribute = resource.custom_attributes + .find_or_initialize_by(key: params[:key]) + + custom_attribute.update(value: params[:value]) + + if custom_attribute.valid? + present custom_attribute, with: Entities::CustomAttribute + else + render_validation_error!(custom_attribute) + end + end + + desc "Delete a custom attribute on a #{attributable_name}" + params do + use :custom_attributes_key + end + delete ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :update_custom_attribute + + resource.custom_attributes.find_by!(key: params[:key]).destroy + + status 204 + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 71253f72533..398a7906dcb 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -57,10 +57,6 @@ module API expose :admin?, as: :is_admin end - class UserWithPrivateDetails < UserWithAdmin - expose :private_token - end - class Email < Grape::Entity expose :id, :email end @@ -89,6 +85,9 @@ module API expose :ssh_url_to_repo, :http_url_to_repo, :web_url expose :name, :name_with_namespace expose :path, :path_with_namespace + expose :avatar_url do |project, options| + project.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :created_at, :last_activity_at end @@ -146,9 +145,7 @@ module API expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } - expose :avatar_url do |user, options| - user.avatar_url(only_path: false) - end + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds, as: :public_jobs @@ -193,8 +190,8 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url do |user, options| - user.avatar_url(only_path: false) + expose :avatar_url do |group, options| + group.avatar_url(only_path: false) end expose :web_url expose :request_access_enabled @@ -219,7 +216,7 @@ module API expose :shared_projects, using: Entities::Project end - class RepoCommit < Grape::Entity + class Commit < Grape::Entity expose :id, :short_id, :title, :created_at expose :parent_ids expose :safe_message, as: :message @@ -227,19 +224,20 @@ module API expose :committer_name, :committer_email, :committed_date end - class RepoCommitStats < Grape::Entity + class CommitStats < Grape::Entity expose :additions, :deletions, :total end - class RepoCommitDetail < RepoCommit - expose :stats, using: Entities::RepoCommitStats + class CommitDetail < Commit + expose :stats, using: Entities::CommitStats expose :status + expose :last_pipeline, using: 'API::Entities::PipelineBasic' end - class RepoBranch < Grape::Entity + class Branch < Grape::Entity expose :name - expose :commit, using: Entities::RepoCommit do |repo_branch, options| + expose :commit, using: Entities::Commit do |repo_branch, options| options[:project].repository.commit(repo_branch.dereferenced_target) end @@ -263,7 +261,7 @@ module API end end - class RepoTreeObject < Grape::Entity + class TreeObject < Grape::Entity expose :id, :name, :type, :path expose :mode do |obj, options| @@ -303,7 +301,7 @@ module API expose :state, :created_at, :updated_at end - class RepoDiff < Grape::Entity + class Diff < Grape::Entity expose :old_path, :new_path, :a_mode, :b_mode expose :new_file?, as: :new_file expose :renamed_file?, as: :renamed_file @@ -366,6 +364,7 @@ module API end expose :due_date expose :confidential + expose :discussion_locked expose :web_url do |issue, options| Gitlab::UrlBuilder.build(issue) @@ -462,6 +461,7 @@ module API expose :diff_head_sha, as: :sha expose :merge_commit_sha expose :user_notes_count + expose :discussion_locked expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch @@ -481,7 +481,7 @@ module API end class MergeRequestChanges < MergeRequest - expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| + expose :diffs, as: :changes, using: Entities::Diff do |compare, _| compare.raw_diffs(limits: false).to_a end end @@ -492,9 +492,9 @@ module API end class MergeRequestDiffFull < MergeRequestDiff - expose :commits, using: Entities::RepoCommit + expose :commits, using: Entities::Commit - expose :diffs, using: Entities::RepoDiff do |compare, _| + expose :diffs, using: Entities::Diff do |compare, _| compare.raw_diffs(limits: false).to_a end end @@ -590,8 +590,7 @@ module API expose :target_type expose :target do |todo, options| - target = todo.target_type == 'Commit' ? 'RepoCommit' : todo.target_type - Entities.const_get(target).represent(todo.target, options) + Entities.const_get(todo.target_type).represent(todo.target, options) end expose :target_url do |todo, options| @@ -727,15 +726,15 @@ module API end class Compare < Grape::Entity - expose :commit, using: Entities::RepoCommit do |compare, options| - Commit.decorate(compare.commits, nil).last + expose :commit, using: Entities::Commit do |compare, options| + ::Commit.decorate(compare.commits, nil).last end - expose :commits, using: Entities::RepoCommit do |compare, options| - Commit.decorate(compare.commits, nil) + expose :commits, using: Entities::Commit do |compare, options| + ::Commit.decorate(compare.commits, nil) end - expose :diffs, using: Entities::RepoDiff do |compare, options| + expose :diffs, using: Entities::Diff do |compare, options| compare.diffs(limits: false).to_a end @@ -771,10 +770,10 @@ module API expose :description end - class RepoTag < Grape::Entity + class Tag < Grape::Entity expose :name, :message - expose :commit, using: Entities::RepoCommit do |repo_tag, options| + expose :commit, using: Entities::Commit do |repo_tag, options| options[:project].repository.commit(repo_tag.dereferenced_target) end @@ -823,9 +822,10 @@ module API class Job < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at + expose :duration expose :user, with: User expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } - expose :commit, with: RepoCommit + expose :commit, with: Commit expose :runner, with: Runner expose :pipeline, with: PipelineBasic end @@ -878,7 +878,7 @@ module API expose :deployable, using: Entities::Job end - class RepoLicense < Grape::Entity + class License < Grape::Entity expose :key, :name, :nickname expose :featured, as: :popular expose :url, as: :html_url @@ -1020,6 +1020,7 @@ module API expose :cache, using: Cache expose :credentials, using: Credentials expose :dependencies, using: Dependency + expose :features end end @@ -1034,5 +1035,27 @@ module API expose :failing_on_hosts expose :total_failures end + + class CustomAttribute < Grape::Entity + expose :key + expose :value + end + + class PagesDomainCertificate < Grape::Entity + expose :subject + expose :expired?, as: :expired + expose :certificate + expose :certificate_text + end + + class PagesDomain < Grape::Entity + expose :domain + expose :url + expose :certificate, + if: ->(pages_domain, _) { pages_domain.certificate? }, + using: PagesDomainCertificate do |pages_domain| + pages_domain + end + end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index abbe2e9ba3e..d6df269486a 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -42,6 +42,8 @@ module API sudo! + validate_access_token!(scopes: scopes_registered_for_endpoint) unless sudo? + @current_user end @@ -140,7 +142,7 @@ module API end def authenticate! - unauthorized! unless current_user && can?(initial_current_user, :access_api) + unauthorized! unless current_user end def authenticate_non_get! @@ -185,6 +187,10 @@ module API end end + def require_pages_enabled! + not_found! unless user_project.pages_available? + end + def can?(object, action, subject = :global) Ability.allowed?(object, action, subject) end @@ -286,7 +292,7 @@ module API if sentry_enabled? && report_exception?(exception) define_params_for_grape_middleware sentry_context - Raven.capture_exception(exception) + Raven.capture_exception(exception, extra: params) end # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 @@ -378,61 +384,36 @@ module API private - def private_token - params[APIGuard::PRIVATE_TOKEN_PARAM] || env[APIGuard::PRIVATE_TOKEN_HEADER] - end - - def warden - env['warden'] - end - - # Check if the request is GET/HEAD, or if CSRF token is valid. - def verified_request? - Gitlab::RequestForgeryProtection.verified?(env) - end - - # Check the Rails session for valid authentication details - def find_user_from_warden - warden.try(:authenticate) if verified_request? - end - - # rubocop:disable Cop/ModuleWithInstanceVariables def initial_current_user return @initial_current_user if defined?(@initial_current_user) - Gitlab::Auth::UniqueIpsLimiter.limit_user! do - @initial_current_user ||= find_user_by_private_token(scopes: scopes_registered_for_endpoint) - @initial_current_user ||= doorkeeper_guard(scopes: scopes_registered_for_endpoint) - @initial_current_user ||= find_user_from_warden - - unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? - @initial_current_user = nil - end - @initial_current_user + begin + @initial_current_user = Gitlab::Auth::UniqueIpsLimiter.limit_user! { find_current_user! } + rescue APIGuard::UnauthorizedError + unauthorized! end end # rubocop:disable Cop/ModuleWithInstanceVariables def sudo! return unless sudo_identifier - return unless initial_current_user + + unauthorized! unless initial_current_user unless initial_current_user.admin? forbidden!('Must be admin to use sudo') end - # Only private tokens should be used for the SUDO feature - unless private_token == initial_current_user.private_token - forbidden!('Private token must be specified in order to use sudo') + unless access_token + forbidden!('Must be authenticated using an OAuth or Personal Access Token to use sudo') end + validate_access_token!(scopes: [:sudo]) + sudoed_user = find_user(sudo_identifier) + not_found!("User with ID or username '#{sudo_identifier}'") unless sudoed_user - if sudoed_user - @current_user = sudoed_user - else - not_found!("No user id or username for: #{sudo_identifier}") - end + @current_user = sudoed_user end def sudo_identifier @@ -457,10 +438,12 @@ module API header(*Gitlab::Workhorse.send_artifacts_entry(build, entry)) end - # The Grape Error Middleware only has access to env but no params. We workaround this by - # defining a method that returns the right value. + # The Grape Error Middleware only has access to `env` but not `params` nor + # `request`. We workaround this by defining methods that returns the right + # values. def define_params_for_grape_middleware - self.define_singleton_method(:params) { Rack::Request.new(env).params.symbolize_keys } + self.define_singleton_method(:request) { Rack::Request.new(env) } + self.define_singleton_method(:params) { request.params.symbolize_keys } end # We could get a Grape or a standard Ruby exception. We should only report anything that @@ -470,22 +453,5 @@ module API exception.status == 500 end - - # An array of scopes that were registered (using `allow_access_with_scope`) - # for the current endpoint class. It also returns scopes registered on - # `API::API`, since these are meant to apply to all API routes. - def scopes_registered_for_endpoint - @scopes_registered_for_endpoint ||= - begin - endpoint_classes = [options[:for].presence, ::API::API].compact - endpoint_classes.reduce([]) do |memo, endpoint| - if endpoint.respond_to?(:allowed_scopes) - memo.concat(endpoint.allowed_scopes) - else - memo - end - end - end - end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index c0fef56378f..6e78ac2c903 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -31,6 +31,12 @@ module API protocol = params[:protocol] actor.update_last_used_at if actor.is_a?(Key) + user = + if actor.is_a?(Key) + actor.user + else + actor + end access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess access_checker = access_checker_klass @@ -47,6 +53,7 @@ module API { status: true, gl_repository: gl_repository, + gl_username: user&.username, repository_path: repository_path, gitaly: gitaly_payload(params[:action]) } @@ -136,7 +143,7 @@ module API codes = nil - ::Users::UpdateService.new(user).execute! do |user| + ::Users::UpdateService.new(current_user, user: user).execute! do |user| codes = user.generate_otp_backup_codes! end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 1729df2aad0..0df41dcc903 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -48,6 +48,7 @@ module API optional :labels, type: String, desc: 'Comma-separated list of label names' optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' + optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" end params :issue_params do @@ -193,7 +194,7 @@ module API desc: 'Date time when the issue was updated. Available only for admins and project owners.' optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' use :issue_params - at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, + at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, :discussion_locked, :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_iid' do diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 8aa1e0216ee..726f09e3669 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -183,13 +183,13 @@ module API end desc 'Get the commits of a merge request' do - success Entities::RepoCommit + success Entities::Commit end get ':id/merge_requests/:merge_request_iid/commits' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) commits = ::Kaminari.paginate_array(merge_request.commits) - present paginate(commits), with: Entities::RepoCommit + present paginate(commits), with: Entities::Commit end desc 'Show the merge request changes' do @@ -214,12 +214,14 @@ module API :remove_source_branch, :state_event, :target_branch, - :title + :title, + :discussion_locked ] optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' optional :state_event, type: String, values: %w[close reopen], desc: 'Status of the merge request' + optional :discussion_locked, type: Boolean, desc: 'Whether the MR discussion is locked' use :optional_params at_least_one_of(*at_least_one_of_ce) @@ -293,7 +295,7 @@ module API unauthorized! unless merge_request.can_cancel_merge_when_pipeline_succeeds?(current_user) - ::MergeRequest::MergeWhenPipelineSucceedsService + ::MergeRequests::MergeWhenPipelineSucceedsService .new(merge_request.target_project, current_user) .cancel(merge_request) end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index d6e7203adaf..0b9ab4eeb05 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -78,6 +78,8 @@ module API } if can?(current_user, noteable_read_ability_name(noteable), noteable) + authorize! :create_note, noteable + if params[:created_at] && (current_user.admin? || user_project.owner == current_user) opts[:created_at] = params[:created_at] end diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index bcc0833aa5c..0266bf2f717 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -35,7 +35,7 @@ module API new_notification_email = params.delete(:notification_email) if new_notification_email - ::Users::UpdateService.new(current_user, notification_email: new_notification_email).execute + ::Users::UpdateService.new(current_user, user: current_user, notification_email: new_notification_email).execute end notification_setting.update(declared_params(include_missing: false)) diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb new file mode 100644 index 00000000000..259f3f34068 --- /dev/null +++ b/lib/api/pages_domains.rb @@ -0,0 +1,117 @@ +module API + class PagesDomains < Grape::API + include PaginationParams + + before do + authenticate! + require_pages_enabled! + end + + after_validation do + normalize_params_file_to_string + end + + helpers do + def find_pages_domain! + user_project.pages_domains.find_by(domain: params[:domain]) || not_found!('PagesDomain') + end + + def pages_domain + @pages_domain ||= find_pages_domain! + end + + def normalize_params_file_to_string + params.each do |k, v| + if v.is_a?(Hash) && v.key?(:tempfile) + params[k] = v[:tempfile].to_a.join('') + end + end + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get all pages domains' do + success Entities::PagesDomain + end + params do + use :pagination + end + get ":id/pages/domains" do + authorize! :read_pages, user_project + + present paginate(user_project.pages_domains.order(:domain)), with: Entities::PagesDomain + end + + desc 'Get a single pages domain' do + success Entities::PagesDomain + end + params do + requires :domain, type: String, desc: 'The domain' + end + get ":id/pages/domains/:domain", requirements: { domain: %r{[^/]+} } do + authorize! :read_pages, user_project + + present pages_domain, with: Entities::PagesDomain + end + + desc 'Create a new pages domain' do + success Entities::PagesDomain + end + params do + requires :domain, type: String, desc: 'The domain' + optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate' + optional :key, allow_blank: false, types: [File, String], desc: 'The key' + all_or_none_of :certificate, :key + end + post ":id/pages/domains" do + authorize! :update_pages, user_project + + pages_domain_params = declared(params, include_parent_namespaces: false) + pages_domain = user_project.pages_domains.create(pages_domain_params) + + if pages_domain.persisted? + present pages_domain, with: Entities::PagesDomain + else + render_validation_error!(pages_domain) + end + end + + desc 'Updates a pages domain' + params do + requires :domain, type: String, desc: 'The domain' + optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate' + optional :key, allow_blank: false, types: [File, String], desc: 'The key' + end + put ":id/pages/domains/:domain", requirements: { domain: %r{[^/]+} } do + authorize! :update_pages, user_project + + pages_domain_params = declared(params, include_parent_namespaces: false) + + # Remove empty private key if certificate is not empty. + if pages_domain_params[:certificate] && !pages_domain_params[:key] + pages_domain_params.delete(:key) + end + + if pages_domain.update(pages_domain_params) + present pages_domain, with: Entities::PagesDomain + else + render_validation_error!(pages_domain) + end + end + + desc 'Delete a pages domain' + params do + requires :domain, type: String, desc: 'The domain' + end + delete ":id/pages/domains/:domain", requirements: { domain: %r{[^/]+} } do + authorize! :update_pages, user_project + + status 204 + pages_domain.destroy + end + end + end +end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 2255fb1b70d..7887b886c03 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -35,7 +35,7 @@ module API end desc 'Get a project repository tree' do - success Entities::RepoTreeObject + success Entities::TreeObject end params do optional :ref, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -52,12 +52,12 @@ module API tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) entries = ::Kaminari.paginate_array(tree.sorted_entries) - present paginate(entries), with: Entities::RepoTreeObject + present paginate(entries), with: Entities::TreeObject end desc 'Get raw blob contents from the repository' params do - requires :sha, type: String, desc: 'The commit, branch name, or tag name' + requires :sha, type: String, desc: 'The commit hash' end get ':id/repository/blobs/:sha/raw' do assign_blob_vars! @@ -67,7 +67,7 @@ module API desc 'Get a blob from the repository' params do - requires :sha, type: String, desc: 'The commit, branch name, or tag name' + requires :sha, type: String, desc: 'The commit hash' end get ':id/repository/blobs/:sha' do assign_blob_vars! diff --git a/lib/api/services.rb b/lib/api/services.rb index 2cbd0517dc3..6454e475036 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -313,13 +313,13 @@ module API desc: 'The base URL to the JIRA instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com' }, { - required: false, + required: true, name: :username, type: String, desc: 'The username of the user created to be used with GitLab/JIRA' }, { - required: false, + required: true, name: :password, type: String, desc: 'The password of the user created to be used with GitLab/JIRA' @@ -374,6 +374,26 @@ module API desc: 'The Slack token' } ], + 'packagist' => [ + { + required: true, + name: :username, + type: String, + desc: 'The username' + }, + { + required: true, + name: :token, + type: String, + desc: 'The Packagist API token' + }, + { + required: false, + name: :server, + type: String, + desc: 'The server' + } + ], 'pipelines-email' => [ { required: true, @@ -551,6 +571,7 @@ module API KubernetesService, MattermostSlashCommandsService, SlackSlashCommandsService, + PackagistService, PipelinesEmailService, PivotaltrackerService, PrometheusService, diff --git a/lib/api/session.rb b/lib/api/session.rb deleted file mode 100644 index 016415c3023..00000000000 --- a/lib/api/session.rb +++ /dev/null @@ -1,20 +0,0 @@ -module API - class Session < Grape::API - desc 'Login to get token' do - success Entities::UserWithPrivateDetails - end - params do - optional :login, type: String, desc: 'The username' - optional :email, type: String, desc: 'The email of the user' - requires :password, type: String, desc: 'The password of the user' - at_least_one_of :login, :email - end - post "/session" do - user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password]) - - return unauthorized! unless user - return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? - present user, with: Entities::UserWithPrivateDetails - end - end -end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 912415e3a7f..0d394a7b441 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -11,18 +11,18 @@ module API end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository tags' do - success Entities::RepoTag + success Entities::Tag end params do use :pagination end get ':id/repository/tags' do tags = ::Kaminari.paginate_array(user_project.repository.tags.sort_by(&:name).reverse) - present paginate(tags), with: Entities::RepoTag, project: user_project + present paginate(tags), with: Entities::Tag, project: user_project end desc 'Get a single repository tag' do - success Entities::RepoTag + success Entities::Tag end params do requires :tag_name, type: String, desc: 'The name of the tag' @@ -31,11 +31,11 @@ module API tag = user_project.repository.find_tag(params[:tag_name]) not_found!('Tag') unless tag - present tag, with: Entities::RepoTag, project: user_project + present tag, with: Entities::Tag, project: user_project end desc 'Create a new repository tag' do - success Entities::RepoTag + success Entities::Tag end params do requires :tag_name, type: String, desc: 'The name of the tag' @@ -51,7 +51,7 @@ module API if result[:status] == :success present result[:tag], - with: Entities::RepoTag, + with: Entities::Tag, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/templates.rb b/lib/api/templates.rb index f70bc0622b7..6550b331fb8 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -49,7 +49,7 @@ module API desc 'Get the list of the available license template' do detail 'This feature was introduced in GitLab 8.7.' - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' @@ -60,12 +60,12 @@ module API featured: declared(params)[:popular].present? ? true : nil } licences = ::Kaminari.paginate_array(Licensee::License.all(options)) - present paginate(licences), with: Entities::RepoLicense + present paginate(licences), with: Entities::License end desc 'Get the text for a specific license' do detail 'This feature was introduced in GitLab 8.7.' - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do requires :name, type: String, desc: 'The name of the template' @@ -75,7 +75,7 @@ module API template = parsed_license_template - present template, with: ::API::Entities::RepoLicense + present template, with: ::API::Entities::License end GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| diff --git a/lib/api/users.rb b/lib/api/users.rb index bdebda58d3f..d80b364bd09 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -6,12 +6,14 @@ module API allow_access_with_scope :read_user, if: -> (request) { request.get? } resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do + include CustomAttributesEndpoints + before do authenticate_non_get! end helpers do - def find_user(params) + def find_user_by_id(params) id = params[:user_id] || params[:id] User.find_by(id: id) || not_found!('User') end @@ -166,7 +168,7 @@ module API user_params[:password_expires_at] = Time.now if user_params[:password].present? - result = ::Users::UpdateService.new(user, user_params.except(:extern_uid, :provider)).execute + result = ::Users::UpdateService.new(current_user, user_params.except(:extern_uid, :provider).merge(user: user)).execute if result[:status] == :success present user, with: Entities::UserPublic @@ -326,10 +328,9 @@ module API user = User.find_by(id: params.delete(:id)) not_found!('User') unless user - email = Emails::CreateService.new(user, declared_params(include_missing: false)).execute + email = Emails::CreateService.new(current_user, declared_params(include_missing: false).merge(user: user)).execute if email.errors.blank? - NotificationService.new.new_email(email) present email, with: Entities::Email else render_validation_error!(email) @@ -367,10 +368,8 @@ module API not_found!('Email') unless email destroy_conditionally!(email) do |email| - Emails::DestroyService.new(current_user, email: email.email).execute + Emails::DestroyService.new(current_user, user: user).execute(email) end - - user.update_secondary_emails! end desc 'Delete a user. Available only for admins.' do @@ -430,7 +429,7 @@ module API resource :impersonation_tokens do helpers do def finder(options = {}) - user = find_user(params) + user = find_user_by_id(params) PersonalAccessTokensFinder.new({ user: user, impersonation: true }.merge(options)) end @@ -508,9 +507,7 @@ module API end get do entity = - if sudo? - Entities::UserWithPrivateDetails - elsif current_user.admin? + if current_user.admin? Entities::UserWithAdmin else Entities::UserPublic @@ -672,10 +669,9 @@ module API requires :email, type: String, desc: 'The new email' end post "emails" do - email = Emails::CreateService.new(current_user, declared_params).execute + email = Emails::CreateService.new(current_user, declared_params.merge(user: current_user)).execute if email.errors.blank? - NotificationService.new.new_email(email) present email, with: Entities::Email else render_validation_error!(email) @@ -691,10 +687,8 @@ module API not_found!('Email') unless email destroy_conditionally!(email) do |email| - Emails::DestroyService.new(current_user, email: email.email).execute + Emails::DestroyService.new(current_user, user: current_user).execute(email) end - - current_user.update_secondary_emails! end desc 'Get a list of user activities' diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb index 81b13249892..69cd12de72c 100644 --- a/lib/api/v3/branches.rb +++ b/lib/api/v3/branches.rb @@ -11,12 +11,12 @@ module API end resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a project repository branches' do - success ::API::Entities::RepoBranch + success ::API::Entities::Branch end get ":id/repository/branches" do branches = user_project.repository.branches.sort_by(&:name) - present branches, with: ::API::Entities::RepoBranch, project: user_project + present branches, with: ::API::Entities::Branch, project: user_project end desc 'Delete a branch' @@ -47,7 +47,7 @@ module API end desc 'Create branch' do - success ::API::Entities::RepoBranch + success ::API::Entities::Branch end params do requires :branch_name, type: String, desc: 'The name of the branch' @@ -60,7 +60,7 @@ module API if result[:status] == :success present result[:branch], - with: ::API::Entities::RepoBranch, + with: ::API::Entities::Branch, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index c189d486f50..f493fd7c7ec 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -8,7 +8,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do params :optional_scope do optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 5936f4700aa..ed206a6def0 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -11,9 +11,9 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository commits' do - success ::API::Entities::RepoCommit + success ::API::Entities::Commit end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -34,11 +34,11 @@ module API after: params[:since], before: params[:until]) - present commits, with: ::API::Entities::RepoCommit + present commits, with: ::API::Entities::Commit end desc 'Commit multiple file changes as one commit' do - success ::API::Entities::RepoCommitDetail + success ::API::Entities::CommitDetail detail 'This feature was introduced in GitLab 8.13' end params do @@ -59,25 +59,25 @@ module API if result[:status] == :success commit_detail = user_project.repository.commits(result[:result], limit: 1).first - present commit_detail, with: ::API::Entities::RepoCommitDetail + present commit_detail, with: ::API::Entities::CommitDetail else render_api_error!(result[:message], 400) end end desc 'Get a specific commit of a project' do - success ::API::Entities::RepoCommitDetail + success ::API::Entities::CommitDetail failure [[404, 'Not Found']] end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ":id/repository/commits/:sha" do + get ":id/repository/commits/:sha", requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! "Commit" unless commit - present commit, with: ::API::Entities::RepoCommitDetail + present commit, with: ::API::Entities::CommitDetail end desc 'Get the diff for a specific commit of a project' do @@ -86,7 +86,7 @@ module API params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ":id/repository/commits/:sha/diff" do + get ":id/repository/commits/:sha/diff", requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! "Commit" unless commit @@ -102,7 +102,7 @@ module API use :pagination requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ':id/repository/commits/:sha/comments' do + get ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -113,13 +113,13 @@ module API desc 'Cherry pick commit into a branch' do detail 'This feature was introduced in GitLab 8.15' - success ::API::Entities::RepoCommit + success ::API::Entities::Commit end params do requires :sha, type: String, desc: 'A commit sha to be cherry picked' requires :branch, type: String, desc: 'The name of the branch' end - post ':id/repository/commits/:sha/cherry_pick' do + post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project commit = user_project.commit(params[:sha]) @@ -138,7 +138,7 @@ module API if result[:status] == :success branch = user_project.repository.find_branch(params[:branch]) - present user_project.repository.commit(branch.dereferenced_target), with: ::API::Entities::RepoCommit + present user_project.repository.commit(branch.dereferenced_target), with: ::API::Entities::Commit else render_api_error!(result[:message], 400) end @@ -156,7 +156,7 @@ module API requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' end end - post ':id/repository/commits/:sha/comments' do + post ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -173,7 +173,7 @@ module API lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] - break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end break if opts[:line_code] diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index c928ce5265b..afdd7b83998 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -220,7 +220,7 @@ module API expose :created_at, :started_at, :finished_at expose :user, with: ::API::Entities::User expose :artifacts_file, using: ::API::Entities::JobArtifactFile, if: -> (build, opts) { build.artifacts? } - expose :commit, with: ::API::Entities::RepoCommit + expose :commit, with: ::API::Entities::Commit expose :runner, with: ::API::Entities::Runner expose :pipeline, with: ::API::Entities::PipelineBasic end @@ -237,7 +237,7 @@ module API end class MergeRequestChanges < MergeRequest - expose :diffs, as: :changes, using: ::API::Entities::RepoDiff do |compare, _| + expose :diffs, as: :changes, using: ::API::Entities::Diff do |compare, _| compare.raw_diffs(limits: false).to_a end end diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index b6b7254ae29..1d6d823f32b 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -135,12 +135,12 @@ module API end desc 'Get the commits of a merge request' do - success ::API::Entities::RepoCommit + success ::API::Entities::Commit end get "#{path}/commits" do merge_request = find_merge_request_with_access(params[:merge_request_id]) - present merge_request.commits, with: ::API::Entities::RepoCommit + present merge_request.commits, with: ::API::Entities::Commit end desc 'Show the merge request changes' do diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb index 0eaa0de2eef..f9a47101e27 100644 --- a/lib/api/v3/repositories.rb +++ b/lib/api/v3/repositories.rb @@ -8,7 +8,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? @@ -19,7 +19,7 @@ module API end desc 'Get a project repository tree' do - success ::API::Entities::RepoTreeObject + success ::API::Entities::TreeObject end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -35,7 +35,7 @@ module API tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) - present tree.sorted_entries, with: ::API::Entities::RepoTreeObject + present tree.sorted_entries, with: ::API::Entities::TreeObject end desc 'Get a raw file contents' @@ -43,7 +43,7 @@ module API requires :sha, type: String, desc: 'The commit, branch name, or tag name' requires :filepath, type: String, desc: 'The path to the file to display' end - get [":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob"] do + get [":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob"], requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do repo = user_project.repository commit = repo.commit(params[:sha]) not_found! "Commit" unless commit @@ -56,7 +56,7 @@ module API params do requires :sha, type: String, desc: 'The commit, branch name, or tag name' end - get ':id/repository/raw_blobs/:sha' do + get ':id/repository/raw_blobs/:sha', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do repo = user_project.repository begin blob = Gitlab::Git::Blob.raw(repo, params[:sha]) diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb index 2d13d6fabfd..44ed94d2869 100644 --- a/lib/api/v3/services.rb +++ b/lib/api/v3/services.rb @@ -395,6 +395,26 @@ module API desc: 'The Slack token' } ], + 'packagist' => [ + { + required: true, + name: :username, + type: String, + desc: 'The username' + }, + { + required: true, + name: :token, + type: String, + desc: 'The Packagist API token' + }, + { + required: false, + name: :server, + type: String, + desc: 'The server' + } + ], 'pipelines-email' => [ { required: true, diff --git a/lib/api/v3/tags.rb b/lib/api/v3/tags.rb index 7e5875cd030..6e37d31d153 100644 --- a/lib/api/v3/tags.rb +++ b/lib/api/v3/tags.rb @@ -8,11 +8,11 @@ module API end resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a project repository tags' do - success ::API::Entities::RepoTag + success ::API::Entities::Tag end get ":id/repository/tags" do tags = user_project.repository.tags.sort_by(&:name).reverse - present tags, with: ::API::Entities::RepoTag, project: user_project + present tags, with: ::API::Entities::Tag, project: user_project end desc 'Delete a repository tag' diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb index 2a2fb59045c..7298203df10 100644 --- a/lib/api/v3/templates.rb +++ b/lib/api/v3/templates.rb @@ -52,7 +52,7 @@ module API detailed_desc = 'This feature was introduced in GitLab 8.7.' detailed_desc << DEPRECATION_MESSAGE unless status == :ok detail detailed_desc - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' @@ -61,7 +61,7 @@ module API options = { featured: declared(params)[:popular].present? ? true : nil } - present Licensee::License.all(options), with: ::API::Entities::RepoLicense + present Licensee::License.all(options), with: ::API::Entities::License end end @@ -70,7 +70,7 @@ module API detailed_desc = 'This feature was introduced in GitLab 8.7.' detailed_desc << DEPRECATION_MESSAGE unless status == :ok detail detailed_desc - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do requires :name, type: String, desc: 'The name of the template' @@ -80,7 +80,7 @@ module API template = parsed_license_template - present template, with: ::API::Entities::RepoLicense + present template, with: ::API::Entities::License end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 3cf3939994a..05aa79dc160 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -101,50 +101,52 @@ module Backup end def unpack - Dir.chdir(backup_path) - - # check for existing backups in the backup dir - if backup_file_list.empty? - $progress.puts "No backups found in #{backup_path}" - $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" - exit 1 - elsif backup_file_list.many? && ENV["BACKUP"].nil? - $progress.puts 'Found more than one backup, please specify which one you want to restore:' - $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' - exit 1 - end + Dir.chdir(backup_path) do + # check for existing backups in the backup dir + if backup_file_list.empty? + $progress.puts "No backups found in #{backup_path}" + $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" + exit 1 + elsif backup_file_list.many? && ENV["BACKUP"].nil? + $progress.puts 'Found more than one backup, please specify which one you want to restore:' + $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' + exit 1 + end - tar_file = if ENV['BACKUP'].present? - "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" - else - backup_file_list.first - end + tar_file = if ENV['BACKUP'].present? + "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" + else + backup_file_list.first + end - unless File.exist?(tar_file) - $progress.puts "The backup file #{tar_file} does not exist!" - exit 1 - end + unless File.exist?(tar_file) + $progress.puts "The backup file #{tar_file} does not exist!" + exit 1 + end - $progress.print 'Unpacking backup ... ' + $progress.print 'Unpacking backup ... ' - unless Kernel.system(*%W(tar -xf #{tar_file})) - $progress.puts 'unpacking backup failed'.color(:red) - exit 1 - else - $progress.puts 'done'.color(:green) - end + unless Kernel.system(*%W(tar -xf #{tar_file})) + $progress.puts 'unpacking backup failed'.color(:red) + exit 1 + else + $progress.puts 'done'.color(:green) + end - ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 - - # restoring mismatching backups can lead to unexpected problems - if settings[:gitlab_version] != Gitlab::VERSION - $progress.puts 'GitLab version mismatch:'.color(:red) - $progress.puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red) - $progress.puts ' Please switch to the following version and try again:'.color(:red) - $progress.puts " version: #{settings[:gitlab_version]}".color(:red) - $progress.puts - $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" - exit 1 + ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 + + # restoring mismatching backups can lead to unexpected problems + if settings[:gitlab_version] != Gitlab::VERSION + $progress.puts(<<~HEREDOC.color(:red)) + GitLab version mismatch: + Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup! + Please switch to the following version and try again: + version: #{settings[:gitlab_version]} + HEREDOC + $progress.puts + $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" + exit 1 + end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 4e92be85110..3ad09a1b421 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -78,7 +78,7 @@ module Backup project.ensure_storage_path_exists cmd = if File.exist?(path_to_project_bundle) - %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) + %W(#{Gitlab.config.git.bin_path} clone --bare --mirror #{path_to_project_bundle} #{path_to_project_repo}) else %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index ef4578aabd6..a0f7e4e5ad5 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -95,7 +95,7 @@ module Banzai end def call - return doc if project.nil? + return doc unless project || group ref_pattern = object_class.reference_pattern link_pattern = object_class.link_reference_pattern @@ -288,10 +288,14 @@ module Banzai end def current_project_path + return unless project + @current_project_path ||= project.full_path end def current_project_namespace_path + return unless project + @current_project_namespace_path ||= project.namespace.full_path end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index ee73fa91589..9cac303e645 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,6 +1,18 @@ module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter + # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use + REDCARPET_OPTIONS = { + fenced_code_blocks: true, + footnotes: true, + lax_spacing: true, + no_intra_emphasis: true, + space_after_headers: true, + strikethrough: true, + superscript: true, + tables: true + }.freeze + def initialize(text, context = nil, result = nil) super text, context, result @text = @text.delete "\r" @@ -13,27 +25,11 @@ module Banzai end def self.renderer - @renderer ||= begin + Thread.current[:banzai_markdown_renderer] ||= begin renderer = Banzai::Renderer::HTML.new - Redcarpet::Markdown.new(renderer, redcarpet_options) + Redcarpet::Markdown.new(renderer, REDCARPET_OPTIONS) end end - - def self.redcarpet_options - # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use - @redcarpet_options ||= { - fenced_code_blocks: true, - footnotes: true, - lax_spacing: true, - no_intra_emphasis: true, - space_after_headers: true, - strikethrough: true, - superscript: true, - tables: true - }.freeze - end - - private_class_method :redcarpet_options end end end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index a6f8650ed3d..c6ae28adf87 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -55,6 +55,10 @@ module Banzai context[:project] end + def group + context[:group] + end + def skip_project_check? context[:skip_project_check] end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 9923ec4e870..6786b9d07b6 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -45,8 +45,9 @@ module Banzai whitelist[:elements].push('abbr') whitelist[:attributes]['abbr'] = %w(title) - # Disallow `name` attribute globally + # Disallow `name` attribute globally, allow on `a` whitelist[:attributes][:all].delete('name') + whitelist[:attributes]['a'].push('name') # Allow any protocol in `a` elements... whitelist[:protocols].delete('a') @@ -72,10 +73,21 @@ module Banzai return unless node.has_attribute?('href') begin + node['href'] = node['href'].strip uri = Addressable::URI.parse(node['href']) - uri.scheme = uri.scheme.strip.downcase if uri.scheme - node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) + return unless uri.scheme + + # Remove all invalid scheme characters before checking against the + # list of unsafe protocols. + # + # See https://tools.ietf.org/html/rfc3986#section-3.1 + scheme = uri.scheme + .strip + .downcase + .gsub(/[^A-Za-z0-9\+\.\-]+/, '') + + node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(scheme) rescue Addressable::URI::InvalidURIError node.remove_attribute('href') end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index f3356d6c51e..afb6e25963c 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -24,7 +24,7 @@ module Banzai end def call - return doc if project.nil? && !skip_project_check? + return doc if project.nil? && group.nil? && !skip_project_check? ref_pattern = User.reference_pattern ref_pattern_start = /\A#{ref_pattern}\z/ @@ -101,19 +101,12 @@ module Banzai end def link_to_all(link_content: nil) - project = context[:project] author = context[:author] - if author && !project.team.member?(author) + if author && !team_member?(author) link_content else - url = urls.project_url(project, - only_path: context[:only_path]) - - data = data_attribute(project: project.id, author: author.try(:id)) - content = link_content || User.reference_prefix + 'all' - - link_tag(url, data, content, 'All Project and Group Members') + parent_url(link_content, author) end end @@ -144,6 +137,35 @@ module Banzai def link_tag(url, data, link_content, title) %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>) end + + def parent + context[:project] || context[:group] + end + + def parent_group? + parent.is_a?(Group) + end + + def team_member?(user) + if parent_group? + parent.member?(user) + else + parent.team.member?(user) + end + end + + def parent_url(link_content, author) + if parent_group? + url = urls.group_url(parent, only_path: context[:only_path]) + data = data_attribute(group: group.id, author: author.try(:id)) + else + url = urls.project_url(parent, only_path: context[:only_path]) + data = data_attribute(project: project.id, author: author.try(:id)) + end + + content = link_content || User.reference_prefix + 'all' + link_tag(url, data, content, 'All Project and Group Members') + end end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index ceca9296851..5f91884a878 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -40,7 +40,7 @@ module Banzai return cacheless_render_field(object, field) end - object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) + object.refresh_markdown_cache! unless object.cached_html_up_to_date?(field) object.cached_html_for(field) end @@ -162,10 +162,5 @@ module Banzai return unless cache_key Rails.cache.__send__(:expanded_key, full_cache_key(cache_key, pipeline_name)) # rubocop:disable GitlabSecurity/PublicSend end - - # GitLab EE needs to disable updates on GET requests in Geo - def self.update_object?(object) - true - end end end diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb index bfcec241489..7cfa82a9a9f 100644 --- a/lib/declarative_policy/rule.rb +++ b/lib/declarative_policy/rule.rb @@ -206,11 +206,13 @@ module DeclarativePolicy end def cached_pass?(context) - passes = @rules.map { |r| r.cached_pass?(context) } - return false if passes.any? { |p| p == false } - return true if passes.all? { |p| p == true } + @rules.each do |rule| + pass = rule.cached_pass?(context) - nil + return pass if pass.nil? || pass == false + end + + true end def repr @@ -245,11 +247,13 @@ module DeclarativePolicy end def cached_pass?(context) - passes = @rules.map { |r| r.cached_pass?(context) } - return true if passes.any? { |p| p == true } - return false if passes.all? { |p| p == false } + @rules.each do |rule| + pass = rule.cached_pass?(context) - nil + return pass if pass.nil? || pass == true + end + + false end def score(context) diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index 56afd1f1392..45ff2ef9ced 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -107,7 +107,7 @@ module DeclarativePolicy end # This is the core spot where all those `#score` methods matter. - # It is critcal for performance to run steps in the correct order, + # It is critical for performance to run steps in the correct order, # so that we don't compute expensive conditions (potentially n times # if we're called on, say, a large list of users). # @@ -139,30 +139,39 @@ module DeclarativePolicy return end - steps = Set.new(@steps) - remaining_enablers = steps.count { |s| s.enable? } + remaining_steps = Set.new(@steps) + remaining_enablers, remaining_preventers = remaining_steps.partition(&:enable?).map { |s| Set.new(s) } loop do - return if steps.empty? + if @state.enabled? + # Once we set this, we never need to unset it, because a single + # prevent will stop this from being enabled + remaining_steps = remaining_preventers + else + # if the permission hasn't yet been enabled and we only have + # prevent steps left, we short-circuit the state here + @state.prevent! if remaining_enablers.empty? + end - # if the permission hasn't yet been enabled and we only have - # prevent steps left, we short-circuit the state here - @state.prevent! if !@state.enabled? && remaining_enablers == 0 + return if remaining_steps.empty? lowest_score = Float::INFINITY next_step = nil - steps.each do |step| + remaining_steps.each do |step| score = step.score + if score < lowest_score next_step = step lowest_score = score end - end - steps.delete(next_step) + break if lowest_score.zero? + end - remaining_enablers -= 1 if next_step.enable? + [remaining_steps, remaining_enablers, remaining_preventers].each do |set| + set.delete(next_step) + end yield next_step, lowest_score end diff --git a/lib/github/client.rb b/lib/github/client.rb index 9c476df7d46..29bd9c1f39e 100644 --- a/lib/github/client.rb +++ b/lib/github/client.rb @@ -1,6 +1,7 @@ module Github class Client TIMEOUT = 60 + DEFAULT_PER_PAGE = 100 attr_reader :connection, :rate_limit @@ -20,7 +21,7 @@ module Github exceed, reset_in = rate_limit.get sleep reset_in if exceed - Github::Response.new(connection.get(url, query)) + Github::Response.new(connection.get(url, { per_page: DEFAULT_PER_PAGE }.merge(query))) end private diff --git a/lib/github/import.rb b/lib/github/import.rb index 9354e142d3d..8cabbdec940 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -1,48 +1,15 @@ require_relative 'error' +require_relative 'import/issue' +require_relative 'import/legacy_diff_note' +require_relative 'import/merge_request' +require_relative 'import/note' module Github class Import include Gitlab::ShellAdapter - class MergeRequest < ::MergeRequest - self.table_name = 'merge_requests' - - self.reset_callbacks :create - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class Issue < ::Issue - self.table_name = 'issues' - - self.reset_callbacks :save - self.reset_callbacks :create - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class Note < ::Note - self.table_name = 'notes' - - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class LegacyDiffNote < ::LegacyDiffNote - self.table_name = 'notes' - - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - attr_reader :project, :repository, :repo, :repo_url, :wiki_url, - :options, :errors, :cached, :verbose + :options, :errors, :cached, :verbose, :last_fetched_at def initialize(project, options = {}) @project = project @@ -54,12 +21,13 @@ module Github @verbose = options.fetch(:verbose, false) @cached = Hash.new { |hash, key| hash[key] = Hash.new } @errors = [] + @last_fetched_at = nil end # rubocop: disable Rails/Output def execute puts 'Fetching repository...'.color(:aqua) if verbose - fetch_repository + setup_and_fetch_repository puts 'Fetching labels...'.color(:aqua) if verbose fetch_labels puts 'Fetching milestones...'.color(:aqua) if verbose @@ -75,7 +43,7 @@ module Github puts 'Expiring repository cache...'.color(:aqua) if verbose expire_repository_cache - true + errors.empty? rescue Github::RepositoryFetchError expire_repository_cache false @@ -85,22 +53,30 @@ module Github private - def fetch_repository + def setup_and_fetch_repository begin project.ensure_repository project.repository.add_remote('github', repo_url) - project.repository.set_remote_as_mirror('github') - project.repository.fetch_remote('github', forced: true) - rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e + project.repository.set_import_remote_as_mirror('github') + project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head') + fetch_remote(forced: true) + rescue Gitlab::Git::Repository::NoRepository, + Gitlab::Git::RepositoryMirroring::RemoteError, + Gitlab::Shell::Error => e error(:project, repo_url, e.message) raise Github::RepositoryFetchError end end + def fetch_remote(forced: false) + @last_fetched_at = Time.now + project.repository.fetch_remote('github', forced: forced) + end + def fetch_wiki_repository return if project.wiki.repository_exists? - wiki_path = "#{project.disk_path}.wiki" + wiki_path = project.wiki.disk_path gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, @@ -125,7 +101,7 @@ module Github label.color = representation.color end - cached[:label_ids][label.title] = label.id + cached[:label_ids][representation.title] = label.id rescue => e error(:label, representation.url, e.message) end @@ -176,7 +152,9 @@ module Github next unless merge_request.new_record? && pull_request.valid? begin - pull_request.restore_branches! + # If the PR has been created/updated after we last fetched the + # remote, we fetch again to get the up-to-date refs. + fetch_remote if pull_request.updated_at > last_fetched_at author_id = user_id(pull_request.author, project.creator_id) description = format_description(pull_request.description, pull_request.author) @@ -185,6 +163,7 @@ module Github iid: pull_request.iid, title: pull_request.title, description: description, + ref_fetched: true, source_project: pull_request.source_project, source_branch: pull_request.source_branch_name, source_branch_sha: pull_request.source_branch_sha, @@ -202,17 +181,10 @@ module Github merge_request.save!(validate: false) merge_request.merge_request_diffs.create - # Fetch review comments review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) - - # Fetch comments - comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments" - fetch_comments(merge_request, :comment, comments_url) rescue => e error(:pull_request, pull_request.url, e.message) - ensure - pull_request.remove_restored_branches! end end @@ -241,12 +213,17 @@ module Github # for both features, like manipulating assignees, labels # and milestones, are provided within the Issues API. if representation.pull_request? - return unless representation.has_labels? + return unless representation.labels? || representation.comments? merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) - merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + + if representation.labels? + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + end + + fetch_comments_conditionally(merge_request, representation) else - return if Issue.where(iid: representation.iid, project_id: project.id).exists? + return if Issue.exists?(iid: representation.iid, project_id: project.id) author_id = user_id(representation.author, project.creator_id) issue = Issue.new @@ -255,25 +232,30 @@ module Github issue.title = representation.title issue.description = format_description(representation.description, representation.author) issue.state = representation.state - issue.label_ids = label_ids(representation.labels) issue.milestone_id = milestone_id(representation.milestone) issue.author_id = author_id - issue.assignee_ids = [user_id(representation.assignee)] issue.created_at = representation.created_at issue.updated_at = representation.updated_at issue.save!(validate: false) - # Fetch comments - if representation.has_comments? - comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" - fetch_comments(issue, :comment, comments_url) - end + issue.update( + label_ids: label_ids(representation.labels), + assignee_ids: assignee_ids(representation.assignees)) + + fetch_comments_conditionally(issue, representation) end rescue => e error(:issue, representation.url, e.message) end end + def fetch_comments_conditionally(issuable, representation) + if representation.comments? + comments_url = "/repos/#{repo}/issues/#{issuable.iid}/comments" + fetch_comments(issuable, :comment, comments_url) + end + end + def fetch_comments(noteable, type, url, klass = Note) while url comments = Github::Client.new(options).get(url) @@ -332,7 +314,11 @@ module Github end def label_ids(labels) - labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact + labels.map { |label| cached[:label_ids][label.title] }.compact + end + + def assignee_ids(assignees) + assignees.map { |assignee| user_id(assignee) }.compact end def milestone_id(milestone) diff --git a/lib/github/import/issue.rb b/lib/github/import/issue.rb new file mode 100644 index 00000000000..171f0872666 --- /dev/null +++ b/lib/github/import/issue.rb @@ -0,0 +1,13 @@ +module Github + class Import + class Issue < ::Issue + self.table_name = 'issues' + + self.reset_callbacks :save + self.reset_callbacks :create + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/legacy_diff_note.rb b/lib/github/import/legacy_diff_note.rb new file mode 100644 index 00000000000..18adff560b6 --- /dev/null +++ b/lib/github/import/legacy_diff_note.rb @@ -0,0 +1,12 @@ +module Github + class Import + class LegacyDiffNote < ::LegacyDiffNote + self.table_name = 'notes' + self.store_full_sti_class = false + + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/merge_request.rb b/lib/github/import/merge_request.rb new file mode 100644 index 00000000000..c258e5d5e0e --- /dev/null +++ b/lib/github/import/merge_request.rb @@ -0,0 +1,13 @@ +module Github + class Import + class MergeRequest < ::MergeRequest + self.table_name = 'merge_requests' + + self.reset_callbacks :create + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/note.rb b/lib/github/import/note.rb new file mode 100644 index 00000000000..8cf4f30e6b7 --- /dev/null +++ b/lib/github/import/note.rb @@ -0,0 +1,13 @@ +module Github + class Import + class Note < ::Note + self.table_name = 'notes' + self.store_full_sti_class = false + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb index 823e8e9a9c4..0087a3d3c4f 100644 --- a/lib/github/representation/branch.rb +++ b/lib/github/representation/branch.rb @@ -7,10 +7,14 @@ module Github raw.dig('user', 'login') || 'unknown' end + def repo? + raw['repo'].present? + end + def repo - return @repo if defined?(@repo) + return unless repo? - @repo = Github::Representation::Repo.new(raw['repo']) if raw['repo'].present? + @repo ||= Github::Representation::Repo.new(raw['repo']) end def ref @@ -25,10 +29,6 @@ module Github Commit.truncate_sha(sha) end - def exists? - @exists ||= branch_exists? && commit_exists? - end - def valid? sha.present? && ref.present? end @@ -47,14 +47,6 @@ module Github private - def branch_exists? - repository.branch_exists?(ref) - end - - def commit_exists? - repository.branch_names_contains(sha).include?(ref) - end - def repository @repository ||= options.fetch(:repository) end diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb index 1b5be91461b..83bf0b5310d 100644 --- a/lib/github/representation/comment.rb +++ b/lib/github/representation/comment.rb @@ -23,7 +23,7 @@ module Github private def generate_line_code(line) - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) end def on_diff? diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb index 9713b82615d..768ba3b993c 100644 --- a/lib/github/representation/issuable.rb +++ b/lib/github/representation/issuable.rb @@ -23,14 +23,14 @@ module Github @author ||= Github::Representation::User.new(raw['user'], options) end - def assignee - return unless assigned? - - @assignee ||= Github::Representation::User.new(raw['assignee'], options) + def labels? + raw['labels'].any? end - def assigned? - raw['assignee'].present? + def labels + @labels ||= Array(raw['labels']).map do |label| + Github::Representation::Label.new(label, options) + end end end end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb index df3540a6e6c..4f1a02cb90f 100644 --- a/lib/github/representation/issue.rb +++ b/lib/github/representation/issue.rb @@ -1,25 +1,27 @@ module Github module Representation class Issue < Representation::Issuable - def labels - raw['labels'] - end - def state raw['state'] == 'closed' ? 'closed' : 'opened' end - def has_comments? + def comments? raw['comments'] > 0 end - def has_labels? - labels.count > 0 - end - def pull_request? raw['pull_request'].present? end + + def assigned? + raw['assignees'].present? + end + + def assignees + @assignees ||= Array(raw['assignees']).map do |user| + Github::Representation::User.new(user, options) + end + end end end end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb index 55461097e8a..0171179bb0f 100644 --- a/lib/github/representation/pull_request.rb +++ b/lib/github/representation/pull_request.rb @@ -1,26 +1,17 @@ module Github module Representation class PullRequest < Representation::Issuable - delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true - delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true + delegate :sha, to: :source_branch, prefix: true + delegate :sha, to: :target_branch, prefix: true def source_project project end def source_branch_name - @source_branch_name ||= - if cross_project? || !source_branch_exists? - source_branch_name_prefixed - else - source_branch_ref - end - end - - def source_branch_exists? - return @source_branch_exists if defined?(@source_branch_exists) - - @source_branch_exists = !cross_project? && source_branch.exists? + # Mimic the "user:branch" displayed in the MR widget, + # i.e. "Request to merge rymai:add-external-mounts into master" + cross_project? ? "#{source_branch.user}:#{source_branch.ref}" : source_branch.ref end def target_project @@ -28,11 +19,7 @@ module Github end def target_branch_name - @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed - end - - def target_branch_exists? - @target_branch_exists ||= target_branch.exists? + target_branch.ref end def state @@ -50,16 +37,14 @@ module Github source_branch.valid? && target_branch.valid? end - def restore_branches! - restore_source_branch! - restore_target_branch! + def assigned? + raw['assignee'].present? end - def remove_restored_branches! - return if opened? + def assignee + return unless assigned? - remove_source_branch! - remove_target_branch! + @assignee ||= Github::Representation::User.new(raw['assignee'], options) end private @@ -72,48 +57,14 @@ module Github @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) end - def source_branch_name_prefixed - "gh-#{target_branch_short_sha}/#{iid}/#{source_branch_user}/#{source_branch_ref}" - end - def target_branch @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) end - def target_branch_name_prefixed - "gl-#{target_branch_short_sha}/#{iid}/#{target_branch_user}/#{target_branch_ref}" - end - def cross_project? - return true if source_branch_repo.nil? - - source_branch_repo.id != target_branch_repo.id - end - - def restore_source_branch! - return if source_branch_exists? - - source_branch.restore!(source_branch_name) - end - - def restore_target_branch! - return if target_branch_exists? - - target_branch.restore!(target_branch_name) - end - - def remove_source_branch! - # We should remove the source/target branches only if they were - # restored. Otherwise, we'll remove branches like 'master' that - # target_branch_exists? returns true. In other words, we need - # to clean up only the restored branches that (source|target)_branch_exists? - # returns false for the first time it has been called, because of - # this that is important to memoize these values. - source_branch.remove!(source_branch_name) unless source_branch_exists? - end + return true unless source_branch.repo? - def remove_target_branch! - target_branch.remove!(target_branch_name) unless target_branch_exists? + source_branch.repo.id != target_branch.repo.id end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 87aeb76b66a..0ad9285c0ea 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,11 +1,11 @@ module Gitlab module Auth - MissingPersonalTokenError = Class.new(StandardError) + MissingPersonalAccessTokenError = Class.new(StandardError) REGISTRY_SCOPES = [:read_registry].freeze # Scopes used for GitLab API access - API_SCOPES = [:api, :read_user].freeze + API_SCOPES = [:api, :read_user, :sudo].freeze # Scopes used for OpenID Connect OPENID_SCOPES = [:openid].freeze @@ -38,7 +38,7 @@ module Gitlab # If sign-in is disabled and LDAP is not configured, recommend a # personal access token on failed auth attempts - raise Gitlab::Auth::MissingPersonalTokenError + raise Gitlab::Auth::MissingPersonalAccessTokenError end def find_with_user_password(login, password) @@ -106,7 +106,7 @@ module Gitlab user = find_with_user_password(login, password) return unless user - raise Gitlab::Auth::MissingPersonalTokenError if user.two_factor_enabled? + raise Gitlab::Auth::MissingPersonalAccessTokenError if user.two_factor_enabled? Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, full_authentication_abilities) end @@ -128,7 +128,7 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) if token && valid_scoped_token?(token, available_scopes) - Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes)) + Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scope(token.scopes)) end end @@ -226,8 +226,10 @@ module Gitlab [] end - def available_scopes - API_SCOPES + registry_scopes + def available_scopes(current_user = nil) + scopes = API_SCOPES + registry_scopes + scopes.delete(:sudo) if current_user && !current_user.admin? + scopes end # Other available scopes diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb new file mode 100644 index 00000000000..c88eb9783ed --- /dev/null +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -0,0 +1,65 @@ +module Gitlab + module BackgroundMigration + class CreateForkNetworkMembershipsRange + RESCHEDULE_DELAY = 15 + + class ForkedProjectLink < ActiveRecord::Base + self.table_name = 'forked_project_links' + end + + def perform(start_id, end_id) + log("Creating memberships for forks: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS + INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id) + + SELECT fork_network_members.fork_network_id, + forked_project_links.forked_to_project_id, + forked_project_links.forked_from_project_id + + FROM forked_project_links + + INNER JOIN fork_network_members + ON forked_project_links.forked_from_project_id = fork_network_members.project_id + + WHERE forked_project_links.id BETWEEN #{start_id} AND #{end_id} + AND NOT EXISTS ( + SELECT true + FROM fork_network_members existing_members + WHERE existing_members.project_id = forked_project_links.forked_to_project_id + ) + INSERT_MEMBERS + + if missing_members?(start_id, end_id) + BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + end + + def missing_members?(start_id, end_id) + count_sql = <<~MISSING_MEMBERS + SELECT COUNT(*) + + FROM forked_project_links + + WHERE NOT EXISTS ( + SELECT true + FROM fork_network_members + WHERE fork_network_members.project_id = forked_project_links.forked_to_project_id + ) + AND EXISTS ( + SELECT true + FROM projects + WHERE forked_project_links.forked_from_project_id = projects.id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + MISSING_MEMBERS + + ForkNetworkMember.count_by_sql(count_sql) > 0 + end + + def log(message) + Rails.logger.info("#{self.class.name} - #{message}") + end + end + end +end diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb new file mode 100644 index 00000000000..e94719db72e --- /dev/null +++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb @@ -0,0 +1,53 @@ +class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys + class GpgKey < ActiveRecord::Base + self.table_name = 'gpg_keys' + + include EachBatch + include ShaAttribute + + sha_attribute :primary_keyid + sha_attribute :fingerprint + + has_many :subkeys, class_name: 'GpgKeySubkey' + end + + class GpgKeySubkey < ActiveRecord::Base + self.table_name = 'gpg_key_subkeys' + + include ShaAttribute + + sha_attribute :keyid + sha_attribute :fingerprint + end + + def perform(gpg_key_id) + gpg_key = GpgKey.find_by(id: gpg_key_id) + + return if gpg_key.nil? + return if gpg_key.subkeys.any? + + create_subkeys(gpg_key) + update_signatures(gpg_key) + end + + private + + def create_subkeys(gpg_key) + gpg_subkeys = Gitlab::Gpg.subkeys_from_key(gpg_key.key) + + gpg_subkeys[gpg_key.primary_keyid.upcase]&.each do |subkey_data| + gpg_key.subkeys.build(keyid: subkey_data[:keyid], fingerprint: subkey_data[:fingerprint]) + end + + # Improve latency by doing all INSERTs in a single call + GpgKey.transaction do + gpg_key.save! + end + end + + def update_signatures(gpg_key) + return unless gpg_key.subkeys.exists? + + InvalidGpgSignatureUpdateWorker.perform_async(gpg_key.id) + end +end diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index 3fde1b09efb..380802258f5 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -3,11 +3,18 @@ module Gitlab class DeserializeMergeRequestDiffsAndCommits attr_reader :diff_ids, :commit_rows, :file_rows + class Error < StandardError + def backtrace + cause.backtrace + end + end + class MergeRequestDiff < ActiveRecord::Base self.table_name = 'merge_request_diffs' end BUFFER_ROWS = 1000 + DIFF_FILE_BUFFER_ROWS = 100 def perform(start_id, stop_id) merge_request_diffs = MergeRequestDiff @@ -26,13 +33,17 @@ module Gitlab if diff_ids.length > BUFFER_ROWS || commit_rows.length > BUFFER_ROWS || - file_rows.length > BUFFER_ROWS + file_rows.length > DIFF_FILE_BUFFER_ROWS flush_buffers! end end flush_buffers! + rescue => e + Rails.logger.info("#{self.class.name}: failed for IDs #{merge_request_diffs.map(&:id)} with #{e.class.name}") + + raise Error.new(e.inspect) end private @@ -45,20 +56,32 @@ module Gitlab def flush_buffers! if diff_ids.any? - MergeRequestDiff.transaction do - Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows) - Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows) + commit_rows.each_slice(BUFFER_ROWS).each do |commit_rows_slice| + bulk_insert('merge_request_diff_commits', commit_rows_slice) + end - MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) + file_rows.each_slice(DIFF_FILE_BUFFER_ROWS).each do |file_rows_slice| + bulk_insert('merge_request_diff_files', file_rows_slice) end + + MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) end reset_buffers! end + def bulk_insert(table, rows) + Gitlab::Database.bulk_insert(table, rows) + rescue ActiveRecord::RecordNotUnique + ids = rows.map { |row| row[:merge_request_diff_id] }.uniq.sort + + Rails.logger.info("#{self.class.name}: rows inserted twice for IDs #{ids}") + end + def single_diff_rows(merge_request_diff) sha_attribute = Gitlab::Database::ShaAttribute.new commits = YAML.load(merge_request_diff.st_commits) rescue [] + commits ||= [] commit_rows = commits.map.with_index do |commit, index| commit_hash = commit.to_hash.with_indifferent_access.except(:parent_ids) diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb new file mode 100644 index 00000000000..bc53e6d7f94 --- /dev/null +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -0,0 +1,313 @@ +module Gitlab + module BackgroundMigration + class NormalizeLdapExternUidsRange + class Identity < ActiveRecord::Base + self.table_name = 'identities' + end + + # Copied this class to make this migration resilient to future code changes. + # And if the normalize behavior is changed in the future, it must be + # accompanied by another migration. + module Gitlab + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) + + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end + + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end + end + + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" + + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" + end + end + + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state + + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end + + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) + + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end + end + + str + end + + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end + + ## + # Return the DN as an escaped string. + def to_s + @dn + end + + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end + + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze + + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") + + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") + + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end + + private + + def initialize_array(args) + buffer = StringIO.new + + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end + end + + @dn = buffer.string + end + + def initialize_string(arg) + @dn = arg.to_s + end + + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end + + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end + end + end + end + + def perform(start_id, end_id) + return unless migrate? + + ldap_identities = Identity.where("provider like 'ldap%'").where(id: start_id..end_id) + ldap_identities.each do |identity| + begin + identity.extern_uid = Gitlab::LDAP::DN.new(identity.extern_uid).to_normalized_s + unless identity.save + Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\". Skipping." + end + rescue Gitlab::LDAP::DN::FormatError => e + Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\" due to \"#{e.message}\". Skipping." + end + end + end + + def migrate? + Identity.table_exists? + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb new file mode 100644 index 00000000000..2ef3a207dd3 --- /dev/null +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -0,0 +1,59 @@ +module Gitlab + module BackgroundMigration + class PopulateForkNetworksRange + def perform(start_id, end_id) + log("Creating fork networks for forked project links: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS + INSERT INTO fork_networks (root_project_id) + SELECT DISTINCT forked_project_links.forked_from_project_id + + FROM forked_project_links + + WHERE NOT EXISTS ( + SELECT true + FROM forked_project_links inner_links + WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id + ) + AND NOT EXISTS ( + SELECT true + FROM fork_networks + WHERE forked_project_links.forked_from_project_id = fork_networks.root_project_id + ) + AND EXISTS ( + SELECT true + FROM projects + WHERE projects.id = forked_project_links.forked_from_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_NETWORKS + + log("Creating memberships for root projects: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_ROOT + INSERT INTO fork_network_members (fork_network_id, project_id) + SELECT DISTINCT fork_networks.id, fork_networks.root_project_id + + FROM fork_networks + + INNER JOIN forked_project_links + ON forked_project_links.forked_from_project_id = fork_networks.root_project_id + + WHERE NOT EXISTS ( + SELECT true + FROM fork_network_members + WHERE fork_network_members.project_id = fork_networks.root_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_ROOT + + delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY + BackgroundMigrationWorker.perform_in(delay, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + + def log(message) + Rails.logger.info("#{self.class.name} - #{message}") + end + end + end +end diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb index 9323bfc7fb2..1d98d187805 100644 --- a/lib/gitlab/bare_repository_importer.rb +++ b/lib/gitlab/bare_repository_importer.rb @@ -56,7 +56,8 @@ module Gitlab name: project_path, path: project_path, repository_storage: storage_name, - namespace_id: group&.id + namespace_id: group&.id, + skip_disk_validation: true } project = Projects::CreateService.new(user, project_params).execute diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 28bbf3b384e..033ecd15749 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -149,16 +149,21 @@ module Gitlab description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) description += pull_request.description + source_branch_sha = pull_request.source_branch_sha + target_branch_sha = pull_request.target_branch_sha + source_branch_sha = project.repository.commit(source_branch_sha)&.sha || source_branch_sha + target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha + merge_request = project.merge_requests.create!( iid: pull_request.iid, title: pull_request.title, description: description, source_project: project, source_branch: pull_request.source_branch_name, - source_branch_sha: pull_request.source_branch_sha, + source_branch_sha: source_branch_sha, target_project: project, target_branch: pull_request.target_branch_name, - target_branch_sha: pull_request.target_branch_sha, + target_branch_sha: target_branch_sha, state: pull_request.state, author_id: gitlab_user_id(project, pull_request.author), assignee_id: nil, @@ -236,7 +241,7 @@ module Gitlab end def generate_line_code(pr_comment) - Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) + Gitlab::Git.diff_line_code(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) end def pull_request_comment_attributes(comment) diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index ad78ae244b2..72b75791bbb 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -155,7 +155,9 @@ module Gitlab stream.each_line do |line| s = StringScanner.new(line) until s.eos? - if s.scan(/\e([@-_])(.*?)([@-~])/) + if s.scan(Gitlab::Regex.build_trace_section_regex) + handle_section(s) + elsif s.scan(/\e([@-_])(.*?)([@-~])/) handle_sequence(s) elsif s.scan(/\e(([@-_])(.*?)?)?$/) break @@ -183,6 +185,15 @@ module Gitlab ) end + def handle_section(s) + action = s[1] + timestamp = s[2] + section = s[3] + line = s.matched()[0...-5] # strips \r\033[0K + + @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>} + end + def handle_sequence(s) indicator = s[1] commands = s[2].split ';' diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb new file mode 100644 index 00000000000..8d82e1b288d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Base + attr_reader :pipeline, :project, :current_user + + def initialize(pipeline, command) + @pipeline = pipeline + @command = command + + @project = command.project + @current_user = command.current_user + end + + def perform! + raise NotImplementedError + end + + def break? + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb new file mode 100644 index 00000000000..d5e17a123df --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Create < Chain::Base + include Chain::Helpers + + def perform! + ::Ci::Pipeline.transaction do + pipeline.save! + + @command.seeds_block&.call(pipeline) + + ::Ci::CreatePipelineStagesService + .new(project, current_user) + .execute(pipeline) + end + rescue ActiveRecord::RecordInvalid => e + error("Failed to persist the pipeline: #{e}") + end + + def break? + !pipeline.persisted? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb new file mode 100644 index 00000000000..02d81286f21 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Helpers + def branch_exists? + return @is_branch if defined?(@is_branch) + + @is_branch = project.repository.branch_exists?(pipeline.ref) + end + + def tag_exists? + return @is_tag if defined?(@is_tag) + + @is_tag = project.repository.tag_exists?(pipeline.ref) + end + + def error(message) + pipeline.errors.add(:base, message) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb new file mode 100644 index 00000000000..015f2988327 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -0,0 +1,36 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Sequence + def initialize(pipeline, command, sequence) + @pipeline = pipeline + @completed = [] + + @sequence = sequence.map do |chain| + chain.new(pipeline, command) + end + end + + def build! + @sequence.each do |step| + step.perform! + + break if step.break? + + @completed << step + end + + @pipeline.tap do + yield @pipeline, self if block_given? + end + end + + def complete? + @completed.size == @sequence.size + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb new file mode 100644 index 00000000000..9a72de87bab --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Skip < Chain::Base + SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i + + def perform! + if skipped? + @pipeline.skip if @command.save_incompleted + end + end + + def skipped? + !@command.ignore_skip_ci && commit_message_skips_ci? + end + + def break? + skipped? + end + + private + + def commit_message_skips_ci? + return false unless @pipeline.git_commit_message + + @skipped ||= !!(@pipeline.git_commit_message =~ SKIP_PATTERN) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb new file mode 100644 index 00000000000..4913a604079 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -0,0 +1,54 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Abilities < Chain::Base + include Gitlab::Allowable + include Chain::Helpers + + def perform! + unless project.builds_enabled? + return error('Pipelines are disabled!') + end + + unless allowed_to_trigger_pipeline? + if can?(current_user, :create_pipeline, project) + return error("Insufficient permissions for protected ref '#{pipeline.ref}'") + else + return error('Insufficient permissions to create a new pipeline') + end + end + end + + def break? + @pipeline.errors.any? + end + + def allowed_to_trigger_pipeline? + if current_user + allowed_to_create? + else # legacy triggers don't have a corresponding user + !project.protected_for?(@pipeline.ref) + end + end + + def allowed_to_create? + return unless can?(current_user, :create_pipeline, project) + + access = Gitlab::UserAccess.new(current_user, project: project) + + if branch_exists? + access.can_update_branch?(@pipeline.ref) + elsif tag_exists? + access.can_create_tag?(@pipeline.ref) + else + true # Allow it for now and we'll reject when we check ref existence + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb new file mode 100644 index 00000000000..075504bcce5 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -0,0 +1,35 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Config < Chain::Base + include Chain::Helpers + + def perform! + unless @pipeline.config_processor + unless @pipeline.ci_yaml_file + return error("Missing #{@pipeline.ci_yaml_file_path} file") + end + + if @command.save_incompleted && @pipeline.has_yaml_errors? + @pipeline.drop!(:config_error) + end + + return error(@pipeline.yaml_errors) + end + + unless @pipeline.has_stage_seeds? + return error('No stages / jobs for this pipeline.') + end + end + + def break? + @pipeline.errors.any? || @pipeline.persisted? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb new file mode 100644 index 00000000000..70a4cfdbdea --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -0,0 +1,30 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Repository < Chain::Base + include Chain::Helpers + + def perform! + unless branch_exists? || tag_exists? + return error('Reference not found') + end + + ## TODO, we check commit in the service, that is why + # there is no repository access here. + # + unless pipeline.sha + return error('Commit not found') + end + end + + def break? + @pipeline.errors.any? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb new file mode 100644 index 00000000000..469fc094cc8 --- /dev/null +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -0,0 +1,143 @@ +module Gitlab + module Ci + module Pipeline + # # Introduction - total running time + # + # The problem this module is trying to solve is finding the total running + # time amongst all the jobs, excluding retries and pending (queue) time. + # We could reduce this problem down to finding the union of periods. + # + # So each job would be represented as a `Period`, which consists of + # `Period#first` as when the job started and `Period#last` as when the + # job was finished. A simple example here would be: + # + # * A (1, 3) + # * B (2, 4) + # * C (6, 7) + # + # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. + # C begins from 6, and ends to 7. Visually it could be viewed as: + # + # 0 1 2 3 4 5 6 7 + # AAAAAAA + # BBBBBBB + # CCCC + # + # The union of A, B, and C would be (1, 4) and (6, 7), therefore the + # total running time should be: + # + # (4 - 1) + (7 - 6) => 4 + # + # # The Algorithm + # + # The algorithm used here for union would be described as follow. + # First we make sure that all periods are sorted by `Period#first`. + # Then we try to merge periods by iterating through the first period + # to the last period. The goal would be merging all overlapped periods + # so that in the end all the periods are discrete. When all periods + # are discrete, we're free to just sum all the periods to get real + # running time. + # + # Here we begin from A, and compare it to B. We could find that + # before A ends, B already started. That is `B.first <= A.last` + # that is `2 <= 3` which means A and B are overlapping! + # + # When we found that two periods are overlapping, we would need to merge + # them into a new period and disregard the old periods. To make a new + # period, we take `A.first` as the new first because remember? we sorted + # them, so `A.first` must be smaller or equal to `B.first`. And we take + # `[A.last, B.last].max` as the new last because we want whoever ended + # later. This could be broken into two cases: + # + # 0 1 2 3 4 + # AAAAAAA + # BBBBBBB + # + # Or: + # + # 0 1 2 3 4 + # AAAAAAAAAA + # BBBB + # + # So that we need to take whoever ends later. Back to our example, + # after merging and discard A and B it could be visually viewed as: + # + # 0 1 2 3 4 5 6 7 + # DDDDDDDDDD + # CCCC + # + # Now we could go on and compare the newly created D and the old C. + # We could figure out that D and C are not overlapping by checking + # `C.first <= D.last` is `false`. Therefore we need to keep both C + # and D. The example would end here because there are no more jobs. + # + # After having the union of all periods, we just need to sum the length + # of all periods to get total time. + # + # (4 - 1) + (7 - 6) => 4 + # + # That is 4 is the answer in the example. + module Duration + extend self + + Period = Struct.new(:first, :last) do + def duration + last - first + end + end + + def from_pipeline(pipeline) + status = %w[success failed running canceled] + builds = pipeline.builds.latest + .where(status: status).where.not(started_at: nil).order(:started_at) + + from_builds(builds) + end + + def from_builds(builds) + now = Time.now + + periods = builds.map do |b| + Period.new(b.started_at, b.finished_at || now) + end + + from_periods(periods) + end + + # periods should be sorted by `first` + def from_periods(periods) + process_duration(process_periods(periods)) + end + + private + + def process_periods(periods) + return periods if periods.empty? + + periods.drop(1).inject([periods.first]) do |result, current| + previous = result.last + + if overlap?(previous, current) + result[-1] = merge(previous, current) + result + else + result << current + end + end + end + + def overlap?(previous, current) + current.first <= previous.last + end + + def merge(previous, current) + Period.new(previous.first, [previous.last, current.last].max) + end + + def process_duration(periods) + periods.sum(&:duration) + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb deleted file mode 100644 index 3208cc2bef6..00000000000 --- a/lib/gitlab/ci/pipeline_duration.rb +++ /dev/null @@ -1,141 +0,0 @@ -module Gitlab - module Ci - # # Introduction - total running time - # - # The problem this module is trying to solve is finding the total running - # time amongst all the jobs, excluding retries and pending (queue) time. - # We could reduce this problem down to finding the union of periods. - # - # So each job would be represented as a `Period`, which consists of - # `Period#first` as when the job started and `Period#last` as when the - # job was finished. A simple example here would be: - # - # * A (1, 3) - # * B (2, 4) - # * C (6, 7) - # - # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. - # C begins from 6, and ends to 7. Visually it could be viewed as: - # - # 0 1 2 3 4 5 6 7 - # AAAAAAA - # BBBBBBB - # CCCC - # - # The union of A, B, and C would be (1, 4) and (6, 7), therefore the - # total running time should be: - # - # (4 - 1) + (7 - 6) => 4 - # - # # The Algorithm - # - # The algorithm used here for union would be described as follow. - # First we make sure that all periods are sorted by `Period#first`. - # Then we try to merge periods by iterating through the first period - # to the last period. The goal would be merging all overlapped periods - # so that in the end all the periods are discrete. When all periods - # are discrete, we're free to just sum all the periods to get real - # running time. - # - # Here we begin from A, and compare it to B. We could find that - # before A ends, B already started. That is `B.first <= A.last` - # that is `2 <= 3` which means A and B are overlapping! - # - # When we found that two periods are overlapping, we would need to merge - # them into a new period and disregard the old periods. To make a new - # period, we take `A.first` as the new first because remember? we sorted - # them, so `A.first` must be smaller or equal to `B.first`. And we take - # `[A.last, B.last].max` as the new last because we want whoever ended - # later. This could be broken into two cases: - # - # 0 1 2 3 4 - # AAAAAAA - # BBBBBBB - # - # Or: - # - # 0 1 2 3 4 - # AAAAAAAAAA - # BBBB - # - # So that we need to take whoever ends later. Back to our example, - # after merging and discard A and B it could be visually viewed as: - # - # 0 1 2 3 4 5 6 7 - # DDDDDDDDDD - # CCCC - # - # Now we could go on and compare the newly created D and the old C. - # We could figure out that D and C are not overlapping by checking - # `C.first <= D.last` is `false`. Therefore we need to keep both C - # and D. The example would end here because there are no more jobs. - # - # After having the union of all periods, we just need to sum the length - # of all periods to get total time. - # - # (4 - 1) + (7 - 6) => 4 - # - # That is 4 is the answer in the example. - module PipelineDuration - extend self - - Period = Struct.new(:first, :last) do - def duration - last - first - end - end - - def from_pipeline(pipeline) - status = %w[success failed running canceled] - builds = pipeline.builds.latest - .where(status: status).where.not(started_at: nil).order(:started_at) - - from_builds(builds) - end - - def from_builds(builds) - now = Time.now - - periods = builds.map do |b| - Period.new(b.started_at, b.finished_at || now) - end - - from_periods(periods) - end - - # periods should be sorted by `first` - def from_periods(periods) - process_duration(process_periods(periods)) - end - - private - - def process_periods(periods) - return periods if periods.empty? - - periods.drop(1).inject([periods.first]) do |result, current| - previous = result.last - - if overlap?(previous, current) - result[-1] = merge(previous, current) - result - else - result << current - end - end - end - - def overlap?(previous, current) - current.first <= previous.last - end - - def merge(previous, current) - Period.new(previous.first, [previous.last, current.last].max) - end - - def process_duration(periods) - periods.sum(&:duration) - end - end - end -end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index e19aae35a81..bc97aa63b02 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -3,7 +3,9 @@ module Gitlab module Stage class Seed attr_reader :pipeline + delegate :project, to: :pipeline + delegate :size, to: :@jobs def initialize(pipeline, stage, jobs) @pipeline = pipeline diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 8ad3e57e59d..2d9166d6bdd 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -8,7 +8,7 @@ module Gitlab end def action_icon - 'icon_action_cancel' + 'cancel' end def action_path diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index e42d3574357..d71e63e73eb 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -8,7 +8,7 @@ module Gitlab end def icon - 'icon_status_warning' + 'warning' end def group diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index c7726543599..b7b45466d3b 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -12,7 +12,7 @@ module Gitlab end def action_icon - 'icon_action_play' + 'play' end def action_title diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 8c8fdc56d75..44ffe783e50 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -8,7 +8,7 @@ module Gitlab end def action_icon - 'icon_action_retry' + 'retry' end def action_title diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index d464738deaf..46e730797e4 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -12,7 +12,7 @@ module Gitlab end def action_icon - 'icon_action_stop' + 'stop' end def action_title diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb index e5fdc1f8136..e6195a60d4f 100644 --- a/lib/gitlab/ci/status/canceled.rb +++ b/lib/gitlab/ci/status/canceled.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_canceled' + 'status_canceled' end def favicon diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb index d188bd286a6..846f00b83dd 100644 --- a/lib/gitlab/ci/status/created.rb +++ b/lib/gitlab/ci/status/created.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_created' + 'status_created' end def favicon diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb index 38e45714c22..27ce85bd3ed 100644 --- a/lib/gitlab/ci/status/failed.rb +++ b/lib/gitlab/ci/status/failed.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_failed' + 'status_failed' end def favicon diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb index a4a7edadac9..fc387e2fd25 100644 --- a/lib/gitlab/ci/status/manual.rb +++ b/lib/gitlab/ci/status/manual.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_manual' + 'status_manual' end def favicon diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb index 5164260b861..6780780db32 100644 --- a/lib/gitlab/ci/status/pending.rb +++ b/lib/gitlab/ci/status/pending.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_pending' + 'status_pending' end def favicon diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb index 993937e98ca..ee13905e46d 100644 --- a/lib/gitlab/ci/status/running.rb +++ b/lib/gitlab/ci/status/running.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_running' + 'status_running' end def favicon diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb index 0c942920b02..0dbdc4de426 100644 --- a/lib/gitlab/ci/status/skipped.rb +++ b/lib/gitlab/ci/status/skipped.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_skipped' + 'status_skipped' end def favicon diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb index d7af98857b0..731013ec017 100644 --- a/lib/gitlab/ci/status/success.rb +++ b/lib/gitlab/ci/status/success.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_success' + 'status_success' end def favicon diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index 4d7d82e04cf..32b4cf43e48 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -15,7 +15,7 @@ module Gitlab end def icon - 'icon_status_warning' + 'status_warning' end def group diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 5b835bb669a..baf55b1fa07 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -27,6 +27,12 @@ module Gitlab end end + def extract_sections + read do |stream| + stream.extract_sections + end + end + def set(data) write do |stream| data = job.hide_secrets(data) diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb new file mode 100644 index 00000000000..9bb0166c9e3 --- /dev/null +++ b/lib/gitlab/ci/trace/section_parser.rb @@ -0,0 +1,97 @@ +module Gitlab + module Ci + class Trace + class SectionParser + def initialize(lines) + @lines = lines + end + + def parse! + @markers = {} + + @lines.each do |line, pos| + parse_line(line, pos) + end + end + + def sections + sanitize_markers.map do |name, markers| + start_, end_ = markers + + { + name: name, + byte_start: start_[:marker], + byte_end: end_[:marker], + date_start: start_[:timestamp], + date_end: end_[:timestamp] + } + end + end + + private + + def parse_line(line, line_start_position) + s = StringScanner.new(line) + until s.eos? + find_next_marker(s) do |scanner| + marker_begins_at = line_start_position + scanner.pointer + + if scanner.scan(Gitlab::Regex.build_trace_section_regex) + marker_ends_at = line_start_position + scanner.pointer + handle_line(scanner[1], scanner[2].to_i, scanner[3], marker_begins_at, marker_ends_at) + true + else + false + end + end + end + end + + def sanitize_markers + @markers.select do |_, markers| + markers.size == 2 && markers[0][:action] == :start && markers[1][:action] == :end + end + end + + def handle_line(action, time, name, marker_start, marker_end) + action = action.to_sym + timestamp = Time.at(time).utc + marker = if action == :start + marker_end + else + marker_start + end + + @markers[name] ||= [] + @markers[name] << { + name: name, + action: action, + timestamp: timestamp, + marker: marker + } + end + + def beginning_of_section_regex + @beginning_of_section_regex ||= /section_/.freeze + end + + def find_next_marker(s) + beginning_of_section_len = 8 + maybe_marker = s.exist?(beginning_of_section_regex) + + if maybe_marker.nil? + s.terminate + else + # repositioning at the beginning of the match + s.pos += maybe_marker - beginning_of_section_len + if block_given? + good_marker = yield(s) + # if not a good marker: Consuming the matched beginning_of_section_regex + s.pos += beginning_of_section_len unless good_marker + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index ab3408f48d6..d52194f688b 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -90,8 +90,25 @@ module Gitlab # so we just silently ignore error for now end + def extract_sections + return [] unless valid? + + lines = to_enum(:each_line_with_pos) + parser = SectionParser.new(lines) + + parser.parse! + parser.sections + end + private + def each_line_with_pos + stream.seek(0, IO::SEEK_SET) + stream.each_line do |line| + yield [line, stream.pos - line.bytesize] + end + end + def read_last_lines(limit) to_enum(:reverse_line).first(limit).reverse.join end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index 243c1f1394d..7e7aaeeaa17 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -23,7 +23,8 @@ module Gitlab @extractor.analyze(closing_statements.join(" ")) @extractor.issues.reject do |issue| - @extractor.project.forked_from?(issue.project) # Don't extract issues on original project + # Don't extract issues from the project this project was forked from + @extractor.project.forked_from?(issue.project) end end end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 98dfe900044..2a0cb640a14 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -4,82 +4,29 @@ module Gitlab include Gitlab::Routing include IconsHelper - MissingResolution = Class.new(ResolutionError) - CONTEXT_LINES = 3 - attr_reader :merge_file_result, :their_path, :our_path, :our_mode, :merge_request, :repository - - def initialize(merge_file_result, conflict, merge_request:) - @merge_file_result = merge_file_result - @their_path = conflict[:theirs][:path] - @our_path = conflict[:ours][:path] - @our_mode = conflict[:ours][:mode] - @merge_request = merge_request - @repository = merge_request.project.repository - @match_line_headers = {} - end - - def content - merge_file_result[:data] - end + attr_reader :merge_request - def our_blob - @our_blob ||= repository.blob_at(merge_request.diff_refs.head_sha, our_path) - end + # 'raw' holds the Gitlab::Git::Conflict::File that this instance wraps + attr_reader :raw - def type - lines unless @type + delegate :type, :content, :their_path, :our_path, :our_mode, :our_blob, :repository, to: :raw - @type.inquiry + def initialize(raw, merge_request:) + @raw = raw + @merge_request = merge_request + @match_line_headers = {} end - # Array of Gitlab::Diff::Line objects def lines return @lines if defined?(@lines) - begin - @type = 'text' - @lines = Gitlab::Conflict::Parser.new.parse(content, - our_path: our_path, - their_path: their_path, - parent_file: self) - rescue Gitlab::Conflict::Parser::ParserError - @type = 'text-editor' - @lines = nil - end + @lines = raw.lines.nil? ? nil : map_raw_lines(raw.lines) end def resolve_lines(resolution) - section_id = nil - - lines.map do |line| - unless line.type - section_id = nil - next line - end - - section_id ||= line_code(line) - - case resolution[section_id] - when 'head' - next unless line.type == 'new' - when 'origin' - next unless line.type == 'old' - else - raise MissingResolution, "Missing resolution for section ID: #{section_id}" - end - - line - end.compact - end - - def resolve_content(resolution) - if resolution == content - raise MissingResolution, "Resolved content has no changes for file #{our_path}" - end - - resolution + map_raw_lines(raw.resolve_lines(resolution)) end def highlight_lines! @@ -163,7 +110,7 @@ module Gitlab end def line_code(line) - Gitlab::Diff::LineCode.generate(our_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(our_path, line.new_pos, line.old_pos) end def create_match_line(line) @@ -227,15 +174,14 @@ module Gitlab new_path: our_path) end - # Don't try to print merge_request or repository. - def inspect - instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode, :type].map do |instance_variable| - value = instance_variable_get("@#{instance_variable}") + private - "#{instance_variable}=\"#{value}\"" + def map_raw_lines(raw_lines) + raw_lines.map do |raw_line| + Gitlab::Diff::Line.new(raw_line[:full_line], raw_line[:type], + raw_line[:line_obj_index], raw_line[:line_old], + raw_line[:line_new], parent_file: self) end - - "#<#{self.class} #{instance_variables.join(' ')}>" end end end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 90f83e0f810..fb28e80ff73 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -1,48 +1,29 @@ module Gitlab module Conflict class FileCollection - ConflictSideMissing = Class.new(StandardError) - - attr_reader :merge_request, :our_commit, :their_commit, :project - - delegate :repository, to: :project - - class << self - # We can only write when getting the merge index from the source - # project, because we will write to that project. We don't use this all - # the time because this fetches a ref into the source project, which - # isn't needed for reading. - def for_resolution(merge_request) - project = merge_request.source_project - - new(merge_request, project).tap do |file_collection| - project - .repository - .with_repo_branch_commit(merge_request.target_project.repository.raw_repository, merge_request.target_branch) do - - yield file_collection - end - end - end - - # We don't need to do `with_repo_branch_commit` here, because the target - # project always fetches source refs when creating merge request diffs. - def read_only(merge_request) - new(merge_request, merge_request.target_project) - end + attr_reader :merge_request, :resolver + + def initialize(merge_request) + source_repo = merge_request.source_project.repository.raw + our_commit = merge_request.source_branch_head.raw + their_commit = merge_request.target_branch_head.raw + target_repo = merge_request.target_project.repository.raw + @resolver = Gitlab::Git::Conflict::Resolver.new(source_repo, our_commit, target_repo, their_commit) + @merge_request = merge_request end - def merge_index - @merge_index ||= repository.rugged.merge_commits(our_commit, their_commit) + def resolve(user, commit_message, files) + args = { + source_branch: merge_request.source_branch, + target_branch: merge_request.target_branch, + commit_message: commit_message || default_commit_message + } + resolver.resolve_conflicts(user, files, args) end def files - @files ||= merge_index.conflicts.map do |conflict| - raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours] - - Gitlab::Conflict::File.new(merge_index.merge_file(conflict[:ours][:path]), - conflict, - merge_request: merge_request) + @files ||= resolver.conflicts.map do |conflict_file| + Gitlab::Conflict::File.new(conflict_file, merge_request: merge_request) end end @@ -61,8 +42,8 @@ module Gitlab end def default_commit_message - conflict_filenames = merge_index.conflicts.map do |conflict| - "# #{conflict[:ours][:path]}" + conflict_filenames = files.map do |conflict| + "# #{conflict.our_path}" end <<EOM.chomp @@ -72,15 +53,6 @@ Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branc #{conflict_filenames.join("\n")} EOM end - - private - - def initialize(merge_request, project) - @merge_request = merge_request - @our_commit = merge_request.source_branch_head.raw.rugged_commit - @their_commit = merge_request.target_branch_head.raw.rugged_commit - @project = project - end end end end diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb deleted file mode 100644 index e3678c914db..00000000000 --- a/lib/gitlab/conflict/parser.rb +++ /dev/null @@ -1,74 +0,0 @@ -module Gitlab - module Conflict - class Parser - UnresolvableError = Class.new(StandardError) - UnmergeableFile = Class.new(UnresolvableError) - UnsupportedEncoding = Class.new(UnresolvableError) - - # Recoverable errors - the conflict can be resolved in an editor, but not with - # sections. - ParserError = Class.new(StandardError) - UnexpectedDelimiter = Class.new(ParserError) - MissingEndDelimiter = Class.new(ParserError) - - def parse(text, our_path:, their_path:, parent_file: nil) - validate_text!(text) - - line_obj_index = 0 - line_old = 1 - line_new = 1 - type = nil - lines = [] - conflict_start = "<<<<<<< #{our_path}" - conflict_middle = '=======' - conflict_end = ">>>>>>> #{their_path}" - - text.each_line.map do |line| - full_line = line.delete("\n") - - if full_line == conflict_start - validate_delimiter!(type.nil?) - - type = 'new' - elsif full_line == conflict_middle - validate_delimiter!(type == 'new') - - type = 'old' - elsif full_line == conflict_end - validate_delimiter!(type == 'old') - - type = nil - elsif line[0] == '\\' - type = 'nonewline' - lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) - else - lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) - line_old += 1 if type != 'new' - line_new += 1 if type != 'old' - - line_obj_index += 1 - end - end - - raise MissingEndDelimiter unless type.nil? - - lines - end - - private - - def validate_text!(text) - raise UnmergeableFile if text.blank? # Typically a binary file - raise UnmergeableFile if text.length > 200.kilobytes - - text.force_encoding('UTF-8') - - raise UnsupportedEncoding unless text.valid_encoding? - end - - def validate_delimiter!(condition) - raise UnexpectedDelimiter unless condition - end - end - end -end diff --git a/lib/gitlab/conflict/resolution_error.rb b/lib/gitlab/conflict/resolution_error.rb deleted file mode 100644 index 0b61256b35a..00000000000 --- a/lib/gitlab/conflict/resolution_error.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Gitlab - module Conflict - ResolutionError = Class.new(StandardError) - end -end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 31a46a738c3..c169c8fe135 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -86,7 +86,7 @@ module Gitlab user_name: user.name, user_username: user.username, user_email: user.email, - user_avatar: user.avatar_url, + user_avatar: user.avatar_url(only_path: false), project_id: project.id, project: project.hook_attrs, commits: commit_attrs, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index a6ec75da385..43a00d6cedb 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -4,6 +4,10 @@ module Gitlab # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html MAX_INT_VALUE = 2147483647 + # The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz: + # https://www.postgresql.org/docs/9.1/static/datatype-datetime.html + # https://dev.mysql.com/doc/refman/5.7/en/datetime.html + MAX_TIMESTAMP_VALUE = Time.at((1 << 31) - 1).freeze def self.config ActiveRecord::Base.configurations[Rails.env] @@ -29,6 +33,15 @@ module Gitlab adapter_name.casecmp('postgresql').zero? end + # Overridden in EE + def self.read_only? + false + end + + def self.read_write? + !self.read_only? + end + def self.version database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end @@ -111,6 +124,10 @@ module Gitlab EOF end + def self.sanitize_timestamp(timestamp) + MAX_TIMESTAMP_VALUE > timestamp ? timestamp : MAX_TIMESTAMP_VALUE.dup + end + # pool_size - The size of the DB pool. # host - An optional host name to use instead of the default one. def self.create_connection_pool(pool_size, host = nil) diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 371cbe04b9b..c98eefbce25 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -13,9 +13,9 @@ module Gitlab def ==(other) other.is_a?(self.class) && - base_sha == other.base_sha && - start_sha == other.start_sha && - head_sha == other.head_sha + shas_equal?(base_sha, other.base_sha) && + shas_equal?(start_sha, other.start_sha) && + shas_equal?(head_sha, other.head_sha) end alias_method :eql?, :== @@ -47,6 +47,22 @@ module Gitlab CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) end end + + private + + def shas_equal?(sha1, sha2) + return true if sha1 == sha2 + return false if sha1.nil? || sha2.nil? + return false unless sha1.class == sha2.class + + length = [sha1.length, sha2.length].min + + # If either of the shas is below the minimum length, we cannot be sure + # that they actually refer to the same commit because of hash collision. + return false if length < Commit::MIN_SHA_LENGTH + + sha1[0, length] == sha2[0, length] + end end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index fcac85ff892..ea5891a028a 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -27,22 +27,29 @@ module Gitlab @fallback_diff_refs = fallback_diff_refs end - def position(line) + def position(position_marker, position_type: :text) return unless diff_refs - Position.new( + data = { + diff_refs: diff_refs, + position_type: position_type.to_s, old_path: old_path, - new_path: new_path, - old_line: line.old_line, - new_line: line.new_line, - diff_refs: diff_refs - ) + new_path: new_path + } + + if position_type == :text + data.merge!(text_position_properties(position_marker)) + else + data.merge!(image_position_properties(position_marker)) + end + + Position.new(data) end def line_code(line) return if line.meta? - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) end def line_for_line_code(code) @@ -228,6 +235,14 @@ module Gitlab private + def text_position_properties(line) + { old_line: line.old_line, new_line: line.new_line } + end + + def image_position_properties(image_point) + image_point.to_h + end + def blobs_changed? old_blob && new_blob && old_blob.id != new_blob.id end diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb new file mode 100644 index 00000000000..5e923b9e602 --- /dev/null +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -0,0 +1,61 @@ +module Gitlab + module Diff + module Formatters + class BaseFormatter + attr_reader :old_path + attr_reader :new_path + attr_reader :base_sha + attr_reader :start_sha + attr_reader :head_sha + attr_reader :position_type + + def initialize(attrs) + if diff_file = attrs[:diff_file] + attrs[:diff_refs] = diff_file.diff_refs + attrs[:old_path] = diff_file.old_path + attrs[:new_path] = diff_file.new_path + end + + if diff_refs = attrs[:diff_refs] + attrs[:base_sha] = diff_refs.base_sha + attrs[:start_sha] = diff_refs.start_sha + attrs[:head_sha] = diff_refs.head_sha + end + + @old_path = attrs[:old_path] + @new_path = attrs[:new_path] + @base_sha = attrs[:base_sha] + @start_sha = attrs[:start_sha] + @head_sha = attrs[:head_sha] + end + + def key + [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || "")] + end + + def to_h + { + base_sha: base_sha, + start_sha: start_sha, + head_sha: head_sha, + old_path: old_path, + new_path: new_path, + position_type: position_type + } + end + + def position_type + raise NotImplementedError + end + + def ==(other) + raise NotImplementedError + end + + def complete? + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/diff/formatters/image_formatter.rb b/lib/gitlab/diff/formatters/image_formatter.rb new file mode 100644 index 00000000000..ccd0d309972 --- /dev/null +++ b/lib/gitlab/diff/formatters/image_formatter.rb @@ -0,0 +1,43 @@ +module Gitlab + module Diff + module Formatters + class ImageFormatter < BaseFormatter + attr_reader :width + attr_reader :height + attr_reader :x + attr_reader :y + + def initialize(attrs) + @x = attrs[:x] + @y = attrs[:y] + @width = attrs[:width] + @height = attrs[:height] + + super(attrs) + end + + def key + @key ||= super.push(x, y) + end + + def complete? + x && y && width && height + end + + def to_h + super.merge(width: width, height: height, x: x, y: y) + end + + def position_type + "image" + end + + def ==(other) + other.is_a?(self.class) && + x == other.x && + y == other.y + end + end + end + end +end diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb new file mode 100644 index 00000000000..01c7e9f51ab --- /dev/null +++ b/lib/gitlab/diff/formatters/text_formatter.rb @@ -0,0 +1,49 @@ +module Gitlab + module Diff + module Formatters + class TextFormatter < BaseFormatter + attr_reader :old_line + attr_reader :new_line + + def initialize(attrs) + @old_line = attrs[:old_line] + @new_line = attrs[:new_line] + + super(attrs) + end + + def key + @key ||= super.push(old_line, new_line) + end + + def complete? + old_line || new_line + end + + def to_h + super.merge(old_line: old_line, new_line: new_line) + end + + def line_age + if old_line && new_line + nil + elsif new_line + 'new' + else + 'old' + end + end + + def position_type + "text" + end + + def ==(other) + other.is_a?(self.class) && + new_line == other.new_line && + old_line == other.old_line + end + end + end + end +end diff --git a/lib/gitlab/diff/image_point.rb b/lib/gitlab/diff/image_point.rb new file mode 100644 index 00000000000..65332dfd239 --- /dev/null +++ b/lib/gitlab/diff/image_point.rb @@ -0,0 +1,23 @@ +module Gitlab + module Diff + class ImagePoint + attr_reader :width, :height, :x, :y + + def initialize(width, height, x, y) + @width = width + @height = height + @x = x + @y = y + end + + def to_h + { + width: width, + height: height, + x: x, + y: y + } + end + end + end +end diff --git a/lib/gitlab/diff/line_code.rb b/lib/gitlab/diff/line_code.rb deleted file mode 100644 index f3578ab3d35..00000000000 --- a/lib/gitlab/diff/line_code.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Gitlab - module Diff - class LineCode - def self.generate(file_path, new_line_position, old_line_position) - "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}" - end - end - end -end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 742f989c50b..7dc9cc7c281 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -17,7 +17,9 @@ module Gitlab # without having to instantiate all the others that come after it. Enumerator.new do |yielder| @lines.each do |line| - next if filename?(line) + # We're expecting a filename parameter only in a meta-part of the diff content + # when type is defined then we're already in a content-part + next if filename?(line) && type.nil? full_line = line.delete("\n") diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index f80afb20f0c..ccfb908bcca 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -1,37 +1,25 @@ -# Defines a specific location, identified by paths and line numbers, +# Defines a specific location, identified by paths line numbers and image coordinates, # within a specific diff, identified by start, head and base commit ids. module Gitlab module Diff class Position - attr_reader :old_path - attr_reader :new_path - attr_reader :old_line - attr_reader :new_line - attr_reader :base_sha - attr_reader :start_sha - attr_reader :head_sha - + attr_accessor :formatter + + delegate :old_path, + :new_path, + :base_sha, + :start_sha, + :head_sha, + :old_line, + :new_line, + :position_type, to: :formatter + + # A position can belong to a text line or to an image coordinate + # it depends of the position_type argument. + # Text position will have: new_line and old_line + # Image position will have: width, height, x, y def initialize(attrs = {}) - if diff_file = attrs[:diff_file] - attrs[:diff_refs] = diff_file.diff_refs - attrs[:old_path] = diff_file.old_path - attrs[:new_path] = diff_file.new_path - end - - if diff_refs = attrs[:diff_refs] - attrs[:base_sha] = diff_refs.base_sha - attrs[:start_sha] = diff_refs.start_sha - attrs[:head_sha] = diff_refs.head_sha - end - - @old_path = attrs[:old_path] - @new_path = attrs[:new_path] - @base_sha = attrs[:base_sha] - @start_sha = attrs[:start_sha] - @head_sha = attrs[:head_sha] - - @old_line = attrs[:old_line] - @new_line = attrs[:new_line] + @formatter = get_formatter_class(attrs[:position_type]).new(attrs) end # `Gitlab::Diff::Position` objects are stored as serialized attributes in @@ -46,27 +34,23 @@ module Gitlab end def encode_with(coder) - coder['attributes'] = self.to_h + coder['attributes'] = formatter.to_h end def key - @key ||= [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || ""), old_line, new_line] + formatter.key end def ==(other) - other.is_a?(self.class) && key == other.key + other.is_a?(self.class) && + other.diff_refs == diff_refs && + other.old_path == old_path && + other.new_path == new_path && + other.formatter == formatter end def to_h - { - old_path: old_path, - new_path: new_path, - old_line: old_line, - new_line: new_line, - base_sha: base_sha, - start_sha: start_sha, - head_sha: head_sha - } + formatter.to_h end def inspect @@ -74,23 +58,15 @@ module Gitlab end def complete? - file_path.present? && - (old_line || new_line) && - diff_refs.complete? + file_path.present? && formatter.complete? && diff_refs.complete? end def to_json(opts = nil) - JSON.generate(self.to_h, opts) + JSON.generate(formatter.to_h, opts) end def type - if old_line && new_line - nil - elsif new_line - 'new' - else - 'old' - end + formatter.line_age end def unchanged? @@ -118,7 +94,9 @@ module Gitlab end def diff_file(repository) - @diff_file ||= begin + return @diff_file if defined?(@diff_file) + + @diff_file = begin if RequestStore.active? key = { project_id: repository.project.id, @@ -146,8 +124,19 @@ module Gitlab def find_diff_file(repository) return unless diff_refs.complete? + return unless comparison = diff_refs.compare_in(repository.project) + comparison.diffs(paths: paths, expanded: true).diff_files.first + end - diff_refs.compare_in(repository.project).diffs(paths: paths, expanded: true).diff_files.first + def get_formatter_class(type) + type ||= "text" + + case type + when 'image' + Gitlab::Diff::Formatters::ImageFormatter + else + Gitlab::Diff::Formatters::TextFormatter + end end end end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index c5a8ea12245..0ea534a5fd0 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -2,8 +2,8 @@ module Gitlab # Checks if a set of migrations requires downtime or not. class EeCompatCheck - CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze - EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze + DEFAULT_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze + EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze PLEASE_READ_THIS_BANNER = %Q{ @@ -17,14 +17,16 @@ module Gitlab ============================================================\n }.freeze - attr_reader :ee_repo_dir, :patches_dir, :ce_repo, :ce_branch, :ee_branch_found - attr_reader :failed_files + attr_reader :ee_repo_dir, :patches_dir, :ce_project_url, :ce_repo_url, :ce_branch, :ee_branch_found + attr_reader :job_id, :failed_files - def initialize(branch:, ce_repo: CE_REPO) + def initialize(branch:, ce_project_url: DEFAULT_CE_PROJECT_URL, job_id: nil) @ee_repo_dir = CHECK_DIR.join('ee-repo') @patches_dir = CHECK_DIR.join('patches') @ce_branch = branch - @ce_repo = ce_repo + @ce_project_url = ce_project_url + @ce_repo_url = "#{ce_project_url}.git" + @job_id = job_id end def check @@ -59,8 +61,8 @@ module Gitlab step("#{ee_repo_dir} already exists") else step( - "Cloning #{EE_REPO} into #{ee_repo_dir}", - %W[git clone --branch master --single-branch --depth=200 #{EE_REPO} #{ee_repo_dir}] + "Cloning #{EE_REPO_URL} into #{ee_repo_dir}", + %W[git clone --branch master --single-branch --depth=200 #{EE_REPO_URL} #{ee_repo_dir}] ) end end @@ -132,7 +134,7 @@ module Gitlab def check_patch(patch_path) step("Checking out master", %w[git checkout master]) step("Resetting to latest master", %w[git reset --hard origin/master]) - step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}]) + step("Fetching CE/#{ce_branch}", %W[git fetch #{ce_repo_url} #{ce_branch}]) step( "Checking if #{patch_path} applies cleanly to EE/master", # Don't use --check here because it can result in a 0-exit status even @@ -237,7 +239,7 @@ module Gitlab end def patch_url - "https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/#{ENV['CI_JOB_ID']}/artifacts/raw/ee_compat_check/patches/#{ce_patch_name}" + "#{ce_project_url}/-/jobs/#{job_id}/artifacts/raw/ee_compat_check/patches/#{ce_patch_name}" end def step(desc, cmd = nil) @@ -304,7 +306,7 @@ module Gitlab # In the EE repo $ git fetch origin $ git checkout -b #{ee_branch_prefix} origin/master - $ git fetch #{ce_repo} #{ce_branch} + $ git fetch #{ce_repo_url} #{ce_branch} $ git cherry-pick SHA # Repeat for all the commits you want to pick You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit. diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 7b3483a7f96..99dfee3dd9b 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -14,9 +14,9 @@ module Gitlab ENCODING_CONFIDENCE_THRESHOLD = 50 def encode!(message) - return nil unless message.respond_to? :force_encoding + return nil unless message.respond_to?(:force_encoding) + return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? - # if message is utf-8 encoding, just return it message.force_encoding("UTF-8") return message if message.valid_encoding? @@ -50,6 +50,9 @@ module Gitlab end def encode_utf8(message) + return nil unless message.is_a?(String) + return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + detect = CharlockHolmes::EncodingDetector.detect(message) if detect && detect[:encoding] begin diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index a8cb7fc3fe7..0e9ef4f897c 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -6,31 +6,33 @@ module Gitlab module FileDetector PATTERNS = { # Project files - readme: /\Areadme/i, - changelog: /\A(changelog|history|changes|news)/i, - license: /\A(licen[sc]e|copying)(\..+|\z)/i, - contributing: /\Acontributing/i, + readme: /\Areadme[^\/]*\z/i, + changelog: /\A(changelog|history|changes|news)[^\/]*\z/i, + license: /\A(licen[sc]e|copying)(\.[^\/]+)?\z/i, + contributing: /\Acontributing[^\/]*\z/i, version: 'version', avatar: /\Alogo\.(png|jpg|gif)\z/, + issue_template: /\A\.gitlab\/issue_templates\/[^\/]+\.md\z/, + merge_request_template: /\A\.gitlab\/merge_request_templates\/[^\/]+\.md\z/, # Configuration files gitignore: '.gitignore', koding: '.koding.yml', gitlab_ci: '.gitlab-ci.yml', - route_map: 'route-map.yml', + route_map: '.gitlab/route-map.yml', # Dependency files - cartfile: /\ACartfile/, + cartfile: /\ACartfile[^\/]*\z/, composer_json: 'composer.json', gemfile: /\A(Gemfile|gems\.rb)\z/, gemfile_lock: 'Gemfile.lock', - gemspec: /\.gemspec\z/, + gemspec: /\A[^\/]*\.gemspec\z/, godeps_json: 'Godeps.json', package_json: 'package.json', podfile: 'Podfile', - podspec_json: /\.podspec\.json\z/, - podspec: /\.podspec\z/, - requirements_txt: /requirements\.txt\z/, + podspec_json: /\A[^\/]*\.podspec\.json\z/, + podspec: /\A[^\/]*\.podspec\z/, + requirements_txt: /\A[^\/]*requirements\.txt\z/, yarn_lock: 'yarn.lock' }.freeze @@ -63,13 +65,11 @@ module Gitlab # type_of('README.md') # => :readme # type_of('VERSION') # => :version def self.type_of(path) - name = File.basename(path) - PATTERNS.each do |type, search| did_match = if search.is_a?(Regexp) - name =~ search + path =~ search else - name.casecmp(search) == 0 + path.casecmp(search) == 0 end return type if did_match diff --git a/lib/gitlab/gcp/model.rb b/lib/gitlab/gcp/model.rb new file mode 100644 index 00000000000..195391f0e3c --- /dev/null +++ b/lib/gitlab/gcp/model.rb @@ -0,0 +1,13 @@ +module Gitlab + module Gcp + module Model + def table_name_prefix + "gcp_" + end + + def model_name + @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) + end + end + end +end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index c78fe63f9b5..1f31cdbc96d 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -66,6 +66,10 @@ module Gitlab end end end + + def diff_line_code(file_path, new_line_position, old_line_position) + "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}" + end end end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index a4336facee5..cc6c7609ec7 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -12,6 +12,12 @@ module Gitlab # blob data should use load_all_data!. MAX_DATA_DISPLAY_SIZE = 10.megabytes + # These limits are used as a heuristic to ignore files which can't be LFS + # pointers. The format of these is described in + # https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md#the-pointer + LFS_POINTER_MIN_SIZE = 120.bytes + LFS_POINTER_MAX_SIZE = 200.bytes + attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary class << self @@ -30,16 +36,7 @@ module Gitlab if is_enabled Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) else - blob = repository.lookup(sha) - - next unless blob.is_a?(Rugged::Blob) - - new( - id: blob.oid, - size: blob.size, - data: blob.content(MAX_DATA_DISPLAY_SIZE), - binary: blob.binary? - ) + rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE) end end end @@ -59,10 +56,25 @@ module Gitlab end end + # Find LFS blobs given an array of sha ids + # Returns array of Gitlab::Git::Blob + # Does not guarantee blob data will be set + def batch_lfs_pointers(repository, blob_ids) + blob_ids.lazy + .select { |sha| possible_lfs_blob?(repository, sha) } + .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } + .select(&:lfs_pointer?) + .force + end + def binary?(data) EncodingHelper.detect_libgit2_binary?(data) end + def size_could_be_lfs?(size) + size.between?(LFS_POINTER_MIN_SIZE, LFS_POINTER_MAX_SIZE) + end + private # Recursive search of blob id by path @@ -167,6 +179,29 @@ module Gitlab end end end + + def rugged_raw(repository, sha, limit:) + blob = repository.lookup(sha) + + return unless blob.is_a?(Rugged::Blob) + + new( + id: blob.oid, + size: blob.size, + data: blob.content(limit), + binary: blob.binary? + ) + end + + # Efficient lookup to determine if object size + # and type make it a possible LFS blob without loading + # blob content into memory with repository.lookup(sha) + def possible_lfs_blob?(repository, sha) + object_header = repository.rugged.read_header(sha) + + object_header[:type] == :blob && + size_could_be_lfs?(object_header[:len]) + end end def initialize(options) @@ -226,7 +261,7 @@ module Gitlab # size # see https://github.com/github/git-lfs/blob/v1.1.0/docs/spec.md#the-pointer def lfs_pointer? - has_lfs_version_key? && lfs_oid.present? && lfs_size.present? + self.class.size_could_be_lfs?(size) && has_lfs_version_key? && lfs_oid.present? && lfs_size.present? end def lfs_oid diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index c53882787f1..3487e099381 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -3,6 +3,14 @@ module Gitlab module Git class Branch < Ref + def self.find(repo, branch_name) + if branch_name.is_a?(Gitlab::Git::Branch) + branch_name + else + repo.find_branch(branch_name) + end + end + def initialize(repository, name, target, target_commit) super(repository, name, target, target_commit) end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 1957c254c28..d5518814483 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -72,7 +72,8 @@ module Gitlab decorate(repo, commit) if commit rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, - Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository + Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository, + Rugged::OdbError, Rugged::TreeError, ArgumentError nil end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb new file mode 100644 index 00000000000..fc1595f1faf --- /dev/null +++ b/lib/gitlab/git/conflict/file.rb @@ -0,0 +1,86 @@ +module Gitlab + module Git + module Conflict + class File + attr_reader :content, :their_path, :our_path, :our_mode, :repository + + def initialize(repository, commit_oid, conflict, content) + @repository = repository + @commit_oid = commit_oid + @their_path = conflict[:theirs][:path] + @our_path = conflict[:ours][:path] + @our_mode = conflict[:ours][:mode] + @content = content + end + + def lines + return @lines if defined?(@lines) + + begin + @type = 'text' + @lines = Gitlab::Git::Conflict::Parser.parse(content, + our_path: our_path, + their_path: their_path) + rescue Gitlab::Git::Conflict::Parser::ParserError + @type = 'text-editor' + @lines = nil + end + end + + def type + lines unless @type + + @type.inquiry + end + + def our_blob + # REFACTOR NOTE: the source of `commit_oid` used to be + # `merge_request.diff_refs.head_sha`. Instead of passing this value + # around the new lib structure, I decided to use `@commit_oid` which is + # equivalent to `merge_request.source_branch_head.raw.rugged_commit.oid`. + # That is what `merge_request.diff_refs.head_sha` is equivalent to when + # `merge_request` is not persisted (see `MergeRequest#diff_head_commit`). + # I think using the same oid is more consistent anyways, but if Conflicts + # start breaking, the change described above is a good place to look at. + @our_blob ||= repository.blob_at(@commit_oid, our_path) + end + + def line_code(line) + Gitlab::Git.diff_line_code(our_path, line[:line_new], line[:line_old]) + end + + def resolve_lines(resolution) + section_id = nil + + lines.map do |line| + unless line[:type] + section_id = nil + next line + end + + section_id ||= line_code(line) + + case resolution[section_id] + when 'head' + next unless line[:type] == 'new' + when 'origin' + next unless line[:type] == 'old' + else + raise Gitlab::Git::Conflict::Resolver::ResolutionError, "Missing resolution for section ID: #{section_id}" + end + + line + end.compact + end + + def resolve_content(resolution) + if resolution == content + raise Gitlab::Git::Conflict::Resolver::ResolutionError, "Resolved content has no changes for file #{our_path}" + end + + resolution + end + end + end + end +end diff --git a/lib/gitlab/git/conflict/parser.rb b/lib/gitlab/git/conflict/parser.rb new file mode 100644 index 00000000000..3effa9d2d31 --- /dev/null +++ b/lib/gitlab/git/conflict/parser.rb @@ -0,0 +1,91 @@ +module Gitlab + module Git + module Conflict + class Parser + UnresolvableError = Class.new(StandardError) + UnmergeableFile = Class.new(UnresolvableError) + UnsupportedEncoding = Class.new(UnresolvableError) + + # Recoverable errors - the conflict can be resolved in an editor, but not with + # sections. + ParserError = Class.new(StandardError) + UnexpectedDelimiter = Class.new(ParserError) + MissingEndDelimiter = Class.new(ParserError) + + class << self + def parse(text, our_path:, their_path:, parent_file: nil) + validate_text!(text) + + line_obj_index = 0 + line_old = 1 + line_new = 1 + type = nil + lines = [] + conflict_start = "<<<<<<< #{our_path}" + conflict_middle = '=======' + conflict_end = ">>>>>>> #{their_path}" + + text.each_line.map do |line| + full_line = line.delete("\n") + + if full_line == conflict_start + validate_delimiter!(type.nil?) + + type = 'new' + elsif full_line == conflict_middle + validate_delimiter!(type == 'new') + + type = 'old' + elsif full_line == conflict_end + validate_delimiter!(type == 'old') + + type = nil + elsif line[0] == '\\' + type = 'nonewline' + lines << { + full_line: full_line, + type: type, + line_obj_index: line_obj_index, + line_old: line_old, + line_new: line_new + } + else + lines << { + full_line: full_line, + type: type, + line_obj_index: line_obj_index, + line_old: line_old, + line_new: line_new + } + + line_old += 1 if type != 'new' + line_new += 1 if type != 'old' + + line_obj_index += 1 + end + end + + raise MissingEndDelimiter unless type.nil? + + lines + end + + private + + def validate_text!(text) + raise UnmergeableFile if text.blank? # Typically a binary file + raise UnmergeableFile if text.length > 200.kilobytes + + text.force_encoding('UTF-8') + + raise UnsupportedEncoding unless text.valid_encoding? + end + + def validate_delimiter!(condition) + raise UnexpectedDelimiter unless condition + end + end + end + end + end +end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb new file mode 100644 index 00000000000..df509c5f4ce --- /dev/null +++ b/lib/gitlab/git/conflict/resolver.rb @@ -0,0 +1,91 @@ +module Gitlab + module Git + module Conflict + class Resolver + ConflictSideMissing = Class.new(StandardError) + ResolutionError = Class.new(StandardError) + + def initialize(repository, our_commit, target_repository, their_commit) + @repository = repository + @our_commit = our_commit.rugged_commit + @target_repository = target_repository + @their_commit = their_commit.rugged_commit + end + + def conflicts + @conflicts ||= begin + target_index = @target_repository.rugged.merge_commits(@our_commit, @their_commit) + + # We don't need to do `with_repo_branch_commit` here, because the target + # project always fetches source refs when creating merge request diffs. + target_index.conflicts.map do |conflict| + raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours] + + Gitlab::Git::Conflict::File.new( + @target_repository, + @our_commit.oid, + conflict, + target_index.merge_file(conflict[:ours][:path])[:data] + ) + end + end + end + + def resolve_conflicts(user, files, source_branch:, target_branch:, commit_message:) + @repository.with_repo_branch_commit(@target_repository, target_branch) do + files.each do |file_params| + conflict_file = conflict_for_path(file_params[:old_path], file_params[:new_path]) + + write_resolved_file_to_index(conflict_file, file_params) + end + + unless index.conflicts.empty? + missing_files = index.conflicts.map { |file| file[:ours][:path] } + + raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" + end + + commit_params = { + message: commit_message, + parents: [@our_commit, @their_commit].map(&:oid) + } + + @repository.commit_index(user, source_branch, index, commit_params) + end + end + + def conflict_for_path(old_path, new_path) + conflicts.find do |conflict| + conflict.their_path == old_path && conflict.our_path == new_path + end + end + + private + + # We can only write when getting the merge index from the source + # project, because we will write to that project. We don't use this all + # the time because this fetches a ref into the source project, which + # isn't needed for reading. + def index + @index ||= @repository.rugged.merge_commits(@our_commit, @their_commit) + end + + def write_resolved_file_to_index(file, params) + if params[:sections] + resolved_lines = file.resolve_lines(params[:sections]) + new_file = resolved_lines.map { |line| line[:full_line] }.join("\n") + + new_file << "\n" if file.our_blob.data.ends_with?("\n") + elsif params[:content] + new_file = file.resolve_content(params[:content]) + end + + our_path = file.our_path + + index.add(path: our_path, oid: @repository.rugged.write(new_file, :blob), mode: file.our_mode) + index.conflict_remove(our_path) + end + end + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 096301d300f..ca94b4baa59 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -24,41 +24,13 @@ module Gitlab SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze - class << self - # The maximum size of a diff to display. - def size_limit - if RequestStore.active? - RequestStore['gitlab_git_diff_size_limit'] ||= find_size_limit - else - find_size_limit - end - end - - # The maximum size before a diff is collapsed. - def collapse_limit - if RequestStore.active? - RequestStore['gitlab_git_diff_collapse_limit'] ||= find_collapse_limit - else - find_collapse_limit - end - end + # The maximum size of a diff to display. + SIZE_LIMIT = 100.kilobytes - def find_size_limit - if Feature.enabled?('gitlab_git_diff_size_limit_increase') - 200.kilobytes - else - 100.kilobytes - end - end - - def find_collapse_limit - if Feature.enabled?('gitlab_git_diff_size_limit_increase') - 100.kilobytes - else - 10.kilobytes - end - end + # The maximum size before a diff is collapsed. + COLLAPSE_LIMIT = 10.kilobytes + class << self def between(repo, head, base, options = {}, *paths) straight = options.delete(:straight) || false @@ -172,7 +144,7 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= self.class.size_limit + @too_large = @diff.bytesize >= SIZE_LIMIT else @too_large end @@ -190,7 +162,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= self.class.collapse_limit + @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT end def collapse! @@ -275,14 +247,14 @@ module Gitlab hunk.each_line do |line| size += line.content.bytesize - if size >= self.class.size_limit + if size >= SIZE_LIMIT too_large! return true end end end - if !expanded && size >= self.class.collapse_limit + if !expanded && size >= COLLAPSE_LIMIT collapse! return true end diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb index f80193ac553..9d0b47a1a6d 100644 --- a/lib/gitlab/git/env.rb +++ b/lib/gitlab/git/env.rb @@ -11,9 +11,11 @@ module Gitlab # # This class is thread-safe via RequestStore. class Env - WHITELISTED_GIT_VARIABLES = %w[ + WHITELISTED_VARIABLES = %w[ GIT_OBJECT_DIRECTORY + GIT_OBJECT_DIRECTORY_RELATIVE GIT_ALTERNATE_OBJECT_DIRECTORIES + GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE ].freeze def self.set(env) @@ -28,12 +30,23 @@ module Gitlab RequestStore.fetch(:gitlab_git_env) { {} } end + def self.to_env_hash + env = {} + + all.compact.each do |key, value| + value = value.join(File::PATH_SEPARATOR) if value.is_a?(Array) + env[key.to_s] = value + end + + env + end + def self.[](key) all[key] end def self.whitelist_git_env(env) - env.select { |key, _| WHITELISTED_GIT_VARIABLES.include?(key.to_s) }.with_indifferent_access + env.select { |key, _| WHITELISTED_VARIABLES.include?(key.to_s) }.with_indifferent_access end end end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 208e4bbaf60..e29a1f7afa1 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -22,22 +22,22 @@ module Gitlab File.exist?(path) end - def trigger(gl_id, oldrev, newrev, ref) + def trigger(gl_id, gl_username, oldrev, newrev, ref) return [true, nil] unless exists? Bundler.with_clean_env do case name when "pre-receive", "post-receive" - call_receive_hook(gl_id, oldrev, newrev, ref) + call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) when "update" - call_update_hook(gl_id, oldrev, newrev, ref) + call_update_hook(gl_id, gl_username, oldrev, newrev, ref) end end end private - def call_receive_hook(gl_id, oldrev, newrev, ref) + def call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") exit_status = false @@ -45,6 +45,7 @@ module Gitlab vars = { 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username, 'PWD' => repo_path, 'GL_PROTOCOL' => GL_PROTOCOL, 'GL_REPOSITORY' => repository.gl_repository @@ -80,9 +81,13 @@ module Gitlab [exit_status, exit_message] end - def call_update_hook(gl_id, oldrev, newrev, ref) + def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) Dir.chdir(repo_path) do - stdout, stderr, status = Open3.capture3({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + env = { + 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username + } + stdout, stderr, status = Open3.capture3(env, path, ref, oldrev, newrev) [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] end end diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb index ea8a87a1290..f302b852b35 100644 --- a/lib/gitlab/git/hooks_service.rb +++ b/lib/gitlab/git/hooks_service.rb @@ -5,12 +5,13 @@ module Gitlab attr_accessor :oldrev, :newrev, :ref - def execute(committer, repository, oldrev, newrev, ref) - @repository = repository - @gl_id = committer.gl_id - @oldrev = oldrev - @newrev = newrev - @ref = ref + def execute(pusher, repository, oldrev, newrev, ref) + @repository = repository + @gl_id = pusher.gl_id + @gl_username = pusher.username + @oldrev = oldrev + @newrev = newrev + @ref = ref %w(pre-receive update).each do |hook_name| status, message = run_hook(hook_name) @@ -29,7 +30,7 @@ module Gitlab def run_hook(name) hook = Gitlab::Git::Hook.new(name, @repository) - hook.trigger(@gl_id, oldrev, newrev, ref) + hook.trigger(@gl_id, @gl_username, oldrev, newrev, ref) end end end diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb new file mode 100644 index 00000000000..732dd5d998a --- /dev/null +++ b/lib/gitlab/git/lfs_changes.rb @@ -0,0 +1,33 @@ +module Gitlab + module Git + class LfsChanges + def initialize(repository, newrev) + @repository = repository + @newrev = newrev + end + + def new_pointers(object_limit: nil, not_in: nil) + @new_pointers ||= begin + rev_list.new_objects(not_in: not_in, require_path: true) do |object_ids| + object_ids = object_ids.take(object_limit) if object_limit + + Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) + end + end + end + + def all_pointers + rev_list.all_objects(require_path: true) do |object_ids| + Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) + end + end + + private + + def rev_list + ::Gitlab::Git::RevList.new(path_to_repo: @repository.path_to_repo, + newrev: @newrev) + end + end + end +end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index 786e2e7e8dc..ab94ba8a73a 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -3,9 +3,17 @@ module Gitlab class OperationService include Gitlab::Git::Popen - WithBranchResult = Struct.new(:newrev, :repo_created, :branch_created) do + BranchUpdate = Struct.new(:newrev, :repo_created, :branch_created) do alias_method :repo_created?, :repo_created alias_method :branch_created?, :branch_created + + def self.from_gitaly(branch_update) + new( + branch_update.commit_id, + branch_update.repo_created, + branch_update.branch_created + ) + end end attr_reader :user, :repository @@ -112,7 +120,7 @@ module Gitlab ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name update_ref_in_hooks(ref, newrev, oldrev) - WithBranchResult.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) + BranchUpdate.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) end def find_oldrev_from_branch(newrev, branch) @@ -152,13 +160,15 @@ module Gitlab # (and have!) accidentally reset the ref to an earlier state, clobbering # commits. See also https://github.com/libgit2/libgit2/issues/1534. command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - _, status = popen( + + output, status = popen( command, repository.path) do |stdin| stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") end unless status.zero? + Gitlab::GitLogger.error("'git update-ref' in #{repository.path}: #{output}") raise Gitlab::Git::CommitError.new( "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ " Please refresh and try again.") diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index 054d45895a5..1ccca13ce2f 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -5,7 +5,9 @@ require 'open3' module Gitlab module Git module Popen - def popen(cmd, path, vars = {}) + FAST_GIT_PROCESS_TIMEOUT = 15.seconds + + def popen(cmd, path, vars = {}, lazy_block: nil) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end @@ -20,13 +22,79 @@ module Gitlab yield(stdin) if block_given? stdin.close - cmd_output << stdout.read + if lazy_block + return lazy_block.call(stdout.lazy) + else + cmd_output << stdout.read + end + cmd_output << stderr.read cmd_status = wait_thr.value.exitstatus end [cmd_output, cmd_status] end + + def popen_with_timeout(cmd, timeout, path, vars = {}) + unless cmd.is_a?(Array) + raise "System commands must be given as an array of strings" + end + + path ||= Dir.pwd + vars['PWD'] = path + + unless File.directory?(path) + FileUtils.mkdir_p(path) + end + + rout, wout = IO.pipe + rerr, werr = IO.pipe + + pid = Process.spawn(vars, *cmd, out: wout, err: werr, chdir: path, pgroup: true) + + begin + status = process_wait_with_timeout(pid, timeout) + + # close write ends so we could read them + wout.close + werr.close + + cmd_output = rout.readlines.join + cmd_output << rerr.readlines.join # Copying the behaviour of `popen` which merges stderr into output + + [cmd_output, status.exitstatus] + rescue Timeout::Error => e + kill_process_group_for_pid(pid) + + raise e + ensure + wout.close unless wout.closed? + werr.close unless werr.closed? + + rout.close + rerr.close + end + end + + def process_wait_with_timeout(pid, timeout) + deadline = timeout.seconds.from_now + wait_time = 0.01 + + while deadline > Time.now + sleep(wait_time) + _, status = Process.wait2(pid, Process::WNOHANG) + + return status unless status.nil? + end + + raise Timeout::Error, "Timeout waiting for process ##{pid}" + end + + def kill_process_group_for_pid(pid) + Process.kill("KILL", -pid) + Process.wait(pid) + rescue Errno::ESRCH + end end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 10ba29acbd1..182ffc96ef9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -6,12 +6,17 @@ require "rubygems/package" module Gitlab module Git class Repository + include Gitlab::Git::RepositoryMirroring include Gitlab::Git::Popen ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES ].freeze + ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY_RELATIVE + GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE + ].freeze SEARCH_CONTEXT_LINES = 3 NoRepository = Class.new(StandardError) @@ -20,13 +25,11 @@ module Gitlab GitError = Class.new(StandardError) DeleteBranchError = Class.new(StandardError) CreateTreeError = Class.new(StandardError) + TagExistsError = Class.new(StandardError) class << self - # Unlike `new`, `create` takes the storage path, not the storage name - def create(storage_path, name, bare: true, symlink_hooks_to: nil) - repo_path = File.join(storage_path, name) - repo_path += '.git' unless repo_path.end_with?('.git') - + # Unlike `new`, `create` takes the repository path + def create(repo_path, bare: true, symlink_hooks_to: nil) FileUtils.mkdir_p(repo_path, mode: 0770) # Equivalent to `git --git-path=#{repo_path} init [--bare]` @@ -55,14 +58,15 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path + attr_reader :storage, :gl_repository, :relative_path, :gitaly_resolver - # 'path' must be the path to a _bare_ git repository, e.g. - # /path/to/my-repo.git + # This initializer method is only used on the client side (gitlab-ce). + # Gitaly-ruby uses a different initializer. def initialize(storage, relative_path, gl_repository) @storage = storage @relative_path = relative_path @gl_repository = gl_repository + @gitaly_resolver = Gitlab::GitalyClient storage_path = Gitlab.config.repositories.storages[@storage]['path'] @path = File.join(storage_path, @relative_path) @@ -73,8 +77,6 @@ module Gitlab delegate :empty?, to: :rugged - delegate :exists?, to: :gitaly_repository_client - def ==(other) path == other.path end @@ -102,6 +104,18 @@ module Gitlab @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end + def exists? + Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| + if enabled + gitaly_repository_client.exists? + else + circuit_breaker.perform do + File.exist?(File.join(@path, 'refs')) + end + end + end + end + # Returns an Array of branch names # sorted by name ASC def branch_names @@ -153,7 +167,7 @@ module Gitlab end def local_branches(sort_by: nil) - gitaly_migrate(:local_branches) do |is_enabled| + gitaly_migrate(:local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_ref_client.local_branches(sort_by: sort_by) else @@ -181,6 +195,28 @@ module Gitlab end end + def has_local_branches? + gitaly_migrate(:has_local_branches) do |is_enabled| + if is_enabled + gitaly_repository_client.has_local_branches? + else + has_local_branches_rugged? + end + end + end + + def has_local_branches_rugged? + rugged.branches.each(:local).any? do |ref| + begin + ref.name && ref.target # ensures the branch is valid + + true + rescue Rugged::ReferenceError + false + end + end + end + # Returns the number of valid tags def tag_count gitaly_migrate(:tag_names) do |is_enabled| @@ -255,6 +291,14 @@ module Gitlab end end + def batch_existence(object_ids, existing: true) + filter_method = existing ? :select : :reject + + object_ids.public_send(filter_method) do |oid| # rubocop:disable GitlabSecurity/PublicSend + rugged.exists?(oid) + end + end + # Returns an Array of branch and tag names def ref_names branch_names + tag_names @@ -386,7 +430,13 @@ module Gitlab options[:limit] ||= 0 options[:offset] ||= 0 - raw_log(options).map { |c| Commit.decorate(self, c) } + gitaly_migrate(:find_commits) do |is_enabled| + if is_enabled + gitaly_commit_client.find_commits(options) + else + raw_log(options).map { |c| Commit.decorate(self, c) } + end + end end # Used in gitaly-ruby @@ -470,6 +520,10 @@ module Gitlab gitaly_commit_client.ancestor?(from, to) end + def merged_branch_names(branch_names = []) + Set.new(git_merged_branch_names(branch_names)) + end + # Return an array of Diff objects that represent the diff # between +from+ and +to+. See Diff::filter_diff_options for the allowed # diff options. The +options+ hash can also include :break_rewrites to @@ -620,49 +674,60 @@ module Gitlab end def add_branch(branch_name, user:, target:) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - OperationService.new(user, self).add_branch(branch_name, target_object.oid) - find_branch(branch_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex + gitaly_migrate(:operation_user_create_branch) do |is_enabled| + if is_enabled + gitaly_add_branch(branch_name, user, target) + else + rugged_add_branch(branch_name, user, target) + end + end end def add_tag(tag_name, user:, target:, message: nil) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - - options = nil # Use nil, not the empty hash. Rugged cares about this. - if message - options = { - message: message, - tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) - } + gitaly_migrate(:operation_user_add_tag) do |is_enabled| + if is_enabled + gitaly_add_tag(tag_name, user: user, target: target, message: message) + else + rugged_add_tag(tag_name, user: user, target: target, message: message) + end end - - OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) - - find_tag(tag_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex end def rm_branch(branch_name, user:) - OperationService.new(user, self).rm_branch(find_branch(branch_name)) + gitaly_migrate(:operation_user_delete_branch) do |is_enabled| + if is_enabled + gitaly_operations_client.user_delete_branch(branch_name, user) + else + OperationService.new(user, self).rm_branch(find_branch(branch_name)) + end + end end def rm_tag(tag_name, user:) - OperationService.new(user, self).rm_tag(find_tag(tag_name)) + gitaly_migrate(:operation_user_delete_tag) do |is_enabled| + if is_enabled + gitaly_operations_client.rm_tag(tag_name, user) + else + Gitlab::Git::OperationService.new(user, self).rm_tag(find_tag(tag_name)) + end + end end def find_tag(name) tags.find { |tag| tag.name == name } end - def merge(user, source_sha, target_branch, message) + def merge(user, source_sha, target_branch, message, &block) + gitaly_migrate(:operation_user_merge_branch) do |is_enabled| + if is_enabled + gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) + else + rugged_merge(user, source_sha, target_branch, message, &block) + end + end + end + + def rugged_merge(user, source_sha, target_branch, message) committer = Gitlab::Git.committer_hash(email: user.email, name: user.name) OperationService.new(user, self).with_branch(target_branch) do |start_commit| @@ -693,6 +758,16 @@ module Gitlab nil end + def ff_merge(user, source_sha, target_branch) + gitaly_migrate(:operation_user_ff_branch) do |is_enabled| + if is_enabled + gitaly_ff_merge(user, source_sha, target_branch) + else + rugged_ff_merge(user, source_sha, target_branch) + end + end + end + def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) OperationService.new(user, self).with_branch( branch_name, @@ -824,16 +899,25 @@ module Gitlab end end - # Delete the specified remote from this repository. - def remote_delete(remote_name) - rugged.remotes.delete(remote_name) - nil + def add_remote(remote_name, url) + rugged.remotes.create(remote_name, url) + rescue Rugged::ConfigError + remote_update(remote_name, url: url) end - # Add a new remote to this repository. - def remote_add(remote_name, url) - rugged.remotes.create(remote_name, url) - nil + def remove_remote(remote_name) + # When a remote is deleted all its remote refs are deleted too, but in + # the case of mirrors we map its refs (that would usualy go under + # [remote_name]/) to the top level namespace. We clean the mapping so + # those don't get deleted. + if rugged.config["remote.#{remote_name}.mirror"] + rugged.config.delete("remote.#{remote_name}.fetch") + end + + rugged.remotes.delete(remote_name) + true + rescue Rugged::ConfigError + false end # Update the specified remote using the values in the +options+ hash @@ -931,7 +1015,11 @@ module Gitlab if start_repository == self yield commit(start_branch_name) else - sha = start_repository.commit(start_branch_name).sha + start_commit = start_repository.commit(start_branch_name) + + return yield nil unless start_commit + + sha = start_commit.sha if branch_commit = commit(sha) yield branch_commit @@ -946,9 +1034,9 @@ module Gitlab def with_repo_tmp_commit(start_repository, start_branch_name, sha) tmp_ref = fetch_ref( - start_repository.path, - "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", - "refs/tmp/#{SecureRandom.hex}/head" + start_repository, + source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + target_ref: "refs/tmp/#{SecureRandom.hex}" ) yield commit(sha) @@ -960,8 +1048,9 @@ module Gitlab with_repo_branch_commit(source_repository, source_branch) do |commit| if commit write_ref(local_ref, commit.sha) + true else - raise Rugged::ReferenceError, 'source repository is empty' + false end end end @@ -979,13 +1068,27 @@ module Gitlab end end - def write_ref(ref_path, sha) - rugged.references.create(ref_path, sha, force: true) + def write_ref(ref_path, ref) + raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') + raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") + + command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z] + input = "update #{ref_path}\x00#{ref}\x00\x00" + output, status = circuit_breaker.perform do + popen(command, path) { |stdin| stdin.write(input) } + end + + raise GitError, output unless status.zero? end - def fetch_ref(source_path, source_ref, target_ref) - args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) - message, status = run_git(args) + def fetch_ref(source_repository, source_ref:, target_ref:) + message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled| + if is_enabled + gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref) + else + local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref) + end + end # Make sure ref was created, and raise Rugged::ReferenceError when not raise Rugged::ReferenceError, message if status != 0 @@ -994,9 +1097,16 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git(args) + def run_git(args, env: {}) + circuit_breaker.perform do + popen([Gitlab.config.git.bin_path, *args], path, env) + end + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb + def run_git_with_timeout(args, timeout, env: {}) circuit_breaker.perform do - popen([Gitlab.config.git.bin_path, *args], path) + popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) end end @@ -1020,11 +1130,41 @@ module Gitlab # This method return true if repository contains some content visible in project page. # def has_visible_content? - branch_count > 0 + return @has_visible_content if defined?(@has_visible_content) + + @has_visible_content = has_local_branches? + end + + def fetch(remote = 'origin') + args = %W(#{Gitlab.config.git.bin_path} fetch #{remote}) + + popen(args, @path).last.zero? + end + + def blob_at(sha, path) + Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) + end + + def commit_index(user, branch_name, index, options) + committer = user_to_committer(user) + + OperationService.new(user, self).with_branch(branch_name) do + commit_params = options.merge( + tree: index.write_tree(rugged), + author: committer, + committer: committer + ) + + create_commit(commit_params) + end end def gitaly_repository - Gitlab::GitalyClient::Util.repository(@storage, @relative_path) + Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) + end + + def gitaly_operations_client + @gitaly_operations_client ||= Gitlab::GitalyClient::OperationService.new(self) end def gitaly_ref_client @@ -1039,10 +1179,16 @@ module Gitlab @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self) end + def gitaly_operation_client + @gitaly_operation_client ||= Gitlab::GitalyClient::OperationService.new(self) + end + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) Gitlab::GitalyClient.migrate(method, status: status, &block) rescue GRPC::NotFound => e raise NoRepository.new(e) + rescue GRPC::InvalidArgument => e + raise ArgumentError.new(e) rescue GRPC::BadStatus => e raise CommandError.new(e) end @@ -1066,6 +1212,13 @@ module Gitlab sort_branches(branches, sort_by) end + def git_merged_branch_names(branch_names = []) + lines = run_git(['branch', '--merged', root_ref] + branch_names) + .first.lines + + lines.map(&:strip) + end + def log_using_shell?(options) options[:path].present? || options[:disable_walk] || @@ -1151,7 +1304,16 @@ module Gitlab end def alternate_object_directories - Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact + relative_paths = Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact + + if relative_paths.any? + relative_paths.map { |d| File.join(path, d) } + else + Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES) + .flatten + .compact + .flat_map { |d| d.split(File::PATH_SEPARATOR) } + end end # Get the content of a blob for a given commit. If the blob is a commit @@ -1361,6 +1523,33 @@ module Gitlab false end + def gitaly_add_tag(tag_name, user:, target:, message: nil) + gitaly_operations_client.add_tag(tag_name, user, target, message) + end + + def rugged_add_tag(tag_name, user:, target:, message: nil) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) + + options = nil # Use nil, not the empty hash. Rugged cares about this. + if message + options = { + message: message, + tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) + } + end + + Gitlab::Git::OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) + + find_tag(tag_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + rescue Rugged::TagError + raise TagExistsError + end + def rugged_create_branch(ref, start_point) rugged_ref = rugged.branches.create(ref, start_point) target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) @@ -1403,6 +1592,62 @@ module Gitlab file.write(gitattributes_content) end end + + def gitaly_add_branch(branch_name, user, target) + gitaly_operation_client.user_create_branch(branch_name, user, target) + rescue GRPC::FailedPrecondition => ex + raise InvalidRef, ex + end + + def rugged_add_branch(branch_name, user, target) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + OperationService.new(user, self).add_branch(branch_name, target_object.oid) + find_branch(branch_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + end + + def local_fetch_ref(source_path, source_ref:, target_ref:) + args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) + run_git(args) + end + + def gitaly_fetch_ref(source_repository, source_ref:, target_ref:) + gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) + gitaly_address = gitaly_resolver.address(source_repository.storage) + gitaly_token = gitaly_resolver.token(source_repository.storage) + + request = Gitaly::SSHUploadPackRequest.new(repository: source_repository.gitaly_repository) + env = { + 'GITALY_ADDRESS' => gitaly_address, + 'GITALY_PAYLOAD' => request.to_json, + 'GITALY_WD' => Dir.pwd, + 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" + } + env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? + + args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref}) + + run_git(args, env: env) + end + + def gitaly_ff_merge(user, source_sha, target_branch) + gitaly_operations_client.user_ff_branch(user, source_sha, target_branch) + rescue GRPC::FailedPrecondition => e + raise CommitError, e + end + + def rugged_ff_merge(user, source_sha, target_branch) + OperationService.new(user, self).with_branch(target_branch) do |our_commit| + raise ArgumentError, 'Invalid merge target' unless our_commit + + source_sha + end + rescue Rugged::ReferenceError + raise ArgumentError, 'Invalid merge source' + end end end end diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb new file mode 100644 index 00000000000..637e7a0659c --- /dev/null +++ b/lib/gitlab/git/repository_mirroring.rb @@ -0,0 +1,95 @@ +module Gitlab + module Git + module RepositoryMirroring + IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze + IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze + MIRROR_REMOTE = 'mirror'.freeze + + RemoteError = Class.new(StandardError) + + def set_remote_as_mirror(remote_name) + # This is used to define repository as equivalent as "git clone --mirror" + rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*' + rugged.config["remote.#{remote_name}.mirror"] = true + rugged.config["remote.#{remote_name}.prune"] = true + end + + def set_import_remote_as_mirror(remote_name) + # Add first fetch with Rugged so it does not create its own. + rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS + + add_remote_fetch_config(remote_name, IMPORT_TAG_REFS) + + rugged.config["remote.#{remote_name}.mirror"] = true + rugged.config["remote.#{remote_name}.prune"] = true + end + + def add_remote_fetch_config(remote_name, refspec) + run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + end + + def fetch_mirror(url) + add_remote(MIRROR_REMOTE, url) + set_remote_as_mirror(MIRROR_REMOTE) + fetch(MIRROR_REMOTE) + remove_remote(MIRROR_REMOTE) + end + + def remote_tags(remote) + # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n" + # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...] + list_remote_tags(remote).map do |line| + target, path = line.strip.split("\t") + + # When the remote repo does not have tags. + if target.nil? || path.nil? + Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}" + return [] + end + + name = path.split('/', 3).last + # We're only interested in tag references + # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name + next if name =~ /\^\{\}\Z/ + + target_commit = Gitlab::Git::Commit.find(self, target) + Gitlab::Git::Tag.new(self, name, target, target_commit) + end.compact + end + + def remote_branches(remote_name) + branches = [] + + rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref| + name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '') + + begin + target_commit = Gitlab::Git::Commit.find(self, ref.target) + branches << Gitlab::Git::Branch.new(self, name, ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end + + branches + end + + private + + def list_remote_tags(remote) + tag_list, exit_code, error = nil + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{full_path} ls-remote --tags #{remote}) + + Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr| + tag_list = stdout.read + error = stderr.read + exit_code = wait_thr.value.exitstatus + end + + raise RemoteError, error unless exit_code.zero? + + tag_list.split('\n') + end + end + end +end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index e0943d3a3eb..4974205b8fd 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -13,11 +13,32 @@ module Gitlab @path_to_repo = path_to_repo end - # This method returns an array of new references + # This method returns an array of new commit references def new_refs execute([*base_args, newrev, '--not', '--all']) end + # Finds newly added objects + # Returns an array of shas + # + # Can skip objects which do not have a path using required_path: true + # This skips commit objects and root trees, which might not be needed when + # looking for blobs + # + # When given a block it will yield objects as a lazy enumerator so + # the caller can limit work done instead of processing megabytes of data + def new_objects(require_path: nil, not_in: nil, &lazy_block) + args = [*base_args, newrev, *not_in_refs(not_in), '--objects'] + + get_objects(args, require_path: require_path, &lazy_block) + end + + def all_objects(require_path: nil, &lazy_block) + args = [*base_args, '--all', '--objects'] + + get_objects(args, require_path: require_path, &lazy_block) + end + # This methods returns an array of missed references # # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. @@ -27,16 +48,27 @@ module Gitlab private + def not_in_refs(references) + return ['--not', '--all'] unless references + return [] if references.empty? + + references.prepend('--not') + end + def execute(args) - output, status = popen(args, nil, Gitlab::Git::Env.all.stringify_keys) + output, status = popen(args, nil, Gitlab::Git::Env.to_env_hash) unless status.zero? - raise "Got a non-zero exit code while calling out `#{args.join(' ')}`." + raise "Got a non-zero exit code while calling out `#{args.join(' ')}`: #{output}" end output.split("\n") end + def lazy_execute(args, &lazy_block) + popen(args, nil, Gitlab::Git::Env.to_env_hash, lazy_block: lazy_block) + end + def base_args [ Gitlab.config.git.bin_path, @@ -44,6 +76,30 @@ module Gitlab 'rev-list' ] end + + def get_objects(args, require_path: nil) + if block_given? + lazy_execute(args) do |lazy_output| + objects = objects_from_output(lazy_output, require_path: require_path) + + yield(objects) + end + else + object_output = execute(args) + + objects_from_output(object_output, require_path: require_path) + end + end + + def objects_from_output(object_output, require_path: nil) + object_output.map do |output_line| + sha, path = output_line.split(' ', 2) + + next if require_path && path.blank? + + sha + end.reject(&:nil?) + end end end end diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb index 08e6c29abad..99518c9b1e4 100644 --- a/lib/gitlab/git/storage.rb +++ b/lib/gitlab/git/storage.rb @@ -12,6 +12,7 @@ module Gitlab CircuitOpen = Class.new(Inaccessible) Misconfiguration = Class.new(Inaccessible) + Failing = Class.new(Inaccessible) REDIS_KEY_PREFIX = 'storage_accessible:'.freeze diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 1eaa2d83fb6..be7598ef011 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -2,15 +2,13 @@ module Gitlab module Git module Storage class CircuitBreaker + include CircuitBreakerSettings + FailureInfo = Struct.new(:last_failure, :failure_count) attr_reader :storage, :hostname, - :storage_path, - :failure_count_threshold, - :failure_wait_time, - :failure_reset_time, - :storage_timeout + :storage_path delegate :last_failure, :failure_count, to: :failure_info @@ -18,7 +16,7 @@ module Gitlab pattern = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}*" Gitlab::Git::Storage.redis.with do |redis| - all_storage_keys = redis.keys(pattern) + all_storage_keys = redis.scan_each(match: pattern).to_a redis.del(*all_storage_keys) unless all_storage_keys.empty? end @@ -53,14 +51,10 @@ module Gitlab config = Gitlab.config.repositories.storages[@storage] @storage_path = config['path'] - @failure_count_threshold = config['failure_count_threshold'] - @failure_wait_time = config['failure_wait_time'] - @failure_reset_time = config['failure_reset_time'] - @storage_timeout = config['storage_timeout'] end def perform - return yield unless Feature.enabled?('git_storage_circuit_breaker') + return yield unless enabled? check_storage_accessible! @@ -70,10 +64,27 @@ module Gitlab def circuit_broken? return false if no_failures? + failure_count > failure_count_threshold + end + + def backing_off? + return false if no_failures? + recent_failure = last_failure > failure_wait_time.seconds.ago - too_many_failures = failure_count > failure_count_threshold + too_many_failures = failure_count > backoff_threshold - recent_failure || too_many_failures + recent_failure && too_many_failures + end + + private + + # The circuitbreaker can be enabled for the entire fleet using a Feature + # flag. + # + # Enabling it for a single host can be done setting the + # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable. + def enabled? + ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker') end def failure_info @@ -89,7 +100,7 @@ module Gitlab return @storage_available if @storage_available if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck - .storage_available?(storage_path, storage_timeout) + .storage_available?(storage_path, storage_timeout, access_retries) track_storage_accessible else track_storage_inaccessible @@ -100,7 +111,11 @@ module Gitlab def check_storage_accessible! if circuit_broken? - raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_wait_time) + raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time) + end + + if backing_off? + raise Gitlab::Git::Storage::Failing.new("Backing off access to #{storage}", failure_wait_time) end unless storage_available? @@ -137,12 +152,6 @@ module Gitlab end end - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" - end - - private - def get_failure_info last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| redis.hmget(cache_key, :last_failure, :failure_count) @@ -152,6 +161,10 @@ module Gitlab FailureInfo.new(last_failure, failure_count.to_i) end + + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" + end end end end diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb new file mode 100644 index 00000000000..257fe8cd8f0 --- /dev/null +++ b/lib/gitlab/git/storage/circuit_breaker_settings.rb @@ -0,0 +1,37 @@ +module Gitlab + module Git + module Storage + module CircuitBreakerSettings + def failure_count_threshold + application_settings.circuitbreaker_failure_count_threshold + end + + def failure_wait_time + application_settings.circuitbreaker_failure_wait_time + end + + def failure_reset_time + application_settings.circuitbreaker_failure_reset_time + end + + def storage_timeout + application_settings.circuitbreaker_storage_timeout + end + + def access_retries + application_settings.circuitbreaker_access_retries + end + + def backoff_threshold + application_settings.circuitbreaker_backoff_threshold + end + + private + + def application_settings + Gitlab::CurrentSettings.current_application_settings + end + end + end + end +end diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb index 91d8241f17b..1307f400700 100644 --- a/lib/gitlab/git/storage/forked_storage_check.rb +++ b/lib/gitlab/git/storage/forked_storage_check.rb @@ -4,8 +4,17 @@ module Gitlab module ForkedStorageCheck extend self - def storage_available?(path, timeout_seconds = 5) - status = timeout_check(path, timeout_seconds) + def storage_available?(path, timeout_seconds = 5, retries = 1) + partial_timeout = timeout_seconds / retries + status = timeout_check(path, partial_timeout) + + # If the status check did not succeed the first time, we retry a few + # more times to avoid one-off failures + current_attempts = 1 + while current_attempts < retries && !status.success? + status = timeout_check(path, partial_timeout) + current_attempts += 1 + end status.success? end diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb index 1564e94b7f7..7049772fe3b 100644 --- a/lib/gitlab/git/storage/health.rb +++ b/lib/gitlab/git/storage/health.rb @@ -23,26 +23,36 @@ module Gitlab end end - def self.all_keys_for_storages(storage_names, redis) + private_class_method def self.all_keys_for_storages(storage_names, redis) keys_per_storage = {} redis.pipelined do storage_names.each do |storage_name| pattern = pattern_for_storage(storage_name) + matched_keys = redis.scan_each(match: pattern) - keys_per_storage[storage_name] = redis.keys(pattern) + keys_per_storage[storage_name] = matched_keys end end - keys_per_storage + # We need to make sure each lazy-loaded `Enumerator` for matched keys + # is loaded into an array. + # + # Otherwise it would be loaded in the second `Redis#pipelined` block + # within `.load_for_keys`. In this pipelined call, the active + # Redis-client changes again, so the values would not be available + # until the end of that pipelined-block. + keys_per_storage.each do |storage_name, key_future| + keys_per_storage[storage_name] = key_future.to_a + end end - def self.load_for_keys(keys_per_storage, redis) + private_class_method def self.load_for_keys(keys_per_storage, redis) info_for_keys = {} redis.pipelined do keys_per_storage.each do |storage_name, keys_future| - info_for_storage = keys_future.value.map do |key| + info_for_storage = keys_future.map do |key| { name: key, failure_count: redis.hget(key, :failure_count) } end diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb index 297c043d054..a12d52d295f 100644 --- a/lib/gitlab/git/storage/null_circuit_breaker.rb +++ b/lib/gitlab/git/storage/null_circuit_breaker.rb @@ -2,15 +2,14 @@ module Gitlab module Git module Storage class NullCircuitBreaker + include CircuitBreakerSettings + # These will have actual values attr_reader :storage, :hostname # These will always have nil values - attr_reader :storage_path, - :failure_wait_time, - :failure_reset_time, - :storage_timeout + attr_reader :storage_path def initialize(storage, hostname, error: nil) @storage = storage @@ -26,8 +25,8 @@ module Gitlab !!@error end - def failure_count_threshold - 1 + def backing_off? + false end def last_failure @@ -35,7 +34,7 @@ module Gitlab end def failure_count - circuit_broken? ? 1 : 0 + circuit_broken? ? failure_count_threshold : 0 end def failure_info diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index ea634d39668..e6b61417de1 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -1,24 +1,29 @@ module Gitlab module Git class User - attr_reader :name, :email, :gl_id + attr_reader :username, :name, :email, :gl_id def self.from_gitlab(gitlab_user) - new(gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) + new(gitlab_user.username, gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) end def self.from_gitaly(gitaly_user) - new(gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) + new(gitaly_user.gl_username, gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) end - def initialize(name, email, gl_id) + def initialize(username, name, email, gl_id) + @username = username @name = name @email = email @gl_id = gl_id end def ==(other) - [name, email, gl_id] == [other.name, other.email, other.gl_id] + [username, name, email, gl_id] == [other.username, other.name, other.email, other.gl_id] + end + + def to_gitaly + Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name, email: email) end end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb new file mode 100644 index 00000000000..fe901d049d4 --- /dev/null +++ b/lib/gitlab/git/wiki.rb @@ -0,0 +1,194 @@ +module Gitlab + module Git + class Wiki + DuplicatePageError = Class.new(StandardError) + + CommitDetails = Struct.new(:name, :email, :message) do + def to_h + { name: name, email: email, message: message } + end + end + PageBlob = Struct.new(:name) + + attr_reader :repository + + def self.default_ref + 'master' + end + + # Initialize with a Gitlab::Git::Repository instance + def initialize(repository) + @repository = repository + end + + def repository_exists? + @repository.exists? + end + + def write_page(name, format, content, commit_details) + @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| + if is_enabled + gitaly_write_page(name, format, content, commit_details) + gollum_wiki.clear_cache + else + gollum_write_page(name, format, content, commit_details) + end + end + end + + def delete_page(page_path, commit_details) + @repository.gitaly_migrate(:wiki_delete_page) do |is_enabled| + if is_enabled + gitaly_delete_page(page_path, commit_details) + gollum_wiki.clear_cache + else + gollum_delete_page(page_path, commit_details) + end + end + end + + def update_page(page_path, title, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) + nil + end + + def pages + gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + end + + def page(title:, version: nil, dir: nil) + @repository.gitaly_migrate(:wiki_find_page) do |is_enabled| + if is_enabled + gitaly_find_page(title: title, version: version, dir: dir) + else + gollum_find_page(title: title, version: version, dir: dir) + end + end + end + + def file(name, version) + @repository.gitaly_migrate(:wiki_find_file) do |is_enabled| + if is_enabled + gitaly_find_file(name, version) + else + gollum_find_file(name, version) + end + end + end + + def page_versions(page_path) + current_page = gollum_page_by_path(page_path) + current_page.versions.map do |gollum_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id) + new_version(gollum_page, gollum_git_commit.id) + end + end + + def preview_slug(title, format) + # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid + # using Rugged through a Gollum::Wiki instance + page_class = Gollum::Page + page = page_class.new(nil) + ext = page_class.format_to_ext(format.to_sym) + name = page_class.cname(title) + '.' + ext + blob = PageBlob.new(name) + page.populate(blob) + page.url_path + end + + private + + def gollum_wiki + @gollum_wiki ||= Gollum::Wiki.new(@repository.path) + end + + def gollum_page_by_path(page_path) + page_name = Gollum::Page.canonicalize_filename(page_path) + page_dir = File.split(page_path).first + + gollum_wiki.paged(page_name, page_dir) + end + + def new_page(gollum_page) + Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id)) + end + + def new_version(gollum_page, commit_id) + commit = Gitlab::Git::Commit.find(@repository, commit_id) + Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format) + end + + def assert_type!(object, klass) + unless object.is_a?(klass) + raise ArgumentError, "expected a #{klass}, got #{object.inspect}" + end + end + + def gitaly_wiki_client + @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) + end + + def gollum_write_page(name, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.write_page(name, format, content, commit_details.to_h) + + nil + rescue Gollum::DuplicatePageError => e + raise Gitlab::Git::Wiki::DuplicatePageError, e.message + end + + def gollum_delete_page(page_path, commit_details) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.delete_page(gollum_page_by_path(page_path), commit_details.to_h) + nil + end + + def gollum_find_page(title:, version: nil, dir: nil) + if version + version = Gitlab::Git::Commit.find(@repository, version).id + end + + gollum_page = gollum_wiki.page(title, version, dir) + return unless gollum_page + + new_page(gollum_page) + end + + def gollum_find_file(name, version) + version ||= self.class.default_ref + gollum_file = gollum_wiki.file(name, version) + return unless gollum_file + + Gitlab::Git::WikiFile.new(gollum_file) + end + + def gitaly_write_page(name, format, content, commit_details) + gitaly_wiki_client.write_page(name, format, content, commit_details) + end + + def gitaly_delete_page(page_path, commit_details) + gitaly_wiki_client.delete_page(page_path, commit_details) + end + + def gitaly_find_page(title:, version: nil, dir: nil) + wiki_page, version = gitaly_wiki_client.find_page(title: title, version: version, dir: dir) + return unless wiki_page + + Gitlab::Git::WikiPage.new(wiki_page, version) + end + + def gitaly_find_file(name, version) + wiki_file = gitaly_wiki_client.find_file(name, version) + return unless wiki_file + + Gitlab::Git::WikiFile.new(wiki_file) + end + end + end +end diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb new file mode 100644 index 00000000000..84335aca4bc --- /dev/null +++ b/lib/gitlab/git/wiki_file.rb @@ -0,0 +1,20 @@ +module Gitlab + module Git + class WikiFile + attr_reader :mime_type, :raw_data, :name, :path + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Because Gollum::File is not serializable we must get all the data from + # 'gollum_file' during initialization, and NOT store it in an instance + # variable. + def initialize(gollum_file) + @mime_type = gollum_file.mime_type + @raw_data = gollum_file.raw_data + @name = gollum_file.name + @path = gollum_file.path + end + end + end +end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb new file mode 100644 index 00000000000..a06bac4414f --- /dev/null +++ b/lib/gitlab/git/wiki_page.rb @@ -0,0 +1,39 @@ +module Gitlab + module Git + class WikiPage + attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Because Gollum::Page is not serializable we must get all the data from + # 'gollum_page' during initialization, and NOT store it in an instance + # variable. + # + # Note that 'version' is a WikiPageVersion instance which it itself + # serializable. That means it's OK to store 'version' in an instance + # variable. + def initialize(gollum_page, version) + @url_path = gollum_page.url_path + @title = gollum_page.title + @format = gollum_page.format + @path = gollum_page.path + @raw_data = gollum_page.raw_data + @name = gollum_page.name + @historical = gollum_page.historical? + + @version = version + end + + def historical? + @historical + end + + def text_data + return @text_data if defined?(@text_data) + + @text_data = @raw_data && Gitlab::EncodingHelper.encode!(@raw_data.dup) + end + end + end +end diff --git a/lib/gitlab/git/wiki_page_version.rb b/lib/gitlab/git/wiki_page_version.rb new file mode 100644 index 00000000000..55f1afedcab --- /dev/null +++ b/lib/gitlab/git/wiki_page_version.rb @@ -0,0 +1,19 @@ +module Gitlab + module Git + class WikiPageVersion + attr_reader :commit, :format + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Both 'commit' (a Gitlab::Git::Commit) and 'format' (a string) are + # serializable. + def initialize(commit, format) + @commit = commit + @format = format + end + + delegate :message, :sha, :id, :author_name, :authored_date, to: :commit + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 62d1ecae676..8998c4b1a83 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -16,7 +16,9 @@ module Gitlab account_blocked: 'Your account has been blocked.', command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', - receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.' + receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', + read_only: 'The repository is temporarily read-only. Please try again later.', + cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -159,6 +161,14 @@ module Gitlab end def check_push_access!(changes) + if project.repository_read_only? + raise UnauthorizedError, ERROR_MESSAGES[:read_only] + end + + if Gitlab::Database.read_only? + raise UnauthorizedError, ERROR_MESSAGES[:cannot_push_to_read_only] + end + if deploy_key check_deploy_key_push_access! elsif user @@ -205,10 +215,6 @@ module Gitlab ).exec end - def matching_merge_request?(newrev, branch_name) - Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? - end - def deploy_key actor if deploy_key? end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 1fe5155c093..98f1f45b338 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,6 +1,7 @@ module Gitlab class GitAccessWiki < GitAccess ERROR_MESSAGES = { + read_only: "You can't push code to a read-only GitLab instance.", write_to_wiki: "You are not allowed to write to this project's wiki." }.freeze @@ -17,6 +18,10 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end + if Gitlab::Database.read_only? + raise UnauthorizedError, ERROR_MESSAGES[:read_only] + end + true end end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index a3c6b21a6a1..2e3e4fc3f1f 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -11,7 +11,7 @@ module Gitlab return false if ref_name.start_with?('refs/remotes/') Gitlab::Utils.system_silent( - %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name})) + %W(#{Gitlab.config.git.bin_path} check-ref-format --branch #{ref_name})) end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index cbd9ff406de..0b35a787e07 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -28,10 +28,18 @@ module Gitlab SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze MAXIMUM_GITALY_CALLS = 30 + CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new private_constant :MUTEX + class << self + attr_accessor :query_time, :migrate_histogram + end + + self.query_time = 0 + self.migrate_histogram = Gitlab::Metrics.histogram(:gitaly_migrate_call_duration, "Gitaly migration call execution timings") + def self.stub(name, storage) MUTEX.synchronize do @stubs ||= {} @@ -69,17 +77,41 @@ module Gitlab # All Gitaly RPC call sites should use GitalyClient.call. This method # makes sure that per-request authentication headers are set. + # + # This method optionally takes a block which receives the keyword + # arguments hash 'kwargs' that will be passed to gRPC. This allows the + # caller to modify or augment the keyword arguments. The block must + # return a hash. + # + # For example: + # + # GitalyClient.call(storage, service, rpc, request) do |kwargs| + # kwargs.merge(deadline: Time.now + 10) + # end + # def self.call(storage, service, rpc, request) + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) enforce_gitaly_request_limits(:call) - metadata = request_metadata(storage) - metadata = yield(metadata) if block_given? - stub(service, storage).__send__(rpc, request, metadata) # rubocop:disable GitlabSecurity/PublicSend + kwargs = request_kwargs(storage) + kwargs = yield(kwargs) if block_given? + stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend + ensure + self.query_time += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start end - def self.request_metadata(storage) + def self.request_kwargs(storage) encoded_token = Base64.strict_encode64(token(storage).to_s) - { metadata: { 'authorization' => "Bearer #{encoded_token}" } } + metadata = { + 'authorization' => "Bearer #{encoded_token}", + 'client_name' => CLIENT_NAME + } + + feature_stack = Thread.current[:gitaly_feature_stack] + feature = feature_stack && feature_stack[0] + metadata['call_site'] = feature.to_s if feature + + { metadata: metadata } end def self.token(storage) @@ -137,7 +169,17 @@ module Gitlab Gitlab::Metrics.measure(metric_name) do # Some migrate calls wrap other migrate calls allow_n_plus_1_calls do - yield is_enabled + feature_stack = Thread.current[:gitaly_feature_stack] ||= [] + feature_stack.unshift(feature) + begin + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + yield is_enabled + ensure + total_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start + migrate_histogram.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time) + feature_stack.shift + Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty? + end end end end @@ -151,7 +193,7 @@ module Gitlab actual_call_count = increment_call_count("gitaly_#{call_site}_actual") # Do no enforce limits in production - return if Rails.env.production? + return if Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"] # Check if this call is nested within a allow_n_plus_1_calls # block and skip check if it is @@ -228,10 +270,20 @@ module Gitlab path.read.chomp end + def self.timestamp(t) + Google::Protobuf::Timestamp.new(seconds: t.to_i) + end + def self.encode(s) + return "" if s.nil? + s.dup.force_encoding(Encoding::ASCII_8BIT) end + def self.encode_repeated(a) + Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) + end + # Count a stack. Used for n+1 detection def self.count_stack return unless RequestStore.active? diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index cf3a3554552..da5505cb2fe 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -18,7 +18,7 @@ module Gitlab response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request) response.flat_map do |msg| - msg.paths.map { |d| d.dup.force_encoding(Encoding::UTF_8) } + msg.paths.map { |d| EncodingHelper.encode!(d.dup) } end end @@ -230,6 +230,26 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request) end + def find_commits(options) + request = Gitaly::FindCommitsRequest.new( + repository: @gitaly_repo, + limit: options[:limit], + offset: options[:offset], + follow: options[:follow], + skip_merges: options[:skip_merges], + disable_walk: options[:disable_walk] + ) + request.after = GitalyClient.timestamp(options[:after]) if options[:after] + request.before = GitalyClient.timestamp(options[:before]) if options[:before] + request.revision = GitalyClient.encode(options[:ref]) if options[:ref] + + request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present? + + response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request) + + consume_commits_response(response) + end + private def call_commit_diff(request_params, options = {}) @@ -254,7 +274,7 @@ module Gitlab repository: @gitaly_repo, left_commit_id: from_id, right_commit_id: to_id, - paths: options.fetch(:paths, []).map { |path| GitalyClient.encode(path) } + paths: options.fetch(:paths, []).compact.map { |path| GitalyClient.encode(path) } } end diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb new file mode 100644 index 00000000000..bd7c345ac01 --- /dev/null +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -0,0 +1,39 @@ +module Gitlab + module GitalyClient + class NamespaceService + def initialize(storage) + @storage = storage + end + + def exists?(name) + request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:namespace_exists, request).exists + end + + def add(name) + request = Gitaly::AddNamespaceRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:add_namespace, request) + end + + def remove(name) + request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:remove_namespace, request) + end + + def rename(from, to) + request = Gitaly::RenameNamespaceRequest.new(storage_name: @storage, from: from, to: to) + + gitaly_client_call(:rename_namespace, request) + end + + private + + def gitaly_client_call(type, request) + GitalyClient.call(@storage, :namespace_service, type, request) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb new file mode 100644 index 00000000000..526d44a8b77 --- /dev/null +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -0,0 +1,127 @@ +module Gitlab + module GitalyClient + class OperationService + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def rm_tag(tag_name, user) + request = Gitaly::UserDeleteTagRequest.new( + repository: @gitaly_repo, + tag_name: GitalyClient.encode(tag_name), + user: Gitlab::Git::User.from_gitlab(user).to_gitaly + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + end + end + + def add_tag(tag_name, user, target, message) + request = Gitaly::UserCreateTagRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + tag_name: GitalyClient.encode(tag_name), + target_revision: GitalyClient.encode(target), + message: GitalyClient.encode(message.to_s) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request) + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + elsif response.exists + raise Gitlab::Git::Repository::TagExistsError + end + + Util.gitlab_tag_from_gitaly_tag(@repository, response.tag) + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::Repository::InvalidRef, e + end + + def user_create_branch(branch_name, user, start_point) + request = Gitaly::UserCreateBranchRequest.new( + repository: @gitaly_repo, + branch_name: GitalyClient.encode(branch_name), + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + start_point: GitalyClient.encode(start_point) + ) + response = GitalyClient.call(@repository.storage, :operation_service, + :user_create_branch, request) + if response.pre_receive_error.present? + raise Gitlab::Git::HooksService::PreReceiveError.new(response.pre_receive_error) + end + + branch = response.branch + return nil unless branch + + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit) + end + + def user_delete_branch(branch_name, user) + request = Gitaly::UserDeleteBranchRequest.new( + repository: @gitaly_repo, + branch_name: GitalyClient.encode(branch_name), + user: Gitlab::Git::User.from_gitlab(user).to_gitaly + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + end + end + + def user_merge_branch(user, source_sha, target_branch, message) + request_enum = QueueEnumerator.new + response_enum = GitalyClient.call( + @repository.storage, + :operation_service, + :user_merge_branch, + request_enum.each + ) + + request_enum.push( + Gitaly::UserMergeBranchRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + commit_id: source_sha, + branch: GitalyClient.encode(target_branch), + message: GitalyClient.encode(message) + ) + ) + + yield response_enum.next.commit_id + + request_enum.push(Gitaly::UserMergeBranchRequest.new(apply: true)) + + branch_update = response_enum.next.branch_update + raise Gitlab::Git::CommitError.new('failed to apply merge to branch') unless branch_update.commit_id.present? + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) + ensure + request_enum.close + end + + def user_ff_branch(user, source_sha, target_branch) + request = Gitaly::UserFFBranchRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + commit_id: source_sha, + branch: GitalyClient.encode(target_branch) + ) + + branch_update = GitalyClient.call( + @repository.storage, + :operation_service, + :user_ff_branch, + request + ).branch_update + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/queue_enumerator.rb b/lib/gitlab/gitaly_client/queue_enumerator.rb new file mode 100644 index 00000000000..b8018029552 --- /dev/null +++ b/lib/gitlab/gitaly_client/queue_enumerator.rb @@ -0,0 +1,28 @@ +module Gitlab + module GitalyClient + class QueueEnumerator + def initialize + @queue = Queue.new + end + + def push(elem) + @queue << elem + end + + def close + push(:close) + end + + def each + return enum_for(:each) unless block_given? + + loop do + elem = @queue.pop + break if elem == :close + + yield elem + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 8ef873d5848..b0c73395cb1 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -155,19 +155,7 @@ module Gitlab def consume_tags_response(response) response.flat_map do |message| - message.tags.map do |gitaly_tag| - if gitaly_tag.target_commit.present? - gitaly_commit = Gitlab::Git::Commit.decorate(@repository, gitaly_tag.target_commit) - end - - Gitlab::Git::Tag.new( - @repository, - encode!(gitaly_tag.name.dup), - gitaly_tag.id, - gitaly_commit, - encode!(gitaly_tag.message.chomp) - ) - end + message.tags.map { |gitaly_tag| Util.gitlab_tag_from_gitaly_tag(@repository, gitaly_tag) } end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 177a1284f38..cef692d3c2a 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -53,6 +53,18 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :fetch_remote, request) end + + def create_repository + request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :create_repository, request) + end + + def has_local_branches? + request = Gitaly::HasLocalBranchesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request) + + response.value + end end end end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index 8fc937496af..b1a033280b4 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -2,12 +2,33 @@ module Gitlab module GitalyClient module Util class << self - def repository(repository_storage, relative_path) + def repository(repository_storage, relative_path, gl_repository) + git_object_directory = Gitlab::Git::Env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence || + Gitlab::Git::Env['GIT_OBJECT_DIRECTORY'].presence + git_alternate_object_directories = + Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE']).presence || + Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES']).flat_map { |d| d.split(File::PATH_SEPARATOR) } + Gitaly::Repository.new( storage_name: repository_storage, relative_path: relative_path, - git_object_directory: Gitlab::Git::Env['GIT_OBJECT_DIRECTORY'].to_s, - git_alternate_object_directories: Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES']) + gl_repository: gl_repository, + git_object_directory: git_object_directory.to_s, + git_alternate_object_directories: git_alternate_object_directories + ) + end + + def gitlab_tag_from_gitaly_tag(repository, gitaly_tag) + if gitaly_tag.target_commit.present? + commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit) + end + + Gitlab::Git::Tag.new( + repository, + Gitlab::EncodingHelper.encode!(gitaly_tag.name.dup), + gitaly_tag.id, + commit, + Gitlab::EncodingHelper.encode!(gitaly_tag.message.chomp) ) end end diff --git a/lib/gitlab/gitaly_client/wiki_file.rb b/lib/gitlab/gitaly_client/wiki_file.rb new file mode 100644 index 00000000000..a2e415864e6 --- /dev/null +++ b/lib/gitlab/gitaly_client/wiki_file.rb @@ -0,0 +1,17 @@ +module Gitlab + module GitalyClient + class WikiFile + FIELDS = %i(name mime_type path raw_data).freeze + + attr_accessor(*FIELDS) + + def initialize(params) + params = params.with_indifferent_access + + FIELDS.each do |field| + instance_variable_set("@#{field}", params[field]) + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb new file mode 100644 index 00000000000..8226278d5f6 --- /dev/null +++ b/lib/gitlab/gitaly_client/wiki_page.rb @@ -0,0 +1,25 @@ +module Gitlab + module GitalyClient + class WikiPage + FIELDS = %i(title format url_path path name historical raw_data).freeze + + attr_accessor(*FIELDS) + + def initialize(params) + params = params.with_indifferent_access + + FIELDS.each do |field| + instance_variable_set("@#{field}", params[field]) + end + end + + def historical? + @historical + end + + def format + @format.to_sym + end + end + end +end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb new file mode 100644 index 00000000000..15f0f30d303 --- /dev/null +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -0,0 +1,120 @@ +require 'stringio' + +module Gitlab + module GitalyClient + class WikiService + MAX_MSG_SIZE = 128.kilobytes.freeze + + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def write_page(name, format, content, commit_details) + request = Gitaly::WikiWritePageRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(name), + format: format.to_s, + commit_details: gitaly_commit_details(commit_details) + ) + + strio = StringIO.new(content) + + enum = Enumerator.new do |y| + until strio.eof? + chunk = strio.read(MAX_MSG_SIZE) + request.content = GitalyClient.encode(chunk) + + y.yield request + + request = Gitaly::WikiWritePageRequest.new + end + end + + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum) + if error = response.duplicate_error.presence + raise Gitlab::Git::Wiki::DuplicatePageError, error + end + end + + def delete_page(page_path, commit_details) + request = Gitaly::WikiDeletePageRequest.new( + repository: @gitaly_repo, + page_path: GitalyClient.encode(page_path), + commit_details: gitaly_commit_details(commit_details) + ) + + GitalyClient.call(@repository.storage, :wiki_service, :wiki_delete_page, request) + end + + def find_page(title:, version: nil, dir: nil) + request = Gitaly::WikiFindPageRequest.new( + repository: @gitaly_repo, + title: GitalyClient.encode(title), + revision: GitalyClient.encode(version), + directory: GitalyClient.encode(dir) + ) + + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request) + wiki_page = version = nil + + response.each do |message| + page = message.page + next unless page + + if wiki_page + wiki_page.raw_data << page.raw_data + else + wiki_page = GitalyClient::WikiPage.new(page.to_h) + # All gRPC strings in a response are frozen, so we get + # an unfrozen version here so appending in the else clause below doesn't blow up. + wiki_page.raw_data = wiki_page.raw_data.dup + + version = Gitlab::Git::WikiPageVersion.new( + Gitlab::Git::Commit.decorate(@repository, page.version.commit), + page.version.format + ) + end + end + + [wiki_page, version] + end + + def find_file(name, revision) + request = Gitaly::WikiFindFileRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(name), + revision: GitalyClient.encode(revision) + ) + + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request) + wiki_file = nil + + response.each do |message| + next unless message.name.present? + + if wiki_file + wiki_file.raw_data << message.raw_data + else + wiki_file = GitalyClient::WikiFile.new(message.to_h) + # All gRPC strings in a response are frozen, so we get + # an unfrozen version here so appending in the else clause below doesn't blow up. + wiki_file.raw_data = wiki_file.raw_data.dup + end + end + + wiki_file + end + + private + + def gitaly_commit_details(commit_details) + Gitaly::WikiCommitDetails.new( + name: GitalyClient.encode(commit_details.name), + email: GitalyClient.encode(commit_details.email), + message: GitalyClient.encode(commit_details.message) + ) + end + end + end +end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb index e21922070c1..8911b81ec9a 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -38,7 +38,7 @@ module Gitlab end def generate_line_code(line) - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) end def on_diff? diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb index 0396122eeb9..ca8d96f5650 100644 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -8,7 +8,7 @@ module Gitlab end def disk_path - "#{project.disk_path}.wiki" + project.wiki.disk_path end def import_url diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 0d5039ddf5f..413872d7e08 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -34,6 +34,21 @@ module Gitlab end end + def subkeys_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + raw_keys = GPGME::Key.find(:public, fingerprints) + + raw_keys.each_with_object({}) do |raw_key, grouped_subkeys| + primary_subkey_id = raw_key.primary_subkey.keyid + + grouped_subkeys[primary_subkey_id] = raw_key.subkeys[1..-1].map do |s| + { keyid: s.keyid, fingerprint: s.fingerprint } + end + end + end + end + def user_infos_from_key(key) using_tmp_keychain do fingerprints = CurrentKeyChain.fingerprints_from_key(key) diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 86bd9f5b125..0f4ba6f83fc 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -43,7 +43,9 @@ module Gitlab # key belonging to the keyid. # This way we can add the key to the temporary keychain and extract # the proper signature. - gpg_key = GpgKey.find_by(primary_keyid: verified_signature.fingerprint) + # NOTE: the invoked method is #fingerprint but it's only returning + # 16 characters (the format used by keyid) instead of 40. + gpg_key = find_gpg_key(verified_signature.fingerprint) if gpg_key Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key) @@ -74,7 +76,7 @@ module Gitlab commit_sha: @commit.sha, project: @commit.project, gpg_key: gpg_key, - gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, + gpg_key_primary_keyid: gpg_key&.keyid || verified_signature.fingerprint, gpg_key_user_name: user_infos[:name], gpg_key_user_email: user_infos[:email], verification_status: verification_status @@ -98,6 +100,10 @@ module Gitlab def user_infos(gpg_key) gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} end + + def find_gpg_key(keyid) + GpgKey.find_by(primary_keyid: keyid) || GpgKeySubkey.find_by(keyid: keyid) + end end end end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index e085eab26c9..1991911ef6a 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -9,8 +9,8 @@ module Gitlab GpgSignature .select(:id, :commit_sha, :project_id) .where('gpg_key_id IS NULL OR verification_status <> ?', GpgSignature.verification_statuses[:verified]) - .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) - .find_each { |sig| sig.gpg_commit.update_signature!(sig) } + .where(gpg_key_primary_keyid: @gpg_key.keyids) + .find_each { |sig| sig.gpg_commit&.update_signature!(sig) } end end end diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb index 635f52131f9..42ded7c286f 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/group_hierarchy.rb @@ -17,12 +17,32 @@ module Gitlab @model = ancestors_base.model end + # Returns the set of descendants of a given relation, but excluding the given + # relation + def descendants + base_and_descendants.where.not(id: descendants_base.select(:id)) + end + + # Returns the set of ancestors of a given relation, but excluding the given + # relation + # + # Passing an `upto` will stop the recursion once the specified parent_id is + # reached. So all ancestors *lower* than the specified ancestor will be + # included. + def ancestors(upto: nil) + base_and_ancestors(upto: upto).where.not(id: ancestors_base.select(:id)) + end + # Returns a relation that includes the ancestors_base set of groups # and all their ancestors (recursively). - def base_and_ancestors + # + # Passing an `upto` will stop the recursion once the specified parent_id is + # reached. So all ancestors *lower* than the specified acestor will be + # included. + def base_and_ancestors(upto: nil) return ancestors_base unless Group.supports_nested_groups? - read_only(base_and_ancestors_cte.apply_to(model.all)) + read_only(base_and_ancestors_cte(upto).apply_to(model.all)) end # Returns a relation that includes the descendants_base set of groups @@ -78,17 +98,19 @@ module Gitlab private - def base_and_ancestors_cte + def base_and_ancestors_cte(stop_id = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) cte << ancestors_base.except(:order) # Recursively get all the ancestors of the base set. - cte << model + parent_query = model .from([groups_table, cte.table]) .where(groups_table[:id].eq(cte.table[:parent_id])) .except(:order) + parent_query = parent_query.where(cte.table[:parent_id].not_eq(stop_id)) if stop_id + cte << parent_query cte end diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb new file mode 100644 index 00000000000..4febb0ab430 --- /dev/null +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -0,0 +1,56 @@ +module Gitlab + module HookData + class IssuableBuilder + CHANGES_KEYS = %i[previous current].freeze + + attr_accessor :issuable + + def initialize(issuable) + @issuable = issuable + end + + def build(user: nil, changes: {}) + hook_data = { + object_kind: issuable.class.name.underscore, + user: user.hook_attrs, + project: issuable.project.hook_attrs, + object_attributes: issuable.hook_attrs, + labels: issuable.labels.map(&:hook_attrs), + changes: final_changes(changes.slice(*safe_keys)), + # DEPRECATED + repository: issuable.project.hook_attrs.slice(:name, :url, :description, :homepage) + } + + if issuable.is_a?(Issue) + hook_data[:assignees] = issuable.assignees.map(&:hook_attrs) if issuable.assignees.any? + else + hook_data[:assignee] = issuable.assignee.hook_attrs if issuable.assignee + end + + hook_data + end + + def safe_keys + issuable_builder::SAFE_HOOK_ATTRIBUTES + issuable_builder::SAFE_HOOK_RELATIONS + end + + private + + def issuable_builder + case issuable + when Issue + Gitlab::HookData::IssueBuilder + when MergeRequest + Gitlab::HookData::MergeRequestBuilder + end + end + + def final_changes(changes_hash) + changes_hash.reduce({}) do |hash, (key, changes_array)| + hash[key] = Hash[CHANGES_KEYS.zip(changes_array)] + hash + end + end + end + end +end diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb new file mode 100644 index 00000000000..de9cab80a02 --- /dev/null +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -0,0 +1,55 @@ +module Gitlab + module HookData + class IssueBuilder + SAFE_HOOK_ATTRIBUTES = %i[ + assignee_id + author_id + branch_name + closed_at + confidential + created_at + deleted_at + description + due_date + id + iid + last_edited_at + last_edited_by_id + milestone_id + moved_to_id + project_id + relative_position + state + time_estimate + title + updated_at + updated_by_id + ].freeze + + SAFE_HOOK_RELATIONS = %i[ + assignees + labels + ].freeze + + attr_accessor :issue + + def initialize(issue) + @issue = issue + end + + def build + attrs = { + url: Gitlab::UrlBuilder.build(issue), + total_time_spent: issue.total_time_spent, + human_total_time_spent: issue.human_total_time_spent, + human_time_estimate: issue.human_time_estimate, + assignee_ids: issue.assignee_ids, + assignee_id: issue.assignee_ids.first # This key is deprecated + } + + issue.attributes.with_indifferent_access.slice(*SAFE_HOOK_ATTRIBUTES) + .merge!(attrs) + end + end + end +end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb new file mode 100644 index 00000000000..eaef19c9d04 --- /dev/null +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -0,0 +1,62 @@ +module Gitlab + module HookData + class MergeRequestBuilder + SAFE_HOOK_ATTRIBUTES = %i[ + assignee_id + author_id + created_at + deleted_at + description + head_pipeline_id + id + iid + last_edited_at + last_edited_by_id + merge_commit_sha + merge_error + merge_params + merge_status + merge_user_id + merge_when_pipeline_succeeds + milestone_id + ref_fetched + source_branch + source_project_id + state + target_branch + target_project_id + time_estimate + title + updated_at + updated_by_id + ].freeze + + SAFE_HOOK_RELATIONS = %i[ + assignee + labels + ].freeze + + attr_accessor :merge_request + + def initialize(merge_request) + @merge_request = merge_request + end + + def build + attrs = { + url: Gitlab::UrlBuilder.build(merge_request), + source: merge_request.source_project.try(:hook_attrs), + target: merge_request.target_project.hook_attrs, + last_commit: merge_request.diff_head_commit&.hook_attrs, + work_in_progress: merge_request.work_in_progress?, + total_time_spent: merge_request.total_time_spent, + human_total_time_spent: merge_request.human_total_time_spent, + human_time_estimate: merge_request.human_time_estimate + } + + merge_request.attributes.with_indifferent_access.slice(*SAFE_HOOK_ATTRIBUTES) + .merge!(attrs) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2171c6c7bbb..561779182bc 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -19,6 +19,7 @@ project_tree: - milestone: - events: - :push_event_payload + - :issue_assignees - snippets: - :award_emoji - notes: @@ -53,6 +54,7 @@ project_tree: - :auto_devops - :triggers - :pipeline_schedules + - :cluster - :services - :hooks - protected_branches: @@ -112,6 +114,7 @@ excluded_attributes: - :milestone_id - :ref_fetched - :merge_jid + - :latest_merge_request_diff_id award_emoji: - :awardable_id statuses: diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 3bc095a99a9..639f4f0c3f0 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport class ProjectTreeRestorer # Relations which cannot have both group_id and project_id at the same time - RESTRICT_PROJECT_AND_GROUP = %i(milestones).freeze + RESTRICT_PROJECT_AND_GROUP = %i(milestone milestones).freeze def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 380b336395d..469b230377d 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -8,6 +8,8 @@ module Gitlab triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', + cluster: 'Gcp::Cluster', + clusters: 'Gcp::Cluster', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', @@ -35,7 +37,7 @@ module Gitlab def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project.id) + @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper @user = user @project = project @@ -56,22 +58,21 @@ module Gitlab private def setup_models - if @relation_name == :notes - set_note_author - - # attachment is deprecated and note uploads are handled by Markdown uploader - @relation_hash['attachment'] = nil + case @relation_name + when :merge_request_diff then setup_st_diff_commits + when :merge_request_diff_files then setup_diff + when :notes then setup_note + when :project_label, :project_labels then setup_label + when :milestone, :milestones then setup_milestone + else + @relation_hash['project_id'] = @project.id end update_user_references update_project_references - handle_group_label if group_label? reset_tokens! remove_encrypted_attributes! - - set_st_diff_commits if @relation_name == :merge_request_diff - set_diff if @relation_name == :merge_request_diff_files end def update_user_references @@ -82,6 +83,12 @@ module Gitlab end end + def setup_note + set_note_author + # attachment is deprecated and note uploads are handled by Markdown uploader + @relation_hash['attachment'] = nil + end + # Sets the author for a note. If the user importing the project # has admin access, an actual mapping with new project members # will be used. Otherwise, a note stating the original author name @@ -134,11 +141,9 @@ module Gitlab @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end - def group_label? - @relation_hash['type'] == 'GroupLabel' - end + def setup_label + return unless @relation_hash['type'] == 'GroupLabel' - def handle_group_label # If there's no group, move the label to a project label if @relation_hash['group_id'] @relation_hash['project_id'] = nil @@ -148,6 +153,14 @@ module Gitlab end end + def setup_milestone + if @relation_hash['group_id'] + @relation_hash['group_id'] = @project.group.id + else + @relation_hash['project_id'] = @project.id + end + end + def reset_tokens! return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s) @@ -196,14 +209,14 @@ module Gitlab relation_class: relation_class) end - def set_st_diff_commits + def setup_st_diff_commits @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs') HashUtil.deep_symbolize_array!(@relation_hash['st_diffs']) HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits']) end - def set_diff + def setup_diff @relation_hash['diff'] = @relation_hash.delete('utf8_diff') end @@ -248,7 +261,13 @@ module Gitlab end def find_or_create_object! - finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] + finder_attributes = if @relation_name == :group_label + %w[title group_id] + elsif parsed_relation_hash['project_id'] + %w[title project_id] + else + %w[title group_id] + end finder_hash = parsed_relation_hash.slice(*finder_attributes) if label? diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index cdbdfa10d0e..da43bd0af4b 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -113,7 +113,7 @@ module Gitlab def kubeconfig_embed_ca_pem(config, ca_pem) cluster = config.dig(:clusters, 0, :cluster) - cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem) + cluster[:'certificate-authority-data'] = Base64.strict_encode64(ca_pem) end end end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index fb68627dedf..e60ceba27c8 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -16,7 +16,7 @@ module Gitlab def self.allowed?(user) self.open(user) do |access| if access.allowed? - Users::UpdateService.new(user, last_credential_check_at: Time.now).execute + Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute true else diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index cd7e4ca7b7e..0afaa2306b5 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -22,8 +22,8 @@ module Gitlab Gitlab::LDAP::Config.new(provider) end - def users(field, value, limit = nil) - options = user_options(field, value, limit) + def users(fields, value, limit = nil) + options = user_options(Array(fields), value, limit) entries = ldap_search(options).select do |entry| entry.respond_to? config.uid @@ -72,20 +72,24 @@ module Gitlab private - def user_options(field, value, limit) - options = { attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq } + def user_options(fields, value, limit) + options = { + attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq, + base: config.base + } + options[:size] = limit if limit - if field.to_sym == :dn + if fields.include?('dn') + raise ArgumentError, 'It is not currently possible to search the DN and other fields at the same time.' if fields.size > 1 + options[:base] = value options[:scope] = Net::LDAP::SearchScope_BaseObject - options[:filter] = user_filter else - options[:base] = config.base - options[:filter] = user_filter(Net::LDAP::Filter.eq(field, value)) + filter = fields.map { |field| Net::LDAP::Filter.eq(field, value) }.inject(:|) end - options + options.merge(filter: user_filter(filter)) end def user_filter(filter = nil) diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index 4fbc5fa5262..1bd0965679a 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -3,6 +3,10 @@ module Gitlab module LDAP class AuthHash < Gitlab::OAuth::AuthHash + def uid + @uid ||= Gitlab::LDAP::Person.normalize_dn(super) + end + private def get_info(key) diff --git a/lib/gitlab/ldap/dn.rb b/lib/gitlab/ldap/dn.rb new file mode 100644 index 00000000000..d6142dc6549 --- /dev/null +++ b/lib/gitlab/ldap/dn.rb @@ -0,0 +1,301 @@ +# -*- ruby encoding: utf-8 -*- + +# Based on the `ruby-net-ldap` gem's `Net::LDAP::DN` +# +# For our purposes, this class is used to normalize DNs in order to allow proper +# comparison. +# +# E.g. DNs should be compared case-insensitively (in basically all LDAP +# implementations or setups), therefore we downcase every DN. + +## +# Objects of this class represent an LDAP DN ("Distinguished Name"). A DN +# ("Distinguished Name") is a unique identifier for an entry within an LDAP +# directory. It is made up of a number of other attributes strung together, +# to identify the entry in the tree. +# +# Each attribute that makes up a DN needs to have its value escaped so that +# the DN is valid. This class helps take care of that. +# +# A fully escaped DN needs to be unescaped when analysing its contents. This +# class also helps take care of that. +module Gitlab + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) + + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end + + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end + end + + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" + + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" + end + end + + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state + + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end + + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) + + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end + end + + str + end + + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end + + ## + # Return the DN as an escaped string. + def to_s + @dn + end + + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end + + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze + + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") + + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") + + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end + + private + + def initialize_array(args) + buffer = StringIO.new + + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end + end + + @dn = buffer.string + end + + def initialize_string(arg) + @dn = arg.to_s + end + + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end + + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end + end + end +end diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 4d6f8ac79de..38d7a9ba2f5 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -17,6 +17,12 @@ module Gitlab adapter.user('dn', dn) end + def self.find_by_email(email, adapter) + email_fields = adapter.config.attributes['email'] + + adapter.user(email_fields, email) + end + def self.disabled_via_active_directory?(dn, adapter) adapter.dn_matches_filter?(dn, AD_USER_DISABLED) end @@ -30,6 +36,26 @@ module Gitlab ] end + def self.normalize_dn(dn) + ::Gitlab::LDAP::DN.new(dn).to_normalized_s + rescue ::Gitlab::LDAP::DN::FormatError => e + Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") + + dn + end + + # Returns the UID in a normalized form. + # + # 1. Excess spaces are stripped + # 2. The string is downcased (for case-insensitivity) + def self.normalize_uid(uid) + ::Gitlab::LDAP::DN.normalize_value(uid) + rescue ::Gitlab::LDAP::DN::FormatError => e + Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") + + uid + end + def initialize(entry, provider) Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @@ -52,7 +78,9 @@ module Gitlab attribute_value(:email) end - delegate :dn, to: :entry + def dn + self.class.normalize_dn(entry.dn) + end private diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 3bf27b37ae6..4d5c67ed892 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -9,49 +9,28 @@ module Gitlab class User < Gitlab::OAuth::User class << self def find_by_uid_and_provider(uid, provider) - # LDAP distinguished name is case-insensitive + uid = Gitlab::LDAP::Person.normalize_dn(uid) + identity = ::Identity .where(provider: provider) - .iwhere(extern_uid: uid).last + .where(extern_uid: uid).last identity && identity.user end end - def initialize(auth_hash) - super - update_user_attributes - end - def save super('LDAP') end # instance methods - def gl_user - @gl_user ||= find_by_uid_and_provider || find_by_email || build_new_user + def find_user + find_by_uid_and_provider || find_by_email || build_new_user end def find_by_uid_and_provider self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) end - def find_by_email - ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_attribute?(:email) - end - - def update_user_attributes - if persisted? - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. - identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } - identity ||= gl_user.identities.build(provider: auth_hash.provider) - - # For a new identity set extern_uid to the LDAP DN - # For an existing identity with matching email but changed DN, update the DN. - # For an existing identity with no change in DN, this line changes nothing. - identity.extern_uid = auth_hash.uid - end - end - def changed? gl_user.changed? || gl_user.identities.any?(&:changed?) end diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index 6bffd410ed0..a42e312b5d3 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -13,7 +13,7 @@ module Gitlab end def self.read_latest - path = Rails.root.join("log", file_name) + path = self.full_log_path return [] unless File.readable?(path) @@ -22,7 +22,15 @@ module Gitlab end def self.build - new(Rails.root.join("log", file_name)) + RequestStore[self.cache_key] ||= new(self.full_log_path) + end + + def self.full_log_path + Rails.root.join("log", file_name) + end + + def self.cache_key + 'logger:'.freeze + self.full_log_path.to_s end end end diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb deleted file mode 100644 index 306923902e0..00000000000 --- a/lib/gitlab/markdown/pipeline.rb +++ /dev/null @@ -1,32 +0,0 @@ -module Gitlab - module Markdown - class Pipeline - def self.[](name) - name ||= :full - const_get("#{name.to_s.camelize}Pipeline") - end - - def self.filters - [] - end - - def self.transform_context(context) - context - end - - def self.html_pipeline - @html_pipeline ||= HTML::Pipeline.new(filters) - end - - class << self - %i(call to_document to_html).each do |meth| - define_method(meth) do |text, context| - context = transform_context(context) - - html_pipeline.__send__(meth, text, context) # rubocop:disable GitlabSecurity/PublicSend - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index f9dd8e41912..b983a40611f 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -11,6 +11,8 @@ module Gitlab # Old gitlad-shell messages don't provide enqueued_at/created_at attributes trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } + + worker.metrics_tags.each { |tag, value| trans.add_tag(tag, value) } if worker.respond_to?(:metrics_tags) rescue Exception => error # rubocop: disable Lint/RescueException trans.add_event(:sidekiq_exception) diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index aba3e0df382..c2cbd3c16a1 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -46,14 +46,14 @@ module Gitlab # Returns the current real time in a given precision. # - # Returns the time as a Float. + # Returns the time as a Fixnum. def self.real_time(precision = :millisecond) Process.clock_gettime(Process::CLOCK_REALTIME, precision) end # Returns the current monotonic clock time in a given precision. # - # Returns the time as a Float. + # Returns the time as a Fixnum. def self.monotonic_time(precision = :millisecond) Process.clock_gettime(Process::CLOCK_MONOTONIC, precision) end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index f42168c720e..cfc6b2a2029 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -4,6 +4,7 @@ module Gitlab module Middleware class Go include ActionView::Helpers::TagHelper + include Gitlab::CurrentSettings PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze @@ -37,10 +38,20 @@ module Gitlab end def go_body(path) - project_url = URI.join(Gitlab.config.gitlab.url, path) + config = Gitlab.config + project_url = URI.join(config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) - meta_tag = tag :meta, name: 'go-import', content: "#{import_prefix} git #{project_url}.git" + repository_url = case current_application_settings.enabled_git_access_protocol + when 'ssh' + shell = config.gitlab_shell + port = ":#{shell.ssh_port}" unless shell.ssh_port == 22 + "ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git" + when 'http', nil + "#{project_url}.git" + end + + meta_tag = tag :meta, name: 'go-import', content: "#{import_prefix} git #{repository_url}" head_tag = content_tag :head, meta_tag content_tag :html, head_tag end diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb new file mode 100644 index 00000000000..8853dfa3d2d --- /dev/null +++ b/lib/gitlab/middleware/read_only.rb @@ -0,0 +1,89 @@ +module Gitlab + module Middleware + class ReadOnly + DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze + APPLICATION_JSON = 'application/json'.freeze + API_VERSIONS = (3..4) + + def initialize(app) + @app = app + @whitelisted = internal_routes + end + + def call(env) + @env = env + @route_hash = nil + + if disallowed_request? && Gitlab::Database.read_only? + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + error_message = 'You cannot do writing operations on a read-only GitLab instance' + + if json_request? + return [403, { 'Content-Type' => 'application/json' }, [{ 'message' => error_message }.to_json]] + else + rack_flash.alert = error_message + rack_session['flash'] = rack_flash.to_session_value + + return [301, { 'Location' => last_visited_url }, []] + end + end + + @app.call(env) + end + + private + + def internal_routes + API_VERSIONS.flat_map { |version| "api/v#{version}/internal" } + end + + def disallowed_request? + DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && !whitelisted_routes + end + + def json_request? + request.media_type == APPLICATION_JSON + end + + def rack_flash + @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) + end + + def rack_session + @env['rack.session'] + end + + def request + @env['rack.request'] ||= Rack::Request.new(@env) + end + + def last_visited_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Rails.application.routes.url_helpers.root_url + end + + def route_hash + @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} + end + + def whitelisted_routes + logout_route || grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route + end + + def logout_route + route_hash[:controller] == 'sessions' && route_hash[:action] == 'destroy' + end + + def sidekiq_route + request.path.start_with?('/admin/sidekiq') + end + + def grack_route + route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' + end + + def lfs_route + route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + end + end + end +end diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb new file mode 100644 index 00000000000..eb3c9002710 --- /dev/null +++ b/lib/gitlab/multi_collection_paginator.rb @@ -0,0 +1,61 @@ +module Gitlab + class MultiCollectionPaginator + attr_reader :first_collection, :second_collection, :per_page + + def initialize(*collections, per_page: nil) + raise ArgumentError.new('Only 2 collections are supported') if collections.size != 2 + + @per_page = per_page || Kaminari.config.default_per_page + @first_collection, @second_collection = collections + end + + def paginate(page) + page = page.to_i + paginated_first_collection(page) + paginated_second_collection(page) + end + + def total_count + @total_count ||= first_collection.size + second_collection.size + end + + private + + def paginated_first_collection(page) + @first_collection_pages ||= Hash.new do |hash, page| + hash[page] = first_collection.page(page).per(per_page) + end + + @first_collection_pages[page] + end + + def paginated_second_collection(page) + @second_collection_pages ||= Hash.new do |hash, page| + second_collection_page = page - first_collection_page_count + + offset = if second_collection_page < 1 || first_collection_page_count.zero? + 0 + else + per_page - first_collection_last_page_size + end + hash[page] = second_collection.page(second_collection_page) + .per(per_page - paginated_first_collection(page).size) + .padding(offset) + end + + @second_collection_pages[page] + end + + def first_collection_page_count + return @first_collection_page_count if defined?(@first_collection_page_count) + + first_collection_page = paginated_first_collection(0) + @first_collection_page_count = first_collection_page.total_pages + end + + def first_collection_last_page_size + return @first_collection_last_page_size if defined?(@first_collection_last_page_size) + + @first_collection_last_page_size = paginated_first_collection(first_collection_page_count).count + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 7704bf715e4..47c2a422387 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -13,6 +13,7 @@ module Gitlab def initialize(auth_hash) self.auth_hash = auth_hash update_profile if sync_profile_from_provider? + add_or_update_user_identities end def persisted? @@ -32,7 +33,7 @@ module Gitlab block_after_save = needs_blocking? - Users::UpdateService.new(gl_user).execute! + Users::UpdateService.new(gl_user, user: gl_user).execute! gl_user.block if block_after_save @@ -44,47 +45,56 @@ module Gitlab end def gl_user - @user ||= find_by_uid_and_provider + return @gl_user if defined?(@gl_user) - if auto_link_ldap_user? - @user ||= find_or_create_ldap_user - end + @gl_user = find_user + end - if signup_enabled? - @user ||= build_new_user - end + def find_user + user = find_by_uid_and_provider - if external_provider? && @user - @user.external = true - end + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? + + user.external = true if external_provider? && user - @user + user end protected - def find_or_create_ldap_user + def add_or_update_user_identities + return unless gl_user + + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + + identity ||= gl_user.identities.build(provider: auth_hash.provider) + identity.extern_uid = auth_hash.uid + + if auto_link_ldap_user? && !gl_user.ldap_user? && ldap_person + log.info "Correct LDAP account has been found. identity to user: #{gl_user.username}." + gl_user.identities.build(provider: ldap_person.provider, extern_uid: ldap_person.dn) + end + end + + def find_or_build_ldap_user return unless ldap_person - # If a corresponding person exists with same uid in a LDAP server, - # check if the user already has a GitLab account. user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) if user - # Case when a LDAP user already exists in Gitlab. Add the OAuth identity to existing account. log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." - user.identities.find_or_initialize_by(extern_uid: auth_hash.uid, provider: auth_hash.provider) - else - log.info "No existing LDAP account was found in GitLab. Checking for #{auth_hash.provider} account." - user = find_by_uid_and_provider - if user.nil? - log.info "No user found using #{auth_hash.provider} provider. Creating a new one." - user = build_new_user - end - log.info "Correct account has been found. Adding LDAP identity to user: #{user.username}." - user.identities.new(provider: ldap_person.provider, extern_uid: ldap_person.dn) + return user end - user + log.info "No user found using #{auth_hash.provider} provider. Creating a new one." + build_new_user + end + + def find_by_email + return unless auth_hash.has_attribute?(:email) + + ::User.find_by(email: auth_hash.email.downcase) end def auto_link_ldap_user? @@ -108,9 +118,9 @@ module Gitlab end def find_ldap_person(auth_hash, adapter) - by_uid = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) - # The `uid` might actually be a DN. Try it next. - by_uid || Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || + Gitlab::LDAP::Person.find_by_email(auth_hash.uid, adapter) || + Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) end def ldap_config @@ -152,7 +162,7 @@ module Gitlab end def build_new_user - user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) + user_params = user_attributes.merge(skip_confirmation: true) Users::BuildService.new(nil, user_params).execute(skip_authorization: true) end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 6df9d60721e..cd8b2eba6c4 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -27,7 +27,6 @@ module Gitlab apple-touch-icon.png assets autocomplete - boards ci dashboard deploy.html @@ -130,7 +129,6 @@ module Gitlab notification_setting pipeline_quota projects - subgroups ].freeze ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index 67fee8c227d..f2825db59ae 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -36,8 +36,8 @@ module Gitlab end def track_query(raw_query, bindings, start, finish) - query = Gitlab::Sherlock::Query.new(raw_query, start, finish) - query_info = { duration: query.duration.round(3), sql: query.formatted_query } + duration = (finish - start) * 1000.0 + query_info = { duration: duration.round(3), sql: raw_query } PEEK_DB_CLIENT.query_details << query_info end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 732fbf68dad..ae136202f0c 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -1,9 +1,9 @@ module Gitlab class ProjectTemplate - attr_reader :title, :name + attr_reader :title, :name, :description, :preview - def initialize(name, title) - @name, @title = name, title + def initialize(name, title, description, preview) + @name, @title, @description, @preview = name, title, description, preview end alias_method :logo, :name @@ -25,9 +25,9 @@ module Gitlab end TEMPLATES_TABLE = [ - ProjectTemplate.new('rails', 'Ruby on Rails'), - ProjectTemplate.new('spring', 'Spring'), - ProjectTemplate.new('express', 'NodeJS Express') + ProjectTemplate.new('rails', 'Ruby on Rails', 'Includes an MVC structure, gemfile, rakefile, and .gitlab-ci.yml file, along with many others, to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/rails'), + ProjectTemplate.new('spring', 'Spring', 'Includes an MVC structure, mvnw, pom.xml, and .gitlab-ci.yml file to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/spring'), + ProjectTemplate.new('express', 'NodeJS Express', 'Includes an MVC structure and .gitlab-ci.yml file to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/express') ].freeze class << self diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb new file mode 100644 index 00000000000..3f52402b31f --- /dev/null +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -0,0 +1,54 @@ +module Gitlab + module QuickActions + # This class takes spend command argument + # and separates date and time from spend command arguments if it present + # example: + # spend_command_time_and_date = "15m 2017-01-02" + # SpendTimeAndDateSeparator.new(spend_command_time_and_date).execute + # => [900, Mon, 02 Jan 2017] + # if date doesn't present return time with current date + # in other cases return nil + class SpendTimeAndDateSeparator + DATE_REGEX = /(\d{2,4}[\/\-.]\d{1,2}[\/\-.]\d{1,2})/ + + def initialize(spend_command_arg) + @spend_arg = spend_command_arg + end + + def execute + return if @spend_arg.blank? + return [get_time, DateTime.now.to_date] unless date_present? + return unless valid_date? + + [get_time, get_date] + end + + private + + def get_time + raw_time = @spend_arg.gsub(DATE_REGEX, '') + Gitlab::TimeTrackingFormatter.parse(raw_time) + end + + def get_date + string_date = @spend_arg.match(DATE_REGEX)[0] + Date.parse(string_date) + end + + def date_present? + DATE_REGEX =~ @spend_arg + end + + def valid_date? + string_date = @spend_arg.match(DATE_REGEX)[0] + date = Date.parse(string_date) rescue nil + + date_past_or_today?(date) + end + + def date_past_or_today?(date) + date&.past? || date&.today? + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 58f6245579a..bd677ec4bf3 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -65,5 +65,9 @@ module Gitlab "can contain only lowercase letters, digits, and '-'. " \ "Must start with a letter, and cannot end with '-'" end + + def build_trace_section_regex + @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([^\r]+)\r\033\[0K/.freeze + end end end diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb index 67a5f368bdb..33d19373098 100644 --- a/lib/gitlab/saml/auth_hash.rb +++ b/lib/gitlab/saml/auth_hash.rb @@ -2,7 +2,7 @@ module Gitlab module Saml class AuthHash < Gitlab::OAuth::AuthHash def groups - get_raw(Gitlab::Saml::Config.groups) + Array.wrap(get_raw(Gitlab::Saml::Config.groups)) end private diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index 0f323a9e8b2..e0a9d1dee77 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -10,41 +10,20 @@ module Gitlab super('SAML') end - def gl_user - if auto_link_ldap_user? - @user ||= find_or_create_ldap_user - end - - @user ||= find_by_uid_and_provider - - if auto_link_saml_user? - @user ||= find_by_email - end + def find_user + user = find_by_uid_and_provider - if signup_enabled? - @user ||= build_new_user - end + user ||= find_by_email if auto_link_saml_user? + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? - if external_users_enabled? && @user + if external_users_enabled? && user # Check if there is overlap between the user's groups and the external groups # setting then set user as external or internal. - @user.external = - if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? - false - else - true - end + user.external = !(auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? end - @user - end - - def find_by_email - if auth_hash.has_attribute?(:email) - user = ::User.find_by(email: auth_hash.email.downcase) - user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user - user - end + user end def changed? diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 81ecdf43ef9..a37112ae5c4 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -65,7 +65,7 @@ module Gitlab # Init new repository # - # storage - project's storage path + # storage - project's storage name # name - project path with namespace # # Ex. @@ -73,7 +73,19 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) - Gitlab::Git::Repository.create(storage, name, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) + relative_path = name.dup + relative_path << '.git' unless relative_path.end_with?('.git') + + gitaly_migrate(:create_repository) do |is_enabled| + if is_enabled + repository = Gitlab::Git::Repository.new(storage, relative_path, '') + repository.gitaly_repository_client.create_repository + true + else + repo_path = File.join(Gitlab.config.repositories.storages[storage]['path'], relative_path) + Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) + end + end rescue => err Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}") false @@ -210,10 +222,18 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) - path = full_path(storage, name) - FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) + Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).add(name) + else + path = full_path(storage, name) + FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) + end + end rescue Errno::EEXIST => e Rails.logger.warn("Directory exists as a file: #{e} at: #{path}") + rescue GRPC::InvalidArgument => e + raise ArgumentError, e.message end # Remove directory from repositories storage @@ -224,7 +244,15 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) - FileUtils.rm_r(full_path(storage, name), force: true) + Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).remove(name) + else + FileUtils.rm_r(full_path(storage, name), force: true) + end + end + rescue GRPC::InvalidArgument => e + raise ArgumentError, e.message end # Move namespace directory inside repositories storage @@ -234,9 +262,17 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) - return false if exists?(storage, new_name) || !exists?(storage, old_name) + Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).rename(old_name, new_name) + else + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) + end + end + rescue GRPC::InvalidArgument + false end def url_to_repo(path) @@ -260,7 +296,13 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) - File.exist?(full_path(storage, dir_name)) + Gitlab::GitalyClient.migrate(:namespace_exists) do |enabled| + if enabled + gitaly_namespace_client(storage).exists?(dir_name) + else + File.exist?(full_path(storage, dir_name)) + end + end end protected @@ -337,6 +379,14 @@ module Gitlab Bundler.with_original_env { Popen.popen(cmd, nil, vars) } end + def gitaly_namespace_client(storage_path) + storage, _value = Gitlab.config.repositories.storages.find do |storage, value| + value['path'] == storage_path + end + + Gitlab::GitalyClient::NamespaceService.new(storage) + end + def gitaly_migrate(method, &block) Gitlab::GitalyClient.migrate(method, &block) rescue GRPC::NotFound, GRPC::BadStatus => e diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb index 3489fb251b6..400a552bf99 100644 --- a/lib/gitlab/sherlock/transaction.rb +++ b/lib/gitlab/sherlock/transaction.rb @@ -89,7 +89,9 @@ module Gitlab ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data| next unless same_thread? - track_query(data[:sql].strip, data[:binds], start, finish) + unless data.fetch(:cached, data[:name] == 'CACHE') + track_query(data[:sql].strip, data[:binds], start, finish) + end end end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 104280f520a..2bfb7caefd9 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -7,7 +7,6 @@ module Gitlab GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i # Wait 30 seconds for running jobs to finish during graceful shutdown SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i - SHUTDOWN_SIGNAL = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL'] || 'SIGKILL').to_s # Create a mutex used to ensure there will be only one thread waiting to # shut Sidekiq down @@ -15,6 +14,7 @@ module Gitlab def call(worker, job, queue) yield + current_rss = get_rss return unless MAX_RSS > 0 && current_rss > MAX_RSS @@ -23,32 +23,45 @@ module Gitlab # Return if another thread is already waiting to shut Sidekiq down return unless MUTEX.try_lock - Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ - "#{MAX_RSS}" - Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"\ - "in #{GRACE_TIME} seconds" - sleep(GRACE_TIME) + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ + " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" - Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" - Process.kill('SIGTERM', Process.pid) + # Wait `GRACE_TIME` to give the memory intensive job time to finish. + # Then, tell Sidekiq to stop fetching new jobs. + wait_and_signal(GRACE_TIME, 'SIGSTP', 'stop fetching new jobs') - Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\ - "#{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" - sleep(SHUTDOWN_WAIT) + # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. + # Then, tell Sidekiq to gracefully shut down by giving jobs a few more + # moments to finish, killing and requeuing them if they didn't, and + # then terminating itself. + wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') - Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" - Process.kill(SHUTDOWN_SIGNAL, Process.pid) + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. + wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') end end private def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{Process.pid})) + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid})) return 0 unless status.zero? output.to_i end + + def wait_and_signal(time, signal, explanation) + Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + sleep(time) + + Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + Process.kill(signal, pid) + end + + def pid + Process.pid + end end end end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index a0a2769cf9e..a1f689d94d9 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -51,6 +51,13 @@ module Gitlab self.num_running(job_ids).zero? end + # Returns true if the given job is running + # + # job_id - The Sidekiq job ID to check. + def self.running?(job_id) + num_running([job_id]) > 0 + end + # Returns the number of jobs that are running. # # job_ids - The Sidekiq job IDs to check. diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 222021e8802..c99b262f1ca 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -12,8 +12,9 @@ module Gitlab # # Project.where("id IN (#{sql})") class Union - def initialize(relations) + def initialize(relations, remove_duplicates: true) @relations = relations + @remove_duplicates = remove_duplicates end def to_sql @@ -25,7 +26,15 @@ module Gitlab @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) end - fragments.join("\nUNION\n") + if fragments.any? + fragments.join("\n#{union_keyword}\n") + else + 'NULL' + end + end + + def union_keyword + @remove_duplicates ? 'UNION' : 'UNION ALL' end end end diff --git a/lib/gitlab/testing/request_blocker_middleware.rb b/lib/gitlab/testing/request_blocker_middleware.rb index aa67fa08577..4a8e3c2eee0 100644 --- a/lib/gitlab/testing/request_blocker_middleware.rb +++ b/lib/gitlab/testing/request_blocker_middleware.rb @@ -7,6 +7,7 @@ module Gitlab class RequestBlockerMiddleware @@num_active_requests = Concurrent::AtomicFixnum.new(0) @@block_requests = Concurrent::AtomicBoolean.new(false) + @@slow_requests = Concurrent::AtomicBoolean.new(false) # Returns the number of requests the server is currently processing. def self.num_active_requests @@ -19,9 +20,15 @@ module Gitlab @@block_requests.value = true end + # Slows down incoming requests (useful for race conditions). + def self.slow_requests! + @@slow_requests.value = true + end + # Allows the server to accept requests again. def self.allow_requests! @@block_requests.value = false + @@slow_requests.value = false end def initialize(app) @@ -33,6 +40,7 @@ module Gitlab if block_requests? block_request(env) else + sleep 0.2 if slow_requests? @app.call(env) end ensure @@ -45,6 +53,10 @@ module Gitlab @@block_requests.true? end + def slow_requests? + @@slow_requests.true? + end + def block_request(env) [503, {}, []] end diff --git a/lib/gitlab/testing/request_inspector_middleware.rb b/lib/gitlab/testing/request_inspector_middleware.rb new file mode 100644 index 00000000000..e387667480d --- /dev/null +++ b/lib/gitlab/testing/request_inspector_middleware.rb @@ -0,0 +1,71 @@ +# rubocop:disable Style/ClassVars + +module Gitlab + module Testing + class RequestInspectorMiddleware + @@log_requests = Concurrent::AtomicBoolean.new(false) + @@logged_requests = Concurrent::Array.new + @@inject_headers = Concurrent::Hash.new + + # Resets the current request log and starts logging requests + def self.log_requests!(headers = {}) + @@inject_headers.replace(headers) + @@logged_requests.replace([]) + @@log_requests.value = true + end + + # Stops logging requests + def self.stop_logging! + @@log_requests.value = false + end + + def self.requests + @@logged_requests + end + + def initialize(app) + @app = app + end + + def call(env) + return @app.call(env) unless @@log_requests.true? + + url = env['REQUEST_URI'] + env.merge! http_headers_env(@@inject_headers) if @@inject_headers.any? + request_headers = env_http_headers(env) + status, headers, body = @app.call(env) + + request = OpenStruct.new( + url: url, + status_code: status, + request_headers: request_headers, + response_headers: headers + ) + log_request request + + [status, headers, body] + end + + private + + def env_http_headers(env) + Hash[*env.select { |k, v| k.start_with? 'HTTP_' } + .collect { |k, v| [k.sub(/^HTTP_/, ''), v] } + .collect { |k, v| [k.split('_').collect(&:capitalize).join('-'), v] } + .sort + .flatten] + end + + def http_headers_env(headers) + Hash[*headers + .collect { |k, v| [k.split('-').collect(&:upcase).join('_'), v] } + .collect { |k, v| [k.prepend('HTTP_'), v] } + .flatten] + end + + def log_request(response) + @@logged_requests.push(response) + end + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 4e1ec1402ea..1caa791c1be 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -1,7 +1,9 @@ module Gitlab class UrlSanitizer + ALLOWED_SCHEMES = %w[http https ssh git].freeze + def self.sanitize(content) - regexp = URI::Parser.new.make_regexp(%w(http https ssh git)) + regexp = URI::Parser.new.make_regexp(ALLOWED_SCHEMES) content.gsub(regexp) { |url| new(url).masked_url } rescue Addressable::URI::InvalidURIError @@ -11,9 +13,9 @@ module Gitlab def self.valid?(url) return false unless url.present? - Addressable::URI.parse(url.strip) + uri = Addressable::URI.parse(url.strip) - true + ALLOWED_SCHEMES.include?(uri.scheme) rescue Addressable::URI::InvalidURIError false end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6857038dba8..70a403652e7 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -48,6 +48,9 @@ module Gitlab deploy_keys: DeployKey.count, deployments: Deployment.count, environments: ::Environment.count, + gcp_clusters: ::Gcp::Cluster.count, + gcp_clusters_enabled: ::Gcp::Cluster.enabled.count, + gcp_clusters_disabled: ::Gcp::Cluster.disabled.count, in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, diff --git a/lib/gitlab/utils/merge_hash.rb b/lib/gitlab/utils/merge_hash.rb new file mode 100644 index 00000000000..385141d44d0 --- /dev/null +++ b/lib/gitlab/utils/merge_hash.rb @@ -0,0 +1,117 @@ +module Gitlab + module Utils + module MergeHash + extend self + # Deep merges an array of hashes + # + # [{ hello: ["world"] }, + # { hello: "Everyone" }, + # { hello: { greetings: ['Bonjour', 'Hello', 'Hallo', 'Dzien dobry'] } }, + # "Goodbye", "Hallo"] + # => [ + # { + # hello: + # [ + # "world", + # "Everyone", + # { greetings: ['Bonjour', 'Hello', 'Hallo', 'Dzien dobry'] } + # ] + # }, + # "Goodbye" + # ] + def merge(elements) + merged, *other_elements = elements + + other_elements.each do |element| + merged = merge_hash_tree(merged, element) + end + + merged + end + + # This extracts all keys and values from a hash into an array + # + # { hello: "world", this: { crushes: ["an entire", "hash"] } } + # => [:hello, "world", :this, :crushes, "an entire", "hash"] + def crush(array_or_hash) + if array_or_hash.is_a?(Array) + crush_array(array_or_hash) + else + crush_hash(array_or_hash) + end + end + + private + + def merge_hash_into_array(array, new_hash) + crushed_new_hash = crush_hash(new_hash) + # Merge the hash into an existing element of the array if there is overlap + if mergeable_index = array.index { |element| crushable?(element) && (crush(element) & crushed_new_hash).any? } + array[mergeable_index] = merge_hash_tree(array[mergeable_index], new_hash) + else + array << new_hash + end + + array + end + + def merge_hash_tree(first_element, second_element) + # If one of the elements is an object, and the other is a Hash or Array + # we can check if the object is already included. If so, we don't need to do anything + # + # Handled cases + # [Hash, Object], [Array, Object] + if crushable?(first_element) && crush(first_element).include?(second_element) + first_element + elsif crushable?(second_element) && crush(second_element).include?(first_element) + second_element + # When the first is an array, we need to go over every element to see if + # we can merge deeper. If no match is found, we add the element to the array + # + # Handled cases: + # [Array, Hash] + elsif first_element.is_a?(Array) && second_element.is_a?(Hash) + merge_hash_into_array(first_element, second_element) + elsif first_element.is_a?(Hash) && second_element.is_a?(Array) + merge_hash_into_array(second_element, first_element) + # If both of them are hashes, we can deep_merge with the same logic + # + # Handled cases: + # [Hash, Hash] + elsif first_element.is_a?(Hash) && second_element.is_a?(Hash) + first_element.deep_merge(second_element) { |key, first, second| merge_hash_tree(first, second) } + # If both elements are arrays, we try to merge each element separatly + # + # Handled cases + # [Array, Array] + elsif first_element.is_a?(Array) && second_element.is_a?(Array) + first_element.map { |child_element| merge_hash_tree(child_element, second_element) } + # If one or both elements are a GroupDescendant, we wrap create an array + # combining them. + # + # Handled cases: + # [Object, Object], [Array, Array] + else + (Array.wrap(first_element) + Array.wrap(second_element)).uniq + end + end + + def crushable?(element) + element.is_a?(Hash) || element.is_a?(Array) + end + + def crush_hash(hash) + hash.flat_map do |key, value| + crushed_value = crushable?(value) ? crush(value) : value + Array.wrap(key) + Array.wrap(crushed_value) + end + end + + def crush_array(array) + array.flat_map do |element| + crushable?(element) ? crush(element) : element + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 17550cf9074..e1219df1b25 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -16,15 +16,16 @@ module Gitlab SECRET_LENGTH = 32 class << self - def git_http_ok(repository, is_wiki, user, action) + def git_http_ok(repository, is_wiki, user, action, show_all_refs: false) project = repository.project repo_path = repository.path_to_repo params = { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), - RepoPath: repo_path + GL_USERNAME: user&.username, + RepoPath: repo_path, + ShowAllRefs: show_all_refs } - server = { address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) @@ -89,6 +90,13 @@ module Gitlab params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) raise "Repository or ref not found" if params.empty? + if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive) + params.merge!( + 'GitalyServer' => gitaly_server_hash(repository), + 'GitalyRepository' => repository.gitaly_repository.to_h + ) + end + [ SEND_DATA_HEADER, "git-archive:#{encode(params)}" @@ -96,11 +104,16 @@ module Gitlab end def send_git_diff(repository, diff_refs) - params = { - 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.base_sha, - 'ShaTo' => diff_refs.head_sha - } + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'RawDiffRequest' => Gitaly::RawDiffRequest.new( + gitaly_diff_or_patch_hash(repository, diff_refs) + ).to_json + } + else + workhorse_diff_or_patch_hash(repository, diff_refs) + end [ SEND_DATA_HEADER, @@ -109,11 +122,16 @@ module Gitlab end def send_git_patch(repository, diff_refs) - params = { - 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.base_sha, - 'ShaTo' => diff_refs.head_sha - } + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'RawPatchRequest' => Gitaly::RawPatchRequest.new( + gitaly_diff_or_patch_hash(repository, diff_refs) + ).to_json + } + else + workhorse_diff_or_patch_hash(repository, diff_refs) + end [ SEND_DATA_HEADER, @@ -209,6 +227,22 @@ module Gitlab token: Gitlab::GitalyClient.token(repository.project.repository_storage) } end + + def workhorse_diff_or_patch_hash(repository, diff_refs) + { + 'RepoPath' => repository.path_to_repo, + 'ShaFrom' => diff_refs.base_sha, + 'ShaTo' => diff_refs.head_sha + } + end + + def gitaly_diff_or_patch_hash(repository, diff_refs) + { + repository: repository.gitaly_repository, + left_commit_id: diff_refs.base_sha, + right_commit_id: diff_refs.head_sha + } + end end end end diff --git a/lib/google_api/auth.rb b/lib/google_api/auth.rb new file mode 100644 index 00000000000..99a82c849e0 --- /dev/null +++ b/lib/google_api/auth.rb @@ -0,0 +1,54 @@ +module GoogleApi + class Auth + attr_reader :access_token, :redirect_uri, :state + + ConfigMissingError = Class.new(StandardError) + + def initialize(access_token, redirect_uri, state: nil) + @access_token = access_token + @redirect_uri = redirect_uri + @state = state + end + + def authorize_url + client.auth_code.authorize_url( + redirect_uri: redirect_uri, + scope: scope, + state: state # This is used for arbitary redirection + ) + end + + def get_token(code) + ret = client.auth_code.get_token(code, redirect_uri: redirect_uri) + return ret.token, ret.expires_at + end + + protected + + def scope + raise NotImplementedError + end + + private + + def config + Gitlab.config.omniauth.providers.find { |provider| provider.name == "google_oauth2" } + end + + def client + return @client if defined?(@client) + + unless config + raise ConfigMissingError + end + + @client = ::OAuth2::Client.new( + config.app_id, + config.app_secret, + site: 'https://accounts.google.com', + token_url: '/o/oauth2/token', + authorize_url: '/o/oauth2/auth' + ) + end + end +end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb new file mode 100644 index 00000000000..a440a3e3562 --- /dev/null +++ b/lib/google_api/cloud_platform/client.rb @@ -0,0 +1,88 @@ +require 'google/apis/container_v1' + +module GoogleApi + module CloudPlatform + class Client < GoogleApi::Auth + DEFAULT_MACHINE_TYPE = 'n1-standard-1'.freeze + SCOPE = 'https://www.googleapis.com/auth/cloud-platform'.freeze + LEAST_TOKEN_LIFE_TIME = 10.minutes + + class << self + def session_key_for_token + :cloud_platform_access_token + end + + def session_key_for_expires_at + :cloud_platform_expires_at + end + + def new_session_key_for_redirect_uri + SecureRandom.hex.tap do |state| + yield session_key_for_redirect_uri(state) + end + end + + def session_key_for_redirect_uri(state) + "cloud_platform_second_redirect_uri_#{state}" + end + end + + def scope + SCOPE + end + + def validate_token(expires_at) + return false unless access_token + return false unless expires_at + + # Making sure that the token will have been still alive during the cluster creation. + return false if token_life_time(expires_at) < LEAST_TOKEN_LIFE_TIME + + true + end + + def projects_zones_clusters_get(project_id, zone, cluster_id) + service = Google::Apis::ContainerV1::ContainerService.new + service.authorization = access_token + + service.get_zone_cluster(project_id, zone, cluster_id) + end + + def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:) + service = Google::Apis::ContainerV1::ContainerService.new + service.authorization = access_token + + request_body = Google::Apis::ContainerV1::CreateClusterRequest.new( + { + "cluster": { + "name": cluster_name, + "initial_node_count": cluster_size, + "node_config": { + "machine_type": machine_type + } + } + } ) + + service.create_cluster(project_id, zone, request_body) + end + + def projects_zones_operations(project_id, zone, operation_id) + service = Google::Apis::ContainerV1::ContainerService.new + service.authorization = access_token + + service.get_zone_operation(project_id, zone, operation_id) + end + + def parse_operation_id(self_link) + m = self_link.match(%r{projects/.*/zones/.*/operations/(.*)}) + m[1] if m + end + + private + + def token_life_time(expires_at) + DateTime.strptime(expires_at, '%s').to_time.utc - Time.now.utc + end + end + end +end diff --git a/lib/omni_auth/strategies/bitbucket.rb b/lib/omni_auth/strategies/bitbucket.rb index 5a7d67c2390..ce1bdfe6ee4 100644 --- a/lib/omni_auth/strategies/bitbucket.rb +++ b/lib/omni_auth/strategies/bitbucket.rb @@ -36,6 +36,10 @@ module OmniAuth email_response = access_token.get('api/2.0/user/emails').parsed @emails ||= email_response && email_response['values'] || nil end + + def callback_url + options[:redirect_uri] || (full_host + script_name + callback_path) + end end end end diff --git a/lib/peek/views/gitaly.rb b/lib/peek/views/gitaly.rb new file mode 100644 index 00000000000..d519d8e86fa --- /dev/null +++ b/lib/peek/views/gitaly.rb @@ -0,0 +1,34 @@ +module Peek + module Views + class Gitaly < View + def duration + ::Gitlab::GitalyClient.query_time + end + + def calls + ::Gitlab::GitalyClient.get_request_count + end + + def results + { duration: formatted_duration, calls: calls } + end + + private + + def formatted_duration + ms = duration * 1000 + if ms >= 1000 + "%.2fms" % ms + else + "%.0fms" % ms + end + end + + def setup_subscribers + subscribe 'start_processing.action_controller' do + ::Gitlab::GitalyClient.query_time = 0 + end + end + end + end +end diff --git a/lib/rspec_flaky/config.rb b/lib/rspec_flaky/config.rb new file mode 100644 index 00000000000..a17ae55910e --- /dev/null +++ b/lib/rspec_flaky/config.rb @@ -0,0 +1,21 @@ +require 'json' + +module RspecFlaky + class Config + def self.generate_report? + ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + end + + def self.suite_flaky_examples_report_path + ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/suite-report.json") + end + + def self.flaky_examples_report_path + ENV['FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/report.json") + end + + def self.new_flaky_examples_report_path + ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/new-report.json") + end + end +end diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb index f81fb90e870..6be24014d89 100644 --- a/lib/rspec_flaky/flaky_example.rb +++ b/lib/rspec_flaky/flaky_example.rb @@ -9,24 +9,21 @@ module RspecFlaky line: example.line, description: example.description, last_attempts_count: example.attempts, - flaky_reports: 1) + flaky_reports: 0) else super end end - def first_flaky_at - self[:first_flaky_at] || Time.now - end - - def last_flaky_at - Time.now - end + def update_flakiness!(last_attempts_count: nil) + self.first_flaky_at ||= Time.now + self.last_flaky_at = Time.now + self.flaky_reports += 1 + self.last_attempts_count = last_attempts_count if last_attempts_count - def last_flaky_job - return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] - - "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" + if ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] + self.last_flaky_job = "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" + end end def to_h diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb new file mode 100644 index 00000000000..973c95b0212 --- /dev/null +++ b/lib/rspec_flaky/flaky_examples_collection.rb @@ -0,0 +1,37 @@ +require 'json' + +module RspecFlaky + class FlakyExamplesCollection < SimpleDelegator + def self.from_json(json) + new(JSON.parse(json)) + end + + def initialize(collection = {}) + unless collection.is_a?(Hash) + raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" + end + + collection_of_flaky_examples = + collection.map do |uid, example| + [ + uid, + example.is_a?(RspecFlaky::FlakyExample) ? example : RspecFlaky::FlakyExample.new(example) + ] + end + + super(Hash[collection_of_flaky_examples]) + end + + def to_report + Hash[map { |uid, example| [uid, example.to_h] }].deep_symbolize_keys + end + + def -(other) + unless other.respond_to?(:key) + raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" + end + + self.class.new(reject { |uid, _| other.key?(uid) }) + end + end +end diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb index ec2fbd9e36c..4a5bfec9967 100644 --- a/lib/rspec_flaky/listener.rb +++ b/lib/rspec_flaky/listener.rb @@ -2,11 +2,15 @@ require 'json' module RspecFlaky class Listener - attr_reader :all_flaky_examples, :new_flaky_examples - - def initialize - @new_flaky_examples = {} - @all_flaky_examples = init_all_flaky_examples + # - suite_flaky_examples: contains all the currently tracked flacky example + # for the whole RSpec suite + # - flaky_examples: contains the examples detected as flaky during the + # current RSpec run + attr_reader :suite_flaky_examples, :flaky_examples + + def initialize(suite_flaky_examples_json = nil) + @flaky_examples = FlakyExamplesCollection.new + @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) end def example_passed(notification) @@ -14,29 +18,21 @@ module RspecFlaky return unless current_example.attempts > 1 - flaky_example_hash = all_flaky_examples[current_example.uid] - - all_flaky_examples[current_example.uid] = - if flaky_example_hash - FlakyExample.new(flaky_example_hash).tap do |ex| - ex.last_attempts_count = current_example.attempts - ex.flaky_reports += 1 - end - else - FlakyExample.new(current_example).tap do |ex| - new_flaky_examples[current_example.uid] = ex - end - end + flaky_example = suite_flaky_examples.fetch(current_example.uid) { FlakyExample.new(current_example) } + flaky_example.update_flakiness!(last_attempts_count: current_example.attempts) + + flaky_examples[current_example.uid] = flaky_example end def dump_summary(_) - write_report_file(all_flaky_examples, all_flaky_examples_report_path) + write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) + new_flaky_examples = flaky_examples - suite_flaky_examples if new_flaky_examples.any? Rails.logger.warn "\nNew flaky examples detected:\n" - Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples)) + Rails.logger.warn JSON.pretty_generate(new_flaky_examples.to_report) - write_report_file(new_flaky_examples, new_flaky_examples_report_path) + write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) end end @@ -46,30 +42,23 @@ module RspecFlaky private - def init_all_flaky_examples - return {} unless File.exist?(all_flaky_examples_report_path) + def init_suite_flaky_examples(suite_flaky_examples_json = nil) + unless suite_flaky_examples_json + return {} unless File.exist?(RspecFlaky::Config.suite_flaky_examples_report_path) - all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path)) + suite_flaky_examples_json = File.read(RspecFlaky::Config.suite_flaky_examples_report_path) + end - Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }] + FlakyExamplesCollection.from_json(suite_flaky_examples_json) end - def write_report_file(examples, file_path) - return unless ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + def write_report_file(examples_collection, file_path) + return unless RspecFlaky::Config.generate_report? report_path_dir = File.dirname(file_path) FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) - File.write(file_path, JSON.pretty_generate(to_report(examples))) - end - - def all_flaky_examples_report_path - @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] || - Rails.root.join("rspec_flaky/all-report.json") - end - def new_flaky_examples_report_path - @new_flaky_examples_report_path ||= ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || - Rails.root.join("rspec_flaky/new-report.json") + File.write(file_path, JSON.pretty_generate(examples_collection.to_report)) end end end diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb index 7b486d78cf0..ad41760dff2 100644 --- a/lib/system_check/app/git_user_default_ssh_config_check.rb +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -5,12 +5,13 @@ module SystemCheck # whitelisted as it may change the SSH client's behaviour dramatically. WHITELIST = %w[ authorized_keys + authorized_keys.lock authorized_keys2 known_hosts ].freeze set_name 'Git user has default SSH configuration?' - set_skip_reason 'skipped (git user is not present or configured)' + set_skip_reason 'skipped (git user is not present / configured)' def skip? !home_dir || !File.directory?(home_dir) diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index c388682dfb4..6ee8c8874ec 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -9,7 +9,7 @@ module SystemCheck end def self.current_version - @current_version ||= Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) + @current_version ||= Gitlab::VersionInfo.parse(Gitlab::TaskHelpers.run_command(%W(#{Gitlab.config.git.bin_path} --version))) end def check? diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb index fd82f5f8a4a..57bbabece1f 100644 --- a/lib/system_check/app/ruby_version_check.rb +++ b/lib/system_check/app/ruby_version_check.rb @@ -5,11 +5,11 @@ module SystemCheck set_check_pass -> { "yes (#{self.current_version})" } def self.required_version - @required_version ||= Gitlab::VersionInfo.new(2, 3, 3) + @required_version ||= Gitlab::VersionInfo.new(2, 3, 5) end def self.current_version - @current_version ||= Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) + @current_version ||= Gitlab::VersionInfo.parse(Gitlab::TaskHelpers.run_command(%w(ruby --version))) end def check? diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 259a755d724..a42f02a84fd 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -3,8 +3,8 @@ namespace :gitlab do desc 'GitLab | Assets | Compile all frontend assets' task compile: [ 'yarn:check', - 'rake:assets:precompile', 'gettext:po_to_json', + 'rake:assets:precompile', 'webpack:compile', 'fix_urls' ] diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index 3eade7bf553..ba221e44e5d 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -4,7 +4,11 @@ namespace :gitlab do task :ee_compat_check, [:branch] => :environment do |_, args| opts = if ENV['CI'] - { branch: ENV['CI_COMMIT_REF_NAME'] } + { + ce_project_url: ENV['CI_PROJECT_URL'], + branch: ENV['CI_COMMIT_REF_NAME'], + job_id: ENV['CI_JOB_ID'] + } else unless args[:branch] puts "Must specify a branch as an argument".color(:red) diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 08677a98fc1..8377fe3269d 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -50,6 +50,8 @@ namespace :gitlab do # only generate a configuration for the most common and simplest case: when # we have exactly one Gitaly process and we are sure it is running locally # because it uses a Unix socket. + # For development and testing purposes, an extra storage is added to gitaly, + # which is not known to Rails, but must be explicitly stubbed. def gitaly_configuration_toml(gitaly_ruby: true) storages = [] address = nil @@ -67,6 +69,11 @@ namespace :gitlab do storages << { name: key, path: val['path'] } end + + if Rails.env.test? + storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } + end + config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } config[:auth] = { token: 'secret' } if Rails.env.test? config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 42825f29e32..0e6aed32c52 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -79,7 +79,7 @@ namespace :gitlab do if File.exist?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.repository_storage_path, + if Gitlab::Shell.new.add_repository(project.repository_storage, project.disk_path) print '.' else diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake new file mode 100644 index 00000000000..e05be4a3405 --- /dev/null +++ b/lib/tasks/gitlab/storage.rake @@ -0,0 +1,85 @@ +namespace :gitlab do + namespace :storage do + desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' + task migrate_to_hashed: :environment do + legacy_projects_count = Project.with_legacy_storage.count + + if legacy_projects_count == 0 + puts 'There are no projects using legacy storage. Nothing to do!' + + next + end + + print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}" + + project_id_batches do |start, finish| + StorageMigratorWorker.perform_async(start, finish) + + print '.' + end + + puts ' Done!' + end + + desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' + task legacy_projects: :environment do + projects_summary(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | List existing projects using Legacy Storage' + task list_legacy_projects: :environment do + projects_list(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage' + task hashed_projects: :environment do + projects_summary(Project.with_hashed_storage) + end + + desc 'Gitlab | Storage | List existing projects using Hashed Storage' + task list_hashed_projects: :environment do + projects_list(Project.with_hashed_storage) + end + + def batch_size + ENV.fetch('BATCH', 200).to_i + end + + def project_id_batches(&block) + Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + ids = relation.pluck(:id) + + yield ids.min, ids.max + end + end + + def projects_summary(relation) + projects_count = relation.count + puts "* Found #{projects_count} projects".color(:green) + + projects_count + end + + def projects_list(relation) + projects_count = projects_summary(relation) + + projects = relation.with_route + limit = ENV.fetch('LIMIT', 500).to_i + + return unless projects_count > 0 + + puts " ! Displaying first #{limit} projects..." if projects_count > limit + + counter = 0 + projects.find_in_batches(batch_size: batch_size) do |batch| + batch.each do |project| + counter += 1 + + puts " - #{project.full_path} (id: #{project.id})".color(:red) + + return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator + end + end + end + end +end diff --git a/lib/tasks/gitlab/users.rake b/lib/tasks/gitlab/users.rake deleted file mode 100644 index 3a16ace60bd..00000000000 --- a/lib/tasks/gitlab/users.rake +++ /dev/null @@ -1,11 +0,0 @@ -namespace :gitlab do - namespace :users do - desc "GitLab | Clear the authentication token for all users" - task clear_all_authentication_tokens: :environment do |t, args| - # Do small batched updates because these updates will be slow and locking - User.select(:id).find_in_batches(batch_size: 100) do |batch| - User.where(id: batch.map(&:id)).update_all(authentication_token: nil) - end - end - end -end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 4d485108cf6..7f86fd7b45e 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -39,13 +39,19 @@ class GithubImport def import! @project.force_import_start + import_success = false + timings = Benchmark.measure do - Github::Import.new(@project, @options).execute + import_success = Github::Import.new(@project, @options).execute end - puts "Import finished. Timings: #{timings}".color(:green) - - @project.import_finish + if import_success + @project.import_finish + puts "Import finished. Timings: #{timings}".color(:green) + else + puts "Import was not successful. Errors were as follows:" + puts @project.import_error + end end def new_project @@ -53,18 +59,23 @@ class GithubImport namespace_path, _sep, name = @project_path.rpartition('/') namespace = find_or_create_namespace(namespace_path) - Projects::CreateService.new( + project = Projects::CreateService.new( @current_user, name: name, path: name, description: @repo['description'], namespace_id: namespace.id, visibility_level: visibility_level, - import_type: 'github', - import_source: @repo['full_name'], - import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@"), skip_wiki: @repo['has_wiki'] ).execute + + project.update!( + import_type: 'github', + import_source: @repo['full_name'], + import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@") + ) + + project end end diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake index ad1818ff1fa..693597afdf8 100644 --- a/lib/tasks/tokens.rake +++ b/lib/tasks/tokens.rake @@ -1,12 +1,7 @@ require_relative '../../app/models/concerns/token_authenticatable.rb' namespace :tokens do - desc "Reset all GitLab user auth tokens" - task reset_all_auth: :environment do - reset_all_users_token(:reset_authentication_token!) - end - - desc "Reset all GitLab email tokens" + desc "Reset all GitLab incoming email tokens" task reset_all_email: :environment do reset_all_users_token(:reset_incoming_email_token!) end @@ -31,11 +26,6 @@ class TmpUser < ActiveRecord::Base self.table_name = 'users' - def reset_authentication_token! - write_new_token(:authentication_token) - save!(validate: false) - end - def reset_incoming_email_token! write_new_token(:incoming_email_token) save!(validate: false) |