diff options
Diffstat (limited to 'lib')
72 files changed, 695 insertions, 181 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 073471b4c4d..5139e869c71 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -154,6 +154,7 @@ module API mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectMilestones + mount ::API::ProjectSnapshots mount ::API::ProjectSnippets mount ::API::ProtectedBranches mount ::API::Repositories diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 6abd575b6ad..7975f35ab1e 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -25,7 +25,7 @@ module API get ":id/#{noteables_str}/:noteable_id/discussions" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) - return not_found!("Discussions") unless can?(current_user, noteable_read_ability_name(noteable), noteable) + break not_found!("Discussions") unless can?(current_user, noteable_read_ability_name(noteable), noteable) notes = noteable.notes .inc_relations_for_view @@ -50,7 +50,7 @@ module API notes = readable_discussion_notes(noteable, params[:discussion_id]) if notes.empty? || !can?(current_user, noteable_read_ability_name(noteable), noteable) - return not_found!("Discussion") + break not_found!("Discussion") end discussion = Discussion.build(notes, noteable) @@ -98,7 +98,7 @@ module API notes = readable_discussion_notes(noteable, params[:discussion_id]) if notes.empty? || !can?(current_user, noteable_read_ability_name(noteable), noteable) - return not_found!("Notes") + break not_found!("Notes") end present notes, with: Entities::Note @@ -117,8 +117,8 @@ module API noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) - return not_found!("Discussion") if notes.empty? - return bad_request!("Discussion is an individual note.") unless notes.first.part_of_discussion? + break not_found!("Discussion") if notes.empty? + break bad_request!("Discussion is an individual note.") unless notes.first.part_of_discussion? opts = { note: params[:body], diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 92800ce6450..55d5c7f1606 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -31,7 +31,7 @@ module API key = params[:key] variable = user_group.variables.find_by(key: key) - return not_found!('GroupVariable') unless variable + break not_found!('GroupVariable') unless variable present variable, with: Entities::Variable end @@ -67,7 +67,7 @@ module API put ':id/variables/:key' do variable = user_group.variables.find_by(key: params[:key]) - return not_found!('GroupVariable') unless variable + break not_found!('GroupVariable') unless variable variable_params = declared_params(include_missing: false).except(:key) diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index cd91df1ecd8..b74b8149834 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -64,8 +64,10 @@ module API authorize! :create_note, noteable parent = noteable_parent(noteable) + if opts[:created_at] - opts.delete(:created_at) unless current_user.admin? || parent.owner == current_user + opts.delete(:created_at) unless + current_user.admin? || parent.owned_by?(current_user) end project = parent if parent.is_a?(Project) diff --git a/lib/api/helpers/project_snapshots_helpers.rb b/lib/api/helpers/project_snapshots_helpers.rb new file mode 100644 index 00000000000..94798a8cb51 --- /dev/null +++ b/lib/api/helpers/project_snapshots_helpers.rb @@ -0,0 +1,25 @@ +module API + module Helpers + module ProjectSnapshotsHelpers + def authorize_read_git_snapshot! + authenticated_with_full_private_access! + end + + def send_git_snapshot(repository) + header(*Gitlab::Workhorse.send_git_snapshot(repository)) + end + + def snapshot_project + user_project + end + + def snapshot_repository + if to_boolean(params[:wiki]) + snapshot_project.wiki.repository + else + snapshot_project.repository + end + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index fcbc248fc3b..6b72caea8fd 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -50,7 +50,7 @@ module API access_checker.check(params[:action], params[:changes]) @project ||= access_checker.project rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e - return { status: false, message: e.message } + break { status: false, message: e.message } end log_user_activity(actor) @@ -142,21 +142,21 @@ module API if key key.update_last_used_at else - return { 'success' => false, 'message' => 'Could not find the given key' } + break { 'success' => false, 'message' => 'Could not find the given key' } end if key.is_a?(DeployKey) - return { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } + break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } end user = key.user unless user - return { success: false, message: 'Could not find a user for the given key' } + break { success: false, message: 'Could not find a user for the given key' } end unless user.two_factor_enabled? - return { success: false, message: 'Two-factor authentication is not enabled for this user' } + break { success: false, message: 'Two-factor authentication is not enabled for this user' } end codes = nil diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 88e7f46c92c..12ff2a1398b 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -310,7 +310,7 @@ module API issue = find_project_issue(params[:issue_iid]) - return not_found!('UserAgentDetail') unless issue.user_agent_detail + break not_found!('UserAgentDetail') unless issue.user_agent_detail present issue.user_agent_detail, with: Entities::UserAgentDetail end diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index b1adef49d46..32379d7c8ab 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -77,7 +77,7 @@ module API build = find_build!(params[:job_id]) authorize!(:update_build, build) - return not_found!(build) unless build.artifacts? + break not_found!(build) unless build.artifacts? build.keep_artifacts! diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 60911c8d733..54d1acbd412 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -120,7 +120,7 @@ module API build = find_build!(params[:job_id]) authorize!(:update_build, build) - return forbidden!('Job is not retryable') unless build.retryable? + break forbidden!('Job is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) @@ -138,7 +138,7 @@ module API build = find_build!(params[:job_id]) authorize!(:erase_build, build) - return forbidden!('Job is not erasable!') unless build.erasable? + break forbidden!('Job is not erasable!') unless build.erasable? build.erase(erased_by: current_user) present build, with: Entities::Job diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index d2b8b832e4e..735591fedd5 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -19,6 +19,7 @@ module API optional :status, type: String, values: HasStatus::AVAILABLE_STATUSES, desc: 'The status of pipelines' optional :ref, type: String, desc: 'The ref of pipelines' + optional :sha, type: String, desc: 'The sha of pipelines' optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations' optional :name, type: String, desc: 'The name of the user who triggered pipelines' optional :username, type: String, desc: 'The username of the user who triggered pipelines' diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb new file mode 100644 index 00000000000..71005acc587 --- /dev/null +++ b/lib/api/project_snapshots.rb @@ -0,0 +1,19 @@ +module API + class ProjectSnapshots < Grape::API + helpers ::API::Helpers::ProjectSnapshotsHelpers + + before { authorize_read_git_snapshot! } + + resource :projects do + desc 'Download a (possibly inconsistent) snapshot of a repository' do + detail 'This feature was introduced in GitLab 10.7' + end + params do + optional :wiki, type: Boolean, desc: 'Set to true to receive the wiki repository' + end + get ':id/snapshot' do + send_git_snapshot(snapshot_repository) + end + end + end +end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 39c03c40bab..1de5551fee9 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -145,7 +145,7 @@ module API snippet = Snippet.find_by!(id: params[:snippet_id], project_id: params[:id]) - return not_found!('UserAgentDetail') unless snippet.user_agent_detail + break not_found!('UserAgentDetail') unless snippet.user_agent_detail present snippet.user_agent_detail, with: Entities::UserAgentDetail end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 3ae6fbd1fa9..8871792060b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -74,6 +74,11 @@ module API present options[:with].prepare_relation(projects, options), options end + + def translate_params_for_compatibility(params) + params[:builds_enabled] = params.delete(:jobs_enabled) if params.key?(:jobs_enabled) + params + end end resource :users, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do @@ -123,7 +128,7 @@ module API end post do attrs = declared_params(include_missing: false) - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) + attrs = translate_params_for_compatibility(attrs) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -155,6 +160,7 @@ module API not_found!('User') unless user attrs = declared_params(include_missing: false) + attrs = translate_params_for_compatibility(attrs) project = ::Projects::CreateService.new(user, attrs).execute if project.saved? @@ -276,7 +282,7 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) + attrs = translate_params_for_compatibility(attrs) result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute @@ -402,7 +408,7 @@ module API end unless user_project.allowed_to_share_with_group? - return render_api_error!("The project sharing with group is disabled", 400) + break render_api_error!("The project sharing with group is disabled", 400) end link = user_project.project_group_links.new(declared_params(include_missing: false)) diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 60aeb69e10a..4d4fbe50f9f 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -29,7 +29,7 @@ module API project.runners.create(attributes) end - return forbidden! unless runner + break forbidden! unless runner if runner.id present runner, with: Entities::RunnerRegistrationDetails @@ -83,7 +83,7 @@ module API if current_runner.runner_queue_value_latest?(params[:last_update]) header 'X-GitLab-Last-Update', params[:last_update] Gitlab::Metrics.add_event(:build_not_found_cached) - return no_content! + break no_content! end new_update = current_runner.ensure_runner_queue_value @@ -152,7 +152,7 @@ module API stream_size = job.trace.append(request.body.read, content_range[0].to_i) if stream_size < 0 - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) + break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) end status 202 diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index c736cc32021..b30305b4bc9 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -94,7 +94,7 @@ module API end put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet + break not_found!('Snippet') unless snippet authorize! :update_personal_snippet, snippet @@ -120,7 +120,7 @@ module API end delete ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet + break not_found!('Snippet') unless snippet authorize! :destroy_personal_snippet, snippet @@ -135,7 +135,7 @@ module API end get ":id/raw" do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet + break not_found!('Snippet') unless snippet env['api.format'] = :txt content_type 'text/plain' @@ -153,7 +153,7 @@ module API snippet = Snippet.find_by!(id: params[:id]) - return not_found!('UserAgentDetail') unless snippet.user_agent_detail + break not_found!('UserAgentDetail') unless snippet.user_agent_detail present snippet.user_agent_detail, with: Entities::UserAgentDetail end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index b3709455bc3..b29e660c6e0 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -62,7 +62,7 @@ module API authorize! :admin_build, user_project trigger = user_project.triggers.find(params.delete(:trigger_id)) - return not_found!('Trigger') unless trigger + break not_found!('Trigger') unless trigger present trigger, with: Entities::Trigger end @@ -99,7 +99,7 @@ module API authorize! :admin_build, user_project trigger = user_project.triggers.find(params.delete(:trigger_id)) - return not_found!('Trigger') unless trigger + break not_found!('Trigger') unless trigger if trigger.update(declared_params(include_missing: false)) present trigger, with: Entities::Trigger @@ -119,7 +119,7 @@ module API authorize! :admin_build, user_project trigger = user_project.triggers.find(params.delete(:trigger_id)) - return not_found!('Trigger') unless trigger + break not_found!('Trigger') unless trigger if trigger.update(owner: current_user) status :ok @@ -140,7 +140,7 @@ module API authorize! :admin_build, user_project trigger = user_project.triggers.find(params.delete(:trigger_id)) - return not_found!('Trigger') unless trigger + break not_found!('Trigger') unless trigger destroy_conditionally!(trigger) end diff --git a/lib/api/users.rb b/lib/api/users.rb index 3920171205f..14b8a796c8e 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -77,7 +77,7 @@ module API authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) unless current_user&.admin? - params.except!(:created_after, :created_before, :order_by, :sort) + params.except!(:created_after, :created_before, :order_by, :sort, :two_factor) end users = UsersFinder.new(current_user, params).execute diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 683b9c993cb..b49448e1e67 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -51,7 +51,7 @@ module API get ':id/repository/commits/:sha/builds' do authorize_read_builds! - return not_found! unless user_project.commit(params[:sha]) + break not_found! unless user_project.commit(params[:sha]) pipelines = user_project.pipelines.where(sha: params[:sha]) builds = user_project.builds.where(pipeline: pipelines).order('id DESC') @@ -153,7 +153,7 @@ module API build = get_build!(params[:build_id]) authorize!(:update_build, build) - return forbidden!('Build is not retryable') unless build.retryable? + break forbidden!('Build is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) @@ -171,7 +171,7 @@ module API build = get_build!(params[:build_id]) authorize!(:erase_build, build) - return forbidden!('Build is not erasable!') unless build.erasable? + break forbidden!('Build is not erasable!') unless build.erasable? build.erase(erased_by: current_user) present build, with: ::API::V3::Entities::Build @@ -188,7 +188,7 @@ module API build = get_build!(params[:build_id]) authorize!(:update_build, build) - return not_found!(build) unless build.artifacts? + break not_found!(build) unless build.artifacts? build.keep_artifacts! diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index a2df969d819..eb3dd113524 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -423,7 +423,7 @@ module API end unless user_project.allowed_to_share_with_group? - return render_api_error!("The project sharing with group is disabled", 400) + break render_api_error!("The project sharing with group is disabled", 400) end link = user_project.project_group_links.new(declared_params(include_missing: false)) diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb index 85613c8ed84..1df8a20e74a 100644 --- a/lib/api/v3/snippets.rb +++ b/lib/api/v3/snippets.rb @@ -90,7 +90,7 @@ module API end put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet + break not_found!('Snippet') unless snippet authorize! :update_personal_snippet, snippet @@ -114,7 +114,7 @@ module API end delete ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet + break not_found!('Snippet') unless snippet authorize! :destroy_personal_snippet, snippet snippet.destroy @@ -129,7 +129,7 @@ module API end get ":id/raw" do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) - return not_found!('Snippet') unless snippet + break not_found!('Snippet') unless snippet env['api.format'] = :txt content_type 'text/plain' diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index 34f07dfb486..969bb2a05de 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -72,7 +72,7 @@ module API authorize! :admin_build, user_project trigger = user_project.triggers.find_by(token: params[:token].to_s) - return not_found!('Trigger') unless trigger + break not_found!('Trigger') unless trigger present trigger, with: ::API::V3::Entities::Trigger end @@ -100,7 +100,7 @@ module API authorize! :admin_build, user_project trigger = user_project.triggers.find_by(token: params[:token].to_s) - return not_found!('Trigger') unless trigger + break not_found!('Trigger') unless trigger trigger.destroy diff --git a/lib/api/variables.rb b/lib/api/variables.rb index d08876ae1b9..a34de9410e8 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -31,7 +31,7 @@ module API key = params[:key] variable = user_project.variables.find_by(key: key) - return not_found!('Variable') unless variable + break not_found!('Variable') unless variable present variable, with: Entities::Variable end @@ -67,7 +67,7 @@ module API put ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key]) - return not_found!('Variable') unless variable + break not_found!('Variable') unless variable variable_params = declared_params(include_missing: false).except(:key) diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 88cb7e7b5a4..9895db9e451 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -53,6 +53,8 @@ module Backup FileUtils.mv(files, timestamped_files_path) rescue Errno::EACCES access_denied_error(app_files_dir) + rescue Errno::EBUSY + resource_busy_error(app_files_dir) end end end diff --git a/lib/backup/helper.rb b/lib/backup/helper.rb index a1ee0faefe9..54b9ce10b4d 100644 --- a/lib/backup/helper.rb +++ b/lib/backup/helper.rb @@ -13,5 +13,19 @@ module Backup EOS raise message end + + def resource_busy_error(path) + message = <<~EOS + + ### NOTICE ### + As part of restore, the task tried to rename `#{path}` before restoring. + This could not be completed, perhaps `#{path}` is a mountpoint? + + To complete the restore, please move the contents of `#{path}` to a + different location and run the restore task again. + + EOS + raise message + end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 89e3f1d9076..65e06fd78c0 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -81,6 +81,8 @@ module Backup FileUtils.mv(files, bk_repos_path) rescue Errno::EACCES access_denied_error(path) + rescue Errno::EBUSY + resource_busy_error(path) end end end diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index 77c91817382..87f14b3b0d2 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -77,7 +77,7 @@ module DeclarativePolicy @state = State.new steps_by_score do |step, score| - return if !debug && @state.prevented? + break if !debug && @state.prevented? passed = nil case step.action diff --git a/lib/gitlab.rb b/lib/gitlab.rb index f6629982512..c5498d0da1a 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,9 +1,19 @@ -require_dependency 'gitlab/git' +require_dependency 'gitlab/popen' module Gitlab + def self.root + Pathname.new(File.expand_path('..', __dir__)) + end + + def self.config + Settings + end + COM_URL = 'https://gitlab.com'.freeze APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))} SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z} + VERSION = File.read(root.join("VERSION")).strip.freeze + REVISION = Gitlab::Popen.popen(%W(#{config.git.bin_path} log --pretty=format:%h -n 1)).first.chomp.freeze def self.com? # Check `gl_subdomain?` as well to keep parity with gitlab.com @@ -19,6 +29,6 @@ module Gitlab end def self.dev_env_or_com? - Rails.env.test? || Rails.env.development? || org? || com? + Rails.env.development? || org? || com? end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 2a44e11efb6..8e5a985edd7 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -51,7 +51,7 @@ module Gitlab Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) - return if user && !user.active? + break if user && !user.active? authenticators = [] diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index 068212d9a21..922d0567d99 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -8,6 +8,8 @@ module Gitlab module Auth module LDAP class User < Gitlab::Auth::OAuth::User + extend ::Gitlab::Utils::Override + class << self def find_by_uid_and_provider(uid, provider) identity = ::Identity.with_extern_uid(provider, uid).take @@ -29,7 +31,8 @@ module Gitlab self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) end - def changed? + override :should_save? + def should_save? gl_user.changed? || gl_user.identities.any?(&:changed?) end @@ -41,6 +44,10 @@ module Gitlab Gitlab::Auth::LDAP::Access.allowed?(gl_user) end + def valid_sign_in? + allowed? && super + end + def ldap_config Gitlab::Auth::LDAP::Config.new(auth_hash.provider) end diff --git a/lib/gitlab/auth/o_auth/identity_linker.rb b/lib/gitlab/auth/o_auth/identity_linker.rb new file mode 100644 index 00000000000..de92d7a214d --- /dev/null +++ b/lib/gitlab/auth/o_auth/identity_linker.rb @@ -0,0 +1,8 @@ +module Gitlab + module Auth + module OAuth + class IdentityLinker < OmniauthIdentityLinkerBase + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index d0c6b0386ba..6c5d0788a0a 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -30,6 +30,10 @@ module Gitlab gl_user.try(:valid?) end + def valid_sign_in? + valid? && persisted? + end + def save(provider = 'OAuth') raise SigninDisabledForProviderError if oauth_provider_disabled? raise SignupDisabledError unless gl_user @@ -64,8 +68,18 @@ module Gitlab user end + def find_and_update! + save if should_save? + + gl_user + end + protected + def should_save? + true + end + def add_or_update_user_identities return unless gl_user diff --git a/lib/gitlab/auth/omniauth_identity_linker_base.rb b/lib/gitlab/auth/omniauth_identity_linker_base.rb new file mode 100644 index 00000000000..ae365fcdfaa --- /dev/null +++ b/lib/gitlab/auth/omniauth_identity_linker_base.rb @@ -0,0 +1,47 @@ +module Gitlab + module Auth + class OmniauthIdentityLinkerBase + attr_reader :current_user, :oauth + + def initialize(current_user, oauth) + @current_user = current_user + @oauth = oauth + @changed = false + end + + def link + save if identity.new_record? + end + + def changed? + @changed + end + + def error_message + identity.validate + + identity.errors.full_messages.join(', ') + end + + private + + def save + @changed = identity.save + end + + def identity + @identity ||= current_user.identities + .with_extern_uid(provider, uid) + .first_or_initialize(extern_uid: uid) + end + + def provider + oauth['provider'] + end + + def uid + oauth['uid'] + end + end + end +end diff --git a/lib/gitlab/auth/saml/identity_linker.rb b/lib/gitlab/auth/saml/identity_linker.rb new file mode 100644 index 00000000000..7e4b191d512 --- /dev/null +++ b/lib/gitlab/auth/saml/identity_linker.rb @@ -0,0 +1,8 @@ +module Gitlab + module Auth + module Saml + class IdentityLinker < OmniauthIdentityLinkerBase + end + end + end +end diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index d4024e9ec39..cb01cd8004c 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -7,6 +7,8 @@ module Gitlab module Auth module Saml class User < Gitlab::Auth::OAuth::User + extend ::Gitlab::Utils::Override + def save super('SAML') end @@ -21,13 +23,14 @@ module Gitlab if external_users_enabled? && user # Check if there is overlap between the user's groups and the external groups # setting then set user as external or internal. - user.external = !(auth_hash.groups & Gitlab::Auth::Saml::Config.external_groups).empty? + user.external = !(auth_hash.groups & saml_config.external_groups).empty? end user end - def changed? + override :should_save? + def should_save? return true unless gl_user gl_user.changed? || gl_user.identities.any?(&:changed?) @@ -35,12 +38,16 @@ module Gitlab protected + def saml_config + Gitlab::Auth::Saml::Config + end + def auto_link_saml_user? Gitlab.config.omniauth.auto_link_saml_user end def external_users_enabled? - !Gitlab::Auth::Saml::Config.external_groups.nil? + !saml_config.external_groups.nil? end def auth_hash=(auth_hash) diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 1a25138e7d6..4ca5a78e068 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -75,10 +75,11 @@ module Gitlab end def mv_repo(project) - FileUtils.mv(repo_path, File.join(project.repository_storage_path, project.disk_path + '.git')) + storage_path = storage_path_for_shard(project.repository_storage) + FileUtils.mv(repo_path, project.repository.path_to_repo) if bare_repo.wiki_exists? - FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git')) + FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git')) end true @@ -88,6 +89,10 @@ module Gitlab false end + def storage_path_for_shard(shard) + Gitlab.config.repositories.storages[shard].legacy_disk_path + end + def find_or_create_groups return nil unless group_path.present? diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb new file mode 100644 index 00000000000..e4227af25d2 --- /dev/null +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -0,0 +1,8 @@ +# This is a base controller for doorkeeper. +# It adds the `can?` helper used in the views. +module Gitlab + class BaseDoorkeeperController < ActionController::Base + include Gitlab::Allowable + helper_method :can? + end +end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index d299a5677de..69b8a8fc68f 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -14,14 +14,10 @@ module Gitlab @command.seeds_block&.call(pipeline) ## - # Populate pipeline with all stages and builds from pipeline seeds. + # Populate pipeline with all stages, and stages with builds. # pipeline.stage_seeds.each do |stage| pipeline.stages << stage.to_resource - - stage.seeds.each do |build| - pipeline.builds << build.to_resource - end end if pipeline.stages.none? diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index cedf4171ab1..47b67930c6d 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -45,7 +45,7 @@ module Gitlab def append(data, offset) write do |stream| current_length = stream.size - return -current_length unless current_length == offset + break -current_length unless current_length == offset data = job.hide_secrets(data) stream.append(data, offset) diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb index ac4308f4e2c..cff924e27ef 100644 --- a/lib/gitlab/ci/trace/http_io.rb +++ b/lib/gitlab/ci/trace/http_io.rb @@ -75,18 +75,28 @@ module Gitlab end end - def read(length = nil) + def read(length = nil, outbuf = "") out = "" - until eof? || (length && out.length >= length) + length ||= size - tell + + until length <= 0 || eof? data = get_chunk break if data.empty? - out << data - @tell += data.bytesize + chunk_bytes = [BUFFER_SIZE - chunk_offset, length].min + chunk_data = data.byteslice(0, chunk_bytes) + + out << chunk_data + @tell += chunk_data.bytesize + length -= chunk_data.bytesize end - out = out[0, length] if length && out.length > length + # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality + if outbuf + outbuf.slice!(0, outbuf.bytesize) + outbuf << out + end out end @@ -158,7 +168,7 @@ module Gitlab # Provider: GCS # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200 - @chunk_range ||= (chunk_start...(chunk_start + @chunk.length)) + @chunk_range ||= (chunk_start...(chunk_start + @chunk.bytesize)) end @chunk[chunk_offset..BUFFER_SIZE] diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 54894a46077..187ad8b833a 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -10,7 +10,9 @@ module Gitlab delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true - delegate :valid?, to: :stream, as: :present?, allow_nil: true + delegate :valid?, to: :stream, allow_nil: true + + alias_method :present?, :valid? def initialize @stream = yield @@ -85,7 +87,7 @@ module Gitlab match = matches.flatten.last coverage = match.gsub(/\d+(\.\d+)?/).first - return coverage if coverage.present? + return coverage if coverage.present? # rubocop:disable Cop/AvoidReturnFromBlocks end nil diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index 633de9f9776..bd14c7eece3 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -30,7 +30,7 @@ module Gitlab return unless enabled? @mutex.synchronize do - return thread if thread? + break thread if thread? @thread = Thread.new { start_working } end @@ -38,7 +38,7 @@ module Gitlab def stop @mutex.synchronize do - return unless thread? + break unless thread? stop_working diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 05b86f32ce2..73971af6a74 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -62,21 +62,20 @@ module Gitlab end def move_repositories(namespace, old_full_path, new_full_path) - repo_paths_for_namespace(namespace).each do |repository_storage_path| + repo_shards_for_namespace(namespace).each do |repository_storage| # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage_path, old_full_path) + gitlab_shell.add_namespace(repository_storage, old_full_path) - unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path) - message = "Exception moving path #{repository_storage_path} \ - from #{old_full_path} to #{new_full_path}" + unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path) + message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}" Rails.logger.error message end end end - def repo_paths_for_namespace(namespace) + def repo_shards_for_namespace(namespace) projects_for_namespace(namespace).distinct.select(:repository_storage) - .map(&:repository_storage_path) + .map(&:repository_storage) end def projects_for_namespace(namespace) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 979225dd216..827aeb12a02 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -51,7 +51,7 @@ module Gitlab end def move_repository(project, old_path, new_path) - unless gitlab_shell.mv_repository(project.repository_storage_path, + unless gitlab_shell.mv_repository(project.repository_storage, old_path, new_path) Rails.logger.error "Error moving #{old_path} to #{new_path}" diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 1b74f735679..b6eeb5d9a2b 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -21,7 +21,7 @@ module Gitlab @text.gsub(@pattern) do |markdown| file = find_file(@source_project, $~[:secret], $~[:file]) - return markdown unless file.try(:exists?) + break markdown unless file.try(:exists?) new_uploader = FileUploader.new(target_project) with_link_in_tmp_dir(file.file) do |open_tmp_file| diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index c9abea90d21..e85e87a54af 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -1,3 +1,5 @@ +require_dependency 'gitlab/encoding_helper' + module Gitlab module Git # The ID of empty tree. diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 0fb82441bf8..fabcd46c8e9 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -486,6 +486,8 @@ module Gitlab end def tree_entry(path) + return unless path.present? + @repository.gitaly_migrate(:commit_tree_entry) do |is_migrated| if is_migrated gitaly_tree_entry(path) diff --git a/lib/gitlab/git/committer_with_hooks.rb b/lib/gitlab/git/committer_with_hooks.rb new file mode 100644 index 00000000000..a8a59f998cd --- /dev/null +++ b/lib/gitlab/git/committer_with_hooks.rb @@ -0,0 +1,47 @@ +module Gitlab + module Git + class CommitterWithHooks < Gollum::Committer + attr_reader :gl_wiki + + def initialize(gl_wiki, options = {}) + @gl_wiki = gl_wiki + super(gl_wiki.gollum_wiki, options) + end + + def commit + # TODO: Remove after 10.8 + return super unless allowed_to_run_hooks? + + result = Gitlab::Git::OperationService.new(git_user, gl_wiki.repository).with_branch( + @wiki.ref, + start_branch_name: @wiki.ref + ) do |start_commit| + super(false) + end + + result[:newrev] + rescue Gitlab::Git::HooksService::PreReceiveError => e + message = "Custom Hook failed: #{e.message}" + raise Gitlab::Git::Wiki::OperationError, message + end + + private + + # TODO: Remove after 10.8 + def allowed_to_run_hooks? + @options[:user_id] != 0 && @options[:username].present? + end + + def git_user + @git_user ||= Gitlab::Git::User.new(@options[:username], + @options[:name], + @options[:email], + gitlab_id) + end + + def gitlab_id + Gitlab::GlId.gl_id_from_id_value(@options[:user_id]) + end + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index a203587aec1..b58296375ef 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -249,7 +249,7 @@ module Gitlab if size >= SIZE_LIMIT too_large! - return true + return true # rubocop:disable Cop/AvoidReturnFromBlocks end end end diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index c1767046ff0..f9f24ecc48d 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -25,7 +25,9 @@ module Gitlab stdin.close if lazy_block - return [lazy_block.call(stdout.lazy), 0] + cmd_output = lazy_block.call(stdout.lazy) + cmd_status = 0 + break else cmd_output << stdout.read end diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb index 6bd6e58feeb..f40e59a8dd0 100644 --- a/lib/gitlab/git/remote_repository.rb +++ b/lib/gitlab/git/remote_repository.rb @@ -12,7 +12,7 @@ module Gitlab # class. # class RemoteRepository - attr_reader :path, :relative_path, :gitaly_repository + attr_reader :relative_path, :gitaly_repository def initialize(repository) @relative_path = repository.relative_path @@ -21,7 +21,6 @@ module Gitlab # These instance variables will not be available in gitaly-ruby, where # we have no disk access to this repository. @repository = repository - @path = repository.path end def empty? @@ -69,6 +68,10 @@ module Gitlab env end + def path + @repository.path + end + private # Must return an object that responds to 'address' and 'storage'. diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 3124c426f97..de0044fc149 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -142,15 +142,7 @@ module Gitlab end def exists? - Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| - if enabled - gitaly_repository_client.exists? - else - circuit_breaker.perform do - File.exist?(File.join(path, 'refs')) - end - end - end + gitaly_repository_client.exists? end # Returns an Array of branch names @@ -399,18 +391,6 @@ module Gitlab nil end - def archive_prefix(ref, sha, append_sha:) - append_sha = (ref != sha) if append_sha.nil? - - project_name = self.name.chomp('.git') - formatted_ref = ref.tr('/', '-') - - prefix_segments = [project_name, formatted_ref] - prefix_segments << sha if append_sha - - prefix_segments.join('-') - end - def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) @@ -421,12 +401,44 @@ module Gitlab { 'RepoPath' => path, 'ArchivePrefix' => prefix, - 'ArchivePath' => archive_file_path(prefix, storage_path, format), + 'ArchivePath' => archive_file_path(storage_path, commit.id, prefix, format), 'CommitId' => commit.id } end - def archive_file_path(name, storage_path, format = "tar.gz") + # This is both the filename of the archive (missing the extension) and the + # name of the top-level member of the archive under which all files go + # + # FIXME: The generated prefix is incorrect for projects with hashed + # storage enabled + def archive_prefix(ref, sha, append_sha:) + append_sha = (ref != sha) if append_sha.nil? + + project_name = self.name.chomp('.git') + formatted_ref = ref.tr('/', '-') + + prefix_segments = [project_name, formatted_ref] + prefix_segments << sha if append_sha + + prefix_segments.join('-') + end + private :archive_prefix + + # The full path on disk where the archive should be stored. This is used + # to cache the archive between requests. + # + # The path is a global namespace, so needs to be globally unique. This is + # achieved by including `gl_repository` in the path. + # + # Archives relating to a particular ref when the SHA is not present in the + # filename must be invalidated when the ref is updated to point to a new + # SHA. This is achieved by including the SHA in the path. + # + # As this is a full path on disk, it is not "cloud native". This should + # be resolved by either removing the cache, or moving the implementation + # into Gitaly and removing the ArchivePath parameter from the git-archive + # senddata response. + def archive_file_path(storage_path, sha, name, format = "tar.gz") # Build file path return nil unless name @@ -444,8 +456,9 @@ module Gitlab end file_name = "#{name}.#{extension}" - File.join(storage_path, self.name, file_name) + File.join(storage_path, self.gl_repository, sha, file_name) end + private :archive_file_path # Return repo size in megabytes def size @@ -1187,6 +1200,8 @@ module Gitlab if is_enabled gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref) else + # When removing this code, also remove source_repository#path + # to remove deprecated method calls local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref) end end @@ -1258,6 +1273,10 @@ module Gitlab true end + def create_from_snapshot(url, auth) + gitaly_repository_client.create_from_snapshot(url, auth) + end + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) gitaly_migrate(:rebase) do |is_enabled| if is_enabled diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index dc424a433fb..8a01f92e2af 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -26,7 +26,7 @@ module Gitlab # When the remote repo does not have tags. if target.nil? || path.nil? Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}" - return [] + break [] end name = path.split('/', 3).last diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 8d82820915d..84a26fe4a6f 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -2,10 +2,11 @@ module Gitlab module Git class Wiki DuplicatePageError = Class.new(StandardError) + OperationError = Class.new(StandardError) - CommitDetails = Struct.new(:name, :email, :message) do + CommitDetails = Struct.new(:user_id, :username, :name, :email, :message) do def to_h - { name: name, email: email, message: message } + { user_id: user_id, username: username, name: name, email: email, message: message } end end PageBlob = Struct.new(:name) @@ -140,6 +141,10 @@ module Gitlab end end + def gollum_wiki + @gollum_wiki ||= Gollum::Wiki.new(@repository.path) + end + private # options: @@ -158,10 +163,6 @@ module Gitlab offset: options[:offset]) end - def gollum_wiki - @gollum_wiki ||= Gollum::Wiki.new(@repository.path) - end - def gollum_page_by_path(page_path) page_name = Gollum::Page.canonicalize_filename(page_path) page_dir = File.split(page_path).first @@ -201,12 +202,12 @@ module Gitlab assert_type!(format, Symbol) assert_type!(commit_details, CommitDetails) - filename = File.basename(name) - dir = (tmp_dir = File.dirname(name)) == '.' ? '' : tmp_dir - - gollum_wiki.write_page(filename, format, content, commit_details.to_h, dir) + with_committer_with_hooks(commit_details) do |committer| + filename = File.basename(name) + dir = (tmp_dir = File.dirname(name)) == '.' ? '' : tmp_dir - nil + gollum_wiki.write_page(filename, format, content, { committer: committer }, dir) + end rescue Gollum::DuplicatePageError => e raise Gitlab::Git::Wiki::DuplicatePageError, e.message end @@ -214,24 +215,23 @@ module Gitlab def gollum_delete_page(page_path, commit_details) assert_type!(commit_details, CommitDetails) - gollum_wiki.delete_page(gollum_page_by_path(page_path), commit_details.to_h) - nil + with_committer_with_hooks(commit_details) do |committer| + gollum_wiki.delete_page(gollum_page_by_path(page_path), committer: committer) + end end def gollum_update_page(page_path, title, format, content, commit_details) assert_type!(format, Symbol) assert_type!(commit_details, CommitDetails) - page = gollum_page_by_path(page_path) - committer = Gollum::Committer.new(page.wiki, commit_details.to_h) - - # Instead of performing two renames if the title has changed, - # the update_page will only update the format and content and - # the rename_page will do anything related to moving/renaming - gollum_wiki.update_page(page, page.name, format, content, committer: committer) - gollum_wiki.rename_page(page, title, committer: committer) - committer.commit - nil + with_committer_with_hooks(commit_details) do |committer| + page = gollum_page_by_path(page_path) + # Instead of performing two renames if the title has changed, + # the update_page will only update the format and content and + # the rename_page will do anything related to moving/renaming + gollum_wiki.update_page(page, page.name, format, content, committer: committer) + gollum_wiki.rename_page(page, title, committer: committer) + end end def gollum_find_page(title:, version: nil, dir: nil) @@ -288,6 +288,20 @@ module Gitlab Gitlab::Git::WikiPage.new(wiki_page, version) end end + + def committer_with_hooks(commit_details) + Gitlab::Git::CommitterWithHooks.new(self, commit_details.to_h) + end + + def with_committer_with_hooks(commit_details, &block) + committer = committer_with_hooks(commit_details) + + yield committer + + committer.commit + + nil + end end end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 39057beefba..498187997e1 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -142,7 +142,7 @@ module Gitlab :repository_service, :is_rebase_in_progress, request, - timeout: GitalyClient.default_timeout + timeout: GitalyClient.fast_timeout ) response.in_progress @@ -159,7 +159,7 @@ module Gitlab :repository_service, :is_squash_in_progress, request, - timeout: GitalyClient.default_timeout + timeout: GitalyClient.fast_timeout ) response.in_progress @@ -235,6 +235,22 @@ module Gitlab ) end + def create_from_snapshot(http_url, http_auth) + request = Gitaly::CreateRepositoryFromSnapshotRequest.new( + repository: @gitaly_repo, + http_url: http_url, + http_auth: http_auth + ) + + GitalyClient.call( + @storage, + :repository_service, + :create_repository_from_snapshot, + request, + timeout: GitalyClient.default_timeout + ) + end + def write_ref(ref_path, ref, old_ref, shell) request = Gitaly::WriteRefRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 7a698e4b3f3..2dfe055a496 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -200,6 +200,8 @@ module Gitlab def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( + user_id: commit_details.user_id, + user_name: encode_binary(commit_details.username), name: encode_binary(commit_details.name), email: encode_binary(commit_details.email), message: encode_binary(commit_details.message) diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb index 624fd00367e..a53d156b41f 100644 --- a/lib/gitlab/gl_id.rb +++ b/lib/gitlab/gl_id.rb @@ -2,10 +2,14 @@ module Gitlab module GlId def self.gl_id(user) if user.present? - "user-#{user.id}" + gl_id_from_id_value(user.id) else - "" + '' end end + + def self.gl_id_from_id_value(id) + "user-#{id}" + end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index af203ff711d..b713fa7e1cd 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.2'.freeze + VERSION = '0.2.3'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index cd840bd5b01..0d1c4f73c6e 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -27,8 +27,6 @@ project_tree: - :releases - project_members: - :user - - lfs_file_locks: - - :user - merge_requests: - notes: - :author @@ -66,6 +64,7 @@ project_tree: - :project_feature - :custom_attributes - :project_badges + - :ci_cd_settings # Only include the following attributes for the models specified. included_attributes: @@ -75,6 +74,8 @@ included_attributes: - :username author: - :name + ci_cd_settings: + - :group_runners_enabled # Do not include the following attributes for the models specified. excluded_attributes: diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 598832fb2df..e3e9f156fb4 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -17,7 +17,8 @@ module Gitlab auto_devops: :project_auto_devops, label: :project_label, custom_attributes: 'ProjectCustomAttribute', - project_badges: 'Badge' }.freeze + project_badges: 'Badge', + ci_cd_settings: 'ProjectCiCdSetting' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 1d9a5d1a20a..d09bce642b0 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -3,18 +3,15 @@ module Gitlab module_function def retry_lock(subject, retries = 100, &block) - loop do - begin - ActiveRecord::Base.transaction do - return yield(subject) - end - rescue ActiveRecord::StaleObjectError - retries -= 1 - raise unless retries >= 0 - - subject.reload - end + ActiveRecord::Base.transaction do + yield(subject) end + rescue ActiveRecord::StaleObjectError + retries -= 1 + raise unless retries >= 0 + + subject.reload + retry end alias_method :retry_optimistic_lock, :retry_lock diff --git a/lib/gitlab/pages_client.rb b/lib/gitlab/pages_client.rb new file mode 100644 index 00000000000..7b358a3bd1b --- /dev/null +++ b/lib/gitlab/pages_client.rb @@ -0,0 +1,117 @@ +module Gitlab + class PagesClient + class << self + attr_reader :certificate, :token + + def call(service, rpc, request, timeout: nil) + kwargs = request_kwargs(timeout) + stub(service).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend + end + + # This function is not thread-safe. Call it from an initializer only. + def read_or_create_token + @token = read_token + rescue Errno::ENOENT + # TODO: uncomment this when omnibus knows how to write the token file for us + # https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/2466 + # + # write_token(SecureRandom.random_bytes(64)) + # + # # Read from disk in case someone else won the race and wrote the file + # # before us. If this fails again let the exception bubble up. + # @token = read_token + end + + # This function is not thread-safe. Call it from an initializer only. + def load_certificate + cert_path = config.certificate + return unless cert_path.present? + + @certificate = File.read(cert_path) + end + + def ping + request = Grpc::Health::V1::HealthCheckRequest.new + call(:health_check, :check, request, timeout: 5.seconds) + end + + private + + def request_kwargs(timeout) + encoded_token = Base64.strict_encode64(token.to_s) + metadata = { + 'authorization' => "Bearer #{encoded_token}" + } + + result = { metadata: metadata } + + return result unless timeout + + # Do not use `Time.now` for deadline calculation, since it + # will be affected by Timecop in some tests, but grpc's c-core + # uses system time instead of timecop's time, so tests will fail + # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will + # circumvent timecop + deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout + result[:deadline] = deadline + + result + end + + def stub(name) + stub_class(name).new(address, grpc_creds) + end + + def stub_class(name) + if name == :health_check + Grpc::Health::V1::Health::Stub + else + # TODO use pages namespace + Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) + end + end + + def address + addr = config.address + addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' + addr + end + + def grpc_creds + if address.start_with?('unix:') + :this_channel_is_insecure + elsif @certificate + GRPC::Core::ChannelCredentials.new(@certificate) + else + # Use system certificate pool + GRPC::Core::ChannelCredentials.new + end + end + + def config + Gitlab.config.pages.admin + end + + def read_token + File.read(token_path) + end + + def token_path + Rails.root.join('.gitlab_pages_secret').to_s + end + + def write_token(new_token) + Tempfile.open(File.basename(token_path), File.dirname(token_path), encoding: 'ascii-8bit') do |f| + f.write(new_token) + f.close + File.link(f.path, token_path) + end + rescue Errno::EACCES => ex + # TODO stop rescuing this exception in GitLab 11.0 https://gitlab.com/gitlab-org/gitlab-ce/issues/45672 + Rails.logger.error("Could not write pages admin token file: #{ex}") + rescue Errno::EEXIST + # Another process wrote the token file concurrently with us. Use their token, not ours. + end + end + end +end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 67407b651a5..156115f8a8f 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -65,11 +65,11 @@ module Gitlab # Init new repository # - # storage - project's storage name + # storage - the shard key # name - project disk path # # Ex. - # create_repository("/path/to/storage", "gitlab/gitlab-ci") + # create_repository("default", "gitlab/gitlab-ci") # def create_repository(storage, name) relative_path = name.dup @@ -291,13 +291,13 @@ module Gitlab # Add empty directory for storing repositories # # Ex. - # add_namespace("/path/to/storage", "gitlab") + # add_namespace("default", "gitlab") # def add_namespace(storage, name) Gitlab::GitalyClient.migrate(:add_namespace, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled - gitaly_namespace_client(storage).add(name) + Gitlab::GitalyClient::NamespaceService.new(storage).add(name) else path = full_path(storage, name) FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) @@ -313,13 +313,13 @@ module Gitlab # Every repository inside this directory will be removed too # # Ex. - # rm_namespace("/path/to/storage", "gitlab") + # rm_namespace("default", "gitlab") # def rm_namespace(storage, name) Gitlab::GitalyClient.migrate(:remove_namespace, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled - gitaly_namespace_client(storage).remove(name) + Gitlab::GitalyClient::NamespaceService.new(storage).remove(name) else FileUtils.rm_r(full_path(storage, name), force: true) end @@ -338,9 +338,10 @@ module Gitlab Gitlab::GitalyClient.migrate(:rename_namespace, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled - gitaly_namespace_client(storage).rename(old_name, new_name) + Gitlab::GitalyClient::NamespaceService.new(storage) + .rename(old_name, new_name) else - return false if exists?(storage, new_name) || !exists?(storage, old_name) + break false if exists?(storage, new_name) || !exists?(storage, old_name) FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) end @@ -374,7 +375,8 @@ module Gitlab Gitlab::GitalyClient.migrate(:namespace_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled - gitaly_namespace_client(storage).exists?(dir_name) + Gitlab::GitalyClient::NamespaceService.new(storage) + .exists?(dir_name) else File.exist?(full_path(storage, dir_name)) end @@ -398,7 +400,7 @@ module Gitlab def full_path(storage, dir_name) raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? - File.join(storage, dir_name) + File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, dir_name) end def gitlab_shell_projects_path @@ -475,14 +477,6 @@ module Gitlab Bundler.with_original_env { Popen.popen(cmd, nil, vars) } end - def gitaly_namespace_client(storage_path) - storage, _value = Gitlab.config.repositories.storages.find do |storage, value| - value.legacy_disk_path == storage_path - end - - Gitlab::GitalyClient::NamespaceService.new(storage) - end - def git_timeout Gitlab.config.gitlab_shell.git_timeout end diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb index c2b8d6de66e..b232ac4da33 100644 --- a/lib/gitlab/sidekiq_middleware/shutdown.rb +++ b/lib/gitlab/sidekiq_middleware/shutdown.rb @@ -25,7 +25,7 @@ module Gitlab # can be only one shutdown thread in the process. def self.create_shutdown_thread mu_synchronize do - return unless @shutdown_thread.nil? + break unless @shutdown_thread.nil? @shutdown_thread = Thread.new { yield } end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 69952cbb47c..8cf5d636743 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -63,10 +63,12 @@ module Gitlab request_cache def can_push_to_branch?(ref) return false unless can_access_git? - return false unless user.can?(:push_code, project) || project.branch_allows_maintainer_push?(user, ref) + return false unless project + + return false if !user.can?(:push_code, project) && !project.branch_allows_maintainer_push?(user, ref) if protected?(ProtectedBranch, project, ref) - project.user_can_push_to_empty_repo?(user) || protected_branch_accessible_to?(ref, action: :push) + protected_branch_accessible_to?(ref, action: :push) else true end @@ -101,6 +103,7 @@ module Gitlab def protected_branch_accessible_to?(ref, action:) ProtectedBranch.protected_ref_accessible_to?( ref, user, + project: project, action: action, protected_refs: project.protected_branches) end @@ -108,6 +111,7 @@ module Gitlab def protected_tag_accessible_to?(ref, action:) ProtectedTag.protected_ref_accessible_to?( ref, user, + project: project, action: action, protected_refs: project.protected_tags) end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index 841fb681435..36162faa1eb 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -20,6 +20,10 @@ module Gitlab subject end + def present(**attributes) + self + end + class_methods do def presenter? true diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 153cb2a8bb1..1f060de657d 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -81,6 +81,20 @@ module Gitlab ] end + def send_git_snapshot(repository) + params = { + 'GitalyServer' => gitaly_server_hash(repository), + 'GetSnapshotRequest' => Gitaly::GetSnapshotRequest.new( + repository: repository.gitaly_repository + ).to_json + } + + [ + SEND_DATA_HEADER, + "git-snapshot:#{encode(params)}" + ] + end + def send_git_diff(repository, diff_refs) params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { diff --git a/lib/omni_auth/strategies/jwt.rb b/lib/omni_auth/strategies/jwt.rb new file mode 100644 index 00000000000..2349b2a28aa --- /dev/null +++ b/lib/omni_auth/strategies/jwt.rb @@ -0,0 +1,62 @@ +require 'omniauth' +require 'jwt' + +module OmniAuth + module Strategies + class JWT + ClaimInvalid = Class.new(StandardError) + + include OmniAuth::Strategy + + args [:secret] + + option :secret, nil + option :algorithm, 'HS256' + option :uid_claim, 'email' + option :required_claims, %w(name email) + option :info_map, { name: "name", email: "email" } + option :auth_url, nil + option :valid_within, nil + + uid { decoded[options.uid_claim] } + + extra do + { raw_info: decoded } + end + + info do + options.info_map.each_with_object({}) do |(k, v), h| + h[k.to_s] = decoded[v.to_s] + end + end + + def request_phase + redirect options.auth_url + end + + def decoded + @decoded ||= ::JWT.decode(request.params['jwt'], options.secret, options.algorithm).first + + (options.required_claims || []).each do |field| + raise ClaimInvalid, "Missing required '#{field}' claim" unless @decoded.key?(field.to_s) + end + + raise ClaimInvalid, "Missing required 'iat' claim" if options.valid_within && !@decoded["iat"] + + if options.valid_within && (Time.now.to_i - @decoded["iat"]).abs > options.valid_within + raise ClaimInvalid, "'iat' timestamp claim is too skewed from present" + end + + @decoded + end + + def callback_phase + super + rescue ClaimInvalid => e + fail! :claim_invalid, e + end + end + + class Jwt < JWT; end + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index abef8cd2bcc..c04dae7446f 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -427,10 +427,7 @@ namespace :gitlab do user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| - File.join( - p.repository_storage_path, - "#{p.disk_path}.git" - ) + p.repository.path_to_repo end repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index d7f28691098..b854c34a8e5 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -10,9 +10,8 @@ namespace :gitlab do end scope.find_each do |project| - base = File.join(project.repository_storage_path, project.disk_path) - puts base + '.git' - puts base + '.wiki.git' + puts project.repository.path_to_repo + puts project.wiki.repository.path_to_repo end end end diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake new file mode 100644 index 00000000000..100e480bd66 --- /dev/null +++ b/lib/tasks/gitlab/pages.rake @@ -0,0 +1,9 @@ +namespace :gitlab do + namespace :pages do + desc 'Ping the pages admin API' + task admin_ping: :gitlab_environment do + Gitlab::PagesClient.ping + puts "OK: gitlab-pages admin API is reachable" + end + end +end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 1d903c81358..f71e69987cb 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -1,9 +1,20 @@ namespace :gitlab do desc "GitLab | Setup production application" task setup: :gitlab_environment do + check_gitaly_connection setup_db end + def check_gitaly_connection + Gitlab.config.repositories.storages.each do |name, _details| + Gitlab::GitalyClient::ServerService.new(name).info + end + rescue GRPC::Unavailable => ex + puts "Failed to connect to Gitaly...".color(:red) + puts "Error: #{ex}" + exit 1 + end + def setup_db warn_user_is_not_gitlab diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index 8ac73bc8ff2..6e8bd9078c8 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -111,7 +111,7 @@ namespace :gitlab do puts " - #{project.full_path} (id: #{project.id})".color(:red) - return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator + return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator, Cop/AvoidReturnFromBlocks end end end @@ -132,7 +132,7 @@ namespace :gitlab do puts " - #{upload.path} (id: #{upload.id})".color(:red) - return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator + return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator, Cop/AvoidReturnFromBlocks end end end |