diff options
Diffstat (limited to 'lib')
89 files changed, 1475 insertions, 700 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 8abb24e6f69..19da0b2c434 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -139,6 +139,7 @@ module API mount ::API::ProjectTemplates mount ::API::ProtectedBranches mount ::API::ProtectedTags + mount ::API::Releases mount ::API::Repositories mount ::API::Runner mount ::API::Runners @@ -149,6 +150,7 @@ module API mount ::API::Snippets mount ::API::Submodules mount ::API::Subscriptions + mount ::API::Suggestions mount ::API::SystemHooks mount ::API::Tags mount ::API::Templates diff --git a/lib/api/branches.rb b/lib/api/branches.rb index e7e58ad0e66..07f529b01bb 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -34,11 +34,11 @@ module API repository = user_project.repository branches = BranchesFinder.new(repository, declared_params(include_missing: false)).execute - + branches = ::Kaminari.paginate_array(branches) merged_branch_names = repository.merged_branch_names(branches.map(&:name)) present( - paginate(::Kaminari.paginate_array(branches)), + paginate(branches), with: Entities::Branch, current_user: current_user, project: user_project, diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 62c966e06b4..08b4f8db8b0 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -116,7 +116,7 @@ module API end MergeRequest.where(source_project: @project, source_branch: ref) - .update_all(head_pipeline_id: pipeline) if pipeline.latest? + .update_all(head_pipeline_id: pipeline.id) if pipeline.latest? present status, with: Entities::CommitStatus rescue StateMachines::InvalidTransition => e diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 5dbfbb85e9e..7116ab2882b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -323,7 +323,7 @@ module API expose :request_access_enabled expose :full_name, :full_path - if ::Group.supports_nested_groups? + if ::Group.supports_nested_objects? expose :parent_id end @@ -1087,11 +1087,20 @@ module API expose :password_authentication_enabled_for_web, as: :signin_enabled end - class Release < Grape::Entity + # deprecated old Release representation + class TagRelease < Grape::Entity expose :tag, as: :tag_name expose :description end + class Release < TagRelease + expose :name + expose :description_html + expose :created_at + expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? } + expose :commit, using: Entities::Commit + end + class Tag < Grape::Entity expose :name, :message, :target @@ -1100,7 +1109,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord - expose :release, using: Entities::Release do |repo_tag, options| + expose :release, using: Entities::TagRelease do |repo_tag, options| options[:project].releases.find_by(tag: repo_tag.name) end # rubocop: enable CodeReuse/ActiveRecord @@ -1495,5 +1504,17 @@ module API expose :label, using: Entities::LabelBasic expose :action end + + class Suggestion < Grape::Entity + expose :id + expose :from_original_line + expose :to_original_line + expose :from_line + expose :to_line + expose :appliable?, as: :appliable + expose :applied + expose :from_content + expose :to_content + end end end diff --git a/lib/api/events.rb b/lib/api/events.rb index 44dae57770d..b98aa9f31e1 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -18,29 +18,15 @@ module API desc: 'Return events sorted in ascending and descending order' end - RedactedEvent = OpenStruct.new(target_title: 'Confidential event').freeze - - def redact_events(events) - events.map do |event| - if event.visible_to_user?(current_user) - event - else - RedactedEvent - end - end - end - - # rubocop: disable CodeReuse/ActiveRecord - def present_events(events, redact: true) - events = events.reorder(created_at: params[:sort]) - .with_associations - + def present_events(events) events = paginate(events) - events = redact_events(events) if redact present events, with: Entities::Event end - # rubocop: enable CodeReuse/ActiveRecord + + def find_events(source) + EventsFinder.new(params.merge(source: source, current_user: current_user, with_associations: true)).execute + end end resource :events do @@ -55,16 +41,14 @@ module API use :event_filter_params use :sort_params end - # rubocop: disable CodeReuse/ActiveRecord + get do authenticate! - events = EventsFinder.new(params.merge(source: current_user, current_user: current_user)).execute.preload(:author, :target) + events = find_events(current_user) - # Since we're viewing our own events, redaction is unnecessary - present_events(events, redact: false) + present_events(events) end - # rubocop: enable CodeReuse/ActiveRecord end params do @@ -82,16 +66,15 @@ module API use :event_filter_params use :sort_params end - # rubocop: disable CodeReuse/ActiveRecord + get ':id/events' do user = find_user(params[:id]) not_found!('User') unless user - events = EventsFinder.new(params.merge(source: user, current_user: current_user)).execute.preload(:author, :target) + events = find_events(user) present_events(events) end - # rubocop: enable CodeReuse/ActiveRecord end params do @@ -106,13 +89,12 @@ module API use :event_filter_params use :sort_params end - # rubocop: disable CodeReuse/ActiveRecord + get ":id/events" do - events = EventsFinder.new(params.merge(source: user_project, current_user: current_user)).execute.preload(:author, :target) + events = find_events(user_project) present_events(events) end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 626a2008dee..64958ff982a 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -113,7 +113,7 @@ module API requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' - if ::Group.supports_nested_groups? + if ::Group.supports_nested_objects? optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 2cceb2ec798..6c1a730935a 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -163,9 +163,11 @@ module API end def find_branch!(branch_name) - user_project.repository.find_branch(branch_name) || not_found!('Branch') - rescue Gitlab::Git::CommandError - render_api_error!('The branch refname is invalid', 400) + if Gitlab::GitRefValidator.validate(branch_name) + user_project.repository.find_branch(branch_name) || not_found!('Branch') + else + render_api_error!('The branch refname is invalid', 400) + end end def find_project_label(id) @@ -291,7 +293,7 @@ module API end end permitted_attrs = ActionController::Parameters.new(attrs).permit! - Gitlab.rails5? ? permitted_attrs.to_h : permitted_attrs + permitted_attrs.to_h end # rubocop: disable CodeReuse/ActiveRecord @@ -494,7 +496,7 @@ module API def send_git_blob(repository, blob) env['api.format'] = :txt content_type 'text/plain' - header['Content-Disposition'] = "attachment; filename=#{blob.name.inspect}" + header['Content-Disposition'] = content_disposition('attachment', blob.name) header(*Gitlab::Workhorse.send_git_blob(repository, blob)) end @@ -527,5 +529,11 @@ module API params[:archived] end + + def content_disposition(disposition, filename) + disposition += %(; filename=#{filename.inspect}) if filename.present? + + disposition + end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index ae40b5f7557..9488b3469d9 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -256,8 +256,9 @@ module API post '/post_receive' do status 200 + PostReceive.perform_async(params[:gl_repository], params[:identifier], - params[:changes]) + params[:changes], params[:push_options].to_a) broadcast_message = BroadcastMessage.current&.last&.message reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 80a5cbd6b19..45c694b6448 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -38,6 +38,8 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get ':id/jobs' do + authorize_read_builds! + builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) @@ -56,7 +58,10 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get ':id/pipelines/:pipeline_id/jobs' do + authorize!(:read_pipeline, user_project) pipeline = user_project.ci_pipelines.find(params[:pipeline_id]) + authorize!(:read_build, pipeline) + builds = pipeline.builds builds = filter_builds(builds, params[:scope]) builds = builds.preload(:job_artifacts_archive, :job_artifacts, project: [:namespace]) diff --git a/lib/api/releases.rb b/lib/api/releases.rb new file mode 100644 index 00000000000..37d06988e64 --- /dev/null +++ b/lib/api/releases.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +module API + class Releases < Grape::API + include PaginationParams + + RELEASE_ENDPOINT_REQUIREMETS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS + .merge(tag_name: API::NO_SLASH_URL_PART_REGEX) + + before { error!('404 Not Found', 404) unless Feature.enabled?(:releases_page, user_project) } + before { authorize_read_releases! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get a project releases' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + use :pagination + end + get ':id/releases' do + releases = ::ReleasesFinder.new(user_project, current_user).execute + + present paginate(releases), with: Entities::Release + end + + desc 'Get a single project release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + end + get ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do + authorize_read_release! + + present release, with: Entities::Release + end + + desc 'Create a new release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + requires :name, type: String, desc: 'The name of the release' + requires :description, type: String, desc: 'The release notes' + optional :ref, type: String, desc: 'The commit sha or branch name' + end + post ':id/releases' do + authorize_create_release! + + result = ::Releases::CreateService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + + desc 'Update a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + optional :name, type: String, desc: 'The name of the release' + optional :description, type: String, desc: 'Release notes with markdown support' + end + put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do + authorize_update_release! + + result = ::Releases::UpdateService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + + desc 'Delete a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + end + delete ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do + authorize_destroy_release! + + result = ::Releases::DestroyService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + end + + helpers do + def authorize_create_release! + authorize! :create_release, user_project + end + + def authorize_read_releases! + authorize! :read_release, user_project + end + + def authorize_read_release! + authorize! :read_release, release + end + + def authorize_update_release! + authorize! :update_release, release + end + + def authorize_destroy_release! + authorize! :destroy_release, release + end + + def release + @release ||= user_project.releases.find_by_tag(params[:tag]) + end + end + end +end diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb new file mode 100644 index 00000000000..d008d1b9e97 --- /dev/null +++ b/lib/api/suggestions.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module API + class Suggestions < Grape::API + before { authenticate! } + + resource :suggestions do + desc 'Apply suggestion patch in the Merge Request it was created' do + success Entities::Suggestion + end + params do + requires :id, type: String, desc: 'The suggestion ID' + end + put ':id/apply' do + suggestion = Suggestion.find_by_id(params[:id]) + + not_found! unless suggestion + authorize! :apply_suggestion, suggestion + + result = ::Suggestions::ApplyService.new(current_user).execute(suggestion) + + if result[:status] == :success + present suggestion, with: Entities::Suggestion, current_user: current_user + else + http_status = result[:http_status] || 400 + render_api_error!(result[:message], http_status) + end + end + end + end +end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index b18eec7d796..aacdca3871a 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -42,21 +42,35 @@ module API end desc 'Create a new repository tag' do + detail 'This optional release_description parameter was deprecated in GitLab 11.7.' success Entities::Tag end params do requires :tag_name, type: String, desc: 'The name of the tag' requires :ref, type: String, desc: 'The commit sha or branch name' optional :message, type: String, desc: 'Specifying a message creates an annotated tag' - optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database' + optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database (deprecated in GitLab 11.7)' end post ':id/repository/tags' do authorize_push_project result = ::Tags::CreateService.new(user_project, current_user) - .execute(params[:tag_name], params[:ref], params[:message], params[:release_description]) + .execute(params[:tag_name], params[:ref], params[:message]) if result[:status] == :success + # Release creation with Tags API was deprecated in GitLab 11.7 + if params[:release_description].present? + release_create_params = { + tag: params[:tag_name], + name: params[:tag_name], # Name can be specified in new API + description: params[:release_description] + } + + ::Releases::CreateService + .new(user_project, current_user, release_create_params) + .execute + end + present result[:tag], with: Entities::Tag, project: user_project @@ -88,44 +102,72 @@ module API end desc 'Add a release note to a tag' do - success Entities::Release + detail 'This feature was deprecated in GitLab 11.7.' + success Entities::TagRelease end params do - requires :tag_name, type: String, desc: 'The name of the tag' + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag requires :description, type: String, desc: 'Release notes with markdown support' end post ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do - authorize_push_project + authorize_create_release! + + ## + # Legacy API does not support tag auto creation. + not_found!('Tag') unless user_project.repository.find_tag(params[:tag]) - result = CreateReleaseService.new(user_project, current_user) - .execute(params[:tag_name], params[:description]) + release_create_params = { + tag: params[:tag], + name: params[:tag], # Name can be specified in new API + description: params[:description] + } + + result = ::Releases::CreateService + .new(user_project, current_user, release_create_params) + .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::TagRelease else render_api_error!(result[:message], result[:http_status]) end end desc "Update a tag's release note" do - success Entities::Release + detail 'This feature was deprecated in GitLab 11.7.' + success Entities::TagRelease end params do - requires :tag_name, type: String, desc: 'The name of the tag' + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag requires :description, type: String, desc: 'Release notes with markdown support' end put ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do - authorize_push_project + authorize_update_release! - result = UpdateReleaseService.new(user_project, current_user) - .execute(params[:tag_name], params[:description]) + result = ::Releases::UpdateService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::TagRelease else render_api_error!(result[:message], result[:http_status]) end end end + + helpers do + def authorize_create_release! + authorize! :create_release, user_project + end + + def authorize_update_release! + authorize! :update_release, release + end + + def release + @release ||= user_project.releases.find_by_tag(params[:tag]) + end + end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index a0434a66ef1..0add2b3f875 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -195,7 +195,7 @@ module Backup if connection.service == ::Fog::Storage::Local connection.directories.create(key: remote_directory) else - connection.directories.get(remote_directory) + connection.directories.new(key: remote_directory) end end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 2e6d742de27..4f60b6f84c6 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -9,11 +9,10 @@ module Banzai def call links.each do |node| uri = uri(node['href'].to_s) - next unless uri - node.set_attribute('href', uri.to_s) + node.set_attribute('href', uri.to_s) if uri - if SCHEMES.include?(uri.scheme) && external_url?(uri) + if SCHEMES.include?(uri&.scheme) && !internal_url?(uri) node.set_attribute('rel', 'nofollow noreferrer noopener') node.set_attribute('target', '_blank') end @@ -35,11 +34,12 @@ module Banzai doc.xpath(query) end - def external_url?(uri) + def internal_url?(uri) + return false if uri.nil? # Relative URLs miss a hostname - return false unless uri.hostname + return true unless uri.hostname - uri.hostname != internal_url.hostname + uri.hostname == internal_url.hostname end def internal_url diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 04ec38209c7..f90a35952e5 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -29,7 +29,7 @@ module Banzai if label yield match, label.id, project, namespace, $~ else - match + escape_html_entities(match) end end end @@ -102,6 +102,10 @@ module Banzai CGI.unescapeHTML(text.to_s) end + def escape_html_entities(text) + CGI.escapeHTML(text.to_s) + end + def object_link_title(object, matches) # use title of wrapped element instead nil diff --git a/lib/banzai/filter/suggestion_filter.rb b/lib/banzai/filter/suggestion_filter.rb new file mode 100644 index 00000000000..307ea449140 --- /dev/null +++ b/lib/banzai/filter/suggestion_filter.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Banzai + module Filter + class SuggestionFilter < HTML::Pipeline::Filter + # Class used for tagging elements that should be rendered + TAG_CLASS = 'js-render-suggestion'.freeze + + def call + return doc unless suggestions_filter_enabled? + + doc.search('pre.suggestion > code').each do |node| + node.add_class(TAG_CLASS) + end + + doc + end + + def suggestions_filter_enabled? + context[:suggestions_filter_enabled] + end + end + end +end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 8a7f9045c24..18e5e9185de 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -69,7 +69,7 @@ module Banzai end def use_rouge?(language) - %w(math mermaid plantuml).exclude?(language) + %w(math mermaid plantuml suggestion).exclude?(language) end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 96bea7ca935..5f13a6d6cde 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -29,6 +29,7 @@ module Banzai Filter::TableOfContentsFilter, Filter::AutolinkFilter, Filter::ExternalLinkFilter, + Filter::SuggestionFilter, *reference_filters, diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 63a998a2c1f..7eaad6d7560 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -14,7 +14,8 @@ module Banzai [ Filter::RedactorFilter, Filter::RelativeLinkFilter, - Filter::IssuableStateFilter + Filter::IssuableStateFilter, + Filter::SuggestionFilter ] end diff --git a/lib/banzai/suggestions_parser.rb b/lib/banzai/suggestions_parser.rb new file mode 100644 index 00000000000..09f36635020 --- /dev/null +++ b/lib/banzai/suggestions_parser.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Banzai + module SuggestionsParser + # Returns the content of each suggestion code block. + # + def self.parse(text) + html = Banzai.render(text, project: nil, no_original_data: true) + doc = Nokogiri::HTML(html) + + doc.search('pre.suggestion').map { |node| node.text } + end + end +end diff --git a/lib/constraints/feature_constrainer.rb b/lib/constraints/feature_constrainer.rb index ca4376a9d38..cd246cf37a4 100644 --- a/lib/constraints/feature_constrainer.rb +++ b/lib/constraints/feature_constrainer.rb @@ -2,14 +2,14 @@ module Constraints class FeatureConstrainer - attr_reader :feature + attr_reader :args - def initialize(feature) - @feature = feature + def initialize(*args) + @args = args end def matches?(_request) - Feature.enabled?(feature) + Feature.enabled?(*args) end end end diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index 5e22523e45a..7ba48ae9c79 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -22,14 +22,10 @@ module DeclarativePolicy key = Cache.policy_key(user, subject) cache[key] ||= - if Gitlab.rails5? - # to avoid deadlocks in multi-threaded environment when - # autoloading is enabled, we allow concurrent loads, - # https://gitlab.com/gitlab-org/gitlab-ce/issues/48263 - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - class_for(subject).new(user, subject, opts) - end - else + # to avoid deadlocks in multi-threaded environment when + # autoloading is enabled, we allow concurrent loads, + # https://gitlab.com/gitlab-org/gitlab-ce/issues/48263 + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do class_for(subject).new(user, subject, opts) end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 6eb5f9e2300..7aa02009aa0 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -199,7 +199,7 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def lfs_token_check(login, password, project) + def lfs_token_check(login, encoded_token, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) actor = @@ -222,7 +222,7 @@ module Gitlab read_authentication_abilities end - if Devise.secure_compare(token_handler.token, password) + if token_handler.token_valid?(encoded_token) Gitlab::Auth::Result.new(actor, nil, token_handler.type, authentication_abilities) end end diff --git a/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb b/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb index 2f76f2f7434..a6194616663 100644 --- a/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb @@ -2,132 +2,13 @@ module Gitlab module BackgroundMigration - # Class that will create fill the project_repositories table - # for all projects that are on hashed storage and an entry is - # is missing in this table. - class BackfillHashedProjectRepositories - # Shard model - class Shard < ActiveRecord::Base - self.table_name = 'shards' - end - - # Class that will find or create the shard by name. - # There is only a small set of shards, which would - # not change quickly, so look them up from memory - # instead of hitting the DB each time. - class ShardFinder - def find_shard_id(name) - shard_id = shards.fetch(name, nil) - return shard_id if shard_id.present? - - Shard.transaction(requires_new: true) do - create!(name) - end - rescue ActiveRecord::RecordNotUnique - reload! - retry - end - - private - - def create!(name) - Shard.create!(name: name).tap { |shard| @shards[name] = shard.id } - end - - def shards - @shards ||= reload! - end - - def reload! - @shards = Hash[*Shard.all.map { |shard| [shard.name, shard.id] }.flatten] - end - end - - # ProjectRegistry model - class ProjectRepository < ActiveRecord::Base - self.table_name = 'project_repositories' - - belongs_to :project, inverse_of: :project_repository - end - - # Project model - class Project < ActiveRecord::Base - self.table_name = 'projects' - - HASHED_PATH_PREFIX = '@hashed' - - HASHED_STORAGE_FEATURES = { - repository: 1, - attachments: 2 - }.freeze - - has_one :project_repository, inverse_of: :project - - class << self - def on_hashed_storage - where(Project.arel_table[:storage_version] - .gteq(HASHED_STORAGE_FEATURES[:repository])) - end - - def without_project_repository - joins(left_outer_join_project_repository) - .where(ProjectRepository.arel_table[:project_id].eq(nil)) - end - - def left_outer_join_project_repository - projects_table = Project.arel_table - repository_table = ProjectRepository.arel_table - - projects_table - .join(repository_table, Arel::Nodes::OuterJoin) - .on(projects_table[:id].eq(repository_table[:project_id])) - .join_sources - end - end - - def hashed_storage? - self.storage_version && self.storage_version >= 1 - end - - def hashed_disk_path - "#{HASHED_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}" - end - - def disk_hash - @disk_hash ||= Digest::SHA2.hexdigest(id.to_s) - end - end - - def perform(start_id, stop_id) - Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) - end - + # Class that will fill the project_repositories table for projects that + # are on hashed storage and an entry is is missing in this table. + class BackfillHashedProjectRepositories < BackfillProjectRepositories private - def project_repositories(start_id, stop_id) + def projects Project.on_hashed_storage - .without_project_repository - .where(id: start_id..stop_id) - .map { |project| build_attributes_for_project(project) } - .compact - end - - def build_attributes_for_project(project) - return unless project.hashed_storage? - - { - project_id: project.id, - shard_id: find_shard_id(project.repository_storage), - disk_path: project.hashed_disk_path - } - end - - def find_shard_id(repository_storage) - shard_finder.find_shard_id(repository_storage) - end - - def shard_finder - @shard_finder ||= ShardFinder.new end end end diff --git a/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb b/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb new file mode 100644 index 00000000000..6dc92672929 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will fill the project_repositories table for projects that + # are on legacy storage and an entry is is missing in this table. + class BackfillLegacyProjectRepositories < BackfillProjectRepositories + private + + def projects + Project.with_parent.on_legacy_storage + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb new file mode 100644 index 00000000000..aaf520d70f6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -0,0 +1,219 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will create fill the project_repositories table + # for projects an entry is is missing in this table. + class BackfillProjectRepositories + OrphanedNamespaceError = Class.new(StandardError) + + # Shard model + class Shard < ActiveRecord::Base + self.table_name = 'shards' + end + + # Class that will find or create the shard by name. + # There is only a small set of shards, which would + # not change quickly, so look them up from memory + # instead of hitting the DB each time. + class ShardFinder + def find_shard_id(name) + shard_id = shards.fetch(name, nil) + return shard_id if shard_id.present? + + Shard.transaction(requires_new: true) do + create!(name) + end + rescue ActiveRecord::RecordNotUnique + reload! + retry + end + + private + + def create!(name) + Shard.create!(name: name).tap { |shard| @shards[name] = shard.id } + end + + def shards + @shards ||= reload! + end + + def reload! + @shards = Hash[*Shard.all.map { |shard| [shard.name, shard.id] }.flatten] + end + end + + module Storage + # Class that returns the disk path for a project using hashed storage + class HashedProject + attr_accessor :project + + ROOT_PATH_PREFIX = '@hashed' + + def initialize(project) + @project = project + end + + def disk_path + "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}" + end + + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) + end + end + + # Class that returns the disk path for a project using legacy storage + class LegacyProject + attr_accessor :project + + def initialize(project) + @project = project + end + + def disk_path + project.full_path + end + end + end + + # Concern used by Project and Namespace to determine the full route to the project + module Routable + extend ActiveSupport::Concern + + def full_path + @full_path ||= build_full_path + end + + def build_full_path + return path unless has_parent? + + raise OrphanedNamespaceError if parent.nil? + + parent.full_path + '/' + path + end + + def has_parent? + read_attribute(association(:parent).reflection.foreign_key) + end + end + + # Namespace model. + class Namespace < ActiveRecord::Base + self.table_name = 'namespaces' + self.inheritance_column = nil + + include Routable + + belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces' + + has_many :projects, inverse_of: :parent + has_many :namespaces, inverse_of: :parent + end + + # ProjectRegistry model + class ProjectRepository < ActiveRecord::Base + self.table_name = 'project_repositories' + + belongs_to :project, inverse_of: :project_repository + end + + # Project model + class Project < ActiveRecord::Base + self.table_name = 'projects' + + include Routable + + HASHED_STORAGE_FEATURES = { + repository: 1, + attachments: 2 + }.freeze + + scope :with_parent, -> { includes(:parent) } + + belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects' + + has_one :project_repository, inverse_of: :project + + delegate :disk_path, to: :storage + + class << self + def on_hashed_storage + where(Project.arel_table[:storage_version] + .gteq(HASHED_STORAGE_FEATURES[:repository])) + end + + def on_legacy_storage + where(Project.arel_table[:storage_version].eq(nil) + .or(Project.arel_table[:storage_version].eq(0))) + end + + def without_project_repository + joins(left_outer_join_project_repository) + .where(ProjectRepository.arel_table[:project_id].eq(nil)) + end + + def left_outer_join_project_repository + projects_table = Project.arel_table + repository_table = ProjectRepository.arel_table + + projects_table + .join(repository_table, Arel::Nodes::OuterJoin) + .on(projects_table[:id].eq(repository_table[:project_id])) + .join_sources + end + end + + def storage + @storage ||= + if hashed_storage? + Storage::HashedProject.new(self) + else + Storage::LegacyProject.new(self) + end + end + + def hashed_storage? + self.storage_version && + self.storage_version >= HASHED_STORAGE_FEATURES[:repository] + end + end + + def perform(start_id, stop_id) + Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) + end + + private + + def projects + raise NotImplementedError, + "#{self.class} does not implement #{__method__}" + end + + def project_repositories(start_id, stop_id) + projects + .without_project_repository + .where(id: start_id..stop_id) + .map { |project| build_attributes_for_project(project) } + .compact + end + + def build_attributes_for_project(project) + { + project_id: project.id, + shard_id: find_shard_id(project.repository_storage), + disk_path: project.disk_path + } + end + + def find_shard_id(repository_storage) + shard_finder.find_shard_id(repository_storage) + end + + def shard_finder + @shard_finder ||= ShardFinder.new + end + end + end +end diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb index 488c1d85387..d3e15a79a8b 100644 --- a/lib/gitlab/blob_helper.rb +++ b/lib/gitlab/blob_helper.rb @@ -12,7 +12,7 @@ module Gitlab end def viewable? - !large? && text? + !large? && text_in_repo? end MEGABYTE = 1024 * 1024 @@ -21,7 +21,7 @@ module Gitlab size.to_i > MEGABYTE end - def binary? + def binary_in_repo? # Large blobs aren't even loaded into memory if data.nil? true @@ -40,8 +40,8 @@ module Gitlab end end - def text? - !binary? + def text_in_repo? + !binary_in_repo? end def image? @@ -113,7 +113,7 @@ module Gitlab def content_type # rubocop:disable Style/MultilineTernaryOperator # rubocop:disable Style/NestedTernaryOperator - @content_type ||= binary_mime_type? || binary? ? mime_type : + @content_type ||= binary_mime_type? || binary_in_repo? ? mime_type : (encoding ? "text/plain; charset=#{encoding.downcase}" : "text/plain") # rubocop:enable Style/NestedTernaryOperator # rubocop:enable Style/MultilineTernaryOperator diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb index f8cda0382fe..09b17b5b76b 100644 --- a/lib/gitlab/checks/base_checker.rb +++ b/lib/gitlab/checks/base_checker.rb @@ -18,12 +18,16 @@ module Gitlab private + def creation? + Gitlab::Git.blank_ref?(oldrev) + end + def deletion? Gitlab::Git.blank_ref?(newrev) end def update? - !Gitlab::Git.blank_ref?(oldrev) && !deletion? + !creation? && !deletion? end def updated_from_web? @@ -33,6 +37,22 @@ module Gitlab def tag_exists? project.repository.tag_exists?(tag_name) end + + def validate_once(resource) + Gitlab::SafeRequestStore.fetch(cache_key_for_resource(resource)) do + yield(resource) + + true + end + end + + def cache_key_for_resource(resource) + "git_access:#{checker_cache_key}:#{resource.cache_key}" + end + + def checker_cache_key + self.class.name.demodulize.underscore + end end end end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 7778d3068cc..8a57a3a6d9a 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader(*ATTRIBUTES) def initialize( - change, user_access:, project:, skip_authorization: false, + change, user_access:, project:, skip_lfs_integrity_check: false, protocol:, logger: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @@ -18,7 +18,6 @@ module Gitlab @tag_name = Gitlab::Git.tag_name(@ref) @user_access = user_access @project = project - @skip_authorization = skip_authorization @skip_lfs_integrity_check = skip_lfs_integrity_check @protocol = protocol @@ -27,8 +26,6 @@ module Gitlab end def exec - return true if skip_authorization - ref_level_checks # Check of commits should happen as the last step # given they're expensive in terms of performance diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index 8ee345ab45a..ea0d8c85a66 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -11,16 +11,20 @@ module Gitlab }.freeze def validate! - return if deletion? || newrev.nil? + return if deletion? return unless should_run_diff_validations? return if commits.empty? - return unless uses_raw_delta_validations? file_paths = [] - process_raw_deltas do |diff| - file_paths << (diff.new_path || diff.old_path) - validate_diff(diff) + process_commits do |commit| + validate_once(commit) do + commit.raw_deltas.each do |diff| + file_paths << (diff.new_path || diff.old_path) + + validate_diff(diff) + end + end end validate_file_paths(file_paths) @@ -28,17 +32,13 @@ module Gitlab private - def should_run_diff_validations? - validate_lfs_file_locks? - end - def validate_lfs_file_locks? strong_memoize(:validate_lfs_file_locks) do project.lfs_enabled? && project.any_lfs_file_locks? end end - def uses_raw_delta_validations? + def should_run_diff_validations? validations_for_diff.present? || path_validations.present? end @@ -59,16 +59,14 @@ module Gitlab validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] end - def process_raw_deltas + def process_commits logger.log_timed(LOG_MESSAGES[:diff_content_check]) do # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 ::Gitlab::GitalyClient.allow_n_plus_1_calls do commits.each do |commit| logger.check_timeout_reached - commit.raw_deltas.each do |diff| - yield(diff) - end + yield(commit) end end end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index e42684e679a..cc6a14d2d9a 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -7,6 +7,7 @@ module Gitlab ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'.freeze def validate! + return unless project.lfs_enabled? return if skip_lfs_integrity_check logger.log_timed(LOG_MESSAGE) do diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb index f3a52f09868..91f8d0bdbc8 100644 --- a/lib/gitlab/checks/push_check.rb +++ b/lib/gitlab/checks/push_check.rb @@ -6,7 +6,7 @@ module Gitlab def validate! logger.log_timed("Checking if you are allowed to push...") do unless can_push? - raise GitAccess::UnauthorizedError, 'You are not allowed to push code to this project.' + raise GitAccess::UnauthorizedError, GitAccess::ERROR_MESSAGES[:push_code] end end end @@ -15,7 +15,7 @@ module Gitlab def can_push? user_access.can_do_action?(:push_code) || - user_access.can_push_to_branch?(branch_name) + project.branch_allows_collaboration?(user_access.user, branch_name) end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 100b9521412..e62d547d862 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -10,7 +10,7 @@ module Gitlab :origin_ref, :checkout_sha, :after_sha, :before_sha, :trigger_request, :schedule, :merge_request, :ignore_skip_ci, :save_incompleted, - :seeds_block, :variables_attributes + :seeds_block, :variables_attributes, :push_options ) do include Gitlab::Utils::StrongMemoize @@ -54,7 +54,13 @@ module Gitlab def protected_ref? strong_memoize(:protected_ref) do - project.protected_for?(ref) + project.protected_for?(origin_ref) + end + end + + def ambiguous_ref? + strong_memoize(:ambiguous_ref) do + project.repository.ambiguous_ref?(origin_ref) end end end diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index b9707d2f8f5..79bbcc1ed1e 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -8,6 +8,7 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i + SKIP_PUSH_OPTION = 'ci.skip' def perform! if skipped? @@ -16,7 +17,7 @@ module Gitlab end def skipped? - !@command.ignore_skip_ci && commit_message_skips_ci? + !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?) end def break? @@ -32,6 +33,10 @@ module Gitlab !!(@pipeline.git_commit_message =~ SKIP_PATTERN) end end + + def push_option_skips_ci? + !!(@command.push_options&.include?(SKIP_PUSH_OPTION)) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb index d88851d8245..9c6c2bc8e25 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/repository.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -16,6 +16,10 @@ module Gitlab unless @command.sha return error('Commit not found') end + + if @command.ambiguous_ref? + return error('Ref is ambiguous') + end end def break? diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index d0613aa59e1..b5350f56f9c 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -149,10 +149,10 @@ performance: only: refs: - branches + kubernetes: active except: variables: - $PERFORMANCE_DISABLED - - $KUBECONFIG == null sast: stage: test @@ -185,7 +185,8 @@ dependency_scanning: - setup_docker - dependency_scanning artifacts: - paths: [gl-dependency-scanning-report.json] + reports: + dependency_scanning: gl-dependency-scanning-report.json only: refs: - branches @@ -228,6 +229,7 @@ dast: only: refs: - branches + kubernetes: active variables: - $GITLAB_FEATURES =~ /\bdast\b/ except: @@ -235,7 +237,6 @@ dast: - master variables: - $DAST_DISABLED - - $KUBECONFIG == null review: stage: review @@ -257,12 +258,12 @@ review: only: refs: - branches + kubernetes: active except: refs: - master variables: - $REVIEW_DISABLED - - $KUBECONFIG == null stop_review: stage: cleanup @@ -280,12 +281,12 @@ stop_review: only: refs: - branches + kubernetes: active except: refs: - master variables: - $REVIEW_DISABLED - - $KUBECONFIG == null # Staging deploys are disabled by default since # continuous deployment to production is enabled by default @@ -309,11 +310,9 @@ staging: only: refs: - master + kubernetes: active variables: - $STAGING_ENABLED - except: - variables: - - $KUBECONFIG == null # Canaries are also disabled by default, but if you want them, # and know what the downsides are, you can enable this by setting @@ -336,11 +335,9 @@ canary: only: refs: - master + kubernetes: active variables: - $CANARY_ENABLED - except: - variables: - - $KUBECONFIG == null .production: &production_template stage: production @@ -366,13 +363,13 @@ production: only: refs: - master + kubernetes: active except: variables: - $STAGING_ENABLED - $CANARY_ENABLED - $INCREMENTAL_ROLLOUT_ENABLED - $INCREMENTAL_ROLLOUT_MODE - - $KUBECONFIG == null production_manual: <<: *production_template @@ -381,6 +378,7 @@ production_manual: only: refs: - master + kubernetes: active variables: - $STAGING_ENABLED - $CANARY_ENABLED @@ -388,7 +386,6 @@ production_manual: variables: - $INCREMENTAL_ROLLOUT_ENABLED - $INCREMENTAL_ROLLOUT_MODE - - $KUBECONFIG == null # This job implements incremental rollout on for every push to `master`. @@ -418,13 +415,13 @@ production_manual: only: refs: - master + kubernetes: active variables: - $INCREMENTAL_ROLLOUT_MODE == "manual" - $INCREMENTAL_ROLLOUT_ENABLED except: variables: - $INCREMENTAL_ROLLOUT_MODE == "timed" - - $KUBECONFIG == null .timed_rollout_template: &timed_rollout_template <<: *rollout_template @@ -433,11 +430,9 @@ production_manual: only: refs: - master + kubernetes: active variables: - $INCREMENTAL_ROLLOUT_MODE == "timed" - except: - variables: - - $KUBECONFIG == null timed rollout 10%: <<: *timed_rollout_template @@ -601,10 +596,55 @@ rollout 100%: fi } + # Extracts variables prefixed with K8S_SECRET_ + # and creates a Kubernetes secret. + # + # e.g. If we have the following environment variables: + # K8S_SECRET_A=value1 + # K8S_SECRET_B=multi\ word\ value + # + # Then we will create a secret with the following key-value pairs: + # data: + # A: dmFsdWUxCg== + # B: bXVsdGkgd29yZCB2YWx1ZQo= + function create_application_secret() { + track="${1-stable}" + export APPLICATION_SECRET_NAME=$(application_secret_name "$track") + + bash -c ' + function k8s_prefixed_variables() { + env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" + } + + kubectl create secret \ + -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \ + --from-env-file <(k8s_prefixed_variables) -o yaml --dry-run | + kubectl replace -n "$KUBE_NAMESPACE" --force -f - + ' + } + + function deploy_name() { + name="$CI_ENVIRONMENT_SLUG" + track="${1-stable}" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + echo $name + } + + function application_secret_name() { + track="${1-stable}" + name=$(deploy_name "$track") + + echo "${name}-secret" + } + function deploy() { track="${1-stable}" percentage="${2:-100}" - name="$CI_ENVIRONMENT_SLUG" + name=$(deploy_name "$track") replicas="1" service_enabled="true" @@ -613,7 +653,6 @@ rollout 100%: # if track is different than stable, # re-use all attached resources if [[ "$track" != "stable" ]]; then - name="$name-$track" service_enabled="false" postgres_enabled="false" fi @@ -626,6 +665,8 @@ rollout 100%: secret_name='' fi + create_application_secret "$track" + if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then echo "Deploying first release with database initialization..." helm upgrade --install \ @@ -638,6 +679,7 @@ rollout 100%: --set image.secrets[0].name="$secret_name" \ --set application.track="$track" \ --set application.database_url="$DATABASE_URL" \ + --set application.secretName="$APPLICATION_SECRET_NAME" \ --set service.url="$CI_ENVIRONMENT_URL" \ --set replicaCount="$replicas" \ --set postgresql.enabled="$postgres_enabled" \ @@ -670,6 +712,7 @@ rollout 100%: --set image.secrets[0].name="$secret_name" \ --set application.track="$track" \ --set application.database_url="$DATABASE_URL" \ + --set application.secretName="$APPLICATION_SECRET_NAME" \ --set service.url="$CI_ENVIRONMENT_URL" \ --set replicaCount="$replicas" \ --set postgresql.enabled="$postgres_enabled" \ @@ -689,11 +732,7 @@ rollout 100%: function scale() { track="${1-stable}" percentage="${2-100}" - name="$CI_ENVIRONMENT_SLUG" - - if [[ "$track" != "stable" ]]; then - name="$name-$track" - fi + name=$(deploy_name "$track") replicas=$(get_replicas "$track" "$percentage") @@ -887,15 +926,14 @@ rollout 100%: function delete() { track="${1-stable}" - name="$CI_ENVIRONMENT_SLUG" - - if [[ "$track" != "stable" ]]; then - name="$name-$track" - fi + name=$(deploy_name "$track") if [[ -n "$(helm ls -q "^$name$")" ]]; then helm delete --purge "$name" fi + + secret_name=$(application_secret_name "$track") + kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name" } before_script: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 93cb31f48c0..0d12cbc6460 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -24,7 +24,6 @@ before_script: - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - - gem install bundler --no-ri --no-rdoc # Bundler is not installed with the image - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby # Optional - Delete if not using `rubocop` diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb index eba1faacc3a..03298d960a4 100644 --- a/lib/gitlab/cleanup/remote_uploads.rb +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -67,7 +67,7 @@ module Gitlab end def remote_directory - connection.directories.get(configuration['remote_directory']) + connection.directories.new(key: configuration['remote_directory']) end def connection diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb index a5e4065cf09..881e5dbc923 100644 --- a/lib/gitlab/color_schemes.rb +++ b/lib/gitlab/color_schemes.rb @@ -12,7 +12,8 @@ module Gitlab Scheme.new(2, 'Dark', 'dark'), Scheme.new(3, 'Solarized Light', 'solarized-light'), Scheme.new(4, 'Solarized Dark', 'solarized-dark'), - Scheme.new(5, 'Monokai', 'monokai') + Scheme.new(5, 'Monokai', 'monokai'), + Scheme.new(6, 'None', 'none') ].freeze # Convenience method to get a space-separated String of all the color scheme diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 477f9101e98..552aad83dd4 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -7,10 +7,6 @@ module Gitlab Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end - def fake_application_settings(attributes = {}) - Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) - end - def clear_in_memory_application_settings! @in_memory_application_settings = nil end @@ -50,28 +46,21 @@ module Gitlab # and other callers from failing, use any loaded settings and return # defaults for missing columns. if ActiveRecord::Migrator.needs_migration? - return fake_application_settings(current_settings&.attributes) - end - - return current_settings if current_settings.present? - - with_fallback_to_fake_application_settings do - ::ApplicationSetting.create_from_defaults || in_memory_application_settings + db_attributes = current_settings&.attributes || {} + ::ApplicationSetting.build_from_defaults(db_attributes) + elsif current_settings.present? + current_settings + else + ::ApplicationSetting.create_from_defaults end end - def in_memory_application_settings - with_fallback_to_fake_application_settings do - @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults - end + def fake_application_settings(attributes = {}) + Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) end - def with_fallback_to_fake_application_settings(&block) - yield - rescue - # In case the application_settings table is not created yet, or if a new - # ApplicationSetting column is not yet migrated we fallback to a simple OpenStruct - fake_application_settings + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults end def connect_to_db? diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 9bf2f9291a8..862127110b9 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -31,7 +31,11 @@ module Gitlab } } ], - total_commits_count: 1 + total_commits_count: 1, + push_options: [ + "ci.skip", + "custom option" + ] }.freeze # Produce a hash of post-receive data @@ -52,10 +56,12 @@ module Gitlab # homepage: String, # }, # commits: Array, - # total_commits_count: Fixnum + # total_commits_count: Fixnum, + # push_options: Array # } # - def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil) + # rubocop:disable Metrics/ParameterLists + def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil, push_options: []) commits = Array(commits) # Total commits count @@ -93,6 +99,7 @@ module Gitlab project: project.hook_attrs, commits: commit_attrs, total_commits_count: commits_count, + push_options: push_options, # DEPRECATED repository: project.hook_attrs.slice(:name, :url, :description, :homepage, :git_http_url, :git_ssh_url, :visibility_level) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 6d40e00c035..b6ca777e029 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -232,11 +232,7 @@ module Gitlab end def self.cached_table_exists?(table_name) - if Gitlab.rails5? - connection.schema_cache.data_source_exists?(table_name) - else - connection.schema_cache.table_exists?(table_name) - end + connection.schema_cache.data_source_exists?(table_name) end private_class_method :connection diff --git a/lib/gitlab/database/arel_methods.rb b/lib/gitlab/database/arel_methods.rb deleted file mode 100644 index 991e4152dcb..00000000000 --- a/lib/gitlab/database/arel_methods.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module ArelMethods - private - - # In Arel 7.0.0 (Arel 7.1.4 is used in Rails 5.0) the `engine` parameter of `Arel::UpdateManager#initializer` - # was removed. - # Remove this file and inline this method when removing rails5? code. - def arel_update_manager - if Gitlab.rails5? - Arel::UpdateManager.new - else - Arel::UpdateManager.new(ActiveRecord::Base) - end - end - end - end -end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 0da5119a3ed..1455e410d4b 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -35,13 +35,7 @@ module Gitlab end def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - arel_from = if Gitlab.rails5? - arel_table.from - else - arel_table - end - - query = arel_from + query = arel_table.from .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) .project(average([arel_table[column_sym]], 'median')) .where( @@ -151,13 +145,8 @@ module Gitlab .order(arel_table[column_sym]) ).as('row_id') - arel_from = if Gitlab.rails5? - arel_table.from.from(arel_table.alias) - else - arel_table.from(arel_table.alias) - end - - count = arel_from.project('COUNT(*)') + count = arel_table.from.from(arel_table.alias) + .project('COUNT(*)') .where(arel_table[partition_column].eq(arel_table.alias[partition_column])) .as('ct') diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index d9578852db6..3abd0600e9d 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -3,8 +3,6 @@ module Gitlab module Database module MigrationHelpers - include Gitlab::Database::ArelMethods - BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time @@ -361,7 +359,7 @@ module Gitlab stop_arel = yield table, stop_arel if block_given? stop_row = exec_query(stop_arel.to_sql).to_hash.first - update_arel = arel_update_manager + update_arel = Arel::UpdateManager.new .table(table) .set([[table[column], value]]) .where(table[:id].gteq(start_id)) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index a5b42bbfdd9..60afa4bcd52 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -5,8 +5,6 @@ module Gitlab module RenameReservedPathsMigration module V1 class RenameBase - include Gitlab::Database::ArelMethods - attr_reader :paths, :migration delegate :update_column_in_batches, @@ -66,7 +64,7 @@ module Gitlab old_full_path, new_full_path) - update = arel_update_manager + update = Arel::UpdateManager.new .table(routes) .set([[routes[:path], replace_statement]]) .where(Arel::Nodes::SqlLiteral.new(filter)) diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index 6516d6e648d..8d97adaff99 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -8,14 +8,7 @@ module Gitlab # behaviour from the default Binary type. ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea else - # In Rails 5.0 `Type` has been moved from `ActiveRecord` to `ActiveModel` - # https://github.com/rails/rails/commit/9cc8c6f3730df3d94c81a55be9ee1b7b4ffd29f6#diff-f8ba7983a51d687976e115adcd95822b - # Remove this method and leave just `ActiveModel::Type::Binary` when removing Gitlab.rails5? code. - if Gitlab.rails5? - ActiveModel::Type::Binary - else - ActiveRecord::Type::Binary - end + ActiveModel::Type::Binary end # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). @@ -26,31 +19,9 @@ module Gitlab class ShaAttribute < BINARY_TYPE PACK_FORMAT = 'H*'.freeze - # It is called from activerecord-4.2.10/lib/active_record internal methods. - # Remove this method when removing Gitlab.rails5? code. - def type_cast_from_database(value) - unpack_sha(super) - end - - # It is called from activerecord-4.2.10/lib/active_record internal methods. - # Remove this method when removing Gitlab.rails5? code. - def type_cast_for_database(value) - serialize(value) - end - - # It is called from activerecord-5.0.6/lib/active_record/attribute.rb - # Remove this method when removing Gitlab.rails5? code.. - def deserialize(value) - value = Gitlab.rails5? ? super : method(:type_cast_from_database).super_method.call(value) - - unpack_sha(value) - end - - # Rename this method to `deserialize(value)` removing Gitlab.rails5? code. # Casts binary data to a SHA1 in hexadecimal. - def unpack_sha(value) - # Uncomment this line when removing Gitlab.rails5? code. - # value = super + def deserialize(value) + value = super(value) value ? value.unpack(PACK_FORMAT)[0] : nil end @@ -58,7 +29,7 @@ module Gitlab def serialize(value) arg = value ? [value].pack(PACK_FORMAT) : nil - Gitlab.rails5? ? super(arg) : method(:type_cast_for_database).super_method.call(arg) + super(arg) end end end diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb index 36e4559b554..10971d2b274 100644 --- a/lib/gitlab/database/subquery.rb +++ b/lib/gitlab/database/subquery.rb @@ -6,15 +6,11 @@ module Gitlab class << self def self_join(relation) t = relation.arel_table - t2 = if !Gitlab.rails5? - relation.arel.as('t2') - else - # Work around a bug in Rails 5, where LIMIT causes trouble - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 - r = relation.limit(nil).arel - r.take(relation.limit_value) if relation.limit_value - r.as('t2') - end + # Work around a bug in Rails 5, where LIMIT causes trouble + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 + r = relation.limit(nil).arel + r.take(relation.limit_value) if relation.limit_value + t2 = r.as('t2') relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first) end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 8ba44dff06f..e410d5a8333 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -3,7 +3,9 @@ module Gitlab module Diff class File - attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs + include Gitlab::Utils::StrongMemoize + + attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs, :unique_identifier delegate :new_file?, :deleted_file?, :renamed_file?, :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, @@ -22,12 +24,20 @@ module Gitlab DiffViewer::Image ].sort_by { |v| v.binary? ? 0 : 1 }.freeze - def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil, stats: nil) + def initialize( + diff, + repository:, + diff_refs: nil, + fallback_diff_refs: nil, + stats: nil, + unique_identifier: nil) + @diff = diff @stats = stats @repository = repository @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + @unique_identifier = unique_identifier @unfolded = false # Ensure items are collected in the the batch @@ -67,7 +77,15 @@ module Gitlab def line_for_position(pos) return nil unless pos.position_type == 'text' - diff_lines.find { |line| line.old_line == pos.old_line && line.new_line == pos.new_line } + # This method is normally used to find which line the diff was + # commented on, and in this context, it's normally the raw diff persisted + # at `note_diff_files`, which is a fraction of the entire diff + # (it goes from the first line, to the commented line, or + # one line below). Therefore it's more performant to fetch + # from bottom to top instead of the other way around. + diff_lines + .reverse_each + .find { |line| line.old_line == pos.old_line && line.new_line == pos.new_line } end def position_for_line_code(code) @@ -122,6 +140,16 @@ module Gitlab old_blob_lazy&.itself end + def new_blob_lines_between(from_line, to_line) + return [] unless new_blob + + from_index = from_line - 1 + to_index = to_line - 1 + + new_blob.load_all_data! + new_blob.data.lines[from_index..to_index] + end + def content_sha new_content_sha || old_content_sha end @@ -156,6 +184,10 @@ module Gitlab @unfolded end + def highlight_loaded? + @highlighted_diff_lines.present? + end + def highlighted_diff_lines @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight @@ -202,12 +234,12 @@ module Gitlab repository.attributes(file_path).fetch('diff') { true } end - def binary? - has_binary_notice? || try_blobs(:binary?) + def binary_in_repo? + has_binary_notice? || try_blobs(:binary_in_repo?) end - def text? - !binary? + def text_in_repo? + !binary_in_repo? end def external_storage_error? @@ -245,12 +277,20 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def raw_binary? - try_blobs(:raw_binary?) + def empty? + valid_blobs.map(&:empty?).all? + end + + def binary? + strong_memoize(:is_binary) do + try_blobs(:binary?) + end end - def raw_text? - !raw_binary? && !different_type? + def text? + strong_memoize(:is_text) do + !binary? && !different_type? + end end def simple_viewer @@ -333,19 +373,19 @@ module Gitlab return DiffViewer::NotDiffable unless diffable? if content_changed? - if raw_text? + if text? DiffViewer::Text else DiffViewer::NoPreview end elsif new_file? - if raw_text? + if text? DiffViewer::Text else DiffViewer::Added end elsif deleted_file? - if raw_text? + if text? DiffViewer::Text else DiffViewer::Deleted diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index f0c4977fc50..001748afb41 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -73,6 +73,10 @@ module Gitlab !meta? end + def suggestible? + !removed? + end + def rich_text @parent_file.try(:highlight_lines!) if @parent_file && !@rich_text diff --git a/lib/gitlab/discussions_diff/file_collection.rb b/lib/gitlab/discussions_diff/file_collection.rb new file mode 100644 index 00000000000..4ab7314f509 --- /dev/null +++ b/lib/gitlab/discussions_diff/file_collection.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module DiscussionsDiff + class FileCollection + include Gitlab::Utils::StrongMemoize + + def initialize(collection) + @collection = collection + end + + # Returns a Gitlab::Diff::File with the given ID (`unique_identifier` in + # Gitlab::Diff::File). + def find_by_id(id) + diff_files_indexed_by_id[id] + end + + # Writes cache and preloads highlighted diff lines for + # object IDs, in @collection. + # + # highlightable_ids - Diff file `Array` responding to ID. The ID will be used + # to generate the cache key. + # + # - Highlight cache is written just for uncached diff files + # - The cache content is not updated (there's no need to do so) + def load_highlight(highlightable_ids) + preload_highlighted_lines(highlightable_ids) + end + + private + + def preload_highlighted_lines(ids) + cached_content = read_cache(ids) + + uncached_ids = ids.select.each_with_index { |_, i| cached_content[i].nil? } + mapping = highlighted_lines_by_ids(uncached_ids) + + HighlightCache.write_multiple(mapping) + + diffs = diff_files_indexed_by_id.values_at(*ids) + + diffs.zip(cached_content).each do |diff, cached_lines| + next unless diff && cached_lines + + diff.highlighted_diff_lines = cached_lines + end + end + + def read_cache(ids) + HighlightCache.read_multiple(ids) + end + + def diff_files_indexed_by_id + strong_memoize(:diff_files_indexed_by_id) do + diff_files.index_by(&:unique_identifier) + end + end + + def diff_files + strong_memoize(:diff_files) do + @collection.map(&:raw_diff_file) + end + end + + # Processes the diff lines highlighting for diff files matching the given + # IDs. + # + # Returns a Hash with { id => [Array of Gitlab::Diff::line], ...] + def highlighted_lines_by_ids(ids) + diff_files_indexed_by_id.slice(*ids).each_with_object({}) do |(id, file), hash| + hash[id] = file.highlighted_diff_lines.map(&:to_hash) + end + end + end + end +end diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb new file mode 100644 index 00000000000..270cfb89488 --- /dev/null +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true +# +module Gitlab + module DiscussionsDiff + class HighlightCache + class << self + VERSION = 1 + EXPIRATION = 1.week + + # Sets multiple keys to a given value. The value + # is serialized as JSON. + # + # mapping - Write multiple cache values at once + def write_multiple(mapping) + Redis::Cache.with do |redis| + redis.multi do |multi| + mapping.each do |raw_key, value| + key = cache_key_for(raw_key) + + multi.set(key, value.to_json, ex: EXPIRATION) + end + end + end + end + + # Reads multiple cache keys at once. + # + # raw_keys - An Array of unique cache keys, without namespaces. + # + # It returns a list of deserialized diff lines. Ex.: + # [[Gitlab::Diff::Line, ...], [Gitlab::Diff::Line]] + def read_multiple(raw_keys) + return [] if raw_keys.empty? + + keys = raw_keys.map { |id| cache_key_for(id) } + + content = + Redis::Cache.with do |redis| + redis.mget(keys) + end + + content.map! do |lines| + next unless lines + + JSON.parse(lines).map! do |line| + line = line.with_indifferent_access + rich_text = line[:rich_text] + line[:rich_text] = rich_text&.html_safe + + Gitlab::Diff::Line.init_from_hash(line) + end + end + end + + def cache_key_for(raw_key) + "#{cache_key_prefix}:#{raw_key}" + end + + private + + def cache_key_prefix + "#{Redis::Cache::CACHE_NAMESPACE}:#{VERSION}:discussion-highlight" + end + end + end + end +end diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 35bb49ad19a..f89d1d15010 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -6,12 +6,14 @@ module Gitlab class BaseHandler attr_reader :mail, :mail_key + HANDLER_ACTION_BASE_REGEX ||= /(?<project_slug>.+)-(?<project_id>\d+)/.freeze + def initialize(mail, mail_key) @mail = mail @mail_key = mail_key end - def can_execute? + def can_handle? raise NotImplementedError end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 69982efbbe6..78a3a9489ac 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -2,21 +2,33 @@ require 'gitlab/email/handler/base_handler' +# handles issue creation emails with these formats: +# incoming+gitlab-org-gitlab-ce-20-Author_Token12345678-issue@incoming.gitlab.com +# incoming+gitlab-org/gitlab-ce+Author_Token12345678@incoming.gitlab.com (legacy) module Gitlab module Email module Handler class CreateIssueHandler < BaseHandler include ReplyProcessing - attr_reader :project_path, :incoming_email_token + + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue\z/.freeze + HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+(?<incoming_email_token>.*)\z/.freeze def initialize(mail, mail_key) super(mail, mail_key) - @project_path, @incoming_email_token = - mail_key && mail_key.split('+', 2) + + if !mail_key&.include?('/') && (matched = HANDLER_REGEX.match(mail_key.to_s)) + @project_slug = matched[:project_slug] + @project_id = matched[:project_id]&.to_i + @incoming_email_token = matched[:incoming_email_token] + elsif matched = HANDLER_REGEX_LEGACY.match(mail_key.to_s) + @project_path = matched[:project_path] + @incoming_email_token = matched[:incoming_email_token] + end end def can_handle? - !incoming_email_token.nil? && !incoming_email_token.include?("+") && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX) + incoming_email_token && (project_id || can_handle_legacy_format?) end def execute @@ -36,10 +48,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def project - @project ||= Project.find_by_full_path(project_path) - end - private def create_issue @@ -50,6 +58,10 @@ module Gitlab description: message_including_reply ).execute end + + def can_handle_legacy_format? + project_path && !incoming_email_token.include?('+') && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY) + end end end end diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index 5772727e855..b3b5063f2ca 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -3,23 +3,33 @@ require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' +# handles merge request creation emails with these formats: +# incoming+gitlab-org-gitlab-ce-20-Author_Token12345678-merge-request@incoming.gitlab.com +# incoming+gitlab-org/gitlab-ce+merge-request+Author_Token12345678@incoming.gitlab.com (legacy) module Gitlab module Email module Handler class CreateMergeRequestHandler < BaseHandler include ReplyProcessing - attr_reader :project_path, :incoming_email_token + + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-merge-request\z/.freeze + HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+merge-request\+(?<incoming_email_token>.*)/.freeze def initialize(mail, mail_key) super(mail, mail_key) - if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s) - @project_path, @incoming_email_token = m.captures + if !mail_key&.include?('/') && (matched = HANDLER_REGEX.match(mail_key.to_s)) + @project_slug = matched[:project_slug] + @project_id = matched[:project_id]&.to_i + @incoming_email_token = matched[:incoming_email_token] + elsif matched = HANDLER_REGEX_LEGACY.match(mail_key.to_s) + @project_path = matched[:project_path] + @incoming_email_token = matched[:incoming_email_token] end end def can_handle? - @project_path && @incoming_email_token + incoming_email_token && (project_id || project_path) end def execute @@ -40,10 +50,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def project - @project ||= Project.find_by_full_path(project_path) - end - def metrics_params super.merge(includes_patches: patch_attachments.any?) end @@ -97,7 +103,7 @@ module Gitlab def remove_patch_attachments patch_attachments.each { |patch| mail.parts.delete(patch) } - # reset the message, so it needs to be reporocessed when the attachments + # reset the message, so it needs to be reprocessed when the attachments # have been modified @message = nil end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index c7c573595fa..b00af15364d 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -3,6 +3,8 @@ require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' +# handles note/reply creation emails with these formats: +# incoming+1234567890abcdef1234567890abcdef@incoming.gitlab.com module Gitlab module Email module Handler diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index ff6b2c729b2..ba9730d2685 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -6,13 +6,26 @@ module Gitlab module ReplyProcessing private + attr_reader :project_id, :project_slug, :project_path, :incoming_email_token + def author raise NotImplementedError end + # rubocop:disable Gitlab/ModuleWithInstanceVariables def project - raise NotImplementedError + return @project if instance_variable_defined?(:@project) + + if project_id + @project = Project.find_by_id(project_id) + @project = nil unless valid_project_slug?(@project) + else + @project = Project.find_by_full_path(project_path) + end + + @project end + # rubocop:enable Gitlab/ModuleWithInstanceVariables def message @message ||= process_message @@ -58,6 +71,10 @@ module Gitlab raise invalid_exception, msg end + + def valid_project_slug?(found_project) + project_slug == found_project.full_path_slug + end end end end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index d2f617b868a..20e4c125626 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -2,14 +2,28 @@ require 'gitlab/email/handler/base_handler' +# handles unsubscribe emails with these formats: +# incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com +# incoming+1234567890abcdef1234567890abcdef+unsubscribe@incoming.gitlab.com (legacy) module Gitlab module Email module Handler class UnsubscribeHandler < BaseHandler delegate :project, to: :sent_notification, allow_nil: true + HANDLER_REGEX_FOR = -> (suffix) { /\A(?<reply_token>\w+)#{Regexp.escape(suffix)}\z/ }.freeze + HANDLER_REGEX = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX).freeze + HANDLER_REGEX_LEGACY = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY).freeze + + def initialize(mail, mail_key) + super(mail, mail_key) + + matched = HANDLER_REGEX.match(mail_key.to_s) || HANDLER_REGEX_LEGACY.match(mail_key.to_s) + @reply_token = matched[:reply_token] if matched + end + def can_handle? - mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ + reply_token.present? end def execute @@ -24,12 +38,10 @@ module Gitlab private - def sent_notification - @sent_notification ||= SentNotification.for(reply_key) - end + attr_reader :reply_token - def reply_key - mail_key.sub(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX, '') + def sent_notification + @sent_notification ||= SentNotification.for(reply_token) end end end diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb index db1aeeea8d3..bd806269bf0 100644 --- a/lib/gitlab/fake_application_settings.rb +++ b/lib/gitlab/fake_application_settings.rb @@ -37,5 +37,9 @@ module Gitlab def pick_repository_storage repository_storages.sample end + + def commit_email_hostname + super.presence || ApplicationSetting.default_commit_email_hostname + end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index c4aac228b2f..44a62586a23 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -54,11 +54,11 @@ module Gitlab end def tag_ref?(ref) - ref.start_with?(TAG_REF_PREFIX) + ref =~ /^#{TAG_REF_PREFIX}.+/ end def branch_ref?(ref) - ref.start_with?(BRANCH_REF_PREFIX) + ref =~ /^#{BRANCH_REF_PREFIX}.+/ end def blank_ref?(ref) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 2d25389594e..259a2b7911a 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -100,7 +100,7 @@ module Gitlab @loaded_all_data = @loaded_size == size end - def binary? + def binary_in_repo? @binary.nil? ? super : @binary == true end @@ -174,7 +174,7 @@ module Gitlab private def has_lfs_version_key? - !empty? && text? && data.start_with?("version https://git-lfs.github.com/spec") + !empty? && text_in_repo? && data.start_with?("version https://git-lfs.github.com/spec") end end end diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb index 558699a6318..1c6242b444a 100644 --- a/lib/gitlab/git/object_pool.rb +++ b/lib/gitlab/git/object_pool.rb @@ -8,7 +8,7 @@ module Gitlab GL_REPOSITORY = "" delegate :exists?, :size, to: :repository - delegate :delete, to: :object_pool_service + delegate :unlink_repository, :delete, to: :object_pool_service attr_reader :storage, :relative_path, :source_repository @@ -23,13 +23,6 @@ module Gitlab end def link(to_link_repo) - remote_name = to_link_repo.object_pool_remote_name - repository.set_config( - "remote.#{remote_name}.url" => relative_path_to(to_link_repo.relative_path), - "remote.#{remote_name}.tagOpt" => "--no-tags", - "remote.#{remote_name}.fetch" => "+refs/*:refs/remotes/#{remote_name}/*" - ) - object_pool_service.link_repository(to_link_repo) end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 802fa65dd63..010bd0e520c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -12,6 +12,10 @@ module Gitlab TimeoutError = Class.new(StandardError) ProjectMovedError = Class.new(NotFoundError) + # Use the magic string '_any' to indicate we do not know what the + # changes are. This is also what gitlab-shell does. + ANY = '_any' + ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', download: 'You are not allowed to download code from this project.', @@ -24,7 +28,8 @@ module Gitlab upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', read_only: 'The repository is temporarily read-only. Please try again later.', - cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." + cannot_push_to_read_only: "You can't push code to a read-only GitLab instance.", + push_code: 'You are not allowed to push code to this project.' }.freeze INTERNAL_TIMEOUT = 50.seconds.freeze @@ -199,7 +204,7 @@ module Gitlab def ensure_project_on_push!(cmd, changes) return if project || deploy_key? - return unless receive_pack?(cmd) && changes == '_any' && authentication_abilities.include?(:push_code) + return unless receive_pack?(cmd) && changes == ANY && authentication_abilities.include?(:push_code) namespace = Namespace.find_by_full_path(namespace_path) @@ -256,24 +261,34 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:upload] end - return if changes.blank? # Allow access this is needed for EE. - check_change_access! end def check_change_access! - # If there are worktrees with a HEAD pointing to a non-existent object, - # calls to `git rev-list --all` will fail in git 2.15+. This should also - # clear stale lock files. - project.repository.clean_stale_repository_files - - # Iterate over all changes to find if user allowed all of them to be applied - changes_list.each.with_index do |change, index| - first_change = index == 0 - - # If user does not have access to make at least one change, cancel all - # push by allowing the exception to bubble up - check_single_change_access(change, skip_lfs_integrity_check: !first_change) + # Deploy keys with write access can push anything + return if deploy_key? + + if changes == ANY + can_push = user_access.can_do_action?(:push_code) || + project.any_branch_allows_collaboration?(user_access.user) + + unless can_push + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] + end + else + # If there are worktrees with a HEAD pointing to a non-existent object, + # calls to `git rev-list --all` will fail in git 2.15+. This should also + # clear stale lock files. + project.repository.clean_stale_repository_files + + # Iterate over all changes to find if user allowed all of them to be applied + changes_list.each.with_index do |change, index| + first_change = index == 0 + + # If user does not have access to make at least one change, cancel all + # push by allowing the exception to bubble up + check_single_change_access(change, skip_lfs_integrity_check: !first_change) + end end end @@ -282,7 +297,6 @@ module Gitlab change, user_access: user_access, project: project, - skip_authorization: deploy_key?, skip_lfs_integrity_check: skip_lfs_integrity_check, protocol: protocol, logger: logger @@ -348,7 +362,7 @@ module Gitlab protected def changes_list - @changes_list ||= Gitlab::ChangesList.new(changes) + @changes_list ||= Gitlab::ChangesList.new(changes == ANY ? [] : changes) end def user diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 3f24001e4ee..0af91957fa8 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -15,7 +15,7 @@ module Gitlab authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) end - def check_single_change_access(change, _options = {}) + def check_change_access! unless user_access.can_do_action?(:create_wiki) raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index cf2329e489d..426436c2164 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -3,12 +3,13 @@ module Gitlab class GitPostReceive include Gitlab::Identifier - attr_reader :project, :identifier, :changes + attr_reader :project, :identifier, :changes, :push_options - def initialize(project, identifier, changes) + def initialize(project, identifier, changes, push_options) @project = project @identifier = identifier @changes = deserialize_changes(changes) + @push_options = push_options end def identify diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 11021ee06b3..8bf8a3b53cd 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -26,6 +26,7 @@ module Gitlab end end + PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 35 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze @@ -50,11 +51,42 @@ module Gitlab @stubs[storage][name] ||= begin klass = stub_class(name) addr = stub_address(storage) - klass.new(addr, :this_channel_is_insecure) + creds = stub_creds(storage) + klass.new(addr, creds) end end end + def self.stub_cert_paths + cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"] + cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE + cert_paths + end + + def self.stub_certs + return @certs if @certs + + @certs = stub_cert_paths.flat_map do |cert_file| + File.read(cert_file).scan(PEM_REGEX).map do |cert| + begin + OpenSSL::X509::Certificate.new(cert).to_pem + rescue OpenSSL::OpenSSLError => e + Rails.logger.error "Could not load certificate #{cert_file} #{e}" + Gitlab::Sentry.track_exception(e, extra: { cert_file: cert_file }) + nil + end + end.compact + end.uniq.join("\n") + end + + def self.stub_creds(storage) + if URI(address(storage)).scheme == 'tls' + GRPC::Core::ChannelCredentials.new stub_certs + else + :this_channel_is_insecure + end + end + def self.stub_class(name) if name == :health_check Grpc::Health::V1::Health::Stub @@ -64,9 +96,7 @@ module Gitlab end def self.stub_address(storage) - addr = address(storage) - addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' - addr + address(storage).sub(%r{^tcp://|^tls://}, '') end def self.clear_stubs! @@ -88,8 +118,8 @@ module Gitlab raise "storage #{storage.inspect} is missing a gitaly_address" end - unless URI(address).scheme.in?(%w(tcp unix)) - raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'" + unless URI(address).scheme.in?(%w(tcp unix tls)) + raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'" end address diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb index 8e412a9b3ef..3e8d6a773ca 100644 --- a/lib/gitlab/gitaly_client/cleanup_service.rb +++ b/lib/gitlab/gitaly_client/cleanup_service.rb @@ -20,6 +20,7 @@ module Gitlab while data = io.read(RepositoryService::MAX_MSG_SIZE) y.yield Gitaly::ApplyBfgObjectMapRequest.new(object_map: data) + break if io&.eof? end end diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb index 272ce73ad64..6e7ede5fd18 100644 --- a/lib/gitlab/gitaly_client/object_pool_service.rb +++ b/lib/gitlab/gitaly_client/object_pool_service.rb @@ -35,7 +35,10 @@ module Gitlab end def unlink_repository(repository) - request = Gitaly::UnlinkRepositoryFromObjectPoolRequest.new(repository: repository.gitaly_repository) + request = Gitaly::UnlinkRepositoryFromObjectPoolRequest.new( + object_pool: object_pool, + repository: repository.gitaly_repository + ) GitalyClient.call(storage, :object_pool_service, :unlink_repository_from_object_pool, request, timeout: GitalyClient.fast_timeout) diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index e53c2d00743..32f61b1d65c 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -73,13 +73,7 @@ module Gitlab if MUTEX.locked? && MUTEX.owned? optimistic_using_tmp_keychain(&block) else - if Gitlab.rails5? - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - MUTEX.synchronize do - optimistic_using_tmp_keychain(&block) - end - end - else + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do MUTEX.synchronize do optimistic_using_tmp_keychain(&block) end diff --git a/lib/gitlab/graphql.rb b/lib/gitlab/graphql.rb index 74c04e5380e..8a59e83974f 100644 --- a/lib/gitlab/graphql.rb +++ b/lib/gitlab/graphql.rb @@ -3,5 +3,9 @@ module Gitlab module Graphql StandardGraphqlError = Class.new(StandardError) + + def self.enabled? + Feature.enabled?(:graphql, default_enabled: true) + end end end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index c9e2a6a78d9..bdecff0931c 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -3,7 +3,8 @@ module Gitlab module ImportExport module CommandLineUtil - DEFAULT_MODE = 0700 + UNTAR_MASK = 'u+rwX,go+rX,go-w' + DEFAULT_DIR_MODE = 0700 def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') @@ -14,8 +15,8 @@ module Gitlab end def mkdir_p(path) - FileUtils.mkdir_p(path, mode: DEFAULT_MODE) - FileUtils.chmod(DEFAULT_MODE, path) + FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE) + FileUtils.chmod(DEFAULT_DIR_MODE, path) end private @@ -41,6 +42,7 @@ module Gitlab def untar_with_options(archive:, dir:, options:) execute(%W(tar -#{options} #{archive} -C #{dir})) + execute(%W(chmod -R #{UNTAR_MASK} #{dir})) end def execute(cmd) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index d10d4f2f746..25cdd5ab121 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -27,7 +27,8 @@ project_tree: - :award_emoji - notes: :author - - :releases + - releases: + :author - project_members: - :user - merge_requests: @@ -62,7 +63,6 @@ project_tree: - :triggers - :pipeline_schedules - :services - - :hooks - protected_branches: - :merge_access_levels - :push_access_levels @@ -155,12 +155,6 @@ excluded_attributes: - :reference - :reference_html - :epic_id - hooks: - - :token - - :encrypted_token - - :encrypted_token_iv - - :encrypted_url - - :encrypted_url_iv runners: - :token - :token_encrypted diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 20fc8226611..cc0c633b943 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -2,8 +2,9 @@ module Gitlab module IncomingEmail - UNSUBSCRIBE_SUFFIX = '+unsubscribe'.freeze - WILDCARD_PLACEHOLDER = '%{key}'.freeze + UNSUBSCRIBE_SUFFIX = '-unsubscribe'.freeze + UNSUBSCRIBE_SUFFIX_LEGACY = '+unsubscribe'.freeze + WILDCARD_PLACEHOLDER = '%{key}'.freeze class << self def enabled? @@ -22,6 +23,7 @@ module Gitlab config.address.sub(WILDCARD_PLACEHOLDER, key) end + # example: incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com def unsubscribe_address(key) config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") end diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb new file mode 100644 index 00000000000..1adf83739ad --- /dev/null +++ b/lib/gitlab/json_cache.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + class JsonCache + attr_reader :backend, :cache_key_with_version, :namespace + + def initialize(options = {}) + @backend = options.fetch(:backend, Rails.cache) + @namespace = options.fetch(:namespace, nil) + @cache_key_with_version = options.fetch(:cache_key_with_version, true) + end + + def active? + if backend.respond_to?(:active?) + backend.active? + else + true + end + end + + def cache_key(key) + expanded_cache_key = [namespace, key].compact + + if cache_key_with_version + expanded_cache_key << Rails.version + end + + expanded_cache_key.join(':') + end + + def expire(key) + backend.delete(cache_key(key)) + end + + def read(key, klass = nil) + value = backend.read(cache_key(key)) + value = parse_value(value, klass) if value + value + end + + def write(key, value, options = nil) + backend.write(cache_key(key), value.to_json, options) + end + + def fetch(key, options = {}, &block) + klass = options.delete(:as) + value = read(key, klass) + + return value unless value.nil? + + value = yield + + write(key, value, options) + + value + end + + private + + def parse_value(raw, klass) + value = ActiveSupport::JSON.decode(raw) + + case value + when Hash then parse_entry(value, klass) + when Array then parse_entries(value, klass) + else + value + end + rescue ActiveSupport::JSON.parse_error + nil + end + + def parse_entry(raw, klass) + klass.new(raw) if valid_entry?(raw, klass) + end + + def valid_entry?(raw, klass) + return false unless klass && raw.is_a?(Hash) + + (raw.keys - klass.attribute_names).empty? + end + + def parse_entries(values, klass) + values.map { |value| parse_entry(value, klass) }.compact + end + end +end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 05d3096a208..c09d3ebc7be 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -2,10 +2,21 @@ module Gitlab class LfsToken - attr_accessor :actor + module LfsTokenHelper + def user? + actor.is_a?(User) + end + + def actor_name + user? ? actor.username : "lfs+deploy-key-#{actor.id}" + end + end + + include LfsTokenHelper - TOKEN_LENGTH = 50 - EXPIRY_TIME = 1800 + DEFAULT_EXPIRE_TIME = 1800 + + attr_accessor :actor def initialize(actor) @actor = @@ -19,36 +30,108 @@ module Gitlab end end - def token - Gitlab::Redis::SharedState.with do |redis| - token = redis.get(redis_shared_state_key) - token ||= Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME) + def token(expire_time: DEFAULT_EXPIRE_TIME) + HMACToken.new(actor).token(expire_time) + end - token - end + def token_valid?(token_to_check) + HMACToken.new(actor).token_valid?(token_to_check) || + LegacyRedisDeviseToken.new(actor).token_valid?(token_to_check) end def deploy_key_pushable?(project) actor.is_a?(DeployKey) && actor.can_push_to?(project) end - def user? - actor.is_a?(User) - end - def type user? ? :lfs_token : :lfs_deploy_token end - def actor_name - actor.is_a?(User) ? actor.username : "lfs+deploy-key-#{actor.id}" + private # rubocop:disable Lint/UselessAccessModifier + + class HMACToken + include LfsTokenHelper + + def initialize(actor) + @actor = actor + end + + def token(expire_time) + hmac_token = JSONWebToken::HMACToken.new(secret) + hmac_token.expire_time = Time.now + expire_time + hmac_token[:data] = { actor: actor_name } + hmac_token.encoded + end + + def token_valid?(token_to_check) + decoded_token = JSONWebToken::HMACToken.decode(token_to_check, secret).first + decoded_token.dig('data', 'actor') == actor_name + rescue JWT::DecodeError + false + end + + private + + attr_reader :actor + + def secret + salt + key + end + + def salt + case actor + when DeployKey, Key + actor.fingerprint.delete(':').first(16) + when User + # Take the last 16 characters as they're more unique than the first 16 + actor.id.to_s + actor.encrypted_password.last(16) + end + end + + def key + # Take 16 characters of attr_encrypted_db_key_base, as that's what the + # cipher needs exactly + Settings.attr_encrypted_db_key_base.first(16) + end end - private + # TODO: LegacyRedisDeviseToken and references need to be removed after + # next released milestone + # + class LegacyRedisDeviseToken + TOKEN_LENGTH = 50 + DEFAULT_EXPIRY_TIME = 1800 * 1000 # 30 mins + + def initialize(actor) + @actor = actor + end + + def token_valid?(token_to_check) + Devise.secure_compare(stored_token, token_to_check) + end + + def stored_token + Gitlab::Redis::SharedState.with { |redis| redis.get(state_key) } + end + + # This method exists purely to facilitate legacy testing to ensure the + # same redis key is used. + # + def store_new_token(expiry_time_in_ms = DEFAULT_EXPIRY_TIME) + Gitlab::Redis::SharedState.with do |redis| + new_token = Devise.friendly_token(TOKEN_LENGTH) + redis.set(state_key, new_token, px: expiry_time_in_ms) + new_token + end + end + + private - def redis_shared_state_key - "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor + attr_reader :actor + + def state_key + "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" + end end end end diff --git a/lib/gitlab/middleware/correlation_id.rb b/lib/gitlab/middleware/correlation_id.rb index 73542dd422e..80dddc41c12 100644 --- a/lib/gitlab/middleware/correlation_id.rb +++ b/lib/gitlab/middleware/correlation_id.rb @@ -20,11 +20,7 @@ module Gitlab private def correlation_id(env) - if Gitlab.rails5? - request(env).request_id - else - request(env).uuid - end + request(env).request_id end def request(env) diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index d1a87c3b3bb..72a788022ef 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -6,6 +6,7 @@ module Gitlab module Middleware class Go include ActionView::Helpers::TagHelper + include ActionController::HttpAuthentication::Basic PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze @@ -14,7 +15,7 @@ module Gitlab end def call(env) - request = Rack::Request.new(env) + request = ActionDispatch::Request.new(env) render_go_doc(request) || @app.call(env) end @@ -110,21 +111,23 @@ module Gitlab def project_for_paths(paths, request) project = Project.where_full_path_in(paths).first - return unless Ability.allowed?(current_user(request), :read_project, project) + return unless Ability.allowed?(current_user(request, project), :read_project, project) project end - def current_user(request) - authenticator = Gitlab::Auth::RequestAuthenticator.new(request) - user = authenticator.find_user_from_access_token || authenticator.find_user_from_warden + def current_user(request, project) + return unless has_basic_credentials?(request) - return unless user&.can?(:access_api) + login, password = user_name_and_password(request) + auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, ip: request.ip) + return unless auth_result.success? - # Right now, the `api` scope is the only one that should be able to determine private project existence. - return unless authenticator.valid_access_token?(scopes: [:api]) + return unless auth_result.actor&.can?(:access_git) - user + return unless auth_result.authentication_abilities.include?(:read_project) + + auth_result.actor end end end diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 97cbdc6cb39..f2772c733c7 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -1,16 +1,16 @@ # frozen_string_literal: true module Gitlab - # Retrieving of parent or child groups based on a base ActiveRecord relation. + # Retrieving of parent or child objects based on a base ActiveRecord relation. # # This class uses recursive CTEs and as a result will only work on PostgreSQL. - class GroupHierarchy + class ObjectHierarchy attr_reader :ancestors_base, :descendants_base, :model # ancestors_base - An instance of ActiveRecord::Relation for which to - # get parent groups. + # get parent objects. # descendants_base - An instance of ActiveRecord::Relation for which to - # get child groups. If omitted, ancestors_base is used. + # get child objects. If omitted, ancestors_base is used. def initialize(ancestors_base, descendants_base = ancestors_base) raise ArgumentError.new("Model of ancestors_base does not match model of descendants_base") if ancestors_base.model != descendants_base.model @@ -39,7 +39,7 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # Returns a relation that includes the ancestors_base set of groups + # Returns a relation that includes the ancestors_base set of objects # and all their ancestors (recursively). # # Passing an `upto` will stop the recursion once the specified parent_id is @@ -47,13 +47,13 @@ module Gitlab # included. # # Passing a `hierarchy_order` with either `:asc` or `:desc` will cause the - # recursive query order from most nested group to root or from the root - # ancestor to most nested group respectively. This uses a `depth` column + # recursive query order from most nested object to root or from the root + # ancestor to most nested object respectively. This uses a `depth` column # where `1` is defined as the depth for the base and increment as we go up # each parent. # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - return ancestors_base unless Group.supports_nested_groups? + return ancestors_base unless hierarchy_supported? recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order @@ -62,16 +62,16 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # Returns a relation that includes the descendants_base set of groups + # Returns a relation that includes the descendants_base set of objects # and all their descendants (recursively). def base_and_descendants - return descendants_base unless Group.supports_nested_groups? + return descendants_base unless hierarchy_supported? read_only(base_and_descendants_cte.apply_to(model.all)) end - # Returns a relation that includes the base groups, their ancestors, - # and the descendants of the base groups. + # Returns a relation that includes the base objects, their ancestors, + # and the descendants of the base objects. # # The resulting query will roughly look like the following: # @@ -91,16 +91,16 @@ module Gitlab # Using this approach allows us to further add criteria to the relation with # Rails thinking it's selecting data the usual way. # - # If nested groups are not supported, ancestors_base is returned. + # If nested objects are not supported, ancestors_base is returned. # rubocop: disable CodeReuse/ActiveRecord - def all_groups - return ancestors_base unless Group.supports_nested_groups? + def all_objects + return ancestors_base unless hierarchy_supported? ancestors = base_and_ancestors_cte descendants = base_and_descendants_cte - ancestors_table = ancestors.alias_to(groups_table) - descendants_table = descendants.alias_to(groups_table) + ancestors_table = ancestors.alias_to(objects_table) + descendants_table = descendants.alias_to(objects_table) relation = model .unscoped @@ -117,23 +117,27 @@ module Gitlab private + def hierarchy_supported? + Gitlab::Database.postgresql? + end + # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) depth_column = :depth base_query = ancestors_base.except(:order) - base_query = base_query.select("1 as #{depth_column}", groups_table[Arel.star]) if hierarchy_order + base_query = base_query.select("1 as #{depth_column}", objects_table[Arel.star]) if hierarchy_order cte << base_query # Recursively get all the ancestors of the base set. parent_query = model - .from([groups_table, cte.table]) - .where(groups_table[:id].eq(cte.table[:parent_id])) + .from([objects_table, cte.table]) + .where(objects_table[:id].eq(cte.table[:parent_id])) .except(:order) - parent_query = parent_query.select(cte.table[depth_column] + 1, groups_table[Arel.star]) if hierarchy_order + parent_query = parent_query.select(cte.table[depth_column] + 1, objects_table[Arel.star]) if hierarchy_order parent_query = parent_query.where(cte.table[:parent_id].not_eq(stop_id)) if stop_id cte << parent_query @@ -149,15 +153,15 @@ module Gitlab # Recursively get all the descendants of the base set. cte << model - .from([groups_table, cte.table]) - .where(groups_table[:parent_id].eq(cte.table[:id])) + .from([objects_table, cte.table]) + .where(objects_table[:parent_id].eq(cte.table[:id])) .except(:order) cte end # rubocop: enable CodeReuse/ActiveRecord - def groups_table + def objects_table model.arel_table end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 8f30cdee232..394556e8708 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -10,9 +10,15 @@ module Gitlab validates :name, :priority, :metrics, presence: true def self.common_metrics - ::PrometheusMetric.common.group_by(&:group_title).map do |name, metrics| - MetricGroup.new(name: name, priority: 0, metrics: metrics.map(&:to_query_metric)) + all_groups = ::PrometheusMetric.common.group_by(&:group_title).map do |name, metrics| + MetricGroup.new( + name: name, + priority: metrics.map(&:priority).max, + metrics: metrics.map(&:to_query_metric) + ) end + + all_groups.sort_by(&:priority).reverse end # EE only diff --git a/lib/gitlab/safe_request_store.rb b/lib/gitlab/safe_request_store.rb index 4e82353adb6..d146913bdb3 100644 --- a/lib/gitlab/safe_request_store.rb +++ b/lib/gitlab/safe_request_store.rb @@ -19,5 +19,13 @@ module Gitlab NULL_STORE end end + + # This method accept an options hash to be compatible with + # ActiveSupport::Cache::Store#write method. The options are + # not passed to the underlying cache implementation because + # RequestStore#write accepts only a key, and value params. + def self.write(key, value, options = nil) + store.write(key, value) + end end end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb deleted file mode 100644 index ccab0e4dd73..00000000000 --- a/lib/gitlab/upgrader.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class Upgrader - def execute - puts "GitLab #{current_version.major} upgrade tool" - puts "Your version is #{current_version}" - puts "Latest available version for GitLab #{current_version.major} is #{latest_version}" - - if latest_version? - puts "You are using the latest GitLab version" - else - puts "Newer GitLab version is available" - - answer = if ARGV.first == "-y" - "yes" - else - prompt("Do you want to upgrade (yes/no)? ", %w{yes no}) - end - - if answer == "yes" - upgrade - else - exit 0 - end - end - end - - def latest_version? - current_version >= latest_version - end - - def current_version - @current_version ||= Gitlab::VersionInfo.parse(current_version_raw) - end - - def latest_version - @latest_version ||= Gitlab::VersionInfo.parse(latest_version_raw) - end - - def current_version_raw - File.read(File.join(gitlab_path, "VERSION")).strip - end - - def latest_version_raw - git_tags = fetch_git_tags - git_tags = git_tags.select { |version| version =~ /v\d+\.\d+\.\d+\Z/ } - git_versions = git_tags.map { |tag| Gitlab::VersionInfo.parse(tag.match(/v\d+\.\d+\.\d+/).to_s) } - "v#{git_versions.sort.last}" - end - - def fetch_git_tags - remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) - remote_tags.split("\n").grep(%r{tags/v#{current_version.major}}) - end - - def update_commands - { - "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash), - "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch), - "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), - "Install gems" => %w(bundle), - "Migrate DB" => %w(bundle exec rake db:migrate), - "Recompile assets" => %w(bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile), - "Clear cache" => %w(bundle exec rake cache:clear) - } - end - - def env - { - 'RAILS_ENV' => 'production', - 'NODE_ENV' => 'production' - } - end - - def upgrade - update_commands.each do |title, cmd| - puts title - puts " -> #{cmd.join(' ')}" - - if system(env, *cmd) - puts " -> OK" - else - puts " -> FAILED" - puts "Failed to upgrade. Try to repeat task or proceed with upgrade manually " - exit 1 - end - end - - puts "Done" - end - - def gitlab_path - File.expand_path(File.join(File.dirname(__FILE__), '../..')) - end - - # Prompt the user to input something - # - # message - the message to display before input - # choices - array of strings of acceptable answers or nil for any answer - # - # Returns the user's answer - def prompt(message, choices = nil) - begin - print(message) - answer = STDIN.gets.chomp - end while !choices.include?(answer) - answer - end - end -end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 008e9cd1d24..083c620267a 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -85,6 +85,8 @@ module Gitlab releases: count(Release), remote_mirrors: count(RemoteMirror), snippets: count(Snippet), + suggestions: count(Suggestion), + todos: count(Todo), uploads: count(Upload), web_hooks: count(WebHook) }.merge(services_usage).merge(approximate_counts) diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index c412961ea3f..c87e97d0213 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -4,16 +4,11 @@ module Gitlab module Utils module Override class Extension - def self.verify_class!(klass, method_name) - instance_method_defined?(klass, method_name) || - raise( - NotImplementedError.new( - "#{klass}\##{method_name} doesn't exist!")) - end - - def self.instance_method_defined?(klass, name, include_super: true) - klass.instance_methods(include_super).include?(name) || - klass.private_instance_methods(include_super).include?(name) + def self.verify_class!(klass, method_name, arity) + extension = new(klass) + parents = extension.parents_for(klass) + extension.verify_method!( + klass: klass, parents: parents, method_name: method_name, sub_method_arity: arity) end attr_reader :subject @@ -22,35 +17,77 @@ module Gitlab @subject = subject end - def add_method_name(method_name) - method_names << method_name - end - - def add_class(klass) - classes << klass + def parents_for(klass) + index = klass.ancestors.index(subject) + klass.ancestors.drop(index + 1) end def verify! classes.each do |klass| - index = klass.ancestors.index(subject) - parents = klass.ancestors.drop(index + 1) - - method_names.each do |method_name| - parents.any? do |parent| - self.class.instance_method_defined?( - parent, method_name, include_super: false) - end || - raise( - NotImplementedError.new( - "#{klass}\##{method_name} doesn't exist!")) + parents = parents_for(klass) + + method_names.each_pair do |method_name, arity| + verify_method!( + klass: klass, + parents: parents, + method_name: method_name, + sub_method_arity: arity) end end end + def verify_method!(klass:, parents:, method_name:, sub_method_arity:) + overridden_parent = parents.find do |parent| + instance_method_defined?(parent, method_name) + end + + raise NotImplementedError.new("#{klass}\##{method_name} doesn't exist!") unless overridden_parent + + super_method_arity = find_direct_method(overridden_parent, method_name).arity + + unless arity_compatible?(sub_method_arity, super_method_arity) + raise NotImplementedError.new("#{subject}\##{method_name} has arity of #{sub_method_arity}, but #{overridden_parent}\##{method_name} has arity of #{super_method_arity}") + end + end + + def add_method_name(method_name, arity = nil) + method_names[method_name] = arity + end + + def add_class(klass) + classes << klass + end + + def verify_override?(method_name) + method_names.has_key?(method_name) + end + private + def instance_method_defined?(klass, name) + klass.instance_methods(false).include?(name) || + klass.private_instance_methods(false).include?(name) + end + + def find_direct_method(klass, name) + method = klass.instance_method(name) + method = method.super_method until method && klass == method.owner + method + end + + def arity_compatible?(sub_method_arity, super_method_arity) + if sub_method_arity >= 0 && super_method_arity >= 0 + # Regular arguments + sub_method_arity == super_method_arity + else + # It's too complex to check this case, just allow sub-method having negative arity + # But we don't allow sub_method_arity > 0 yet super_method_arity < 0 + sub_method_arity < 0 + end + end + def method_names - @method_names ||= [] + @method_names ||= {} end def classes @@ -80,11 +117,21 @@ module Gitlab def override(method_name) return unless ENV['STATIC_VERIFICATION'] + Override.extensions[self] ||= Extension.new(self) + Override.extensions[self].add_method_name(method_name) + end + + def method_added(method_name) + super + + return unless ENV['STATIC_VERIFICATION'] + return unless Override.extensions[self]&.verify_override?(method_name) + + method_arity = instance_method(method_name).arity if is_a?(Class) - Extension.verify_class!(self, method_name) + Extension.verify_class!(self, method_name, method_arity) else # We delay the check for modules - Override.extensions[self] ||= Extension.new(self) - Override.extensions[self].add_method_name(method_name) + Override.extensions[self].add_method_name(method_name, method_arity) end end diff --git a/lib/json_web_token/hmac_token.rb b/lib/json_web_token/hmac_token.rb index ceb1b9c913f..ec0917ab49d 100644 --- a/lib/json_web_token/hmac_token.rb +++ b/lib/json_web_token/hmac_token.rb @@ -18,7 +18,7 @@ module JSONWebToken end def encoded - JWT.encode(payload, secret, JWT_ALGORITHM) + JWT.encode(payload, secret, JWT_ALGORITHM, { typ: 'JWT' }) end private diff --git a/lib/json_web_token/rsa_token.rb b/lib/json_web_token/rsa_token.rb index 160e1e506f1..bcce811cd28 100644 --- a/lib/json_web_token/rsa_token.rb +++ b/lib/json_web_token/rsa_token.rb @@ -11,7 +11,8 @@ module JSONWebToken def encoded headers = { - kid: kid + kid: kid, + typ: 'JWT' } JWT.encode(payload, key, 'RS256', headers) end diff --git a/lib/mysql_zero_date.rb b/lib/mysql_zero_date.rb index 216560148fa..f36610abf8f 100644 --- a/lib/mysql_zero_date.rb +++ b/lib/mysql_zero_date.rb @@ -17,4 +17,4 @@ module MysqlZeroDate end end -ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate) if Gitlab.rails5? +ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate) diff --git a/lib/rails4_migration_version.rb b/lib/rails4_migration_version.rb deleted file mode 100644 index ae48734dfad..00000000000 --- a/lib/rails4_migration_version.rb +++ /dev/null @@ -1,16 +0,0 @@ -# rubocop:disable Naming/FileName -# frozen_string_literal: true - -# When switching to rails 5, we added migration version to all migration -# classes. This patch makes it possible to run versioned migrations -# also with rails 4 - -unless Gitlab.rails5? - module ActiveRecord - class Migration - def self.[](version) - Migration - end - end - end -end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index f539b1df955..09dc3aa9882 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -2,6 +2,12 @@ namespace :gitlab do namespace :storage do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' task migrate_to_hashed: :environment do + if Gitlab::Database.read_only? + warn 'This task requires database write access. Exiting.' + + next + end + storage_migrator = Gitlab::HashedStorage::Migrator.new helper = Gitlab::HashedStorage::RakeHelper @@ -9,7 +15,7 @@ namespace :gitlab do project = Project.with_unmigrated_storage.find_by(id: helper.range_from) unless project - puts "There are no projects requiring storage migration with ID=#{helper.range_from}" + warn "There are no projects requiring storage migration with ID=#{helper.range_from}" next end @@ -23,7 +29,7 @@ namespace :gitlab do legacy_projects_count = Project.with_unmigrated_storage.count if legacy_projects_count == 0 - puts 'There are no projects requiring storage migration. Nothing to do!' + warn 'There are no projects requiring storage migration. Nothing to do!' next end diff --git a/lib/version_check.rb b/lib/version_check.rb index ccf7bb493db..c9f102f6b19 100644 --- a/lib/version_check.rb +++ b/lib/version_check.rb @@ -5,16 +5,17 @@ require "base64" # This class is used to build image URL to # check if it is a new version for update class VersionCheck - def data + def self.data { version: Gitlab::VERSION } end - def url + def self.url encoded_data = Base64.urlsafe_encode64(data.to_json) + "#{host}?gitlab_info=#{encoded_data}" end - def host + def self.host 'https://version.gitlab.com/check.svg' end end |