diff options
| author | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 21:28:43 +0100 |
|---|---|---|
| committer | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 21:28:43 +0100 |
| commit | a2f375e8f74870dcdcfa1c7886bd1c14c80a684e (patch) | |
| tree | 6b6e3a4f7554f4671edc17d87869dd6916984404 /lib | |
| parent | a22f6fa6e50bb31921415b01fd345d6802581390 (diff) | |
| parent | 81852d1f902c2923c239e9c33cab77f5fd6ca8d8 (diff) | |
| download | gitlab-ce-a2f375e8f74870dcdcfa1c7886bd1c14c80a684e.tar.gz | |
Merge remote-tracking branch 'origin/master' into object-storage-ee-to-ce-backportobject-storage-ee-to-ce-backport
Diffstat (limited to 'lib')
90 files changed, 1256 insertions, 446 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index e953f3d2eca..754549f72f0 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -138,6 +138,7 @@ module API mount ::API::PagesDomains mount ::API::Pipelines mount ::API::PipelineSchedules + mount ::API::ProjectImport mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectMilestones diff --git a/lib/api/commits.rb b/lib/api/commits.rb index d8fd6a6eb06..3d6e78d2d80 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -97,13 +97,16 @@ module API end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + use :pagination end get ':id/repository/commits/:sha/diff', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - present commit.raw_diffs.to_a, with: Entities::Diff + raw_diffs = ::Kaminari.paginate_array(commit.raw_diffs.to_a) + + present paginate(raw_diffs), with: Entities::Diff end desc "Get a commit's comments" do @@ -156,6 +159,27 @@ module API end end + desc 'Get all references a commit is pushed to' do + detail 'This feature was introduced in GitLab 10.6' + success Entities::BasicRef + end + params do + requires :sha, type: String, desc: 'A commit sha' + optional :type, type: String, values: %w[branch tag all], default: 'all', desc: 'Scope' + use :pagination + end + get ':id/repository/commits/:sha/refs', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + commit = user_project.commit(params[:sha]) + not_found!('Commit') unless commit + + refs = [] + refs.concat(user_project.repository.branch_names_contains(commit.id).map {|name| { type: 'branch', name: name }}) unless params[:type] == 'tag' + refs.concat(user_project.repository.tag_names_contains(commit.id).map {|name| { type: 'tag', name: name }}) unless params[:type] == 'branch' + refs = Kaminari.paginate_array(refs) + + present paginate(refs), with: Entities::BasicRef + end + desc 'Post comment to commit' do success Entities::CommitNote end @@ -165,7 +189,7 @@ module API optional :path, type: String, desc: 'The file path' given :path do requires :line, type: Integer, desc: 'The line number' - requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' + requires :line_type, type: String, values: %w[new old], default: 'new', desc: 'The type of the line' end end post ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 7838de13c56..167878ba600 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -22,6 +22,7 @@ module API end expose :avatar_path, if: ->(user, options) { options.fetch(:only_path, false) && user.avatar_path } + expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes expose :web_url do |user, options| Gitlab::Routing.url_helpers.user_url(user) @@ -90,6 +91,13 @@ module API expose :created_at end + class ProjectImportStatus < ProjectIdentity + expose :import_status + + # TODO: Use `expose_nil` once we upgrade the grape-entity gem + expose :import_error, if: lambda { |status, _ops| status.import_error } + end + class BasicProjectDetails < ProjectIdentity include ::API::ProjectsRelationBuilder @@ -109,6 +117,8 @@ module API expose :star_count, :forks_count expose :last_activity_at + expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes + def self.preload_relation(projects_relation, options = {}) projects_relation.preload(:project_feature, :route) .preload(namespace: [:route, :owner], @@ -230,6 +240,8 @@ module API expose :parent_id end + expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes + expose :statistics, if: :statistics do with_options format_with: -> (value) { value.to_i } do expose :storage_size @@ -274,6 +286,11 @@ module API expose :stats, using: Entities::CommitStats, if: :stats expose :status expose :last_pipeline, using: 'API::Entities::PipelineBasic' + expose :project_id + end + + class BasicRef < Grape::Entity + expose :type, :name end class Branch < Grape::Entity @@ -1137,6 +1154,10 @@ module API expose :domain expose :url expose :project_id + expose :verified?, as: :verified + expose :verification_code, as: :verification_code + expose :enabled_until + expose :certificate, as: :certificate_expiration, if: ->(pages_domain, _) { pages_domain.certificate? }, @@ -1148,6 +1169,10 @@ module API class PagesDomain < Grape::Entity expose :domain expose :url + expose :verified?, as: :verified + expose :verification_code, as: :verification_code + expose :enabled_until + expose :certificate, if: ->(pages_domain, _) { pages_domain.certificate? }, using: PagesDomainCertificate do |pages_domain| @@ -1172,6 +1197,7 @@ module API expose :id expose :ref expose :startline + expose :project_id end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index b81f07a1770..4a4df1b8b9e 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -1,6 +1,7 @@ module API class Groups < Grape::API include PaginationParams + include Helpers::CustomAttributes before { authenticate_non_get! } @@ -67,6 +68,8 @@ module API } groups = groups.with_statistics if options[:statistics] + groups, options = with_custom_attributes(groups, options) + present paginate(groups), options end end @@ -79,6 +82,7 @@ module API end params do use :group_list_params + use :with_custom_attributes end get do groups = find_groups(params) @@ -142,9 +146,20 @@ module API desc 'Get a single group, with containing projects.' do success Entities::GroupDetail end + params do + use :with_custom_attributes + end get ":id" do group = find_group!(params[:id]) - present group, with: Entities::GroupDetail, current_user: current_user + + options = { + with: Entities::GroupDetail, + current_user: current_user + } + + group, options = with_custom_attributes(group, options) + + present group, options end desc 'Remove a group.' @@ -175,12 +190,19 @@ module API optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' use :pagination + use :with_custom_attributes end get ":id/projects" do projects = find_group_projects(params) - entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project - present entity.prepare_relation(projects), with: entity, current_user: current_user + options = { + with: params[:simple] ? Entities::BasicProjectDetails : Entities::Project, + current_user: current_user + } + + projects, options = with_custom_attributes(projects, options) + + present options[:with].prepare_relation(projects), options end desc 'Get a list of subgroups in this group.' do @@ -188,6 +210,7 @@ module API end params do use :group_list_params + use :with_custom_attributes end get ":id/subgroups" do groups = find_groups(params) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index e75d8f1e6eb..de9058ce71f 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -172,7 +172,7 @@ module API def find_project_snippet(id) finder_params = { project: user_project } - SnippetsFinder.new(current_user, finder_params).execute.find(id) + SnippetsFinder.new(current_user, finder_params).find(id) end def find_merge_request_with_access(iid, access_level = :read_merge_request) diff --git a/lib/api/helpers/custom_attributes.rb b/lib/api/helpers/custom_attributes.rb new file mode 100644 index 00000000000..70e4eda95f8 --- /dev/null +++ b/lib/api/helpers/custom_attributes.rb @@ -0,0 +1,28 @@ +module API + module Helpers + module CustomAttributes + extend ActiveSupport::Concern + + included do + helpers do + params :with_custom_attributes do + optional :with_custom_attributes, type: Boolean, default: false, desc: 'Include custom attributes in the response' + end + + def with_custom_attributes(collection_or_resource, options = {}) + options = options.merge( + with_custom_attributes: params[:with_custom_attributes] && + can?(current_user, :read_custom_attribute) + ) + + if options[:with_custom_attributes] && collection_or_resource.is_a?(ActiveRecord::Relation) + collection_or_resource = collection_or_resource.includes(:custom_attributes) + end + + [collection_or_resource, options] + end + end + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 9285fb90cdc..b3660e4a1d0 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -13,7 +13,7 @@ module API # key_id - ssh key id for Git over SSH # user_id - user id for Git over HTTP # protocol - Git access protocol being used, e.g. HTTP or SSH - # project - project path with namespace + # project - project full_path (not path on disk) # action - git action (git-upload-pack or git-receive-pack) # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList post "/allowed" do diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index d7b613a717e..ba33993d852 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -2,6 +2,8 @@ module API class PagesDomains < Grape::API include PaginationParams + PAGES_DOMAINS_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(domain: API::NO_SLASH_URL_PART_REGEX) + before do authenticate! end @@ -48,7 +50,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do before do require_pages_enabled! end @@ -71,7 +73,7 @@ module API params do requires :domain, type: String, desc: 'The domain' end - get ":id/pages/domains/:domain", requirements: { domain: %r{[^/]+} } do + get ":id/pages/domains/:domain", requirements: PAGES_DOMAINS_ENDPOINT_REQUIREMENTS do authorize! :read_pages, user_project present pages_domain, with: Entities::PagesDomain @@ -105,7 +107,7 @@ module API optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate' optional :key, allow_blank: false, types: [File, String], desc: 'The key' end - put ":id/pages/domains/:domain", requirements: { domain: %r{[^/]+} } do + put ":id/pages/domains/:domain", requirements: PAGES_DOMAINS_ENDPOINT_REQUIREMENTS do authorize! :update_pages, user_project pages_domain_params = declared(params, include_parent_namespaces: false) @@ -126,7 +128,7 @@ module API params do requires :domain, type: String, desc: 'The domain' end - delete ":id/pages/domains/:domain", requirements: { domain: %r{[^/]+} } do + delete ":id/pages/domains/:domain", requirements: PAGES_DOMAINS_ENDPOINT_REQUIREMENTS do authorize! :update_pages, user_project status 204 diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb new file mode 100644 index 00000000000..a509c1f32c1 --- /dev/null +++ b/lib/api/project_import.rb @@ -0,0 +1,69 @@ +module API + class ProjectImport < Grape::API + include PaginationParams + + helpers do + def import_params + declared_params(include_missing: false) + end + + def file_is_valid? + import_params[:file] && import_params[:file]['tempfile'].respond_to?(:read) + end + + def validate_file! + render_api_error!('The file is invalid', 400) unless file_is_valid? + end + end + + before do + forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project') + end + + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + params do + requires :path, type: String, desc: 'The new project path and name' + requires :file, type: File, desc: 'The project export file to be imported' + optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace." + end + desc 'Create a new project import' do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::ProjectImportStatus + end + post 'import' do + validate_file! + + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42437') + + namespace = if import_params[:namespace] + find_namespace!(import_params[:namespace]) + else + current_user.namespace + end + + project_params = { + path: import_params[:path], + namespace_id: namespace.id, + file: import_params[:file]['tempfile'] + } + + project = ::Projects::GitlabProjectsImportService.new(current_user, project_params).execute + + render_api_error!(project.errors.full_messages&.first, 400) unless project.saved? + + present project, with: Entities::ProjectImportStatus + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + desc 'Get a project export status' do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::ProjectImportStatus + end + get ':id/import' do + present user_project, with: Entities::ProjectImportStatus + end + end + end +end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 5b481121a10..b552b0e0c5d 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -3,6 +3,7 @@ require_dependency 'declarative_policy' module API class Projects < Grape::API include PaginationParams + include Helpers::CustomAttributes before { authenticate_non_get! } @@ -80,6 +81,7 @@ module API projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] projects = projects.with_statistics if params[:statistics] projects = paginate(projects) + projects, options = with_custom_attributes(projects, options) if current_user project_members = current_user.project_members.preload(:source, user: [notification_settings: :source]) @@ -107,6 +109,7 @@ module API requires :user_id, type: String, desc: 'The ID or username of the user' use :collection_params use :statistics_params + use :with_custom_attributes end get ":user_id/projects" do user = find_user(params[:user_id]) @@ -127,6 +130,7 @@ module API params do use :collection_params use :statistics_params + use :with_custom_attributes end get do present_projects load_projects @@ -196,11 +200,19 @@ module API end params do use :statistics_params + use :with_custom_attributes end get ":id" do - entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present user_project, with: entity, current_user: current_user, - user_can_admin_project: can?(current_user, :admin_project, user_project), statistics: params[:statistics] + options = { + with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, + current_user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project), + statistics: params[:statistics] + } + + project, options = with_custom_attributes(user_project, options) + + present project, options end desc 'Fork new project for the current user or provided namespace.' do @@ -242,6 +254,7 @@ module API end params do use :collection_params + use :with_custom_attributes end get ':id/forks' do forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute @@ -258,6 +271,7 @@ module API [ :jobs_enabled, :resolve_outdated_diff_discussions, + :ci_config_path, :container_registry_enabled, :default_branch, :description, diff --git a/lib/api/search.rb b/lib/api/search.rb index 9f08fd96a3b..3556ad98c52 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -11,7 +11,7 @@ module API projects: Entities::BasicProjectDetails, milestones: Entities::Milestone, notes: Entities::Note, - commits: Entities::Commit, + commits: Entities::CommitDetail, blobs: Entities::Blob, wiki_blobs: Entities::Blob, snippet_titles: Entities::Snippet, @@ -35,7 +35,7 @@ module API def process_results(results) case params[:scope] when 'wiki_blobs' - paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob) } + paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob, user_project) } when 'blobs' paginate(results).map { |blob| blob[1] } else @@ -85,9 +85,7 @@ module API use :pagination end get ':id/-/search' do - find_group!(params[:id]) - - present search(group_id: params[:id]), with: entity + present search(group_id: user_group.id), with: entity end end @@ -106,9 +104,7 @@ module API use :pagination end get ':id/-/search' do - find_project!(params[:id]) - - present search(project_id: params[:id]), with: entity + present search(project_id: user_project.id), with: entity end end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index cee4d309816..152df23a327 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -147,7 +147,7 @@ module API attrs[:password_authentication_enabled_for_web] = attrs.delete(:password_authentication_enabled) end - if current_settings.update_attributes(attrs) + if ApplicationSettings::UpdateService.new(current_settings, current_user, attrs).execute present current_settings, with: Entities::ApplicationSetting else render_validation_error!(current_settings) diff --git a/lib/api/todos.rb b/lib/api/todos.rb index ffccfebe752..c6dbcf84e3a 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -60,7 +60,7 @@ module API end post ':id/mark_as_done' do TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user) - todo = Todo.find(params[:id]) + todo = current_user.todos.find(params[:id]) present todo, with: Entities::Todo, current_user: current_user end diff --git a/lib/api/users.rb b/lib/api/users.rb index 3cc12724b8a..3920171205f 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -2,6 +2,7 @@ module API class Users < Grape::API include PaginationParams include APIGuard + include Helpers::CustomAttributes allow_access_with_scope :read_user, if: -> (request) { request.get? } @@ -70,6 +71,7 @@ module API use :sort_params use :pagination + use :with_custom_attributes end get do authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) @@ -94,8 +96,9 @@ module API entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic users = users.preload(:identities, :u2f_registrations) if entity == Entities::UserWithAdmin + users, options = with_custom_attributes(users, with: entity) - present paginate(users), with: entity + present paginate(users), options end desc 'Get a single user' do @@ -103,12 +106,16 @@ module API end params do requires :id, type: Integer, desc: 'The ID of the user' + + use :with_custom_attributes end get ":id" do user = User.find_by(id: params[:id]) not_found!('User') unless user && can?(current_user, :read_user, user) opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : { with: Entities::User } + user, opts = with_custom_attributes(user, opts) + present user, opts end diff --git a/lib/api/v3/todos.rb b/lib/api/v3/todos.rb index 2f2cf259987..3e2c61f6dbd 100644 --- a/lib/api/v3/todos.rb +++ b/lib/api/v3/todos.rb @@ -12,7 +12,7 @@ module API end delete ':id' do TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user) - todo = Todo.find(params[:id]) + todo = current_user.todos.find(params[:id]) present todo, with: ::API::Entities::Todo, current_user: current_user end diff --git a/lib/banzai/filter/html_entity_filter.rb b/lib/banzai/filter/html_entity_filter.rb index f3bd587c28b..e008fd428b0 100644 --- a/lib/banzai/filter/html_entity_filter.rb +++ b/lib/banzai/filter/html_entity_filter.rb @@ -5,7 +5,7 @@ module Banzai # Text filter that escapes these HTML entities: & " < > class HtmlEntityFilter < HTML::Pipeline::TextFilter def call - ERB::Util.html_escape_once(text) + ERB::Util.html_escape(text) end end end diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index 327ea9449a1..77299abe324 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -15,6 +15,8 @@ module Banzai issuables = extractor.extract([doc]) issuables.each do |node, issuable| + next if !can_read_cross_project? && issuable.project != project + if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project) node.content += " (#{issuable.state})" end @@ -25,6 +27,10 @@ module Banzai private + def can_read_cross_project? + Ability.allowed?(current_user, :read_cross_project) + end + def current_user context[:current_user] end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 2a6b0964ac5..8ec696ce5fc 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -64,7 +64,7 @@ module Banzai finder_params[:group_ids] = [project.group.id] end - MilestonesFinder.new(finder_params).execute.find_by(params) + MilestonesFinder.new(finder_params).find_by(params) end def url_for_object(milestone, project) diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index a79a0154846..0ac7e231b5b 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,23 +14,33 @@ module Banzai end def highlight_node(node) - code = node.text css_classes = 'code highlight js-syntax-highlight' - language = node.attr('lang') + lang = node.attr('lang') + retried = false - if use_rouge?(language) - lexer = lexer_for(language) + if use_rouge?(lang) + lexer = lexer_for(lang) language = lexer.tag + else + lexer = Rouge::Lexers::PlainText.new + language = lang + end + + begin + code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, node.text), tag: language) + css_classes << " #{language}" if language + rescue + # Gracefully handle syntax highlighter bugs/errors to ensure users can + # still access an issue/comment/etc. First, retry with the plain text + # filter. If that fails, then just skip this entirely, but that would + # be a pretty bad upstream bug. + return if retried - begin - code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code), tag: language) - css_classes << " #{language}" - rescue - # Gracefully handle syntax highlighter bugs/errors to ensure - # users can still access an issue/comment/etc. + language = nil + lexer = Rouge::Lexers::PlainText.new + retried = true - language = nil - end + retry end highlighted = %(<pre class="#{css_classes}" lang="#{language}" v-pre="true"><code>#{code}</code></pre>) diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index de3ebe72720..827df7c08ae 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -19,8 +19,9 @@ module Banzai # # Returns the documents passed as the first argument. def redact(documents) - all_document_nodes = document_nodes(documents) + redact_cross_project_references(documents) unless can_read_cross_project? + all_document_nodes = document_nodes(documents) redact_document_nodes(all_document_nodes) end @@ -51,6 +52,18 @@ module Banzai metadata end + def redact_cross_project_references(documents) + extractor = Banzai::IssuableExtractor.new(project, user) + issuables = extractor.extract(documents) + + issuables.each do |node, issuable| + next if issuable.project == project + + node['class'] = node['class'].gsub('has-tooltip', '') + node['title'] = nil + end + end + # Returns the nodes visible to the current user. # # nodes - The input nodes to check. @@ -78,5 +91,11 @@ module Banzai { document: document, nodes: Querying.css(document, 'a.gfm[data-reference-type]') } end end + + private + + def can_read_cross_project? + Ability.allowed?(user, :read_cross_project) + end end end diff --git a/lib/banzai/reference_parser/issuable_parser.rb b/lib/banzai/reference_parser/issuable_parser.rb index 3953867eb83..fad127d7e5b 100644 --- a/lib/banzai/reference_parser/issuable_parser.rb +++ b/lib/banzai/reference_parser/issuable_parser.rb @@ -18,7 +18,7 @@ module Banzai end def can_read_reference?(user, issuable) - can?(user, "read_#{issuable.class.to_s.underscore}".to_sym, issuable) + can?(user, "read_#{issuable.class.to_s.underscore}_iid".to_sym, issuable) end end end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 38d4e3f3e44..230827129b6 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -5,12 +5,31 @@ module Banzai def nodes_visible_to_user(user, nodes) issues = records_for_nodes(nodes) + issues_to_check = issues.values - readable_issues = Ability - .issues_readable_by_user(issues.values, user).to_set + unless can?(user, :read_cross_project) + issues_to_check, cross_project_issues = issues_to_check.partition do |issue| + issue.project == project + end + end + + readable_issues = Ability.issues_readable_by_user(issues_to_check, user).to_set nodes.select do |node| - readable_issues.include?(issues[node]) + issue_in_node = issues[node] + + # We check the inclusion of readable issues first because it's faster. + # + # But we need to fall back to `read_issue_iid` if the user cannot read + # cross project, since it might be possible the user can see the IID + # but not the issue. + if readable_issues.include?(issue_in_node) + true + elsif cross_project_issues&.include?(issue_in_node) + can_read_reference?(user, issue_in_node) + else + false + end end end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index ee7f4be6b9f..62c41801d75 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -8,7 +8,8 @@ module Gitlab module Asciidoc DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', - 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' + 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font', + 'outfilesuffix=.adoc' ].freeze # Public: Converts the provided Asciidoc markup into HTML. diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index 46ec040ce92..a0b5cd868c3 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -20,6 +20,14 @@ module Gitlab rescue Gitlab::Auth::AuthenticationError nil end + + def valid_access_token?(scopes: []) + validate_access_token!(scopes: scopes) + + true + rescue Gitlab::Auth::AuthenticationError + false + end end end end diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index 03b17b319fa..1b4a9e8a194 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -14,6 +14,14 @@ module Gitlab def perform(start_id, end_id) log("Creating memberships for forks: #{start_id} - #{end_id}") + insert_members(start_id, end_id) + + if missing_members?(start_id, end_id) + BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + end + + def insert_members(start_id, end_id) ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id) @@ -33,10 +41,9 @@ module Gitlab WHERE existing_members.project_id = forked_project_links.forked_to_project_id ) INSERT_MEMBERS - - if missing_members?(start_id, end_id) - BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) - end + rescue ActiveRecord::RecordNotUnique => e + # `fork_network_member` was created concurrently in another migration + log(e.message) end def missing_members?(start_id, end_id) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 8a8e770940e..9232f20a063 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -5,157 +5,10 @@ module Gitlab # This class processes a batch of rows in `untracked_files_for_uploads` by # adding each file to the `uploads` table if it does not exist. class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength - # This class is responsible for producing the attributes necessary to - # track an uploaded file in the `uploads` table. - class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength - self.table_name = 'untracked_files_for_uploads' - - # Ends with /:random_hex/:filename - FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} - FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/ - - # These regex patterns are tested against a relative path, relative to - # the upload directory. - # For convenience, if there exists a capture group in the pattern, then - # it indicates the model_id. - PATH_PATTERNS = [ - { - pattern: %r{\A-/system/appearance/logo/(\d+)/}, - uploader: 'AttachmentUploader', - model_type: 'Appearance' - }, - { - pattern: %r{\A-/system/appearance/header_logo/(\d+)/}, - uploader: 'AttachmentUploader', - model_type: 'Appearance' - }, - { - pattern: %r{\A-/system/note/attachment/(\d+)/}, - uploader: 'AttachmentUploader', - model_type: 'Note' - }, - { - pattern: %r{\A-/system/user/avatar/(\d+)/}, - uploader: 'AvatarUploader', - model_type: 'User' - }, - { - pattern: %r{\A-/system/group/avatar/(\d+)/}, - uploader: 'AvatarUploader', - model_type: 'Namespace' - }, - { - pattern: %r{\A-/system/project/avatar/(\d+)/}, - uploader: 'AvatarUploader', - model_type: 'Project' - }, - { - pattern: FILE_UPLOADER_PATH, - uploader: 'FileUploader', - model_type: 'Project' - } - ].freeze - - def to_h - @upload_hash ||= { - path: upload_path, - uploader: uploader, - model_type: model_type, - model_id: model_id, - size: file_size, - checksum: checksum - } - end - - def upload_path - # UntrackedFile#path is absolute, but Upload#path depends on uploader - @upload_path ||= - if uploader == 'FileUploader' - # Path relative to project directory in uploads - matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH) - matchd[0].sub(%r{\A/}, '') # remove leading slash - else - path - end - end - - def uploader - matching_pattern_map[:uploader] - end - - def model_type - matching_pattern_map[:model_type] - end - - def model_id - return @model_id if defined?(@model_id) - - pattern = matching_pattern_map[:pattern] - matchd = path_relative_to_upload_dir.match(pattern) - - # If something is captured (matchd[1] is not nil), it is a model_id - # Only the FileUploader pattern will not match an ID - @model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id - end - - def file_size - File.size(absolute_path) - end - - def checksum - Digest::SHA256.file(absolute_path).hexdigest - end - - private - - def matching_pattern_map - @matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map| - path_relative_to_upload_dir.match(path_pattern_map[:pattern]) - end - - unless @matching_pattern_map - raise "Unknown upload path pattern \"#{path}\"" - end - - @matching_pattern_map - end - - def file_uploader_model_id - matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE) - not_found_msg = <<~MSG - Could not capture project full_path from a FileUploader path: - "#{path_relative_to_upload_dir}" - MSG - raise not_found_msg unless matchd - - full_path = matchd[1] - project = Project.find_by_full_path(full_path) - return nil unless project - - project.id - end - - # Not including a leading slash - def path_relative_to_upload_dir - upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength - base = %r{\A#{Regexp.escape(upload_dir)}/} - @path_relative_to_upload_dir ||= path.sub(base, '') - end - - def absolute_path - File.join(Gitlab.config.uploads.storage_path, path) - end - end - - # This class is used to query the `uploads` table. - class Upload < ActiveRecord::Base - self.table_name = 'uploads' - end - def perform(start_id, end_id) return unless migrate? - files = UntrackedFile.where(id: start_id..end_id) + files = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.where(id: start_id..end_id) processed_files = insert_uploads_if_needed(files) processed_files.delete_all @@ -165,7 +18,8 @@ module Gitlab private def migrate? - UntrackedFile.table_exists? && Upload.table_exists? + Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.table_exists? && + Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.table_exists? end def insert_uploads_if_needed(files) @@ -197,7 +51,7 @@ module Gitlab def filter_existing_uploads(files) paths = files.map(&:upload_path) - existing_paths = Upload.where(path: paths).pluck(:path).to_set + existing_paths = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.where(path: paths).pluck(:path).to_set files.reject do |file| existing_paths.include?(file.upload_path) @@ -229,7 +83,7 @@ module Gitlab end ids.each do |model_type, model_ids| - model_class = Object.const_get(model_type) + model_class = "Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::#{model_type}".constantize found_ids = model_class.where(id: model_ids.uniq).pluck(:id) deleted_ids = ids[model_type] - found_ids ids[model_type] = deleted_ids @@ -249,8 +103,8 @@ module Gitlab end def drop_temp_table_if_finished - if UntrackedFile.all.empty? - UntrackedFile.connection.drop_table(:untracked_files_for_uploads, + if Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.all.empty? && !Rails.env.test? # Dropping a table intermittently breaks test cleanup + Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.connection.drop_table(:untracked_files_for_uploads, if_exists: true) end end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb new file mode 100644 index 00000000000..a2c5acbde71 --- /dev/null +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -0,0 +1,201 @@ +# frozen_string_literal: true +module Gitlab + module BackgroundMigration + module PopulateUntrackedUploadsDependencies + # This class is responsible for producing the attributes necessary to + # track an uploaded file in the `uploads` table. + class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength + self.table_name = 'untracked_files_for_uploads' + + # Ends with /:random_hex/:filename + FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} + FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/ + + # These regex patterns are tested against a relative path, relative to + # the upload directory. + # For convenience, if there exists a capture group in the pattern, then + # it indicates the model_id. + PATH_PATTERNS = [ + { + pattern: %r{\A-/system/appearance/logo/(\d+)/}, + uploader: 'AttachmentUploader', + model_type: 'Appearance' + }, + { + pattern: %r{\A-/system/appearance/header_logo/(\d+)/}, + uploader: 'AttachmentUploader', + model_type: 'Appearance' + }, + { + pattern: %r{\A-/system/note/attachment/(\d+)/}, + uploader: 'AttachmentUploader', + model_type: 'Note' + }, + { + pattern: %r{\A-/system/user/avatar/(\d+)/}, + uploader: 'AvatarUploader', + model_type: 'User' + }, + { + pattern: %r{\A-/system/group/avatar/(\d+)/}, + uploader: 'AvatarUploader', + model_type: 'Namespace' + }, + { + pattern: %r{\A-/system/project/avatar/(\d+)/}, + uploader: 'AvatarUploader', + model_type: 'Project' + }, + { + pattern: FILE_UPLOADER_PATH, + uploader: 'FileUploader', + model_type: 'Project' + } + ].freeze + + def to_h + @upload_hash ||= { + path: upload_path, + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size, + checksum: checksum + } + end + + def upload_path + # UntrackedFile#path is absolute, but Upload#path depends on uploader + @upload_path ||= + if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH) + matchd[0].sub(%r{\A/}, '') # remove leading slash + else + path + end + end + + def uploader + matching_pattern_map[:uploader] + end + + def model_type + matching_pattern_map[:model_type] + end + + def model_id + return @model_id if defined?(@model_id) + + pattern = matching_pattern_map[:pattern] + matchd = path_relative_to_upload_dir.match(pattern) + + # If something is captured (matchd[1] is not nil), it is a model_id + # Only the FileUploader pattern will not match an ID + @model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id + end + + def file_size + File.size(absolute_path) + end + + def checksum + Digest::SHA256.file(absolute_path).hexdigest + end + + private + + def matching_pattern_map + @matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map| + path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + end + + unless @matching_pattern_map + raise "Unknown upload path pattern \"#{path}\"" + end + + @matching_pattern_map + end + + def file_uploader_model_id + matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE) + not_found_msg = <<~MSG + Could not capture project full_path from a FileUploader path: + "#{path_relative_to_upload_dir}" + MSG + raise not_found_msg unless matchd + + full_path = matchd[1] + project = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Project.find_by_full_path(full_path) + return nil unless project + + project.id + end + + # Not including a leading slash + def path_relative_to_upload_dir + upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength + base = %r{\A#{Regexp.escape(upload_dir)}/} + @path_relative_to_upload_dir ||= path.sub(base, '') + end + + def absolute_path + File.join(Gitlab.config.uploads.storage_path, path) + end + end + + # Avoid using application code + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + end + + # Avoid using application code + class Appearance < ActiveRecord::Base + self.table_name = 'appearances' + end + + # Avoid using application code + class Namespace < ActiveRecord::Base + self.table_name = 'namespaces' + end + + # Avoid using application code + class Note < ActiveRecord::Base + self.table_name = 'notes' + end + + # Avoid using application code + class User < ActiveRecord::Base + self.table_name = 'users' + end + + # Since project Markdown upload paths don't contain the project ID, we have to find the + # project by its full_path. Due to MySQL/PostgreSQL differences, and historical reasons, + # the logic is somewhat complex, so I've mostly copied it in here. + class Project < ActiveRecord::Base + self.table_name = 'projects' + + def self.find_by_full_path(path) + binary = Gitlab::Database.mysql? ? 'BINARY' : '' + order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" + where_full_path_in(path).reorder(order_sql).take + end + + def self.where_full_path_in(path) + cast_lower = Gitlab::Database.postgresql? + + path = connection.quote(path) + + where = + if cast_lower + "(LOWER(routes.path) = LOWER(#{path}))" + else + "(routes.path = #{path})" + end + + joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index a7a1bbe1752..914a9e48a2f 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -43,7 +43,11 @@ module Gitlab store_untracked_file_paths - schedule_populate_untracked_uploads_jobs + if UntrackedFile.all.empty? + drop_temp_table + else + schedule_populate_untracked_uploads_jobs + end end private @@ -92,7 +96,7 @@ module Gitlab end end - yield(paths) + yield(paths) if paths.any? end def build_find_command(search_dir) @@ -165,6 +169,13 @@ module Gitlab bulk_queue_background_migration_jobs_by_range( UntrackedFile, FOLLOW_UP_MIGRATION) end + + def drop_temp_table + unless Rails.env.test? # Dropping a table intermittently breaks test cleanup + UntrackedFile.connection.drop_table(:untracked_files_for_uploads, + if_exists: true) + end + end end end end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index d75e73dac10..3ce5f807989 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -16,11 +16,11 @@ module Gitlab lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' }.freeze - attr_reader :user_access, :project, :skip_authorization, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name + attr_reader :user_access, :project, :skip_authorization, :skip_lfs_integrity_check, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name def initialize( change, user_access:, project:, skip_authorization: false, - protocol: + skip_lfs_integrity_check: false, protocol: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @@ -28,6 +28,7 @@ module Gitlab @user_access = user_access @project = project @skip_authorization = skip_authorization + @skip_lfs_integrity_check = skip_lfs_integrity_check @protocol = protocol end @@ -37,7 +38,7 @@ module Gitlab push_checks branch_checks tag_checks - lfs_objects_exist_check + lfs_objects_exist_check unless skip_lfs_integrity_check commits_check unless skip_commits_check true @@ -120,6 +121,7 @@ module Gitlab def commits_check return if deletion? || newrev.nil? + return unless should_run_commit_validations? # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 ::Gitlab::GitalyClient.allow_n_plus_1_calls do @@ -138,6 +140,10 @@ module Gitlab private + def should_run_commit_validations? + commit_check.validate_lfs_file_locks? + end + def updated_from_web? protocol == 'web' end @@ -175,7 +181,7 @@ module Gitlab end def commits - project.repository.new_commits(newrev) + @commits ||= project.repository.new_commits(newrev) end end end diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb index ae0cd142378..43a52b493bb 100644 --- a/lib/gitlab/checks/commit_check.rb +++ b/lib/gitlab/checks/commit_check.rb @@ -35,14 +35,14 @@ module Gitlab end end - private - def validate_lfs_file_locks? strong_memoize(:validate_lfs_file_locks) do project.lfs_enabled? && project.lfs_file_locks.any? && newrev && oldrev end end + private + def lfs_file_locks_validation lambda do |paths| lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 0735243e021..9576d5a3fd8 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -34,6 +34,8 @@ module Gitlab end def events_by_date(date) + return Event.none unless can_read_cross_project? + events = Event.contributions.where(author_id: contributor.id) .where(created_at: date.beginning_of_day..date.end_of_day) .where(project_id: projects) @@ -53,6 +55,10 @@ module Gitlab private + def can_read_cross_project? + Ability.allowed?(current_user, :read_cross_project) + end + def event_counts(date_from, feature) t = Event.arel_table diff --git a/lib/gitlab/cross_project_access.rb b/lib/gitlab/cross_project_access.rb new file mode 100644 index 00000000000..6eaed51b64c --- /dev/null +++ b/lib/gitlab/cross_project_access.rb @@ -0,0 +1,67 @@ +module Gitlab + class CrossProjectAccess + class << self + delegate :add_check, :find_check, :checks, + to: :instance + end + + def self.instance + @instance ||= new + end + + attr_reader :checks + + def initialize + @checks = {} + end + + def add_check( + klass, + actions: {}, + positive_condition: nil, + negative_condition: nil, + skip: false) + + new_check = CheckInfo.new(actions, + positive_condition, + negative_condition, + skip + ) + + @checks[klass] ||= Gitlab::CrossProjectAccess::CheckCollection.new + @checks[klass].add_check(new_check) + recalculate_checks_for_class(klass) + + @checks[klass] + end + + def find_check(object) + @cached_checks ||= Hash.new do |cache, new_class| + parent_classes = @checks.keys.select { |existing_class| new_class <= existing_class } + closest_class = closest_parent(parent_classes, new_class) + cache[new_class] = @checks[closest_class] + end + + @cached_checks[object.class] + end + + private + + def recalculate_checks_for_class(klass) + new_collection = @checks[klass] + + @checks.each do |existing_class, existing_check_collection| + if existing_class < klass + existing_check_collection.add_collection(new_collection) + elsif klass < existing_class + new_collection.add_collection(existing_check_collection) + end + end + end + + def closest_parent(classes, subject) + relevant_ancestors = subject.ancestors & classes + relevant_ancestors.first + end + end +end diff --git a/lib/gitlab/cross_project_access/check_collection.rb b/lib/gitlab/cross_project_access/check_collection.rb new file mode 100644 index 00000000000..88376232065 --- /dev/null +++ b/lib/gitlab/cross_project_access/check_collection.rb @@ -0,0 +1,47 @@ +module Gitlab + class CrossProjectAccess + class CheckCollection + attr_reader :checks + + def initialize + @checks = [] + end + + def add_collection(collection) + @checks |= collection.checks + end + + def add_check(check) + @checks << check + end + + def should_run?(object) + skips, runs = arranged_checks + + # If one rule tells us to skip, we skip the cross project check + return false if skips.any? { |check| check.should_skip?(object) } + + # If the rule isn't skipped, we run it if any of the checks says we + # should run + runs.any? { |check| check.should_run?(object) } + end + + def arranged_checks + return [@skips, @runs] if @skips && @runs + + @skips = [] + @runs = [] + + @checks.each do |check| + if check.skip + @skips << check + else + @runs << check + end + end + + [@skips, @runs] + end + end + end +end diff --git a/lib/gitlab/cross_project_access/check_info.rb b/lib/gitlab/cross_project_access/check_info.rb new file mode 100644 index 00000000000..e8a845c7f1e --- /dev/null +++ b/lib/gitlab/cross_project_access/check_info.rb @@ -0,0 +1,66 @@ +module Gitlab + class CrossProjectAccess + class CheckInfo + attr_accessor :actions, :positive_condition, :negative_condition, :skip + + def initialize(actions, positive_condition, negative_condition, skip) + @actions = actions + @positive_condition = positive_condition + @negative_condition = negative_condition + @skip = skip + end + + def should_skip?(object) + return !should_run?(object) unless @skip + + skip_for_action = @actions[current_action(object)] + skip_for_action = false if @actions[current_action(object)].nil? + + # We need to do the opposite of what was defined in the following cases: + # - skip_cross_project_access_check index: true, if: -> { false } + # - skip_cross_project_access_check index: true, unless: -> { true } + if positive_condition_is_false?(object) + skip_for_action = !skip_for_action + end + + if negative_condition_is_true?(object) + skip_for_action = !skip_for_action + end + + skip_for_action + end + + def should_run?(object) + return !should_skip?(object) if @skip + + run_for_action = @actions[current_action(object)] + run_for_action = true if @actions[current_action(object)].nil? + + # We need to do the opposite of what was defined in the following cases: + # - requires_cross_project_access index: true, if: -> { false } + # - requires_cross_project_access index: true, unless: -> { true } + if positive_condition_is_false?(object) + run_for_action = !run_for_action + end + + if negative_condition_is_true?(object) + run_for_action = !run_for_action + end + + run_for_action + end + + def positive_condition_is_false?(object) + @positive_condition && !object.instance_exec(&@positive_condition) + end + + def negative_condition_is_true?(object) + @negative_condition && object.instance_exec(&@negative_condition) + end + + def current_action(object) + object.respond_to?(:action_name) ? object.action_name.to_sym : nil + end + end + end +end diff --git a/lib/gitlab/cross_project_access/class_methods.rb b/lib/gitlab/cross_project_access/class_methods.rb new file mode 100644 index 00000000000..90eac94800c --- /dev/null +++ b/lib/gitlab/cross_project_access/class_methods.rb @@ -0,0 +1,48 @@ +module Gitlab + class CrossProjectAccess + module ClassMethods + def requires_cross_project_access(*args) + positive_condition, negative_condition, actions = extract_params(args) + + Gitlab::CrossProjectAccess.add_check( + self, + actions: actions, + positive_condition: positive_condition, + negative_condition: negative_condition + ) + end + + def skip_cross_project_access_check(*args) + positive_condition, negative_condition, actions = extract_params(args) + + Gitlab::CrossProjectAccess.add_check( + self, + actions: actions, + positive_condition: positive_condition, + negative_condition: negative_condition, + skip: true + ) + end + + private + + def extract_params(args) + actions = {} + positive_condition = nil + negative_condition = nil + + args.each do |argument| + if argument.is_a?(Hash) + positive_condition = argument.delete(:if) + negative_condition = argument.delete(:unless) + actions.merge!(argument) + else + actions[argument] = true + end + end + + [positive_condition, negative_condition, actions] + end + end + end +end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 0f897e6316c..269016daac2 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -27,7 +27,17 @@ module Gitlab rich_line = highlight_line(diff_line) || diff_line.text if line_inline_diffs = inline_diffs[i] - rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) + begin + rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) + # This should only happen when the encoding of the diff doesn't + # match the blob, which is a bug. But we shouldn't fail to render + # completely in that case, even though we want to report the error. + rescue RangeError => e + if Gitlab::Sentry.enabled? + Gitlab::Sentry.context + Raven.capture_exception(e) + end + end end diff_line.text = rich_line diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index c0edcabc6fd..6659efa0961 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -28,9 +28,9 @@ module Gitlab # encode and clean the bad chars message.replace clean(message) - rescue ArgumentError - return nil - rescue + rescue ArgumentError => e + return unless e.message.include?('unknown encoding name') + encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" end diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 10ffc345bd5..8c082c0c336 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -28,7 +28,7 @@ module Gitlab def find_by_content(query) results = repository.search_files_by_content(query, ref).first(BATCH_SIZE) - results.map { |result| Gitlab::ProjectSearchResults.parse_search_result(result) } + results.map { |result| Gitlab::ProjectSearchResults.parse_search_result(result, project) } end def find_by_filename(query, except: []) @@ -45,7 +45,8 @@ module Gitlab basename: File.basename(blob.path), ref: ref, startline: 1, - data: blob.data + data: blob.data, + project: project ) end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index d95561fe1b2..ae27a138b7c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -508,7 +508,7 @@ module Gitlab @committed_date = Time.at(commit.committer.date.seconds).utc @committer_name = commit.committer.name.dup @committer_email = commit.committer.email.dup - @parent_ids = commit.parent_ids + @parent_ids = Array(commit.parent_ids) end def serialize_keys diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 6761fb0937a..ddb9cf433eb 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1234,7 +1234,13 @@ module Gitlab end def squash_in_progress?(squash_id) - fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) + gitaly_migrate(:squash_in_progress) do |is_enabled| + if is_enabled + gitaly_repository_client.squash_in_progress?(squash_id) + else + fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) + end + end end def push_remote_branches(remote_name, branch_names, forced: true) @@ -1349,7 +1355,7 @@ module Gitlab if is_enabled gitaly_ref_client.branch_names_contains_sha(sha) else - refs_contains_sha(:branch, sha) + refs_contains_sha('refs/heads/', sha) end end end @@ -1359,7 +1365,7 @@ module Gitlab if is_enabled gitaly_ref_client.tag_names_contains_sha(sha) else - refs_contains_sha(:tag, sha) + refs_contains_sha('refs/tags/', sha) end end end @@ -1458,19 +1464,25 @@ module Gitlab end end - def refs_contains_sha(ref_type, sha) - args = %W(#{ref_type} --contains #{sha}) - names = run_git(args).first + def refs_contains_sha(refs_prefix, sha) + refs_prefix << "/" unless refs_prefix.ends_with?('/') - return [] unless names.respond_to?(:split) + # By forcing the output to %(refname) each line wiht a ref will start with + # the ref prefix. All other lines can be discarded. + args = %W(for-each-ref --contains=#{sha} --format=%(refname) #{refs_prefix}) + names, code = run_git(args) - names = names.split("\n").map(&:strip) + return [] unless code.zero? - names.each do |name| - name.slice! '* ' + refs = [] + left_slice_count = refs_prefix.length + names.lines.each do |line| + next unless line.start_with?(refs_prefix) + + refs << line.rstrip[left_slice_count..-1] end - names + refs end def rugged_write_config(full_path:) @@ -1614,17 +1626,14 @@ module Gitlab # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. def branches_filter(filter: nil, sort_by: nil) - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464 - branches = Gitlab::GitalyClient.allow_n_plus_1_calls do - rugged.branches.each(filter).map do |rugged_ref| - begin - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end.compact - end + branches = rugged.branches.each(filter).map do |rugged_ref| + begin + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact sort_branches(branches, sort_by) end @@ -2191,13 +2200,14 @@ module Gitlab ) diff_range = "#{start_sha}...#{end_sha}" diff_files = run_git!( - %W(diff --name-only --diff-filter=a --binary #{diff_range}) + %W(diff --name-only --diff-filter=ar --binary #{diff_range}) ).chomp with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do # Apply diff of the `diff_range` to the worktree diff = run_git!(%W(diff --binary #{diff_range})) - run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| + run_git!(%w(apply --index --whitespace=nowarn), chdir: squash_path, env: env) do |stdin| + stdin.binmode stdin.write(diff) end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index ba6058fd3c9..b6ceb542dd1 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -14,14 +14,14 @@ module Gitlab # Uses rugged for raw objects # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 - def where(repository, sha, path = nil) + def where(repository, sha, path = nil, recursive = false) path = nil if path == '' || path == '/' Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled| if is_enabled - repository.gitaly_commit_client.tree_entries(repository, sha, path) + repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) else - tree_entries_from_rugged(repository, sha, path) + tree_entries_from_rugged(repository, sha, path, recursive) end end end @@ -57,7 +57,22 @@ module Gitlab end end - def tree_entries_from_rugged(repository, sha, path) + def tree_entries_from_rugged(repository, sha, path, recursive) + current_path_entries = get_tree_entries_from_rugged(repository, sha, path) + ordered_entries = [] + + current_path_entries.each do |entry| + ordered_entries << entry + + if recursive && entry.dir? + ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) + end + end + + ordered_entries + end + + def get_tree_entries_from_rugged(repository, sha, path) commit = repository.lookup(sha) root_tree = commit.tree diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 8ec3386184a..6400089a22f 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -198,8 +198,8 @@ module Gitlab end def check_repository_existence! - unless project.repository.exists? - raise UnauthorizedError, ERROR_MESSAGES[:no_repo] + unless repository.exists? + raise NotFoundError, ERROR_MESSAGES[:no_repo] end end @@ -238,19 +238,22 @@ module Gitlab changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied - changes_list.each do |change| + changes_list.each.with_index do |change, index| + first_change = index == 0 + # If user does not have access to make at least one change, cancel all # push by allowing the exception to bubble up - check_single_change_access(change) + check_single_change_access(change, skip_lfs_integrity_check: !first_change) end end - def check_single_change_access(change) + def check_single_change_access(change, skip_lfs_integrity_check: false) Checks::ChangeAccess.new( change, user_access: user_access, project: project, skip_authorization: deploy_key?, + skip_lfs_integrity_check: skip_lfs_integrity_check, protocol: protocol ).exec end @@ -324,5 +327,9 @@ module Gitlab def push_to_read_only_message ERROR_MESSAGES[:cannot_push_to_read_only] end + + def repository + project.repository + end end end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 1c9477e84b2..a5b3902ebf4 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -13,7 +13,7 @@ module Gitlab authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) end - def check_single_change_access(change) + def check_single_change_access(change, _options = {}) unless user_access.can_do_action?(:create_wiki) raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end @@ -28,5 +28,11 @@ module Gitlab def push_to_read_only_message ERROR_MESSAGES[:read_only] end + + private + + def repository + project.wiki.repository + end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c5d3e944f7d..9cd76630484 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -125,6 +125,8 @@ module Gitlab kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend + rescue GRPC::Unavailable => ex + handle_grpc_unavailable!(ex) ensure duration = Gitlab::Metrics::System.monotonic_time - start @@ -135,6 +137,27 @@ module Gitlab duration) end + def self.handle_grpc_unavailable!(ex) + status = ex.to_status + raise ex unless status.details == 'Endpoint read failed' + + # There is a bug in grpc 1.8.x that causes a client process to get stuck + # always raising '14:Endpoint read failed'. The only thing that we can + # do to recover is to restart the process. + # + # See https://gitlab.com/gitlab-org/gitaly/issues/1029 + + if Sidekiq.server? + raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s) + else + # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request. + Process.kill('QUIT', Process.pid) + end + + raise ex + end + private_class_method :handle_grpc_unavailable! + def self.current_transaction_labels Gitlab::Metrics::Transaction.current&.labels || {} end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 269a048cf5d..d60f57717b5 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -105,11 +105,12 @@ module Gitlab entry unless entry.oid.blank? end - def tree_entries(repository, revision, path) + def tree_entries(repository, revision, path, recursive) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, revision: encode_binary(revision), - path: path.present? ? encode_binary(path) : '.' + path: path.present? ? encode_binary(path) : '.', + recursive: recursive ) response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 60706b4f0d8..603457d0664 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -134,6 +134,23 @@ module Gitlab response.in_progress end + def squash_in_progress?(squash_id) + request = Gitaly::IsSquashInProgressRequest.new( + repository: @gitaly_repo, + squash_id: squash_id.to_s + ) + + response = GitalyClient.call( + @storage, + :repository_service, + :is_squash_in_progress, + request, + timeout: GitalyClient.default_timeout + ) + + response.in_progress + end + def fetch_source_branch(source_repository, source_branch, local_ref) request = Gitaly::FetchSourceBranchRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 7dd68a0d1cd..ab0b751fe24 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -63,6 +63,7 @@ module Gitlab true rescue Gitlab::Shell::Error => e if e.message !~ /repository not exported/ + project.create_wiki fail_import("Failed to import the wiki: #{e.message}") else true diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 86a90d57d9c..ba04387022d 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -13,8 +13,6 @@ module Gitlab gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class - gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') - gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 672b5579dfd..90dd569aaf8 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -60,7 +60,9 @@ module Gitlab def create_cached_signature! using_keychain do |gpg_key| - GpgSignature.create!(attributes(gpg_key)) + signature = GpgSignature.new(attributes(gpg_key)) + signature.save! unless Gitlab::Database.read_only? + signature end end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index c14646b0611..a00795f553e 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -50,9 +50,10 @@ module Gitlab end def wiki_restorer - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + Gitlab::ImportExport::WikiRestorer.new(path_to_bundle: wiki_repo_path, shared: @shared, - project: ProjectWiki.new(project_tree.restored_project)) + project: ProjectWiki.new(project_tree.restored_project), + wiki_enabled: @project.wiki_enabled?) end def uploads_restorer diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb deleted file mode 100644 index 77bb3ca6581..00000000000 --- a/lib/gitlab/import_export/project_creator.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module ImportExport - class ProjectCreator - def initialize(namespace_id, current_user, file, project_path) - @namespace_id = namespace_id - @current_user = current_user - @file = file - @project_path = project_path - end - - def execute - ::Projects::CreateService.new( - @current_user, - name: @project_path, - path: @project_path, - namespace_id: @namespace_id, - import_type: "gitlab_project", - import_source: @file - ).execute - end - end - end -end diff --git a/lib/gitlab/import_export/wiki_restorer.rb b/lib/gitlab/import_export/wiki_restorer.rb new file mode 100644 index 00000000000..f33bfb332ab --- /dev/null +++ b/lib/gitlab/import_export/wiki_restorer.rb @@ -0,0 +1,23 @@ +module Gitlab + module ImportExport + class WikiRestorer < RepoRestorer + def initialize(project:, shared:, path_to_bundle:, wiki_enabled:) + super(project: project, shared: shared, path_to_bundle: path_to_bundle) + + @wiki_enabled = wiki_enabled + end + + def restore + @project.wiki if create_empty_wiki? + + super + end + + private + + def create_empty_wiki? + !File.exist?(@path_to_bundle) && @wiki_enabled + end + end + end +end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index f654508c391..f7a8eae0be4 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -15,16 +15,22 @@ module Gitlab # push to that array when done. Once the waiter has popped `count` items, it # knows all the jobs are done. class JobWaiter + KEY_PREFIX = "gitlab:job_waiter".freeze + def self.notify(key, jid) Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } end + def self.key?(key) + key.is_a?(String) && key =~ /\A#{KEY_PREFIX}:\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/ + end + attr_reader :key, :finished attr_accessor :jobs_remaining # jobs_remaining - the number of jobs left to wait for # key - The key of this waiter. - def initialize(jobs_remaining = 0, key = "gitlab:job_waiter:#{SecureRandom.uuid}") + def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}") @key = key @jobs_remaining = jobs_remaining @finished = [] diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 47b3fce3e7a..a6bea98d631 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -15,7 +15,7 @@ module Gitlab end def self.servers - Gitlab.config.ldap.servers.values + Gitlab.config.ldap['servers']&.values || [] end def self.available_servers diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index b91757c2a4b..c59df556247 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -63,8 +63,6 @@ module Gitlab Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @provider = provider - - validate_entry end def name @@ -117,19 +115,6 @@ module Gitlab entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend end - - def validate_entry - allowed_attrs = self.class.ldap_attributes(config).map(&:downcase) - - # Net::LDAP::Entry transforms keys to symbols. Change to strings to compare. - entry_attrs = entry.attribute_names.map { |n| n.to_s.downcase } - invalid_attrs = entry_attrs - allowed_attrs - - if invalid_attrs.any? - raise InvalidEntryError, - "#{self.class.name} initialized with Net::LDAP::Entry containing invalid attributes(s): #{invalid_attrs}" - end - end end end end diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb index 5980a4ded2b..db8bdde74b2 100644 --- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -23,7 +23,7 @@ module Gitlab end def stop_working - server.shutdown + server.shutdown if server @server = nil end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 45b9e14ba55..f3e48083c19 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -73,7 +73,7 @@ module Gitlab # event_name - The name of the event (e.g. "git_push"). # tags - A set of tags to attach to the event. def add_event(event_name, tags = {}) - self.class.transaction_metric(event_name, :counter, prefix: 'event_', tags: tags).increment(tags.merge(labels)) + self.class.transaction_metric(event_name, :counter, prefix: 'event_', use_feature_flag: true, tags: tags).increment(tags.merge(labels)) @metrics << Metric.new(EVENT_SERIES, { count: 1 }, tags.merge(event: event_name), :event) end @@ -150,11 +150,12 @@ module Gitlab with_feature :prometheus_metrics_transaction_allocated_memory end - def self.transaction_metric(name, type, prefix: nil, tags: {}) + def self.transaction_metric(name, type, prefix: nil, use_feature_flag: false, tags: {}) metric_name = "gitlab_transaction_#{prefix}#{name}_total".to_sym fetch_metric(type, metric_name) do docstring "Transaction #{prefix}#{name} #{type}" base_labels tags.merge(BASE_LABELS) + with_feature "prometheus_transaction_#{prefix}#{name}_total".to_sym if use_feature_flag if type == :gauge multiprocess_mode :livesum diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 1a570f480c6..1fd8f147b44 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -114,7 +114,15 @@ module Gitlab end def current_user(request) - request.env['warden']&.authenticate + authenticator = Gitlab::Auth::RequestAuthenticator.new(request) + user = authenticator.find_user_from_access_token || authenticator.find_user_from_warden + + return unless user&.can?(:access_api) + + # Right now, the `api` scope is the only one that should be able to determine private project existence. + return unless authenticator.valid_access_token?(scopes: [:api]) + + user end end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index cc1e92480be..d4c54049b74 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -42,7 +42,7 @@ module Gitlab key, value = parsed_field.first if value.nil? - value = open_file(tmp_path) + value = open_file(tmp_path, @request.params["#{key}.name"]) @open_files << value else value = decorate_params_value(value, @request.params[key], tmp_path) @@ -70,7 +70,7 @@ module Gitlab case path_value when nil - value_hash[path_key] = open_file(tmp_path) + value_hash[path_key] = open_file(tmp_path, value_hash.dig(path_key, '.name')) @open_files << value_hash[path_key] value_hash when Hash @@ -81,8 +81,8 @@ module Gitlab end end - def open_file(path) - ::UploadedFile.new(path, File.basename(path), 'application/octet-stream') + def open_file(path, name) + ::UploadedFile.new(path, name || File.basename(path), 'application/octet-stream') end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index a3e1c66c19f..28ebac1776e 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -198,9 +198,11 @@ module Gitlab end def update_profile + clear_user_synced_attributes_metadata + return unless sync_profile_from_provider? || creating_linked_ldap_user? - metadata = gl_user.user_synced_attributes_metadata || gl_user.build_user_synced_attributes_metadata + metadata = gl_user.build_user_synced_attributes_metadata if sync_profile_from_provider? UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| @@ -221,6 +223,10 @@ module Gitlab end end + def clear_user_synced_attributes_metadata + gl_user&.user_synced_attributes_metadata&.destroy + end + def log Gitlab::AppLogger end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 95d94b3cc68..98a168b43bb 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -45,6 +45,7 @@ module Gitlab if user private_token ||= user.personal_access_tokens.active.pluck(:token).first + raise 'Your user must have a personal_access_token' unless private_token end headers['Private-Token'] = private_token if private_token diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 9e2fa07a205..cf0935dbd9a 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -41,7 +41,7 @@ module Gitlab @commits_count ||= commits.count end - def self.parse_search_result(result) + def self.parse_search_result(result, project = nil) ref = nil filename = nil basename = nil @@ -66,7 +66,8 @@ module Gitlab basename: basename, ref: ref, startline: startline, - data: data + data: data, + project_id: project ? project.id : nil ) end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 729fef34b35..e91c6fb2e27 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -6,9 +6,14 @@ module Gitlab attr_accessor :name, :priority, :metrics validates :name, :priority, :metrics, presence: true - def self.all + def self.common_metrics AdditionalMetricsParser.load_groups_from_yaml end + + # EE only + def self.for_project(_) + common_metrics + end end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 294a6ae34ca..972ab75d1d5 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -7,6 +7,7 @@ module Gitlab def query(environment_id, deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( + deployment.project, common_query_context( deployment.environment, timeframe_start: (deployment.created_at - 30.minutes).to_f, diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb index 32fe8201a8d..9273e69e158 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -7,6 +7,7 @@ module Gitlab def query(environment_id) ::Environment.find_by(id: environment_id).try do |environment| query_metrics( + environment.project, common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f) ) end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index 4c3edccc71a..5710ad47c1a 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -18,7 +18,7 @@ module Gitlab private def groups_data - metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all) + metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.common_metrics) lookup = active_series_lookup(metrics_groups) groups = {} diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index 5cddc96a643..0c280dc9a3c 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -2,10 +2,10 @@ module Gitlab module Prometheus module Queries module QueryAdditionalMetrics - def query_metrics(query_context) + def query_metrics(project, query_context) query_processor = method(:process_query).curry[query_context] - groups = matched_metrics.map do |group| + groups = matched_metrics(project).map do |group| metrics = group.metrics.map do |metric| { title: metric.title, @@ -60,8 +60,8 @@ module Gitlab @available_metrics ||= client_label_values || [] end - def matched_metrics - result = Gitlab::Prometheus::MetricGroup.all.map do |group| + def matched_metrics(project) + result = Gitlab::Prometheus::MetricGroup.for_project(project).map do |group| group.metrics.select! do |metric| metric.required_metrics.all?(&available_metrics.method(:include?)) end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 10527972663..659021c9ac9 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -1,8 +1,9 @@ module Gitlab - PrometheusError = Class.new(StandardError) - # Helper methods to interact with Prometheus network services & resources class PrometheusClient + Error = Class.new(StandardError) + QueryError = Class.new(Gitlab::PrometheusClient::Error) + attr_reader :rest_client, :headers def initialize(rest_client) @@ -22,10 +23,10 @@ module Gitlab def query_range(query, start: 8.hours.ago, stop: Time.now) get_result('matrix') do json_api_get('query_range', - query: query, - start: start.to_f, - end: stop.to_f, - step: 1.minute.to_i) + query: query, + start: start.to_f, + end: stop.to_f, + step: 1.minute.to_i) end end @@ -43,22 +44,22 @@ module Gitlab path = ['api', 'v1', type].join('/') get(path, args) rescue JSON::ParserError - raise PrometheusError, 'Parsing response failed' + raise PrometheusClient::Error, 'Parsing response failed' rescue Errno::ECONNREFUSED - raise PrometheusError, 'Connection refused' + raise PrometheusClient::Error, 'Connection refused' end def get(path, args) response = rest_client[path].get(params: args) handle_response(response) rescue SocketError - raise PrometheusError, "Can't connect to #{rest_client.url}" + raise PrometheusClient::Error, "Can't connect to #{rest_client.url}" rescue OpenSSL::SSL::SSLError - raise PrometheusError, "#{rest_client.url} contains invalid SSL data" + raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data" rescue RestClient::ExceptionWithResponse => ex handle_exception_response(ex.response) rescue RestClient::Exception - raise PrometheusError, "Network connection error" + raise PrometheusClient::Error, "Network connection error" end def handle_response(response) @@ -66,16 +67,18 @@ module Gitlab if response.code == 200 && json_data['status'] == 'success' json_data['data'] || {} else - raise PrometheusError, "#{response.code} - #{response.body}" + raise PrometheusClient::Error, "#{response.code} - #{response.body}" end end def handle_exception_response(response) - if response.code == 400 + if response.code == 200 && response['status'] == 'success' + response['data'] || {} + elsif response.code == 400 json_data = JSON.parse(response.body) - raise PrometheusError, json_data['error'] || 'Bad data received' + raise PrometheusClient::QueryError, json_data['error'] || 'Bad data received' else - raise PrometheusError, "#{response.code} - #{response.body}" + raise PrometheusClient::Error, "#{response.code} - #{response.body}" end end diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb index f64f1757144..9f69a9e4a39 100644 --- a/lib/gitlab/query_limiting.rb +++ b/lib/gitlab/query_limiting.rb @@ -6,7 +6,7 @@ module Gitlab # This ensures we don't produce any errors that users can't do anything # about themselves. def self.enable? - Gitlab.com? || Rails.env.development? || Rails.env.test? + Rails.env.development? || Rails.env.test? end # Allows the current request to execute any number of SQL queries. diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb index 66049c94ec6..4c83581c4b1 100644 --- a/lib/gitlab/query_limiting/active_support_subscriber.rb +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -3,8 +3,10 @@ module Gitlab class ActiveSupportSubscriber < ActiveSupport::Subscriber attach_to :active_record - def sql(*) - Transaction.current&.increment + def sql(event) + unless event.payload[:name] == 'CACHE' + Transaction.current&.increment + end end end end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 3cbafadb0d0..66d7d9275cf 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -51,13 +51,7 @@ module Gitlab error = ThresholdExceededError.new(error_message) - if raise_error? - raise(error) - else - # Raven automatically logs to the Rails log if disabled, thus we don't - # need to manually log anything in case Sentry support is not enabled. - Raven.capture_exception(error) - end + raise(error) if raise_error? end def increment diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 3937d9c153a..96415271316 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -24,15 +24,14 @@ module Gitlab action_block.nil? end - def available?(opts) + def available?(context) return true unless condition_block - context = OpenStruct.new(opts) context.instance_exec(&condition_block) end - def explain(context, opts, arg) - return unless available?(opts) + def explain(context, arg) + return unless available?(context) if explanation.respond_to?(:call) execute_block(explanation, context, arg) @@ -41,15 +40,13 @@ module Gitlab end end - def execute(context, opts, arg) - return if noop? || !available?(opts) + def execute(context, arg) + return if noop? || !available?(context) execute_block(action_block, context, arg) end - def to_h(opts) - context = OpenStruct.new(opts) - + def to_h(context) desc = description if desc.respond_to?(:call) desc = context.instance_exec(&desc) rescue '' diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index 536765305e1..d82dccd0db5 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -62,9 +62,8 @@ module Gitlab # Allows to define conditions that must be met in order for the command # to be returned by `.command_names` & `.command_definitions`. - # It accepts a block that will be evaluated with the context given to - # `CommandDefintion#to_h`. - # + # It accepts a block that will be evaluated with the context + # of a QuickActions::InterpretService instance # Example: # # condition do diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index c0878a34fb1..075ff91700c 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -29,7 +29,7 @@ module Gitlab # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] # msg #=> "hello\nworld" # ``` - def extract_commands(content, opts = {}) + def extract_commands(content) return [content, []] unless content content = content.dup @@ -37,7 +37,7 @@ module Gitlab commands = [] content.delete!("\r") - content.gsub!(commands_regex(opts)) do + content.gsub!(commands_regex) do if $~[:cmd] commands << [$~[:cmd], $~[:arg]].reject(&:blank?) '' @@ -60,8 +60,8 @@ module Gitlab # It looks something like: # # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ - def commands_regex(opts) - names = command_names(opts).map(&:to_s) + def commands_regex + names = command_names.map(&:to_s) @commands_regex ||= %r{ (?<code> @@ -133,7 +133,7 @@ module Gitlab [content, commands] end - def command_names(opts) + def command_names command_definitions.flat_map do |command| next if command.noop? diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 7ab85e1c35c..ac3de2a8f71 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -40,12 +40,16 @@ module Gitlab 'a-zA-Z0-9_/\\$\\{\\}\\. \\-' end + def environment_name_regex_chars_without_slash + 'a-zA-Z0-9_\\$\\{\\}\\. -' + end + def environment_name_regex - @environment_name_regex ||= /\A[#{environment_name_regex_chars}]+\z/.freeze + @environment_name_regex ||= /\A[#{environment_name_regex_chars_without_slash}]([#{environment_name_regex_chars}]*[#{environment_name_regex_chars_without_slash}])?\z/.freeze end def environment_name_regex_message - "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces" + "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces, but it cannot start or end with '/'" end def kubernetes_namespace_regex diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 5ad219179f3..5a5ae7f19d4 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -1,7 +1,7 @@ module Gitlab class SearchResults class FoundBlob - attr_reader :id, :filename, :basename, :ref, :startline, :data + attr_reader :id, :filename, :basename, :ref, :startline, :data, :project_id def initialize(opts = {}) @id = opts.fetch(:id, nil) @@ -11,6 +11,7 @@ module Gitlab @startline = opts.fetch(:startline, nil) @data = opts.fetch(:data, nil) @per_page = opts.fetch(:per_page, 20) + @project_id = opts.fetch(:project_id, nil) end def path diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index e90a90508a2..07d7c91cb5d 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -37,7 +37,7 @@ module Gitlab config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } config[:bin_dir] = Gitlab.config.gitaly.client_path - TOML.dump(config) + TomlRB.dump(config) end # rubocop:disable Rails/Output diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index f4a41dc3eda..4ba44e0feef 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -294,7 +294,8 @@ module Gitlab # add_namespace("/path/to/storage", "gitlab") # def add_namespace(storage, name) - Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:add_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).add(name) else @@ -315,7 +316,8 @@ module Gitlab # rm_namespace("/path/to/storage", "gitlab") # def rm_namespace(storage, name) - Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:remove_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).remove(name) else @@ -333,7 +335,8 @@ module Gitlab # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # def mv_namespace(storage, old_name, new_name) - Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:rename_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).rename(old_name, new_name) else @@ -368,7 +371,8 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) - Gitlab::GitalyClient.migrate(:namespace_exists) do |enabled| + Gitlab::GitalyClient.migrate(:namespace_exists, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).exists?(dir_name) else diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb deleted file mode 100644 index b89ae2505c9..00000000000 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ /dev/null @@ -1,67 +0,0 @@ -module Gitlab - module SidekiqMiddleware - class MemoryKiller - # Default the RSS limit to 0, meaning the MemoryKiller is disabled - MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i - # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i - # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i - - # Create a mutex used to ensure there will be only one thread waiting to - # shut Sidekiq down - MUTEX = Mutex.new - - def call(worker, job, queue) - yield - - current_rss = get_rss - - return unless MAX_RSS > 0 && current_rss > MAX_RSS - - Thread.new do - # Return if another thread is already waiting to shut Sidekiq down - return unless MUTEX.try_lock - - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" - - # Wait `GRACE_TIME` to give the memory intensive job time to finish. - # Then, tell Sidekiq to stop fetching new jobs. - wait_and_signal(GRACE_TIME, 'SIGSTP', 'stop fetching new jobs') - - # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. - # Then, tell Sidekiq to gracefully shut down by giving jobs a few more - # moments to finish, killing and requeuing them if they didn't, and - # then terminating itself. - wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') - - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. - wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') - end - end - - private - - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status.zero? - - output.to_i - end - - def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - sleep(time) - - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - Process.kill(signal, pid) - end - - def pid - Process.pid - end - end - end -end diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb new file mode 100644 index 00000000000..c2b8d6de66e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/shutdown.rb @@ -0,0 +1,133 @@ +require 'mutex_m' + +module Gitlab + module SidekiqMiddleware + class Shutdown + extend Mutex_m + + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + # Wait 30 seconds for running jobs to finish during graceful shutdown + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # This exception can be used to request that the middleware start shutting down Sidekiq + WantShutdown = Class.new(StandardError) + + ShutdownWithoutRaise = Class.new(WantShutdown) + private_constant :ShutdownWithoutRaise + + # For testing only, to avoid race conditions (?) in Rspec mocks. + attr_reader :trace + + # We store the shutdown thread in a class variable to ensure that there + # can be only one shutdown thread in the process. + def self.create_shutdown_thread + mu_synchronize do + return unless @shutdown_thread.nil? + + @shutdown_thread = Thread.new { yield } + end + end + + # For testing only: so we can wait for the shutdown thread to finish. + def self.shutdown_thread + mu_synchronize { @shutdown_thread } + end + + # For testing only: so that we can reset the global state before each test. + def self.clear_shutdown_thread + mu_synchronize { @shutdown_thread = nil } + end + + def initialize + @trace = Queue.new if Rails.env.test? + end + + def call(worker, job, queue) + shutdown_exception = nil + + begin + yield + check_rss! + rescue WantShutdown => ex + shutdown_exception = ex + end + + return unless shutdown_exception + + self.class.create_shutdown_thread do + do_shutdown(worker, job, shutdown_exception) + end + + raise shutdown_exception unless shutdown_exception.is_a?(ShutdownWithoutRaise) + end + + private + + def do_shutdown(worker, job, shutdown_exception) + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\ + "#{worker.class} JID-#{job['jid']}" + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + + # Wait `GRACE_TIME` to give the memory intensive job time to finish. + # Then, tell Sidekiq to stop fetching new jobs. + wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') + + # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. + # Then, tell Sidekiq to gracefully shut down by giving jobs a few more + # moments to finish, killing and requeuing them if they didn't, and + # then terminating itself. + wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') + + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. + wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') + end + + def check_rss! + return unless MAX_RSS > 0 + + current_rss = get_rss + return unless current_rss > MAX_RSS + + raise ShutdownWithoutRaise.new("current RSS #{current_rss} exceeds maximum RSS #{MAX_RSS}") + end + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) + return 0 unless status.zero? + + output.to_i + end + + def wait_and_signal(time, signal, explanation) + Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + sleep(time) + + Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + kill(signal, pid) + end + + def pid + Process.pid + end + + def sleep(time) + if Rails.env.test? + @trace << [:sleep, time] + else + Kernel.sleep(time) + end + end + + def kill(signal, pid) + if Rails.env.test? + @trace << [:kill, signal, pid] + else + Process.kill(signal, pid) + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb index cc3c9a50555..466554e398c 100644 --- a/lib/gitlab/slash_commands/base_command.rb +++ b/lib/gitlab/slash_commands/base_command.rb @@ -31,10 +31,11 @@ module Gitlab raise NotImplementedError end - attr_accessor :project, :current_user, :params + attr_accessor :project, :current_user, :params, :chat_name - def initialize(project, user, params = {}) - @project, @current_user, @params = project, user, params.dup + def initialize(project, chat_name, params = {}) + @project, @current_user, @params = project, chat_name.user, params.dup + @chat_name = chat_name end private diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index a78408b0519..85aaa6b0eba 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -13,12 +13,13 @@ module Gitlab if command if command.allowed?(project, current_user) - command.new(project, current_user, params).execute(match) + command.new(project, chat_name, params).execute(match) else Gitlab::SlashCommands::Presenters::Access.new.access_denied end else - Gitlab::SlashCommands::Help.new(project, current_user, params).execute(available_commands, params[:text]) + Gitlab::SlashCommands::Help.new(project, chat_name, params) + .execute(available_commands, params[:text]) end end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 5f0c98cb5a4..53744bad1f4 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -25,7 +25,11 @@ module Gitlab query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING end - def fuzzy_arel_match(column, query) + # column - The column name to search in. + # query - The text to search for. + # lower_exact_match - When set to `true` we'll fall back to using + # `LOWER(column) = query` instead of using `ILIKE`. + def fuzzy_arel_match(column, query, lower_exact_match: false) query = query.squish return nil unless query.present? @@ -36,7 +40,13 @@ module Gitlab else # No words of at least 3 chars, but we can search for an exact # case insensitive match with the query as a whole - arel_table[column].matches(sanitize_sql_like(query)) + if lower_exact_match + Arel::Nodes::NamedFunction + .new('LOWER', [arel_table[column]]) + .eq(query) + else + arel_table[column].matches(sanitize_sql_like(query)) + end end end diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 545e7c74f7e..6f63ea91ae8 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -53,7 +53,7 @@ module Gitlab end def valid? - key.present? && bits && technology.supported_sizes.include?(bits) + SSHKey.valid_ssh_public_key?(key_text) end def type diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 15eb1c41213..ff4dc29efea 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -65,7 +65,7 @@ module Gitlab return false unless can_access_git? if protected?(ProtectedBranch, project, ref) - return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) + return true if project.user_can_push_to_empty_repo?(user) protected_branch_accessible_to?(ref, action: :push) else diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index ff638c07755..f30dd995695 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -76,9 +76,13 @@ module GoogleApi "initial_node_count": cluster_size, "node_config": { "machine_type": machine_type + }, + "legacy_abac": { + "enabled": true } } - } ) + } + ) service.create_cluster(project_id, zone, request_body, options: user_agent_header) end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index c2d3a6b6950..c6942d22926 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -115,7 +115,7 @@ namespace :gemojione do end end - style_path = Rails.root.join(*%w(app assets stylesheets framework emoji-sprites.scss)) + style_path = Rails.root.join(*%w(app assets stylesheets framework emoji_sprites.scss)) # Combine the resized assets into a packed sprite and re-generate the SCSS SpriteFactory.cssurl = "image-url('$IMAGE')" diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 107ff1d8aeb..e9ca6404fe8 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -2,7 +2,7 @@ namespace :gitlab do namespace :gitaly do desc "GitLab | Install or upgrade gitaly" task :install, [:dir, :repo] => :gitlab_environment do |t, args| - require 'toml' + require 'toml-rb' warn_user_is_not_gitlab @@ -38,7 +38,7 @@ namespace :gitlab do desc "GitLab | Print storage configuration in TOML format" task storage_config: :environment do - require 'toml' + require 'toml-rb' puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}" puts "# This is in TOML format suitable for use in Gitaly's config.toml file." diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 3ab406eff2c..fe5032cae18 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -16,5 +16,54 @@ unless Rails.env.production? task :javascript do Rake::Task['eslint'].invoke end + + desc "GitLab | lint | Run several lint checks" + task :all do + status = 0 + + %w[ + config_lint + haml_lint + scss_lint + flay + gettext:lint + lint:static_verification + ].each do |task| + pid = Process.fork do + rd, wr = IO.pipe + stdout = $stdout.dup + stderr = $stderr.dup + $stdout.reopen(wr) + $stderr.reopen(wr) + + begin + begin + Rake::Task[task].invoke + rescue RuntimeError # The haml_lint tasks raise a RuntimeError + exit(1) + end + rescue SystemExit => ex + msg = "*** Rake task #{task} failed with the following error(s):" + raise ex + ensure + $stdout.reopen(stdout) + $stderr.reopen(stderr) + wr.close + + if msg + warn "\n#{msg}\n\n" + IO.copy_stream(rd, $stderr) + else + IO.copy_stream(rd, $stdout) + end + end + end + + Process.waitpid(pid) + status += $?.exitstatus + end + + exit(status) + end end end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 31cbd651edb..1c7a8a90f5c 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -8,6 +8,7 @@ task setup_postgresql: :environment do require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb') + require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb') NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up @@ -17,4 +18,5 @@ task setup_postgresql: :environment do IndexRedirectRoutesPathForLike.new.up AddIndexOnNamespacesLowerName.new.up ReworkRedirectRoutesIndexes.new.up + UsersNameLowerIndex.new.up end |