diff options
Diffstat (limited to 'lib')
59 files changed, 781 insertions, 254 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index f3f64244589..754549f72f0 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -138,6 +138,7 @@ module API mount ::API::PagesDomains mount ::API::Pipelines mount ::API::PipelineSchedules + mount ::API::ProjectImport mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectMilestones @@ -146,6 +147,7 @@ module API mount ::API::Repositories mount ::API::Runner mount ::API::Runners + mount ::API::Search mount ::API::Services mount ::API::Settings mount ::API::SidekiqMetrics diff --git a/lib/api/commits.rb b/lib/api/commits.rb index d8fd6a6eb06..d83c43ee49b 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -156,6 +156,27 @@ module API end end + desc 'Get all references a commit is pushed to' do + detail 'This feature was introduced in GitLab 10.6' + success Entities::BasicRef + end + params do + requires :sha, type: String, desc: 'A commit sha' + optional :type, type: String, values: %w[branch tag all], default: 'all', desc: 'Scope' + use :pagination + end + get ':id/repository/commits/:sha/refs', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + commit = user_project.commit(params[:sha]) + not_found!('Commit') unless commit + + refs = [] + refs.concat(user_project.repository.branch_names_contains(commit.id).map {|name| { type: 'branch', name: name }}) unless params[:type] == 'tag' + refs.concat(user_project.repository.tag_names_contains(commit.id).map {|name| { type: 'tag', name: name }}) unless params[:type] == 'branch' + refs = Kaminari.paginate_array(refs) + + present paginate(refs), with: Entities::BasicRef + end + desc 'Post comment to commit' do success Entities::CommitNote end @@ -165,7 +186,7 @@ module API optional :path, type: String, desc: 'The file path' given :path do requires :line, type: Integer, desc: 'The line number' - requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' + requires :line_type, type: String, values: %w[new old], default: 'new', desc: 'The type of the line' end end post ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e13463ec66b..45c737c6c29 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -22,6 +22,7 @@ module API end expose :avatar_path, if: ->(user, options) { options.fetch(:only_path, false) && user.avatar_path } + expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes expose :web_url do |user, options| Gitlab::Routing.url_helpers.user_url(user) @@ -90,6 +91,13 @@ module API expose :created_at end + class ProjectImportStatus < ProjectIdentity + expose :import_status + + # TODO: Use `expose_nil` once we upgrade the grape-entity gem + expose :import_error, if: lambda { |status, _ops| status.import_error } + end + class BasicProjectDetails < ProjectIdentity include ::API::ProjectsRelationBuilder @@ -109,6 +117,8 @@ module API expose :star_count, :forks_count expose :last_activity_at + expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes + def self.preload_relation(projects_relation, options = {}) projects_relation.preload(:project_feature, :route) .preload(namespace: [:route, :owner], @@ -230,6 +240,8 @@ module API expose :parent_id end + expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes + expose :statistics, if: :statistics do with_options format_with: -> (value) { value.to_i } do expose :storage_size @@ -274,6 +286,11 @@ module API expose :stats, using: Entities::CommitStats, if: :stats expose :status expose :last_pipeline, using: 'API::Entities::PipelineBasic' + expose :project_id + end + + class BasicRef < Grape::Entity + expose :type, :name end class Branch < Grape::Entity @@ -314,24 +331,20 @@ module API end end - class ProjectSnippet < Grape::Entity + class Snippet < Grape::Entity expose :id, :title, :file_name, :description expose :author, using: Entities::UserBasic expose :updated_at, :created_at - - expose :web_url do |snippet, options| + expose :project_id + expose :web_url do |snippet| Gitlab::UrlBuilder.build(snippet) end end - class PersonalSnippet < Grape::Entity - expose :id, :title, :file_name, :description - expose :author, using: Entities::UserBasic - expose :updated_at, :created_at + class ProjectSnippet < Snippet + end - expose :web_url do |snippet| - Gitlab::UrlBuilder.build(snippet) - end + class PersonalSnippet < Snippet expose :raw_url do |snippet| Gitlab::UrlBuilder.build(snippet) + "/raw" end @@ -1168,5 +1181,15 @@ module API class ApplicationWithSecret < Application expose :secret end + + class Blob < Grape::Entity + expose :basename + expose :data + expose :filename + expose :id + expose :ref + expose :startline + expose :project_id + end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index b81f07a1770..4a4df1b8b9e 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -1,6 +1,7 @@ module API class Groups < Grape::API include PaginationParams + include Helpers::CustomAttributes before { authenticate_non_get! } @@ -67,6 +68,8 @@ module API } groups = groups.with_statistics if options[:statistics] + groups, options = with_custom_attributes(groups, options) + present paginate(groups), options end end @@ -79,6 +82,7 @@ module API end params do use :group_list_params + use :with_custom_attributes end get do groups = find_groups(params) @@ -142,9 +146,20 @@ module API desc 'Get a single group, with containing projects.' do success Entities::GroupDetail end + params do + use :with_custom_attributes + end get ":id" do group = find_group!(params[:id]) - present group, with: Entities::GroupDetail, current_user: current_user + + options = { + with: Entities::GroupDetail, + current_user: current_user + } + + group, options = with_custom_attributes(group, options) + + present group, options end desc 'Remove a group.' @@ -175,12 +190,19 @@ module API optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' use :pagination + use :with_custom_attributes end get ":id/projects" do projects = find_group_projects(params) - entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project - present entity.prepare_relation(projects), with: entity, current_user: current_user + options = { + with: params[:simple] ? Entities::BasicProjectDetails : Entities::Project, + current_user: current_user + } + + projects, options = with_custom_attributes(projects, options) + + present options[:with].prepare_relation(projects), options end desc 'Get a list of subgroups in this group.' do @@ -188,6 +210,7 @@ module API end params do use :group_list_params + use :with_custom_attributes end get ":id/subgroups" do groups = find_groups(params) diff --git a/lib/api/helpers/custom_attributes.rb b/lib/api/helpers/custom_attributes.rb new file mode 100644 index 00000000000..70e4eda95f8 --- /dev/null +++ b/lib/api/helpers/custom_attributes.rb @@ -0,0 +1,28 @@ +module API + module Helpers + module CustomAttributes + extend ActiveSupport::Concern + + included do + helpers do + params :with_custom_attributes do + optional :with_custom_attributes, type: Boolean, default: false, desc: 'Include custom attributes in the response' + end + + def with_custom_attributes(collection_or_resource, options = {}) + options = options.merge( + with_custom_attributes: params[:with_custom_attributes] && + can?(current_user, :read_custom_attribute) + ) + + if options[:with_custom_attributes] && collection_or_resource.is_a?(ActiveRecord::Relation) + collection_or_resource = collection_or_resource.includes(:custom_attributes) + end + + [collection_or_resource, options] + end + end + end + end + end +end diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index bb70370ba77..09805049169 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -12,13 +12,16 @@ module API private def add_pagination_headers(paginated_data) - header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', total_pages(paginated_data).to_s header 'X-Per-Page', paginated_data.limit_value.to_s header 'X-Page', paginated_data.current_page.to_s header 'X-Next-Page', paginated_data.next_page.to_s header 'X-Prev-Page', paginated_data.prev_page.to_s header 'Link', pagination_links(paginated_data) + + return if data_without_counts?(paginated_data) + + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', total_pages(paginated_data).to_s end def pagination_links(paginated_data) @@ -37,8 +40,10 @@ module API request_params[:page] = 1 links << %(<#{request_url}?#{request_params.to_query}>; rel="first") - request_params[:page] = total_pages(paginated_data) - links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + unless data_without_counts?(paginated_data) + request_params[:page] = total_pages(paginated_data) + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + end links.join(', ') end @@ -55,6 +60,10 @@ module API relation end + + def data_without_counts?(paginated_data) + paginated_data.is_a?(Kaminari::PaginatableWithoutCount) + end end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 3d0d1287407..fbe30192a16 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -3,7 +3,6 @@ module API module Runner JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 10 * 60 def runner_registration_token_valid? ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], @@ -18,30 +17,14 @@ module API def authenticate_runner! forbidden! unless current_runner + + current_runner.update_cached_info(get_runner_version_from_params) end def current_runner @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s) end - def update_runner_info - return unless update_runner? - - current_runner.contacted_at = Time.now - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - - def update_runner? - # Use a random threshold to prevent beating DB updates. - # It generates a distribution between [40m, 80m]. - # - contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - - current_runner.contacted_at.nil? || - (Time.now - current_runner.contacted_at) >= contacted_at_max_age - end - def validate_job!(job) not_found! unless job diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 9285fb90cdc..b3660e4a1d0 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -13,7 +13,7 @@ module API # key_id - ssh key id for Git over SSH # user_id - user id for Git over HTTP # protocol - Git access protocol being used, e.g. HTTP or SSH - # project - project path with namespace + # project - project full_path (not path on disk) # action - git action (git-upload-pack or git-receive-pack) # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList post "/allowed" do diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb new file mode 100644 index 00000000000..a509c1f32c1 --- /dev/null +++ b/lib/api/project_import.rb @@ -0,0 +1,69 @@ +module API + class ProjectImport < Grape::API + include PaginationParams + + helpers do + def import_params + declared_params(include_missing: false) + end + + def file_is_valid? + import_params[:file] && import_params[:file]['tempfile'].respond_to?(:read) + end + + def validate_file! + render_api_error!('The file is invalid', 400) unless file_is_valid? + end + end + + before do + forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project') + end + + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + params do + requires :path, type: String, desc: 'The new project path and name' + requires :file, type: File, desc: 'The project export file to be imported' + optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace." + end + desc 'Create a new project import' do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::ProjectImportStatus + end + post 'import' do + validate_file! + + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42437') + + namespace = if import_params[:namespace] + find_namespace!(import_params[:namespace]) + else + current_user.namespace + end + + project_params = { + path: import_params[:path], + namespace_id: namespace.id, + file: import_params[:file]['tempfile'] + } + + project = ::Projects::GitlabProjectsImportService.new(current_user, project_params).execute + + render_api_error!(project.errors.full_messages&.first, 400) unless project.saved? + + present project, with: Entities::ProjectImportStatus + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + desc 'Get a project export status' do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::ProjectImportStatus + end + get ':id/import' do + present user_project, with: Entities::ProjectImportStatus + end + end + end +end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 5b481121a10..e90892a90f7 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -3,6 +3,7 @@ require_dependency 'declarative_policy' module API class Projects < Grape::API include PaginationParams + include Helpers::CustomAttributes before { authenticate_non_get! } @@ -80,6 +81,7 @@ module API projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] projects = projects.with_statistics if params[:statistics] projects = paginate(projects) + projects, options = with_custom_attributes(projects, options) if current_user project_members = current_user.project_members.preload(:source, user: [notification_settings: :source]) @@ -107,6 +109,7 @@ module API requires :user_id, type: String, desc: 'The ID or username of the user' use :collection_params use :statistics_params + use :with_custom_attributes end get ":user_id/projects" do user = find_user(params[:user_id]) @@ -127,6 +130,7 @@ module API params do use :collection_params use :statistics_params + use :with_custom_attributes end get do present_projects load_projects @@ -196,11 +200,19 @@ module API end params do use :statistics_params + use :with_custom_attributes end get ":id" do - entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present user_project, with: entity, current_user: current_user, - user_can_admin_project: can?(current_user, :admin_project, user_project), statistics: params[:statistics] + options = { + with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, + current_user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project), + statistics: params[:statistics] + } + + project, options = with_custom_attributes(user_project, options) + + present project, options end desc 'Fork new project for the current user or provided namespace.' do @@ -242,6 +254,7 @@ module API end params do use :collection_params + use :with_custom_attributes end get ':id/forks' do forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 1f80646a2ea..5469cba69a6 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -78,7 +78,6 @@ module API post '/request' do authenticate_runner! no_content! unless current_runner.active? - update_runner_info if current_runner.runner_queue_value_latest?(params[:last_update]) header 'X-GitLab-Last-Update', params[:last_update] diff --git a/lib/api/search.rb b/lib/api/search.rb new file mode 100644 index 00000000000..3556ad98c52 --- /dev/null +++ b/lib/api/search.rb @@ -0,0 +1,111 @@ +module API + class Search < Grape::API + include PaginationParams + + before { authenticate! } + + helpers do + SCOPE_ENTITY = { + merge_requests: Entities::MergeRequestBasic, + issues: Entities::IssueBasic, + projects: Entities::BasicProjectDetails, + milestones: Entities::Milestone, + notes: Entities::Note, + commits: Entities::CommitDetail, + blobs: Entities::Blob, + wiki_blobs: Entities::Blob, + snippet_titles: Entities::Snippet, + snippet_blobs: Entities::Snippet + }.freeze + + def search(additional_params = {}) + search_params = { + scope: params[:scope], + search: params[:search], + snippets: snippets?, + page: params[:page], + per_page: params[:per_page] + }.merge(additional_params) + + results = SearchService.new(current_user, search_params).search_objects + + process_results(results) + end + + def process_results(results) + case params[:scope] + when 'wiki_blobs' + paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob, user_project) } + when 'blobs' + paginate(results).map { |blob| blob[1] } + else + paginate(results) + end + end + + def snippets? + %w(snippet_blobs snippet_titles).include?(params[:scope]).to_s + end + + def entity + SCOPE_ENTITY[params[:scope].to_sym] + end + end + + resource :search do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs', + values: %w(projects issues merge_requests milestones snippet_titles snippet_blobs) + use :pagination + end + get do + present search, with: entity + end + end + + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :id, type: String, desc: 'The ID of a group' + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + projects, issues, merge_requests, milestones', + values: %w(projects issues merge_requests milestones) + use :pagination + end + get ':id/-/search' do + present search(group_id: user_group.id), with: entity + end + end + + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs', + values: %w(issues merge_requests milestones notes wiki_blobs commits blobs) + use :pagination + end + get ':id/-/search' do + present search(project_id: user_project.id), with: entity + end + end + end +end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index ffccfebe752..c6dbcf84e3a 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -60,7 +60,7 @@ module API end post ':id/mark_as_done' do TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user) - todo = Todo.find(params[:id]) + todo = current_user.todos.find(params[:id]) present todo, with: Entities::Todo, current_user: current_user end diff --git a/lib/api/users.rb b/lib/api/users.rb index 3cc12724b8a..3920171205f 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -2,6 +2,7 @@ module API class Users < Grape::API include PaginationParams include APIGuard + include Helpers::CustomAttributes allow_access_with_scope :read_user, if: -> (request) { request.get? } @@ -70,6 +71,7 @@ module API use :sort_params use :pagination + use :with_custom_attributes end get do authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) @@ -94,8 +96,9 @@ module API entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic users = users.preload(:identities, :u2f_registrations) if entity == Entities::UserWithAdmin + users, options = with_custom_attributes(users, with: entity) - present paginate(users), with: entity + present paginate(users), options end desc 'Get a single user' do @@ -103,12 +106,16 @@ module API end params do requires :id, type: Integer, desc: 'The ID of the user' + + use :with_custom_attributes end get ":id" do user = User.find_by(id: params[:id]) not_found!('User') unless user && can?(current_user, :read_user, user) opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : { with: Entities::User } + user, opts = with_custom_attributes(user, opts) + present user, opts end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index c856ba99f09..7d8b1f369fe 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -174,7 +174,7 @@ module API use :pagination end get "/search/:query", requirements: { query: %r{[^/]+} } do - search_service = Search::GlobalService.new(current_user, search: params[:query]).execute + search_service = ::Search::GlobalService.new(current_user, search: params[:query]).execute projects = search_service.objects('projects', params[:page], false) projects = projects.reorder(params[:order_by] => params[:sort]) diff --git a/lib/api/v3/todos.rb b/lib/api/v3/todos.rb index 2f2cf259987..3e2c61f6dbd 100644 --- a/lib/api/v3/todos.rb +++ b/lib/api/v3/todos.rb @@ -12,7 +12,7 @@ module API end delete ':id' do TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user) - todo = Todo.find(params[:id]) + todo = current_user.todos.find(params[:id]) present todo, with: ::API::Entities::Todo, current_user: current_user end diff --git a/lib/banzai/filter/html_entity_filter.rb b/lib/banzai/filter/html_entity_filter.rb index f3bd587c28b..e008fd428b0 100644 --- a/lib/banzai/filter/html_entity_filter.rb +++ b/lib/banzai/filter/html_entity_filter.rb @@ -5,7 +5,7 @@ module Banzai # Text filter that escapes these HTML entities: & " < > class HtmlEntityFilter < HTML::Pipeline::TextFilter def call - ERB::Util.html_escape_once(text) + ERB::Util.html_escape(text) end end end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index a79a0154846..0ac7e231b5b 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,23 +14,33 @@ module Banzai end def highlight_node(node) - code = node.text css_classes = 'code highlight js-syntax-highlight' - language = node.attr('lang') + lang = node.attr('lang') + retried = false - if use_rouge?(language) - lexer = lexer_for(language) + if use_rouge?(lang) + lexer = lexer_for(lang) language = lexer.tag + else + lexer = Rouge::Lexers::PlainText.new + language = lang + end + + begin + code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, node.text), tag: language) + css_classes << " #{language}" if language + rescue + # Gracefully handle syntax highlighter bugs/errors to ensure users can + # still access an issue/comment/etc. First, retry with the plain text + # filter. If that fails, then just skip this entirely, but that would + # be a pretty bad upstream bug. + return if retried - begin - code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code), tag: language) - css_classes << " #{language}" - rescue - # Gracefully handle syntax highlighter bugs/errors to ensure - # users can still access an issue/comment/etc. + language = nil + lexer = Rouge::Lexers::PlainText.new + retried = true - language = nil - end + retry end highlighted = %(<pre class="#{css_classes}" lang="#{language}" v-pre="true"><code>#{code}</code></pre>) diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index ee7f4be6b9f..62c41801d75 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -8,7 +8,8 @@ module Gitlab module Asciidoc DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', - 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' + 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font', + 'outfilesuffix=.adoc' ].freeze # Public: Converts the provided Asciidoc markup into HTML. diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index 03b17b319fa..1b4a9e8a194 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -14,6 +14,14 @@ module Gitlab def perform(start_id, end_id) log("Creating memberships for forks: #{start_id} - #{end_id}") + insert_members(start_id, end_id) + + if missing_members?(start_id, end_id) + BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + end + + def insert_members(start_id, end_id) ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id) @@ -33,10 +41,9 @@ module Gitlab WHERE existing_members.project_id = forked_project_links.forked_to_project_id ) INSERT_MEMBERS - - if missing_members?(start_id, end_id) - BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) - end + rescue ActiveRecord::RecordNotUnique => e + # `fork_network_member` was created concurrently in another migration + log(e.message) end def missing_members?(start_id, end_id) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 8a8e770940e..ee55fabd6f0 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -249,7 +249,7 @@ module Gitlab end def drop_temp_table_if_finished - if UntrackedFile.all.empty? + if UntrackedFile.all.empty? && !Rails.env.test? # Dropping a table intermittently breaks test cleanup UntrackedFile.connection.drop_table(:untracked_files_for_uploads, if_exists: true) end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index a7a1bbe1752..914a9e48a2f 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -43,7 +43,11 @@ module Gitlab store_untracked_file_paths - schedule_populate_untracked_uploads_jobs + if UntrackedFile.all.empty? + drop_temp_table + else + schedule_populate_untracked_uploads_jobs + end end private @@ -92,7 +96,7 @@ module Gitlab end end - yield(paths) + yield(paths) if paths.any? end def build_find_command(search_dir) @@ -165,6 +169,13 @@ module Gitlab bulk_queue_background_migration_jobs_by_range( UntrackedFile, FOLLOW_UP_MIGRATION) end + + def drop_temp_table + unless Rails.env.test? # Dropping a table intermittently breaks test cleanup + UntrackedFile.connection.drop_table(:untracked_files_for_uploads, + if_exists: true) + end + end end end end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 945d70e7a24..521680b8708 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -16,11 +16,11 @@ module Gitlab lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' }.freeze - attr_reader :user_access, :project, :skip_authorization, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name + attr_reader :user_access, :project, :skip_authorization, :skip_lfs_integrity_check, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name def initialize( change, user_access:, project:, skip_authorization: false, - protocol: + skip_lfs_integrity_check: false, protocol: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @@ -28,16 +28,18 @@ module Gitlab @user_access = user_access @project = project @skip_authorization = skip_authorization + @skip_lfs_integrity_check = skip_lfs_integrity_check @protocol = protocol end - def exec + def exec(skip_commits_check: false) return true if skip_authorization push_checks branch_checks tag_checks - lfs_objects_exist_check + lfs_objects_exist_check unless skip_lfs_integrity_check + commits_check unless skip_commits_check true end @@ -117,6 +119,24 @@ module Gitlab end end + def commits_check + return if deletion? || newrev.nil? + + # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + commits.each do |commit| + commit_check.validate(commit, validations_for_commit(commit)) + end + end + + commit_check.validate_file_paths + end + + # Method overwritten in EE to inject custom validations + def validations_for_commit(_) + [] + end + private def updated_from_web? @@ -150,6 +170,14 @@ module Gitlab raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] end end + + def commit_check + @commit_check ||= Gitlab::Checks::CommitCheck.new(project, user_access.user, newrev, oldrev) + end + + def commits + project.repository.new_commits(newrev) + end end end end diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb new file mode 100644 index 00000000000..ae0cd142378 --- /dev/null +++ b/lib/gitlab/checks/commit_check.rb @@ -0,0 +1,61 @@ +module Gitlab + module Checks + class CommitCheck + include Gitlab::Utils::StrongMemoize + + attr_reader :project, :user, :newrev, :oldrev + + def initialize(project, user, newrev, oldrev) + @project = project + @user = user + @newrev = user + @oldrev = user + @file_paths = [] + end + + def validate(commit, validations) + return if validations.empty? && path_validations.empty? + + commit.raw_deltas.each do |diff| + @file_paths << (diff.new_path || diff.old_path) + + validations.each do |validation| + if error = validation.call(diff) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + end + + def validate_file_paths + path_validations.each do |validation| + if error = validation.call(@file_paths) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + + private + + def validate_lfs_file_locks? + strong_memoize(:validate_lfs_file_locks) do + project.lfs_enabled? && project.lfs_file_locks.any? && newrev && oldrev + end + end + + def lfs_file_locks_validation + lambda do |paths| + lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first + + if lfs_lock + return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" + end + end + end + + def path_validations + validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] + end + end + end +end diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/ci/config/loader.rb index e7d9f6a7761..141d2714cb6 100644 --- a/lib/gitlab/ci/config/loader.rb +++ b/lib/gitlab/ci/config/loader.rb @@ -6,6 +6,8 @@ module Gitlab def initialize(config) @config = YAML.safe_load(config, [Symbol], [], true) + rescue Psych::Exception => e + raise FormatError, e.message end def valid? diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 0bd78b03448..a7285ac8f9d 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -85,7 +85,7 @@ module Gitlab begin Gitlab::Ci::YamlProcessor.new(content) nil - rescue ValidationError, Psych::SyntaxError => e + rescue ValidationError => e e.message end end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index c0edcabc6fd..6659efa0961 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -28,9 +28,9 @@ module Gitlab # encode and clean the bad chars message.replace clean(message) - rescue ArgumentError - return nil - rescue + rescue ArgumentError => e + return unless e.message.include?('unknown encoding name') + encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" end diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 10ffc345bd5..8c082c0c336 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -28,7 +28,7 @@ module Gitlab def find_by_content(query) results = repository.search_files_by_content(query, ref).first(BATCH_SIZE) - results.map { |result| Gitlab::ProjectSearchResults.parse_search_result(result) } + results.map { |result| Gitlab::ProjectSearchResults.parse_search_result(result, project) } end def find_by_filename(query, except: []) @@ -45,7 +45,8 @@ module Gitlab basename: File.basename(blob.path), ref: ref, startline: 1, - data: blob.data + data: blob.data, + project: project ) end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 4828301dbb9..b2fca2c16de 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -53,11 +53,7 @@ module Gitlab def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| if is_enabled - Gitlab::GitalyClient.allow_n_plus_1_calls do - blob_references.map do |sha, path| - find_by_gitaly(repository, sha, path, limit: blob_size_limit) - end - end + repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a else blob_references.map do |sha, path| find_by_rugged(repository, sha, path, limit: blob_size_limit) diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 768617e2cae..d95561fe1b2 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -402,15 +402,6 @@ module Gitlab end end - # Get a collection of Rugged::Reference objects for this commit. - # - # Ex. - # commit.ref(repo) - # - def refs(repo) - repo.refs_hash[id] - end - # Get ref names collection # # Ex. @@ -418,7 +409,7 @@ module Gitlab # def ref_names(repo) refs(repo).map do |ref| - ref.name.sub(%r{^refs/(heads|remotes|tags)/}, "") + ref.sub(%r{^refs/(heads|remotes|tags)/}, "") end end @@ -553,6 +544,15 @@ module Gitlab date: Google::Protobuf::Timestamp.new(seconds: author_or_committer[:time].to_i) ) end + + # Get a collection of Gitlab::Git::Ref objects for this commit. + # + # Ex. + # commit.ref(repo) + # + def refs(repo) + repo.refs_hash[id] + end end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d7510061def..5f014e43c6f 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -631,21 +631,18 @@ module Gitlab end end - # Get refs hash which key is SHA1 - # and value is a Rugged::Reference + # Get refs hash which key is is the commit id + # and value is a Gitlab::Git::Tag or Gitlab::Git::Branch + # Note that both inherit from Gitlab::Git::Ref def refs_hash - # Initialize only when first call - if @refs_hash.nil? - @refs_hash = Hash.new { |h, k| h[k] = [] } - - rugged.references.each do |r| - # Symbolic/remote references may not have an OID; skip over them - target_oid = r.target.try(:oid) - if target_oid - sha = rev_parse_target(target_oid).oid - @refs_hash[sha] << r - end - end + return @refs_hash if @refs_hash + + @refs_hash = Hash.new { |h, k| h[k] = [] } + + (tags + branches).each do |ref| + next unless ref.target && ref.name + + @refs_hash[ref.dereferenced_target.id] << ref.name end @refs_hash @@ -1617,17 +1614,14 @@ module Gitlab # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. def branches_filter(filter: nil, sort_by: nil) - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464 - branches = Gitlab::GitalyClient.allow_n_plus_1_calls do - rugged.branches.each(filter).map do |rugged_ref| - begin - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end.compact - end + branches = rugged.branches.each(filter).map do |rugged_ref| + begin + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact sort_branches(branches, sort_by) end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 39040d56971..ac12271a87e 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -25,9 +25,8 @@ module Gitlab @repository.exists? end - # Disabled because of https://gitlab.com/gitlab-org/gitaly/merge_requests/539 def write_page(name, format, content, commit_details) - @repository.gitaly_migrate(:wiki_write_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| if is_enabled gitaly_write_page(name, format, content, commit_details) gollum_wiki.clear_cache @@ -48,9 +47,8 @@ module Gitlab end end - # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42094 def update_page(page_path, title, format, content, commit_details) - @repository.gitaly_migrate(:wiki_update_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| if is_enabled gitaly_update_page(page_path, title, format, content, commit_details) gollum_wiki.clear_cache diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 8ec3386184a..9ec3858b493 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -238,19 +238,22 @@ module Gitlab changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied - changes_list.each do |change| + changes_list.each.with_index do |change, index| + first_change = index == 0 + # If user does not have access to make at least one change, cancel all # push by allowing the exception to bubble up - check_single_change_access(change) + check_single_change_access(change, skip_lfs_integrity_check: !first_change) end end - def check_single_change_access(change) + def check_single_change_access(change, skip_lfs_integrity_check: false) Checks::ChangeAccess.new( change, user_access: user_access, project: project, skip_authorization: deploy_key?, + skip_lfs_integrity_check: skip_lfs_integrity_check, protocol: protocol ).exec end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 1c9477e84b2..84d6e1490c3 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -13,7 +13,7 @@ module Gitlab authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) end - def check_single_change_access(change) + def check_single_change_access(change, _options = {}) unless user_access.can_do_action?(:create_wiki) raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index d70a1a7665e..dfa0fa43b0f 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class BlobService + include Gitlab::EncodingHelper + def initialize(repository) @gitaly_repo = repository.gitaly_repository end @@ -54,6 +56,30 @@ module Gitlab end end end + + def get_blobs(revision_paths, limit = -1) + return [] if revision_paths.empty? + + revision_paths.map! do |rev, path| + Gitaly::GetBlobsRequest::RevisionPath.new(revision: rev, path: encode_binary(path)) + end + + request = Gitaly::GetBlobsRequest.new( + repository: @gitaly_repo, + revision_paths: revision_paths, + limit: limit + ) + + response = GitalyClient.call( + @gitaly_repo.storage_name, + :blob_service, + :get_blobs, + request, + timeout: GitalyClient.default_timeout + ) + + GitalyClient::BlobsStitcher.new(response) + end end end end diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb new file mode 100644 index 00000000000..5ca592ff812 --- /dev/null +++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb @@ -0,0 +1,47 @@ +module Gitlab + module GitalyClient + class BlobsStitcher + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + current_blob_data = nil + + @rpc_response.each do |msg| + begin + if msg.oid.blank? && msg.data.blank? + next + elsif msg.oid.present? + yield new_blob(current_blob_data) if current_blob_data + + current_blob_data = msg.to_h.slice(:oid, :path, :size, :revision, :mode) + current_blob_data[:data] = msg.data.dup + else + current_blob_data[:data] << msg.data + end + end + end + + yield new_blob(current_blob_data) if current_blob_data + end + + private + + def new_blob(blob_data) + Gitlab::Git::Blob.new( + id: blob_data[:oid], + mode: blob_data[:mode].to_s(8), + name: File.basename(blob_data[:path]), + path: blob_data[:path], + size: blob_data[:size], + commit_id: blob_data[:revision], + data: blob_data[:data], + binary: Gitlab::Git::Blob.binary?(blob_data[:data]) + ) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 5767f06b0ce..269a048cf5d 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -222,14 +222,25 @@ module Gitlab end def find_commit(revision) - request = Gitaly::FindCommitRequest.new( - repository: @gitaly_repo, - revision: encode_binary(revision) - ) - - response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) - - response.commit + if RequestStore.active? + # We don't use RequeStstore.fetch(key) { ... } directly because `revision` + # can be a branch name, so we can't use it as a key as it could point + # to another commit later on (happens a lot in tests). + key = { + storage: @gitaly_repo.storage_name, + relative_path: @gitaly_repo.relative_path, + commit_id: revision + } + return RequestStore[key] if RequestStore.exist?(key) + + commit = call_find_commit(revision) + return unless commit + + key[:commit_id] = commit.id + RequestStore[key] = commit + else + call_find_commit(revision) + end end def patch(revision) @@ -346,6 +357,17 @@ module Gitlab def encode_repeated(a) Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| encode_binary(s) } ) end + + def call_find_commit(revision) + request = Gitaly::FindCommitRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision) + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) + + response.commit + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 86a90d57d9c..ba04387022d 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -13,8 +13,6 @@ module Gitlab gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class - gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') - gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2daed10f678..9f404003125 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -27,6 +27,8 @@ project_tree: - :releases - project_members: - :user + - lfs_file_locks: + - :user - merge_requests: - notes: - :author diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index c14646b0611..a00795f553e 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -50,9 +50,10 @@ module Gitlab end def wiki_restorer - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + Gitlab::ImportExport::WikiRestorer.new(path_to_bundle: wiki_repo_path, shared: @shared, - project: ProjectWiki.new(project_tree.restored_project)) + project: ProjectWiki.new(project_tree.restored_project), + wiki_enabled: @project.wiki_enabled?) end def uploads_restorer diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb deleted file mode 100644 index 77bb3ca6581..00000000000 --- a/lib/gitlab/import_export/project_creator.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module ImportExport - class ProjectCreator - def initialize(namespace_id, current_user, file, project_path) - @namespace_id = namespace_id - @current_user = current_user - @file = file - @project_path = project_path - end - - def execute - ::Projects::CreateService.new( - @current_user, - name: @project_path, - path: @project_path, - namespace_id: @namespace_id, - import_type: "gitlab_project", - import_source: @file - ).execute - end - end - end -end diff --git a/lib/gitlab/import_export/wiki_restorer.rb b/lib/gitlab/import_export/wiki_restorer.rb new file mode 100644 index 00000000000..f33bfb332ab --- /dev/null +++ b/lib/gitlab/import_export/wiki_restorer.rb @@ -0,0 +1,23 @@ +module Gitlab + module ImportExport + class WikiRestorer < RepoRestorer + def initialize(project:, shared:, path_to_bundle:, wiki_enabled:) + super(project: project, shared: shared, path_to_bundle: path_to_bundle) + + @wiki_enabled = wiki_enabled + end + + def restore + @project.wiki if create_empty_wiki? + + super + end + + private + + def create_empty_wiki? + !File.exist?(@path_to_bundle) && @wiki_enabled + end + end + end +end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 47b3fce3e7a..a6bea98d631 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -15,7 +15,7 @@ module Gitlab end def self.servers - Gitlab.config.ldap.servers.values + Gitlab.config.ldap['servers']&.values || [] end def self.available_servers diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index b91757c2a4b..c59df556247 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -63,8 +63,6 @@ module Gitlab Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @provider = provider - - validate_entry end def name @@ -117,19 +115,6 @@ module Gitlab entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend end - - def validate_entry - allowed_attrs = self.class.ldap_attributes(config).map(&:downcase) - - # Net::LDAP::Entry transforms keys to symbols. Change to strings to compare. - entry_attrs = entry.attribute_names.map { |n| n.to_s.downcase } - invalid_attrs = entry_attrs - allowed_attrs - - if invalid_attrs.any? - raise InvalidEntryError, - "#{self.class.name} initialized with Net::LDAP::Entry containing invalid attributes(s): #{invalid_attrs}" - end - end end end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index cc1e92480be..d4c54049b74 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -42,7 +42,7 @@ module Gitlab key, value = parsed_field.first if value.nil? - value = open_file(tmp_path) + value = open_file(tmp_path, @request.params["#{key}.name"]) @open_files << value else value = decorate_params_value(value, @request.params[key], tmp_path) @@ -70,7 +70,7 @@ module Gitlab case path_value when nil - value_hash[path_key] = open_file(tmp_path) + value_hash[path_key] = open_file(tmp_path, value_hash.dig(path_key, '.name')) @open_files << value_hash[path_key] value_hash when Hash @@ -81,8 +81,8 @@ module Gitlab end end - def open_file(path) - ::UploadedFile.new(path, File.basename(path), 'application/octet-stream') + def open_file(path, name) + ::UploadedFile.new(path, name || File.basename(path), 'application/octet-stream') end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index a3e1c66c19f..28ebac1776e 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -198,9 +198,11 @@ module Gitlab end def update_profile + clear_user_synced_attributes_metadata + return unless sync_profile_from_provider? || creating_linked_ldap_user? - metadata = gl_user.user_synced_attributes_metadata || gl_user.build_user_synced_attributes_metadata + metadata = gl_user.build_user_synced_attributes_metadata if sync_profile_from_provider? UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| @@ -221,6 +223,10 @@ module Gitlab end end + def clear_user_synced_attributes_metadata + gl_user&.user_synced_attributes_metadata&.destroy + end + def log Gitlab::AppLogger end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 95d94b3cc68..98a168b43bb 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -45,6 +45,7 @@ module Gitlab if user private_token ||= user.personal_access_tokens.active.pluck(:token).first + raise 'Your user must have a personal_access_token' unless private_token end headers['Private-Token'] = private_token if private_token diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 4823f703ba4..cf0935dbd9a 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -2,11 +2,12 @@ module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref - def initialize(current_user, project, query, repository_ref = nil) + def initialize(current_user, project, query, repository_ref = nil, per_page: 20) @current_user = current_user @project = project @repository_ref = repository_ref.presence || project.default_branch @query = query + @per_page = per_page end def objects(scope, page = nil) @@ -40,7 +41,7 @@ module Gitlab @commits_count ||= commits.count end - def self.parse_search_result(result) + def self.parse_search_result(result, project = nil) ref = nil filename = nil basename = nil @@ -65,7 +66,8 @@ module Gitlab basename: basename, ref: ref, startline: startline, - data: data + data: data, + project_id: project ? project.id : nil ) end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 69d055c901c..294a6ae34ca 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -4,7 +4,7 @@ module Gitlab class AdditionalMetricsDeploymentQuery < BaseQuery include QueryAdditionalMetrics - def query(deployment_id) + def query(environment_id, deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( common_query_context( diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 170f483540e..6e6da593178 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -2,7 +2,7 @@ module Gitlab module Prometheus module Queries class DeploymentQuery < BaseQuery - def query(deployment_id) + def query(environment_id, deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| environment_slug = deployment.environment.slug diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index aa94614bf18..10527972663 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -3,10 +3,10 @@ module Gitlab # Helper methods to interact with Prometheus network services & resources class PrometheusClient - attr_reader :api_url + attr_reader :rest_client, :headers - def initialize(api_url:) - @api_url = api_url + def initialize(rest_client) + @rest_client = rest_client end def ping @@ -40,37 +40,40 @@ module Gitlab private def json_api_get(type, args = {}) - get(join_api_url(type, args)) + path = ['api', 'v1', type].join('/') + get(path, args) + rescue JSON::ParserError + raise PrometheusError, 'Parsing response failed' rescue Errno::ECONNREFUSED raise PrometheusError, 'Connection refused' end - def join_api_url(type, args = {}) - url = URI.parse(api_url) - rescue URI::Error - raise PrometheusError, "Invalid API URL: #{api_url}" - else - url.path = [url.path.sub(%r{/+\z}, ''), 'api', 'v1', type].join('/') - url.query = args.to_query - - url.to_s - end - - def get(url) - handle_response(HTTParty.get(url)) + def get(path, args) + response = rest_client[path].get(params: args) + handle_response(response) rescue SocketError - raise PrometheusError, "Can't connect to #{url}" + raise PrometheusError, "Can't connect to #{rest_client.url}" rescue OpenSSL::SSL::SSLError - raise PrometheusError, "#{url} contains invalid SSL data" - rescue HTTParty::Error + raise PrometheusError, "#{rest_client.url} contains invalid SSL data" + rescue RestClient::ExceptionWithResponse => ex + handle_exception_response(ex.response) + rescue RestClient::Exception raise PrometheusError, "Network connection error" end def handle_response(response) - if response.code == 200 && response['status'] == 'success' - response['data'] || {} - elsif response.code == 400 - raise PrometheusError, response['error'] || 'Bad data received' + json_data = JSON.parse(response.body) + if response.code == 200 && json_data['status'] == 'success' + json_data['data'] || {} + else + raise PrometheusError, "#{response.code} - #{response.body}" + end + end + + def handle_exception_response(response) + if response.code == 400 + json_data = JSON.parse(response.body) + raise PrometheusError, json_data['error'] || 'Bad data received' else raise PrometheusError, "#{response.code} - #{response.body}" end diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb index f64f1757144..9f69a9e4a39 100644 --- a/lib/gitlab/query_limiting.rb +++ b/lib/gitlab/query_limiting.rb @@ -6,7 +6,7 @@ module Gitlab # This ensures we don't produce any errors that users can't do anything # about themselves. def self.enable? - Gitlab.com? || Rails.env.development? || Rails.env.test? + Rails.env.development? || Rails.env.test? end # Allows the current request to execute any number of SQL queries. diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 3cbafadb0d0..66d7d9275cf 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -51,13 +51,7 @@ module Gitlab error = ThresholdExceededError.new(error_message) - if raise_error? - raise(error) - else - # Raven automatically logs to the Rails log if disabled, thus we don't - # need to manually log anything in case Sentry support is not enabled. - Raven.capture_exception(error) - end + raise(error) if raise_error? end def increment diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 7ab85e1c35c..ac3de2a8f71 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -40,12 +40,16 @@ module Gitlab 'a-zA-Z0-9_/\\$\\{\\}\\. \\-' end + def environment_name_regex_chars_without_slash + 'a-zA-Z0-9_\\$\\{\\}\\. -' + end + def environment_name_regex - @environment_name_regex ||= /\A[#{environment_name_regex_chars}]+\z/.freeze + @environment_name_regex ||= /\A[#{environment_name_regex_chars_without_slash}]([#{environment_name_regex_chars}]*[#{environment_name_regex_chars_without_slash}])?\z/.freeze end def environment_name_regex_message - "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces" + "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces, but it cannot start or end with '/'" end def kubernetes_namespace_regex diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 7362514167f..5a5ae7f19d4 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -1,7 +1,7 @@ module Gitlab class SearchResults class FoundBlob - attr_reader :id, :filename, :basename, :ref, :startline, :data + attr_reader :id, :filename, :basename, :ref, :startline, :data, :project_id def initialize(opts = {}) @id = opts.fetch(:id, nil) @@ -10,6 +10,8 @@ module Gitlab @ref = opts.fetch(:ref, nil) @startline = opts.fetch(:startline, nil) @data = opts.fetch(:data, nil) + @per_page = opts.fetch(:per_page, 20) + @project_id = opts.fetch(:project_id, nil) end def path @@ -21,7 +23,7 @@ module Gitlab end end - attr_reader :current_user, :query + attr_reader :current_user, :query, :per_page # Limit search results by passed projects # It allows us to search only for projects user has access to @@ -33,11 +35,12 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, limit_projects, query, default_project_filter: false) + def initialize(current_user, limit_projects, query, default_project_filter: false, per_page: 20) @current_user = current_user @limit_projects = limit_projects || Project.all @query = query @default_project_filter = default_project_filter + @per_page = per_page end def objects(scope, page = nil, without_count = true) @@ -153,10 +156,6 @@ module Gitlab 'projects' end - def per_page - 20 - end - def project_ids_relation limit_projects.select(:id).reorder(nil) end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index f4a41dc3eda..4ba44e0feef 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -294,7 +294,8 @@ module Gitlab # add_namespace("/path/to/storage", "gitlab") # def add_namespace(storage, name) - Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:add_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).add(name) else @@ -315,7 +316,8 @@ module Gitlab # rm_namespace("/path/to/storage", "gitlab") # def rm_namespace(storage, name) - Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:remove_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).remove(name) else @@ -333,7 +335,8 @@ module Gitlab # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # def mv_namespace(storage, old_name, new_name) - Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:rename_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).rename(old_name, new_name) else @@ -368,7 +371,8 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) - Gitlab::GitalyClient.migrate(:namespace_exists) do |enabled| + Gitlab::GitalyClient.migrate(:namespace_exists, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).exists?(dir_name) else diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 545e7c74f7e..89ca1298120 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -21,22 +21,6 @@ module Gitlab technology(name)&.supported_sizes end - def self.sanitize(key_content) - ssh_type, *parts = key_content.strip.split - - return key_content if parts.empty? - - parts.each_with_object("#{ssh_type} ").with_index do |(part, content), index| - content << part - - if Gitlab::SSHPublicKey.new(content).valid? - break [content, parts[index + 1]].compact.join(' ') # Add the comment part if present - elsif parts.size == index + 1 # return original content if we've reached the last element - break key_content - end - end - end - attr_reader :key_text, :key # Unqualified MD5 fingerprint for compatibility @@ -53,23 +37,23 @@ module Gitlab end def valid? - key.present? && bits && technology.supported_sizes.include?(bits) + key.present? end def type - technology.name if key.present? + technology.name if valid? end def bits - return if key.blank? + return unless valid? case type when :rsa - key.n&.num_bits + key.n.num_bits when :dsa - key.p&.num_bits + key.p.num_bits when :ecdsa - key.group.order&.num_bits + key.group.order.num_bits when :ed25519 256 else diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index c2d3a6b6950..c6942d22926 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -115,7 +115,7 @@ namespace :gemojione do end end - style_path = Rails.root.join(*%w(app assets stylesheets framework emoji-sprites.scss)) + style_path = Rails.root.join(*%w(app assets stylesheets framework emoji_sprites.scss)) # Combine the resized assets into a packed sprite and re-generate the SCSS SpriteFactory.cssurl = "image-url('$IMAGE')" diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 3ab406eff2c..fe5032cae18 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -16,5 +16,54 @@ unless Rails.env.production? task :javascript do Rake::Task['eslint'].invoke end + + desc "GitLab | lint | Run several lint checks" + task :all do + status = 0 + + %w[ + config_lint + haml_lint + scss_lint + flay + gettext:lint + lint:static_verification + ].each do |task| + pid = Process.fork do + rd, wr = IO.pipe + stdout = $stdout.dup + stderr = $stderr.dup + $stdout.reopen(wr) + $stderr.reopen(wr) + + begin + begin + Rake::Task[task].invoke + rescue RuntimeError # The haml_lint tasks raise a RuntimeError + exit(1) + end + rescue SystemExit => ex + msg = "*** Rake task #{task} failed with the following error(s):" + raise ex + ensure + $stdout.reopen(stdout) + $stderr.reopen(stderr) + wr.close + + if msg + warn "\n#{msg}\n\n" + IO.copy_stream(rd, $stderr) + else + IO.copy_stream(rd, $stdout) + end + end + end + + Process.waitpid(pid) + status += $?.exitstatus + end + + exit(status) + end end end |