diff options
Diffstat (limited to 'lib')
64 files changed, 1617 insertions, 408 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 52cd7cbe3db..bbdd2039f43 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -45,9 +45,9 @@ module API end before { allow_access_with_scope :api } - before { Gitlab::I18n.set_locale(current_user) } + before { Gitlab::I18n.locale = current_user&.preferred_language } - after { Gitlab::I18n.reset_locale } + after { Gitlab::I18n.use_default_locale } rescue_from Gitlab::Access::AccessDeniedError do rack_response({ 'message' => '403 Forbidden' }.to_json, 403) @@ -110,6 +110,7 @@ module API mount ::API::Notes mount ::API::NotificationSettings mount ::API::Pipelines + mount ::API::PipelineSchedules mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectSnippets diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 01cc8e8e1ca..e10bd230ae2 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -152,7 +152,10 @@ module API expose :web_url expose :request_access_enabled expose :full_name, :full_path - expose :parent_id + + if ::Group.supports_nested_groups? + expose :parent_id + end expose :statistics, if: :statistics do with_options format_with: -> (value) { value.to_i } do @@ -252,7 +255,9 @@ module API class RepoDiff < Grape::Entity expose :old_path, :new_path, :a_mode, :b_mode, :diff - expose :new_file, :renamed_file, :deleted_file + expose :new_file?, as: :new_file + expose :renamed_file?, as: :renamed_file + expose :deleted_file?, as: :deleted_file end class Milestone < ProjectEntity @@ -681,6 +686,17 @@ module API expose :coverage end + class PipelineSchedule < Grape::Entity + expose :id + expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active + expose :created_at, :updated_at + expose :owner, using: Entities::UserBasic + end + + class PipelineScheduleDetails < PipelineSchedule + expose :last_pipeline, using: Entities::PipelineBasic + end + class EnvironmentBasic < Grape::Entity expose :id, :name, :slug, :external_url end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 3da7d735da8..e14a988a153 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -70,7 +70,11 @@ module API params do requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' - optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' + + if ::Group.supports_nested_groups? + optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' + end + use :optional_params end post do @@ -147,8 +151,8 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute - projects = filter_projects(projects) + projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute + projects = reorder_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 226a7ddd50e..d61450f8258 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -256,31 +256,21 @@ module API # project helpers - def filter_projects(projects) - if params[:membership] - projects = projects.merge(current_user.authorized_projects) - end - - if params[:owned] - projects = projects.merge(current_user.owned_projects) - end - - if params[:starred] - projects = projects.merge(current_user.starred_projects) - end - - if params[:search].present? - projects = projects.search(params[:search]) - end - - if params[:visibility].present? - projects = projects.search_by_visibility(params[:visibility]) - end - - projects = projects.where(archived: params[:archived]) + def reorder_projects(projects) projects.reorder(params[:order_by] => params[:sort]) end + def project_finder_params + finder_params = {} + finder_params[:owned] = true if params[:owned].present? + finder_params[:non_public] = true if params[:membership].present? + finder_params[:starred] = true if params[:starred].present? + finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] + finder_params[:archived] = params[:archived] + finder_params[:search] = params[:search] if params[:search] + finder_params + end + # file helpers def uploaded_file(field, uploads_path) diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 96aaaf868ea..9ebd4841296 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -136,14 +136,15 @@ module API post "/notify_post_receive" do status 200 - return unless Gitlab::GitalyClient.enabled? - - begin - repository = wiki? ? project.wiki.repository : project.repository - Gitlab::GitalyClient::Notifications.new(repository.raw_repository).post_receive - rescue GRPC::Unavailable => e - render_api_error!(e, 500) - end + # TODO: Re-enable when Gitaly is processing the post-receive notification + # return unless Gitlab::GitalyClient.enabled? + # + # begin + # repository = wiki? ? project.wiki.repository : project.repository + # Gitlab::GitalyClient::Notifications.new(repository.raw_repository).post_receive + # rescue GRPC::Unavailable => e + # render_api_error!(e, 500) + # end end end end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb new file mode 100644 index 00000000000..93d89209934 --- /dev/null +++ b/lib/api/pipeline_schedules.rb @@ -0,0 +1,131 @@ +module API + class PipelineSchedules < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get all pipeline schedules' do + success Entities::PipelineSchedule + end + params do + use :pagination + optional :scope, type: String, values: %w[active inactive], + desc: 'The scope of pipeline schedules' + end + get ':id/pipeline_schedules' do + authorize! :read_pipeline_schedule, user_project + + schedules = PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope]) + .preload([:owner, :last_pipeline]) + present paginate(schedules), with: Entities::PipelineSchedule + end + + desc 'Get a single pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + end + get ':id/pipeline_schedules/:pipeline_schedule_id' do + authorize! :read_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + present pipeline_schedule, with: Entities::PipelineScheduleDetails + end + + desc 'Create a new pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :description, type: String, desc: 'The description of pipeline schedule' + requires :ref, type: String, desc: 'The branch/tag name will be triggered' + requires :cron, type: String, desc: 'The cron' + optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone' + optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule' + end + post ':id/pipeline_schedules' do + authorize! :create_pipeline_schedule, user_project + + pipeline_schedule = Ci::CreatePipelineScheduleService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if pipeline_schedule.persisted? + present pipeline_schedule, with: Entities::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Edit a pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + optional :description, type: String, desc: 'The description of pipeline schedule' + optional :ref, type: String, desc: 'The branch/tag name will be triggered' + optional :cron, type: String, desc: 'The cron' + optional :cron_timezone, type: String, desc: 'The timezone' + optional :active, type: Boolean, desc: 'The activation of pipeline schedule' + end + put ':id/pipeline_schedules/:pipeline_schedule_id' do + authorize! :update_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + if pipeline_schedule.update(declared_params(include_missing: false)) + present pipeline_schedule, with: Entities::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Take ownership of a pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + end + post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do + authorize! :update_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + if pipeline_schedule.own!(current_user) + present pipeline_schedule, with: Entities::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Delete a pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + end + delete ':id/pipeline_schedules/:pipeline_schedule_id' do + authorize! :admin_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + status :accepted + present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails + end + end + + helpers do + def pipeline_schedule + @pipeline_schedule ||= + user_project.pipeline_schedules + .preload(:owner, :last_pipeline) + .find_by(id: params.delete(:pipeline_schedule_id)) + end + end + end +end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index ed5004e8d1a..d00d4fe1737 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -21,6 +21,7 @@ module API optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + optional :tag_list, type: Array[String], desc: 'The list of tags for a project' end params :optional_params do @@ -58,6 +59,8 @@ module API optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' + optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' + optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' end params :create_params do @@ -65,16 +68,19 @@ module API optional :import_url, type: String, desc: 'URL from which the project is imported' end - def present_projects(projects, options = {}) + def present_projects(options = {}) + projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute + projects = reorder_projects(projects) + projects = projects.with_statistics if params[:statistics] + projects = projects.with_issues_enabled if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] + options = options.reverse_merge( - with: Entities::Project, - current_user: current_user, - simple: params[:simple] + with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, + statistics: params[:statistics], + current_user: current_user ) - - projects = filter_projects(projects) - projects = projects.with_statistics if options[:statistics] - options[:with] = Entities::BasicProjectDetails if options[:simple] + options[:with] = Entities::BasicProjectDetails if params[:simple] present paginate(projects), options end @@ -88,8 +94,7 @@ module API use :statistics_params end get do - entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics] + present_projects end desc 'Create new project' do @@ -225,6 +230,7 @@ module API :request_access_enabled, :shared_runners_enabled, :snippets_enabled, + :tag_list, :visibility, :wiki_enabled ] diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 8f16e532ecb..14d2bff9cb5 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -85,7 +85,7 @@ module API optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded' optional :format, type: String, desc: 'The archive format' end - get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do + get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do begin send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue diff --git a/lib/api/services.rb b/lib/api/services.rb index cb07df9e249..47bd9940f77 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -304,7 +304,13 @@ module API required: true, name: :url, type: String, - desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com' + desc: 'The base URL to the JIRA instance web interface which is being linked to this GitLab project. E.g., https://jira.example.com' + }, + { + required: false, + name: :api_url, + type: String, + desc: 'The base URL to the JIRA instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com' }, { required: true, diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 332f233bf5e..2e1b243c2db 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -137,7 +137,10 @@ module API expose :web_url expose :request_access_enabled expose :full_name, :full_path - expose :parent_id + + if ::Group.supports_nested_groups? + expose :parent_id + end expose :statistics, if: :statistics do with_options format_with: -> (value) { value.to_i } do diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb index 6187445fc8d..2c52d21fa1c 100644 --- a/lib/api/v3/groups.rb +++ b/lib/api/v3/groups.rb @@ -74,7 +74,11 @@ module API params do requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' - optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' + + if ::Group.supports_nested_groups? + optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' + end + use :optional_params end post do diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb index 0f234d4cdad..d9e76560d03 100644 --- a/lib/api/v3/helpers.rb +++ b/lib/api/v3/helpers.rb @@ -14,6 +14,33 @@ module API authorize! access_level, merge_request merge_request end + + # project helpers + + def filter_projects(projects) + if params[:membership] + projects = projects.merge(current_user.authorized_projects) + end + + if params[:owned] + projects = projects.merge(current_user.owned_projects) + end + + if params[:starred] + projects = projects.merge(current_user.starred_projects) + end + + if params[:search].present? + projects = projects.search(params[:search]) + end + + if params[:visibility].present? + projects = projects.where(visibility_level: Gitlab::VisibilityLevel.level_value(params[:visibility])) + end + + projects = projects.where(archived: params[:archived]) + projects.reorder(params[:order_by] => params[:sort]) + end end end end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 164612cb8dd..896c00b88e7 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -147,7 +147,7 @@ module API get '/starred' do authenticate! - present_projects current_user.viewable_starred_projects + present_projects ProjectsFinder.new(current_user: current_user, params: { starred: true }).execute end desc 'Get all projects for admin user' do diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb index e4d14bc8168..0eaa0de2eef 100644 --- a/lib/api/v3/repositories.rb +++ b/lib/api/v3/repositories.rb @@ -72,7 +72,7 @@ module API optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded' optional :format, type: String, desc: 'The archive format' end - get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do + get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do begin send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 330cd963626..f755c99ea4a 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -84,7 +84,11 @@ module Backup Dir.chdir(backup_path) do backup_file_list.each do |file| - next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/ + # For backward compatibility, there are 3 names the backups can have: + # - 1495527122_gitlab_backup.tar + # - 1495527068_2017_05_23_gitlab_backup.tar + # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar + next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+.*)?)?_gitlab_backup\.tar$/ timestamp = $1.to_i diff --git a/lib/banzai/filter/ascii_doc_post_processing_filter.rb b/lib/banzai/filter/ascii_doc_post_processing_filter.rb new file mode 100644 index 00000000000..c9fcf057c5f --- /dev/null +++ b/lib/banzai/filter/ascii_doc_post_processing_filter.rb @@ -0,0 +1,13 @@ +module Banzai + module Filter + class AsciiDocPostProcessingFilter < HTML::Pipeline::Filter + def call + doc.search('[data-math-style]').each do |node| + node.set_attribute('class', 'code math js-render-math') + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 522217deae4..2d6e8ffc90f 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -31,6 +31,10 @@ module Banzai # Allow span elements whitelist[:elements].push('span') + # Allow data-math-style attribute in order to support LaTeX formatting + whitelist[:attributes]['code'] = %w(data-math-style) + whitelist[:attributes]['pre'] = %w(data-math-style) + # Allow html5 details/summary elements whitelist[:elements].push('details') whitelist[:elements].push('summary') diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb new file mode 100644 index 00000000000..1048b927cd3 --- /dev/null +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -0,0 +1,14 @@ +module Banzai + module Pipeline + class AsciiDocPipeline < BasePipeline + def self.filters + FilterArray[ + Filter::SanitizationFilter, + Filter::ExternalLinkFilter, + Filter::PlantumlFilter, + Filter::AsciiDocPostProcessingFilter + ] + end + end + end +end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index c2503fa2adc..d99a3bfa625 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -163,14 +163,15 @@ module Banzai # been queried the object is returned from the cache. def collection_objects_for_ids(collection, ids) if RequestStore.active? + ids = ids.map(&:to_i) cache = collection_cache[collection_cache_key(collection)] - to_query = ids.map(&:to_i) - cache.keys + to_query = ids - cache.keys unless to_query.empty? collection.where(id: to_query).each { |row| cache[row.id] = row } end - cache.values + cache.values_at(*ids) else collection.where(id: ids) end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index 5f379756c11..6fc1d56d7a0 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -1,9 +1,9 @@ class GroupUrlConstrainer def matches?(request) - id = request.params[:id] + full_path = request.params[:group_id] || request.params[:id] - return false unless DynamicPathValidator.valid?(id) + return false unless DynamicPathValidator.valid_group_path?(full_path) - Group.find_by_full_path(id, follow_redirects: request.get?).present? + Group.find_by_full_path(full_path, follow_redirects: request.get?).present? end end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index 6f542f63f98..4c0aee6c48f 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -2,9 +2,9 @@ class ProjectUrlConstrainer def matches?(request) namespace_path = request.params[:namespace_id] project_path = request.params[:project_id] || request.params[:id] - full_path = namespace_path + '/' + project_path + full_path = [namespace_path, project_path].join('/') - return false unless DynamicPathValidator.valid?(full_path) + return false unless DynamicPathValidator.valid_project_path?(full_path) Project.find_by_full_path(full_path, follow_redirects: request.get?).present? end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index 28159dc0dec..d16ae7f3f40 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -1,5 +1,9 @@ class UserUrlConstrainer def matches?(request) - User.find_by_full_path(request.params[:username], follow_redirects: request.get?).present? + full_path = request.params[:username] + + return false unless DynamicPathValidator.valid_user_path?(full_path) + + User.find_by_full_path(full_path, follow_redirects: request.get?).present? end end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 96d38f6daa0..3d41ac76406 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -20,21 +20,20 @@ module Gitlab backend: :gitlab_html5, attributes: DEFAULT_ADOC_ATTRS } - context[:pipeline] = :markup + context[:pipeline] = :ascii_doc plantuml_setup html = ::Asciidoctor.convert(input, asciidoc_opts) html = Banzai.render(html, context) - html.html_safe end def self.plantuml_setup Asciidoctor::PlantUml.configure do |conf| - conf.url = ApplicationSetting.current.plantuml_url - conf.svg_enable = ApplicationSetting.current.plantuml_enabled - conf.png_enable = ApplicationSetting.current.plantuml_enabled + conf.url = current_application_settings.plantuml_url + conf.svg_enable = current_application_settings.plantuml_enabled + conf.png_enable = current_application_settings.plantuml_enabled conf.txt_enable = false end end @@ -47,13 +46,13 @@ module Gitlab def stem(node) return super unless node.style.to_sym == :latexmath - %(<pre#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="display"><code>#{node.content}</code></pre>) + %(<pre#{id_attribute(node)} data-math-style="display"><code>#{node.content}</code></pre>) end def inline_quoted(node) return super unless node.type.to_sym == :latexmath - %(<code#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="inline">#{node.text}</code>) + %(<code#{id_attribute(node)} data-math-style="inline">#{node.text}</code>) end private diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index fa462cbe095..c4c0623df6c 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -73,7 +73,7 @@ module Gitlab match = "" - stream.each_line do |line| + reverse_line do |line| matches = line.scan(regex) next unless matches.is_a?(Array) next if matches.empty? @@ -86,34 +86,39 @@ module Gitlab nil rescue # if bad regex or something goes wrong we dont want to interrupt transition - # so we just silentrly ignore error for now + # so we just silently ignore error for now end private - def read_last_lines(last_lines) - chunks = [] - pos = lines = 0 - max = stream.size - - # We want an extra line to make sure fist line has full contents - while lines <= last_lines && pos < max - pos += BUFFER_SIZE - - buf = - if pos <= max - stream.seek(-pos, IO::SEEK_END) - stream.read(BUFFER_SIZE) - else # Reached the head, read only left - stream.seek(0) - stream.read(BUFFER_SIZE - (pos - max)) - end - - lines += buf.count("\n") - chunks.unshift(buf) + def read_last_lines(limit) + to_enum(:reverse_line).first(limit).reverse.join + end + + def reverse_line + stream.seek(0, IO::SEEK_END) + debris = '' + + until (buf = read_backward(BUFFER_SIZE)).empty? + buf += debris + debris, *lines = buf.each_line.to_a + lines.reverse_each do |line| + yield(line.force_encoding('UTF-8')) + end end - chunks.join.lines.last(last_lines).join + yield(debris.force_encoding('UTF-8')) unless debris.empty? + end + + def read_backward(length) + cur_offset = stream.tell + start = cur_offset - length + start = 0 if start < 0 + + stream.seek(start, IO::SEEK_SET) + stream.read(cur_offset - start).tap do + stream.seek(start, IO::SEEK_SET) + end end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index e76c9abbe04..a412bb6dbd2 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -42,7 +42,7 @@ module Gitlab 'in the body of your migration class' end - if Database.postgresql? + if supports_drop_index_concurrently? options = options.merge({ algorithm: :concurrently }) disable_statement_timeout end @@ -50,6 +50,39 @@ module Gitlab remove_index(table_name, options.merge({ column: column_name })) end + # Removes an existing index, concurrently when supported + # + # On PostgreSQL this method removes an index concurrently. + # + # Example: + # + # remove_concurrent_index :users, "index_X_by_Y" + # + # See Rails' `remove_index` for more info on the available arguments. + def remove_concurrent_index_by_name(table_name, index_name, options = {}) + if transaction_open? + raise 'remove_concurrent_index_by_name can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if supports_drop_index_concurrently? + options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout + end + + remove_index(table_name, options.merge({ name: index_name })) + end + + # Only available on Postgresql >= 9.2 + def supports_drop_index_concurrently? + return false unless Database.postgresql? + + version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i + + version >= 90200 + end + # Adds a foreign key with only minimal locking on the tables involved. # # This method only requires minimal locking when using PostgreSQL. When diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb index c45ae8feb2c..3192bf6f667 100644 --- a/lib/gitlab/dependency_linker.rb +++ b/lib/gitlab/dependency_linker.rb @@ -1,7 +1,16 @@ module Gitlab module DependencyLinker LINKERS = [ - GemfileLinker + GemfileLinker, + GemspecLinker, + PackageJsonLinker, + ComposerJsonLinker, + PodfileLinker, + PodspecLinker, + PodspecJsonLinker, + CartfileLinker, + GodepsJsonLinker, + RequirementsTxtLinker ].freeze def self.linker(blob_name) diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index 5f4027e7e81..7bbd154eb03 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -1,6 +1,9 @@ module Gitlab module DependencyLinker class BaseLinker + URL_REGEX = %r{https?://[^'" ]+}.freeze + REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze + class_attribute :file_type def self.support?(blob_name) @@ -26,59 +29,20 @@ module Gitlab private - def package_url(name) - raise NotImplementedError - end - def link_dependencies raise NotImplementedError end - def package_link(name, url = package_url(name)) - return name unless url - - %{<a href="#{ERB::Util.html_escape_once(url)}" rel="noopener noreferrer" target="_blank">#{ERB::Util.html_escape_once(name)}</a>} + def license_url(name) + Licensee::License.find(name)&.url end - # Links package names in a method call or assignment string argument. - # - # Example: - # link_method_call("gem") - # # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"` - # - # link_method_call("gem", "specific_package") - # # Will link `specific_package` in `gem "specific_package"` - # - # link_method_call("github", /[^\/]+\/[^\/]+/) - # # Will link `user/repo` in `github "user/repo"`, but not `github "package"` - # - # link_method_call(%w[add_dependency add_development_dependency]) - # # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"` - # - # link_method_call("name") - # # Will link `package` in `self.name = "package"` - def link_method_call(method_names, value = nil, &url_proc) - value = - case value - when String - Regexp.escape(value) - when nil - /[^'"]+/ - else - value - end - - method_names = Array(method_names).map { |name| Regexp.escape(name) } - - regex = %r{ - #{Regexp.union(method_names)} # Method name - \s* # Whitespace - [(=]? # Opening brace or equals sign - \s* # Whitespace - ['"](?<name>#{value})['"] # Package name in quotes - }x + def github_url(name) + "https://github.com/#{name}" + end - link_regex(regex, &url_proc) + def link_tag(name, url) + %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>} end # Links package names based on regex. @@ -86,13 +50,13 @@ module Gitlab # Example: # link_regex(/(github:|:github =>)\s*['"](?<name>[^'"]+)['"]/) # # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"` - def link_regex(regex) + def link_regex(regex, &url_proc) highlighted_lines.map!.with_index do |rich_line, i| marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe) marker.mark(regex, group: :name) do |text, left:, right:| - url = block_given? ? yield(text) : package_url(text) - package_link(text, url) + url = yield(text) + url ? link_tag(text, url) : text end end end @@ -104,6 +68,19 @@ module Gitlab def highlighted_lines @highlighted_lines ||= highlighted_text.lines end + + def regexp_for_value(value, default: /[^'" ]+/) + case value + when Array + Regexp.union(value.map { |v| regexp_for_value(v, default: default) }) + when String + Regexp.escape(value) + when Regexp + value + else + default + end + end end end end diff --git a/lib/gitlab/dependency_linker/cartfile_linker.rb b/lib/gitlab/dependency_linker/cartfile_linker.rb new file mode 100644 index 00000000000..4f69f2c4ab2 --- /dev/null +++ b/lib/gitlab/dependency_linker/cartfile_linker.rb @@ -0,0 +1,14 @@ +module Gitlab + module DependencyLinker + class CartfileLinker < MethodLinker + self.file_type = :cartfile + + private + + def link_dependencies + link_method_call('github', REPO_REGEX, &method(:github_url)) + link_method_call(%w[github git binary], URL_REGEX, &:itself) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/cocoapods.rb b/lib/gitlab/dependency_linker/cocoapods.rb new file mode 100644 index 00000000000..2fbde7da1b4 --- /dev/null +++ b/lib/gitlab/dependency_linker/cocoapods.rb @@ -0,0 +1,10 @@ +module Gitlab + module DependencyLinker + module Cocoapods + def package_url(name) + package = name.split("/", 2).first + "https://cocoapods.org/pods/#{package}" + end + end + end +end diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb new file mode 100644 index 00000000000..0245bf4077a --- /dev/null +++ b/lib/gitlab/dependency_linker/composer_json_linker.rb @@ -0,0 +1,18 @@ +module Gitlab + module DependencyLinker + class ComposerJsonLinker < PackageJsonLinker + self.file_type = :composer_json + + private + + def link_packages + link_packages_at_key("require", &method(:package_url)) + link_packages_at_key("require-dev", &method(:package_url)) + end + + def package_url(name) + "https://packagist.org/packages/#{name}" if name =~ %r{\A#{REPO_REGEX}\z} + end + end + end +end diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb index 9b82e126528..d034ea67387 100644 --- a/lib/gitlab/dependency_linker/gemfile_linker.rb +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -1,28 +1,31 @@ module Gitlab module DependencyLinker - class GemfileLinker < BaseLinker + class GemfileLinker < MethodLinker self.file_type = :gemfile private def link_dependencies - # Link `gem "package_name"` to https://rubygems.org/gems/package_name - link_method_call("gem") + link_urls + link_packages + end + def link_urls # Link `github: "user/repo"` to https://github.com/user/repo - link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/) do |name| - "https://github.com/#{name}" - end + link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url)) # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo - link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>https?://[^'"]+)['"]}) { |url| url } + link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) # Link `source "https://rubygems.org"` to https://rubygems.org - link_method_call("source", %r{https?://[^'"]+}) { |url| url } + link_method_call('source', URL_REGEX, &:itself) end - def package_url(name) - "https://rubygems.org/gems/#{name}" + def link_packages + # Link `gem "package_name"` to https://rubygems.org/gems/package_name + link_method_call('gem') do |name| + "https://rubygems.org/gems/#{name}" + end end end end diff --git a/lib/gitlab/dependency_linker/gemspec_linker.rb b/lib/gitlab/dependency_linker/gemspec_linker.rb new file mode 100644 index 00000000000..f1783ee2ab4 --- /dev/null +++ b/lib/gitlab/dependency_linker/gemspec_linker.rb @@ -0,0 +1,18 @@ +module Gitlab + module DependencyLinker + class GemspecLinker < MethodLinker + self.file_type = :gemspec + + private + + def link_dependencies + link_method_call('homepage', URL_REGEX, &:itself) + link_method_call('license', &method(:license_url)) + + link_method_call(%w[name add_dependency add_runtime_dependency add_development_dependency]) do |name| + "https://rubygems.org/gems/#{name}" + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/godeps_json_linker.rb b/lib/gitlab/dependency_linker/godeps_json_linker.rb new file mode 100644 index 00000000000..fe091baee6d --- /dev/null +++ b/lib/gitlab/dependency_linker/godeps_json_linker.rb @@ -0,0 +1,26 @@ +module Gitlab + module DependencyLinker + class GodepsJsonLinker < JsonLinker + NESTED_REPO_REGEX = %r{([^/]+/)+[^/]+?}.freeze + + self.file_type = :godeps_json + + private + + def link_dependencies + link_json('ImportPath') do |path| + case path + when %r{\A(?<repo>gitlab\.com/#{NESTED_REPO_REGEX})\.git/(?<path>.+)\z}, + %r{\A(?<repo>git(lab|hub)\.com/#{REPO_REGEX})/(?<path>.+)\z} + + "https://#{$~[:repo]}/tree/master/#{$~[:path]}" + when /\Agolang\.org/ + "https://godoc.org/#{path}" + else + "https://#{path}" + end + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/json_linker.rb b/lib/gitlab/dependency_linker/json_linker.rb new file mode 100644 index 00000000000..a8ef25233d8 --- /dev/null +++ b/lib/gitlab/dependency_linker/json_linker.rb @@ -0,0 +1,44 @@ +module Gitlab + module DependencyLinker + class JsonLinker < BaseLinker + def link + return highlighted_text unless json + + super + end + + private + + # Links package names in a JSON key or values. + # + # Example: + # link_json('name') + # # Will link `package` in `"name": "package"` + # + # link_json('name', 'specific_package') + # # Will link `specific_package` in `"name": "specific_package"` + # + # link_json('name', /[^\/]+\/[^\/]+/) + # # Will link `user/repo` in `"name": "user/repo"`, but not `"name": "package"` + # + # link_json('specific_package', '1.0.1', link: :key) + # # Will link `specific_package` in `"specific_package": "1.0.1"` + def link_json(key, value = nil, link: :value, &url_proc) + key = regexp_for_value(key, default: /[^" ]+/) + value = regexp_for_value(value, default: /[^" ]+/) + + if link == :value + value = /(?<name>#{value})/ + else + key = /(?<name>#{key})/ + end + + link_regex(/"#{key}":\s*"#{value}"/, &url_proc) + end + + def json + @json ||= JSON.parse(plain_text) rescue nil + end + end + end +end diff --git a/lib/gitlab/dependency_linker/method_linker.rb b/lib/gitlab/dependency_linker/method_linker.rb new file mode 100644 index 00000000000..0ffa2a83c93 --- /dev/null +++ b/lib/gitlab/dependency_linker/method_linker.rb @@ -0,0 +1,39 @@ +module Gitlab + module DependencyLinker + class MethodLinker < BaseLinker + private + + # Links package names in a method call or assignment string argument. + # + # Example: + # link_method_call('gem') + # # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"` + # + # link_method_call('gem', 'specific_package') + # # Will link `specific_package` in `gem "specific_package"` + # + # link_method_call('github', /[^\/"]+\/[^\/"]+/) + # # Will link `user/repo` in `github "user/repo"`, but not `github "package"` + # + # link_method_call(%w[add_dependency add_development_dependency]) + # # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"` + # + # link_method_call('name') + # # Will link `package` in `self.name = "package"` + def link_method_call(method_name, value = nil, &url_proc) + method_name = regexp_for_value(method_name) + value = regexp_for_value(value) + + regex = %r{ + #{method_name} # Method name + \s* # Whitespace + [(=]? # Opening brace or equals sign + \s* # Whitespace + ['"](?<name>#{value})['"] # Package name in quotes + }x + + link_regex(regex, &url_proc) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/package_json_linker.rb b/lib/gitlab/dependency_linker/package_json_linker.rb new file mode 100644 index 00000000000..330c95f0880 --- /dev/null +++ b/lib/gitlab/dependency_linker/package_json_linker.rb @@ -0,0 +1,44 @@ +module Gitlab + module DependencyLinker + class PackageJsonLinker < JsonLinker + self.file_type = :package_json + + private + + def link_dependencies + link_json('name', json["name"], &method(:package_url)) + link_json('license', &method(:license_url)) + link_json(%w[homepage url], URL_REGEX, &:itself) + + link_packages + end + + def link_packages + link_packages_at_key("dependencies", &method(:package_url)) + link_packages_at_key("devDependencies", &method(:package_url)) + end + + def link_packages_at_key(key, &url_proc) + dependencies = json[key] + return unless dependencies + + dependencies.each do |name, version| + link_json(name, version, link: :key, &url_proc) + + link_json(name) do |value| + case value + when /\A#{URL_REGEX}\z/ + value + when /\A#{REPO_REGEX}\z/ + github_url(value) + end + end + end + end + + def package_url(name) + "https://npmjs.com/package/#{name}" + end + end + end +end diff --git a/lib/gitlab/dependency_linker/podfile_linker.rb b/lib/gitlab/dependency_linker/podfile_linker.rb new file mode 100644 index 00000000000..60ad166ea17 --- /dev/null +++ b/lib/gitlab/dependency_linker/podfile_linker.rb @@ -0,0 +1,15 @@ +module Gitlab + module DependencyLinker + class PodfileLinker < GemfileLinker + include Cocoapods + + self.file_type = :podfile + + private + + def link_packages + link_method_call('pod', &method(:package_url)) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/podspec_json_linker.rb b/lib/gitlab/dependency_linker/podspec_json_linker.rb new file mode 100644 index 00000000000..d82237ed3f1 --- /dev/null +++ b/lib/gitlab/dependency_linker/podspec_json_linker.rb @@ -0,0 +1,32 @@ +module Gitlab + module DependencyLinker + class PodspecJsonLinker < JsonLinker + include Cocoapods + + self.file_type = :podspec_json + + private + + def link_dependencies + link_json('name', json["name"], &method(:package_url)) + link_json('license', &method(:license_url)) + link_json(%w[homepage git], URL_REGEX, &:itself) + + link_packages_at_key("dependencies", &method(:package_url)) + + json["subspecs"]&.each do |subspec| + link_packages_at_key("dependencies", subspec, &method(:package_url)) + end + end + + def link_packages_at_key(key, root = json, &url_proc) + dependencies = root[key] + return unless dependencies + + dependencies.each do |name, _| + link_regex(/"(?<name>#{Regexp.escape(name)})":\s*\[/, &url_proc) + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb new file mode 100644 index 00000000000..a52c7a02439 --- /dev/null +++ b/lib/gitlab/dependency_linker/podspec_linker.rb @@ -0,0 +1,24 @@ +module Gitlab + module DependencyLinker + class PodspecLinker < MethodLinker + include Cocoapods + + STRING_REGEX = /['"](?<name>[^'"]+)['"]/.freeze + + self.file_type = :podspec + + private + + def link_dependencies + link_method_call('homepage', URL_REGEX, &:itself) + + link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) + + link_method_call('license', &method(:license_url)) + link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url)) + + link_method_call(%w[name dependency], &method(:package_url)) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb new file mode 100644 index 00000000000..2e197e5cd94 --- /dev/null +++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb @@ -0,0 +1,17 @@ +module Gitlab + module DependencyLinker + class RequirementsTxtLinker < BaseLinker + self.file_type = :requirements_txt + + private + + def link_dependencies + link_regex(/^(?<name>(?![a-z+]+:)[^#.-][^ ><=;\[]+)/) do |name| + "https://pypi.python.org/pypi/#{name}" + end + + link_regex(%r{^(?<name>https?://[^ ]+)}, &:itself) + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index c6bf25b5874..2aef7fdaa35 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,16 +1,17 @@ module Gitlab module Diff class File - attr_reader :diff, :repository, :diff_refs + attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs - delegate :new_file, :deleted_file, :renamed_file, - :old_path, :new_path, :a_mode, :b_mode, + delegate :new_file?, :deleted_file?, :renamed_file?, + :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, :submodule?, :too_large?, :collapsed?, to: :diff, prefix: false - def initialize(diff, repository:, diff_refs: nil) + def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil) @diff = diff @repository = repository @diff_refs = diff_refs + @fallback_diff_refs = fallback_diff_refs end def position(line) @@ -49,24 +50,60 @@ module Gitlab line_code(line) if line end + def old_sha + diff_refs&.base_sha + end + + def new_sha + diff_refs&.head_sha + end + + def content_sha + return old_content_sha if deleted_file? + return @content_sha if defined?(@content_sha) + + refs = diff_refs || fallback_diff_refs + @content_sha = refs&.head_sha + end + def content_commit - return unless diff_refs + return @content_commit if defined?(@content_commit) + + sha = content_sha + @content_commit = repository.commit(sha) if sha + end + + def old_content_sha + return if new_file? + return @old_content_sha if defined?(@old_content_sha) - repository.commit(deleted_file ? old_ref : new_ref) + refs = diff_refs || fallback_diff_refs + @old_content_sha = refs&.base_sha end def old_content_commit - return unless diff_refs + return @old_content_commit if defined?(@old_content_commit) - repository.commit(old_ref) + sha = old_content_sha + @old_content_commit = repository.commit(sha) if sha end - def old_ref - diff_refs.try(:base_sha) + def blob + return @blob if defined?(@blob) + + sha = content_sha + return @blob = nil unless sha + + repository.blob_at(sha, file_path) end - def new_ref - diff_refs.try(:head_sha) + def old_blob + return @old_blob if defined?(@old_blob) + + sha = old_content_sha + return @old_blob = nil unless sha + + @old_blob = repository.blob_at(sha, old_path) end attr_writer :highlighted_diff_lines @@ -85,10 +122,6 @@ module Gitlab @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize end - def mode_changed? - a_mode && b_mode && a_mode != b_mode - end - def raw_diff diff.diff.to_s end @@ -117,20 +150,8 @@ module Gitlab diff_lines.count(&:removed?) end - def old_blob(commit = old_content_commit) - return unless commit - - repository.blob_at(commit.id, old_path) - end - - def blob(commit = content_commit) - return unless commit - - repository.blob_at(commit.id, file_path) - end - def file_identifier - "#{file_path}-#{new_file}-#{deleted_file}-#{renamed_file}" + "#{file_path}-#{new_file?}-#{deleted_file?}-#{renamed_file?}" end end end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 2b9fc65b985..79836a2fbab 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -2,7 +2,7 @@ module Gitlab module Diff module FileCollection class Base - attr_reader :project, :diff_options, :diff_view, :diff_refs + attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs delegate :count, :size, :real_size, to: :diff_files @@ -10,24 +10,33 @@ module Gitlab ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false) end - def initialize(diffable, project:, diff_options: nil, diff_refs: nil) + def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil) diff_options = self.class.default_options.merge(diff_options || {}) - @diffable = diffable - @diffs = diffable.raw_diffs(diff_options) - @project = project + @diffable = diffable + @diffs = diffable.raw_diffs(diff_options) + @project = project @diff_options = diff_options - @diff_refs = diff_refs + @diff_refs = diff_refs + @fallback_diff_refs = fallback_diff_refs end def diff_files @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } end + def diff_file_with_old_path(old_path) + diff_files.find { |diff_file| diff_file.old_path == old_path } + end + + def diff_file_with_new_path(new_path) + diff_files.find { |diff_file| diff_file.new_path == new_path } + end + private def decorate_diff!(diff) - Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs) + Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs) end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 0bd226ef050..9a58b500a2c 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -8,7 +8,8 @@ module Gitlab super(merge_request_diff, project: merge_request_diff.project, diff_options: diff_options, - diff_refs: merge_request_diff.diff_refs) + diff_refs: merge_request_diff.diff_refs, + fallback_diff_refs: merge_request_diff.fallback_diff_refs) end def diff_files diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 7db896522a9..ed2f541977a 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,7 +3,7 @@ module Gitlab class Highlight attr_reader :diff_file, :diff_lines, :raw_lines, :repository - delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff + delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff def initialize(diff_lines, repository: nil) @repository = repository @@ -61,12 +61,12 @@ module Gitlab def old_lines return unless diff_file - @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_ref, diff_old_path) + @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_sha, diff_old_path) end def new_lines return unless diff_file - @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_ref, diff_new_path) + @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_sha, diff_new_path) end end end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index fc728123c97..4d96778a2b2 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -12,20 +12,26 @@ module Gitlab attr_reader :head_sha def initialize(attrs = {}) + if diff_file = attrs[:diff_file] + attrs[:diff_refs] = diff_file.diff_refs + attrs[:old_path] = diff_file.old_path + attrs[:new_path] = diff_file.new_path + end + + if diff_refs = attrs[:diff_refs] + attrs[:base_sha] = diff_refs.base_sha + attrs[:start_sha] = diff_refs.start_sha + attrs[:head_sha] = diff_refs.head_sha + end + @old_path = attrs[:old_path] @new_path = attrs[:new_path] + @base_sha = attrs[:base_sha] + @start_sha = attrs[:start_sha] + @head_sha = attrs[:head_sha] + @old_line = attrs[:old_line] @new_line = attrs[:new_line] - - if attrs[:diff_refs] - @base_sha = attrs[:diff_refs].base_sha - @start_sha = attrs[:diff_refs].start_sha - @head_sha = attrs[:diff_refs].head_sha - else - @base_sha = attrs[:base_sha] - @start_sha = attrs[:start_sha] - @head_sha = attrs[:head_sha] - end end # `Gitlab::Diff::Position` objects are stored as serialized attributes in @@ -129,11 +135,11 @@ module Gitlab end def diff_line(repository) - @diff_line ||= diff_file(repository).line_for_position(self) + @diff_line ||= diff_file(repository)&.line_for_position(self) end def line_code(repository) - @line_code ||= diff_file(repository).line_code_for_position(self) + @line_code ||= diff_file(repository)&.line_code_for_position(self) end private diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index e89ff238ec7..dcabb5f7fe5 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -3,21 +3,21 @@ module Gitlab module Diff class PositionTracer - attr_accessor :repository + attr_accessor :project attr_accessor :old_diff_refs attr_accessor :new_diff_refs attr_accessor :paths - def initialize(repository:, old_diff_refs:, new_diff_refs:, paths: nil) - @repository = repository + def initialize(project:, old_diff_refs:, new_diff_refs:, paths: nil) + @project = project @old_diff_refs = old_diff_refs @new_diff_refs = new_diff_refs @paths = paths end - def trace(old_position) + def trace(ab_position) return unless old_diff_refs&.complete? && new_diff_refs&.complete? - return unless old_position.diff_refs == old_diff_refs + return unless ab_position.diff_refs == old_diff_refs # Suppose we have an MR with source branch `feature` and target branch `master`. # When the MR was created, the head of `master` was commit A, and the @@ -44,14 +44,16 @@ module Gitlab # # For diff notes for diff A->B, the position looks like this: # Position - # base_sha - ID of commit A + # start_sha - ID of commit A # head_sha - ID of commit B + # base_sha - ID of base commit of A and B # old_path - path as of A (nil if file was newly created) # new_path - path as of B (nil if file was deleted) # old_line - line number as of A (nil if file was newly created) # new_line - line number as of B (nil if file was deleted) # - # We can easily update `base_sha` and `head_sha` to hold the IDs of commits C and D, + # We can easily update `start_sha` and `head_sha` to hold the IDs of + # commits C and D, and can trivially determine `base_sha` based on those, # but need to find the paths and line numbers as of C and D. # # If the file was unchanged or newly created in A->B, the path as of D can be found @@ -68,107 +70,161 @@ module Gitlab # by generating diff A->C ("base to base"), finding the diff file with # `diff_file.old_path == position.old_path`, and taking `diff_file.new_path`. # The path as of D can be found by taking diff C->D, finding the diff file - # with that same `old_path` and taking `diff_file.new_path`. + # with `old_path` set to that `diff_file.new_path` and taking `diff_file.new_path`. # The line number as of C can be found by using the LineMapper on diff A->C # and providing the line number as of A. # The line number as of D can be found by using the LineMapper on diff C->D # and providing the line number as of C. - results = nil - results ||= trace_added_line(old_position) if old_position.added? || old_position.unchanged? - results ||= trace_removed_line(old_position) if old_position.removed? || old_position.unchanged? - - return unless results - - file_diff, old_line, new_line = results - - new_position = Position.new( - old_path: file_diff.old_path, - new_path: file_diff.new_path, - head_sha: new_diff_refs.head_sha, - start_sha: new_diff_refs.start_sha, - base_sha: new_diff_refs.base_sha, - old_line: old_line, - new_line: new_line - ) - - # If a position is found, but is not actually contained in the diff, for example - # because it was an unchanged line in the context of a change that was undone, - # we cannot return this as a successful trace. - return unless new_position.diff_line(repository) - - new_position + if ab_position.added? + trace_added_line(ab_position) + elsif ab_position.removed? + trace_removed_line(ab_position) + else # unchanged + trace_unchanged_line(ab_position) + end end private - def trace_added_line(old_position) - file_path = old_position.new_path - - return unless diff_head_to_head - - file_head_to_head = diff_head_to_head.find { |diff_file| diff_file.old_path == file_path } - - file_path = file_head_to_head.new_path if file_head_to_head - - new_line = LineMapper.new(file_head_to_head).old_to_new(old_position.new_line) - - return unless new_line - - file_diff = new_diffs.find { |diff_file| diff_file.new_path == file_path } - return unless file_diff - - old_line = LineMapper.new(file_diff).new_to_old(new_line) - - [file_diff, old_line, new_line] + def trace_added_line(ab_position) + b_path = ab_position.new_path + b_line = ab_position.new_line + + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + d_path = bd_diff&.new_path || b_path + d_line = LineMapper.new(bd_diff).old_to_new(b_line) + + if d_line + cd_diff = cd_diffs.diff_file_with_new_path(d_path) + + c_path = cd_diff&.old_path || d_path + c_line = LineMapper.new(cd_diff).new_to_old(d_line) + + if c_line + # If the line is still in D but also in C, it has turned from an + # added line into an unchanged one. + new_position = position(cd_diff, c_line, d_line) + if valid_position?(new_position) + # If the line is still in the MR, we don't treat this as outdated. + { position: new_position, outdated: false } + else + # If the line is no longer in the MR, we unfortunately cannot show + # the current state on the CD diff, so we treat it as outdated. + ac_diff = ac_diffs.diff_file_with_new_path(c_path) + + { position: position(ac_diff, nil, c_line), outdated: true } + end + else + # If the line is still in D and not in C, it is still added. + { position: position(cd_diff, nil, d_line), outdated: false } + end + else + # If the line is no longer in D, it has been removed from the MR. + { position: position(bd_diff, b_line, nil), outdated: true } + end end - def trace_removed_line(old_position) - file_path = old_position.old_path + def trace_removed_line(ab_position) + a_path = ab_position.old_path + a_line = ab_position.old_line - return unless diff_base_to_base + ac_diff = ac_diffs.diff_file_with_old_path(a_path) - file_base_to_base = diff_base_to_base.find { |diff_file| diff_file.old_path == file_path } + c_path = ac_diff&.new_path || a_path + c_line = LineMapper.new(ac_diff).old_to_new(a_line) - file_path = file_base_to_base.old_path if file_base_to_base + if c_line + cd_diff = cd_diffs.diff_file_with_old_path(c_path) - old_line = LineMapper.new(file_base_to_base).old_to_new(old_position.old_line) + d_path = cd_diff&.new_path || c_path + d_line = LineMapper.new(cd_diff).old_to_new(c_line) - return unless old_line + if d_line + # If the line is still in C but also in D, it has turned from a + # removed line into an unchanged one. + bd_diff = bd_diffs.diff_file_with_new_path(d_path) - file_diff = new_diffs.find { |diff_file| diff_file.old_path == file_path } - return unless file_diff - - new_line = LineMapper.new(file_diff).old_to_new(old_line) + { position: position(bd_diff, nil, d_line), outdated: true } + else + # If the line is still in C and not in D, it is still removed. + { position: position(cd_diff, c_line, nil), outdated: false } + end + else + # If the line is no longer in C, it has been removed outside of the MR. + { position: position(ac_diff, a_line, nil), outdated: true } + end + end - [file_diff, old_line, new_line] + def trace_unchanged_line(ab_position) + a_path = ab_position.old_path + a_line = ab_position.old_line + b_path = ab_position.new_path + b_line = ab_position.new_line + + ac_diff = ac_diffs.diff_file_with_old_path(a_path) + + c_path = ac_diff&.new_path || a_path + c_line = LineMapper.new(ac_diff).old_to_new(a_line) + + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + d_line = LineMapper.new(bd_diff).old_to_new(b_line) + + cd_diff = cd_diffs.diff_file_with_old_path(c_path) + + if c_line && d_line + # If the line is still in C and D, it is still unchanged. + new_position = position(cd_diff, c_line, d_line) + if valid_position?(new_position) + # If the line is still in the MR, we don't treat this as outdated. + { position: new_position, outdated: false } + else + # If the line is no longer in the MR, we unfortunately cannot show + # the current state on the CD diff or any change on the BD diff, + # so we treat it as outdated. + { position: nil, outdated: true } + end + elsif d_line # && !c_line + # If the line is still in D but no longer in C, it has turned from + # an unchanged line into an added one. + # We don't treat this as outdated since the line is still in the MR. + { position: position(cd_diff, nil, d_line), outdated: false } + else # !d_line && (c_line || !c_line) + # If the line is no longer in D, it has turned from an unchanged line + # into a removed one. + { position: position(bd_diff, b_line, nil), outdated: true } + end end - def diff_base_to_base - @diff_base_to_base ||= diff_files(old_diff_refs.base_sha || old_diff_refs.start_sha, new_diff_refs.base_sha || new_diff_refs.start_sha) + def ac_diffs + @ac_diffs ||= compare( + old_diff_refs.base_sha || old_diff_refs.start_sha, + new_diff_refs.base_sha || new_diff_refs.start_sha, + straight: true + ) end - def diff_head_to_head - @diff_head_to_head ||= diff_files(old_diff_refs.head_sha, new_diff_refs.head_sha) + def bd_diffs + @bd_diffs ||= compare(old_diff_refs.head_sha, new_diff_refs.head_sha, straight: true) end - def new_diffs - @new_diffs ||= diff_files(new_diff_refs.start_sha, new_diff_refs.head_sha, use_base: true) + def cd_diffs + @cd_diffs ||= compare(new_diff_refs.start_sha, new_diff_refs.head_sha) end - def diff_files(start_sha, head_sha, use_base: false) - base_sha = self.repository.merge_base(start_sha, head_sha) || start_sha + def compare(start_sha, head_sha, straight: false) + compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) + compare.diffs(paths: paths) + end - diffs = self.repository.raw_repository.diff( - use_base ? base_sha : start_sha, - head_sha, - {}, - *paths - ) + def position(diff_file, old_line, new_line) + Position.new(diff_file: diff_file, old_line: old_line, new_line: new_line) + end - diffs.decorate! do |diff| - Gitlab::Diff::File.new(diff, repository: self.repository) - end + def valid_position?(position) + !!position.diff_line(project.repository) end end end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index ba31041d0c1..cc285162b44 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -10,7 +10,7 @@ module Gitlab # - Ending in `issues/id`/realtime_changes` for the `issue_title` route USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes commit pipelines merge_requests new].freeze - RESERVED_WORDS = DynamicPathValidator::WILDCARD_ROUTES - USED_IN_ROUTES + RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) ROUTES = [ Gitlab::EtagCaching::Router::Route.new( @@ -38,7 +38,7 @@ module Gitlab 'project_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines/\d+\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines/\d+\.json\z), 'project_pipeline' ) ].freeze diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 31d1b66b4f7..deade337354 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -11,6 +11,10 @@ module Gitlab # Stats properties attr_accessor :new_file, :renamed_file, :deleted_file + alias_method :new_file?, :new_file + alias_method :deleted_file?, :deleted_file + alias_method :renamed_file?, :renamed_file + attr_accessor :too_large # The maximum size of a diff to display. @@ -208,6 +212,10 @@ module Gitlab hash end + def mode_changed? + a_mode && b_mode && a_mode != b_mode + end + def submodule? a_mode == '160000' || b_mode == '160000' end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index bcbad8ec829..898a5ae15f2 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -19,22 +19,19 @@ module Gitlab @line_count = 0 @byte_count = 0 @overflow = false + @empty = true @array = Array.new end def each(&block) - if @populated - # @iterator.each is slower than just iterating the array in place - @array.each(&block) - else - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do - each_patch(&block) - end + Gitlab::GitalyClient.migrate(:commit_raw_diffs) do + each_patch(&block) end end def empty? - !@iterator.any? + any? # Make sure the iterator has been exercised + @empty end def overflow? @@ -60,17 +57,17 @@ module Gitlab collection = each_with_index do |element, i| @array[i] = yield(element) end - @populated = true collection end + alias_method :to_ary, :to_a + private def populate! return if @populated each { nil } # force a loop through all diffs - @populated = true nil end @@ -79,15 +76,17 @@ module Gitlab end def each_patch - @iterator.each_with_index do |raw, i| - # First yield cached Diff instances from @array - if @array[i] - yield @array[i] - next - end + i = 0 + @array.each do |diff| + yield diff + i += 1 + end + + return if @overflow + return if @iterator.nil? - # We have exhausted @array, time to create new Diff instances or stop. - break if @overflow + @iterator.each do |raw| + @empty = false if !@all_diffs && i >= @max_files @overflow = true @@ -113,7 +112,13 @@ module Gitlab end yield @array[i] = diff + i += 1 end + + @populated = true + + # Allow iterator to be garbage-collected. It cannot be reused anyway. + @iterator = nil end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 6200bd460ea..319633656ff 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -1,3 +1,5 @@ +# rubocop:disable Metrics/AbcSize + module Gitlab module GonHelper def add_gon_variables @@ -12,6 +14,8 @@ module Gitlab gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url + gon.revision = Gitlab::REVISION + gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb new file mode 100644 index 00000000000..e9d5d52cabb --- /dev/null +++ b/lib/gitlab/group_hierarchy.rb @@ -0,0 +1,104 @@ +module Gitlab + # Retrieving of parent or child groups based on a base ActiveRecord relation. + # + # This class uses recursive CTEs and as a result will only work on PostgreSQL. + class GroupHierarchy + attr_reader :base, :model + + # base - An instance of ActiveRecord::Relation for which to get parent or + # child groups. + def initialize(base) + @base = base + @model = base.model + end + + # Returns a relation that includes the base set of groups and all their + # ancestors (recursively). + def base_and_ancestors + return model.none unless Group.supports_nested_groups? + + base_and_ancestors_cte.apply_to(model.all) + end + + # Returns a relation that includes the base set of groups and all their + # descendants (recursively). + def base_and_descendants + return model.none unless Group.supports_nested_groups? + + base_and_descendants_cte.apply_to(model.all) + end + + # Returns a relation that includes the base groups, their ancestors, and the + # descendants of the base groups. + # + # The resulting query will roughly look like the following: + # + # WITH RECURSIVE ancestors AS ( ... ), + # descendants AS ( ... ) + # SELECT * + # FROM ( + # SELECT * + # FROM ancestors namespaces + # + # UNION + # + # SELECT * + # FROM descendants namespaces + # ) groups; + # + # Using this approach allows us to further add criteria to the relation with + # Rails thinking it's selecting data the usual way. + def all_groups + return base unless Group.supports_nested_groups? + + ancestors = base_and_ancestors_cte + descendants = base_and_descendants_cte + + ancestors_table = ancestors.alias_to(groups_table) + descendants_table = descendants.alias_to(groups_table) + + union = SQL::Union.new([model.unscoped.from(ancestors_table), + model.unscoped.from(descendants_table)]) + + model. + unscoped. + with. + recursive(ancestors.to_arel, descendants.to_arel). + from("(#{union.to_sql}) #{model.table_name}") + end + + private + + def base_and_ancestors_cte + cte = SQL::RecursiveCTE.new(:base_and_ancestors) + + cte << base.except(:order) + + # Recursively get all the ancestors of the base set. + cte << model. + from([groups_table, cte.table]). + where(groups_table[:id].eq(cte.table[:parent_id])). + except(:order) + + cte + end + + def base_and_descendants_cte + cte = SQL::RecursiveCTE.new(:base_and_descendants) + + cte << base.except(:order) + + # Recursively get all the descendants of the base set. + cte << model. + from([groups_table, cte.table]). + where(groups_table[:parent_id].eq(cte.table[:id])). + except(:order) + + cte + end + + def groups_table + model.arel_table + end + end +end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index df962d203b7..e78b7f22e03 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -2,6 +2,9 @@ module Gitlab module HealthChecks class FsShardsCheck extend BaseAbstractCheck + RANDOM_STRING = SecureRandom.hex(1000).freeze + COMMAND_TIMEOUT = '1'.freeze + TIMEOUT_EXECUTABLE = 'timeout'.freeze class << self def readiness @@ -41,8 +44,6 @@ module Gitlab private - RANDOM_STRING = SecureRandom.hex(1000).freeze - def operation_metrics(ok_metric, latency_metric, operation, **labels) with_timing operation do |result, elapsed| [ @@ -63,8 +64,8 @@ module Gitlab @storage_paths ||= Gitlab.config.repositories.storages end - def with_timeout(args) - %w{timeout 1}.concat(args) + def exec_with_timeout(cmd_args, *args, &block) + Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) end def tmp_file_path(storage_name) @@ -78,7 +79,7 @@ module Gitlab def storage_stat_test(storage_name) stat_path = File.join(path(storage_name), '.') begin - _, status = Gitlab::Popen.popen(with_timeout(%W{ stat #{stat_path} })) + _, status = exec_with_timeout(%W{ stat #{stat_path} }) status == 0 rescue Errno::ENOENT File.exist?(stat_path) && File::Stat.new(stat_path).readable? @@ -86,7 +87,7 @@ module Gitlab end def storage_write_test(tmp_path) - _, status = Gitlab::Popen.popen(with_timeout(%W{ tee #{tmp_path} })) do |stdin| + _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin| stdin.write(RANDOM_STRING) end status == 0 @@ -96,7 +97,7 @@ module Gitlab end def storage_read_test(tmp_path) - _, status = Gitlab::Popen.popen(with_timeout(%W{ diff #{tmp_path} - })) do |stdin| + _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin| stdin.write(RANDOM_STRING) end status == 0 @@ -106,7 +107,7 @@ module Gitlab end def delete_test_file(tmp_path) - _, status = Gitlab::Popen.popen(with_timeout(%W{ rm -f #{tmp_path} })) + _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) status == 0 rescue Errno::ENOENT File.delete(tmp_path) rescue Errno::ENOENT diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 3411516319f..5ab3eeb3aff 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -12,15 +12,36 @@ module Gitlab AVAILABLE_LANGUAGES.keys end - def set_locale(current_user) - requested_locale = current_user&.preferred_language || ::I18n.default_locale - locale = FastGettext.set_locale(requested_locale) - ::I18n.locale = locale + def locale + FastGettext.locale end - def reset_locale + def locale=(locale_string) + requested_locale = locale_string || ::I18n.default_locale + new_locale = FastGettext.set_locale(requested_locale) + ::I18n.locale = new_locale + end + + def use_default_locale FastGettext.set_locale(::I18n.default_locale) ::I18n.locale = ::I18n.default_locale end + + def with_locale(locale_string) + original_locale = locale + + self.locale = locale_string + yield + ensure + self.locale = original_locale + end + + def with_user_locale(user, &block) + with_locale(user&.preferred_language, &block) + end + + def with_default_locale(&block) + with_locale(::I18n.default_locale, &block) + end end end diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb index 9ad7a38d505..ac9d66c836d 100644 --- a/lib/gitlab/o_auth/provider.rb +++ b/lib/gitlab/o_auth/provider.rb @@ -22,7 +22,11 @@ module Gitlab def self.config_for(name) name = name.to_s if ldap_provider?(name) - Gitlab::LDAP::Config.new(name).options + if Gitlab::LDAP::Config.valid_provider?(name) + Gitlab::LDAP::Config.new(name).options + else + nil + end else Gitlab.config.omniauth.providers.find { |provider| provider.name == name } end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb new file mode 100644 index 00000000000..1c0abc9f7cf --- /dev/null +++ b/lib/gitlab/path_regex.rb @@ -0,0 +1,264 @@ +module Gitlab + module PathRegex + extend self + + # All routes that appear on the top level must be listed here. + # This will make sure that groups cannot be created with these names + # as these routes would be masked by the paths already in place. + # + # Example: + # /api/api-project + # + # the path `api` shouldn't be allowed because it would be masked by `api/*` + # + TOP_LEVEL_ROUTES = %w[ + - + .well-known + abuse_reports + admin + all + api + assets + autocomplete + ci + dashboard + explore + files + groups + health_check + help + hooks + import + invites + issues + jwt + koding + member + merge_requests + new + notes + notification_settings + oauth + profile + projects + public + repository + robots.txt + s + search + sent_notifications + services + snippets + teams + u + unicorn_test + unsubscribes + uploads + users + ].freeze + + # This list should contain all words following `/*namespace_id/:project_id` in + # routes that contain a second wildcard. + # + # Example: + # /*namespace_id/:project_id/badges/*ref/build + # + # If `badges` was allowed as a project/group name, we would not be able to access the + # `badges` route for those projects: + # + # Consider a namespace with path `foo/bar` and a project called `badges`. + # The route to the build badge would then be `/foo/bar/badges/badges/master/build.svg` + # + # When accessing this path the route would be matched to the `badges` path + # with the following params: + # - namespace_id: `foo` + # - project_id: `bar` + # - ref: `badges/master` + # + # Failing to find the project, this would result in a 404. + # + # By rejecting `badges` the router can _count_ on the fact that `badges` will + # be preceded by the `namespace/project`. + PROJECT_WILDCARD_ROUTES = %w[ + badges + blame + blob + builds + commits + create + create_dir + edit + environments/folders + files + find_file + gitlab-lfs/objects + info/lfs/objects + new + preview + raw + refs + tree + update + wikis + ].freeze + + # These are all the paths that follow `/groups/*id/ or `/groups/*group_id` + # We need to reject these because we have a `/groups/*id` page that is the same + # as the `/*id`. + # + # If we would allow a subgroup to be created with the name `activity` then + # this group would not be accessible through `/groups/parent/activity` since + # this would map to the activity-page of its parent. + GROUP_ROUTES = %w[ + activity + analytics + audit_events + avatar + edit + group_members + hooks + issues + labels + ldap + ldap_group_links + merge_requests + milestones + notification_setting + pipeline_quota + projects + subgroups + ].freeze + + ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES + ILLEGAL_GROUP_PATH_WORDS = (PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze + + # The namespace regex is used in JavaScript to validate usernames in the "Register" form. However, Javascript + # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`. + # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to + # allow non-regex validations, etc), `NAMESPACE_FORMAT_REGEX_JS` serves as a Javascript-compatible version of + # `NAMESPACE_FORMAT_REGEX`, with the negative lookbehind assertion removed. This means that the client-side validation + # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. + PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze + NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze + + NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze + NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/.freeze + PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/.freeze + FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/)*#{NAMESPACE_FORMAT_REGEX}}.freeze + + def root_namespace_route_regex + @root_namespace_route_regex ||= begin + illegal_words = Regexp.new(Regexp.union(TOP_LEVEL_ROUTES).source, Regexp::IGNORECASE) + + single_line_regexp %r{ + (?!(#{illegal_words})/) + #{NAMESPACE_FORMAT_REGEX} + }x + end + end + + def full_namespace_route_regex + @full_namespace_route_regex ||= begin + illegal_words = Regexp.new(Regexp.union(ILLEGAL_GROUP_PATH_WORDS).source, Regexp::IGNORECASE) + + single_line_regexp %r{ + #{root_namespace_route_regex} + (?: + / + (?!#{illegal_words}/) + #{NAMESPACE_FORMAT_REGEX} + )* + }x + end + end + + def project_route_regex + @project_route_regex ||= begin + illegal_words = Regexp.new(Regexp.union(ILLEGAL_PROJECT_PATH_WORDS).source, Regexp::IGNORECASE) + + single_line_regexp %r{ + (?!(#{illegal_words})/) + #{PROJECT_PATH_FORMAT_REGEX} + }x + end + end + + def project_git_route_regex + @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze + end + + def root_namespace_path_regex + @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z} + end + + def full_namespace_path_regex + @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} + end + + def project_path_regex + @project_path_regex ||= %r{\A#{project_route_regex}/\z} + end + + def full_project_path_regex + @full_project_path_regex ||= %r{\A#{full_namespace_route_regex}/#{project_route_regex}/\z} + end + + def full_namespace_format_regex + @namespace_format_regex ||= /A#{FULL_NAMESPACE_FORMAT_REGEX}\z/.freeze + end + + def namespace_format_regex + @namespace_format_regex ||= /\A#{NAMESPACE_FORMAT_REGEX}\z/.freeze + end + + def namespace_format_message + "can contain only letters, digits, '_', '-' and '.'. " \ + "Cannot start with '-' or end in '.', '.git' or '.atom'." \ + end + + def project_path_format_regex + @project_path_format_regex ||= /\A#{PROJECT_PATH_FORMAT_REGEX}\z/.freeze + end + + def project_path_format_message + "can contain only letters, digits, '_', '-' and '.'. " \ + "Cannot start with '-', end in '.git' or end in '.atom'" \ + end + + def archive_formats_regex + # |zip|tar| tar.gz | tar.bz2 | + @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze + end + + def git_reference_regex + # Valid git ref regex, see: + # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html + + @git_reference_regex ||= single_line_regexp %r{ + (?! + (?# doesn't begins with) + \/| (?# rule #6) + (?# doesn't contain) + .*(?: + [\/.]\.| (?# rule #1,3) + \/\/| (?# rule #6) + @\{| (?# rule #8) + \\ (?# rule #9) + ) + ) + [^\000-\040\177~^:?*\[]+ (?# rule #4-5) + (?# doesn't end with) + (?<!\.lock) (?# rule #1) + (?<![\/.]) (?# rule #6-7) + }x + end + + private + + def single_line_regexp(regex) + # Turns a multiline extended regexp into a single line one, + # beacuse `rake routes` breaks on multiline regexes. + Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze + end + end +end diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb new file mode 100644 index 00000000000..bb0df1e3dad --- /dev/null +++ b/lib/gitlab/project_authorizations/with_nested_groups.rb @@ -0,0 +1,125 @@ +module Gitlab + module ProjectAuthorizations + # Calculating new project authorizations when supporting nested groups. + # + # This class relies on Common Table Expressions to efficiently get all data, + # including data for nested groups. As a result this class can only be used + # on PostgreSQL. + class WithNestedGroups + attr_reader :user + + # user - The User object for which to calculate the authorizations. + def initialize(user) + @user = user + end + + def calculate + cte = recursive_cte + cte_alias = cte.table.alias(Group.table_name) + projects = Project.arel_table + links = ProjectGroupLink.arel_table + + relations = [ + # The project a user has direct access to. + user.projects.select_for_project_authorization, + + # The personal projects of the user. + user.personal_projects.select_as_master_for_project_authorization, + + # Projects that belong directly to any of the groups the user has + # access to. + Namespace. + unscoped. + select([alias_as_column(projects[:id], 'project_id'), + cte_alias[:access_level]]). + from(cte_alias). + joins(:projects), + + # Projects shared with any of the namespaces the user has access to. + Namespace. + unscoped. + select([links[:project_id], + least(cte_alias[:access_level], + links[:group_access], + 'access_level')]). + from(cte_alias). + joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id'). + joins('INNER JOIN projects ON projects.id = project_group_links.project_id'). + joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id'). + where('p_ns.share_with_group_lock IS FALSE') + ] + + union = Gitlab::SQL::Union.new(relations) + + ProjectAuthorization. + unscoped. + with. + recursive(cte.to_arel). + select_from_union(union) + end + + private + + # Builds a recursive CTE that gets all the groups the current user has + # access to, including any nested groups. + def recursive_cte + cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) + members = Member.arel_table + namespaces = Namespace.arel_table + + # Namespaces the user is a member of. + cte << user.groups. + select([namespaces[:id], members[:access_level]]). + except(:order) + + # Sub groups of any groups the user is a member of. + cte << Group.select([namespaces[:id], + greatest(members[:access_level], + cte.table[:access_level], 'access_level')]). + joins(join_cte(cte)). + joins(join_members). + except(:order) + + cte + end + + # Builds a LEFT JOIN to join optional memberships onto the CTE. + def join_members + members = Member.arel_table + namespaces = Namespace.arel_table + + cond = members[:source_id]. + eq(namespaces[:id]). + and(members[:source_type].eq('Namespace')). + and(members[:requested_at].eq(nil)). + and(members[:user_id].eq(user.id)) + + Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) + end + + # Builds an INNER JOIN to join namespaces onto the CTE. + def join_cte(cte) + namespaces = Namespace.arel_table + cond = cte.table[:id].eq(namespaces[:parent_id]) + + Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) + end + + def greatest(left, right, column_alias) + sql_function('GREATEST', [left, right], column_alias) + end + + def least(left, right, column_alias) + sql_function('LEAST', [left, right], column_alias) + end + + def sql_function(name, args, column_alias) + alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) + end + + def alias_as_column(value, alias_to) + Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) + end + end + end +end diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb new file mode 100644 index 00000000000..627e8c5fba2 --- /dev/null +++ b/lib/gitlab/project_authorizations/without_nested_groups.rb @@ -0,0 +1,35 @@ +module Gitlab + module ProjectAuthorizations + # Calculating new project authorizations when not supporting nested groups. + class WithoutNestedGroups + attr_reader :user + + # user - The User object for which to calculate the authorizations. + def initialize(user) + @user = user + end + + def calculate + relations = [ + # Projects the user is a direct member of + user.projects.select_for_project_authorization, + + # Personal projects + user.personal_projects.select_as_master_for_project_authorization, + + # Projects of groups the user is a member of + user.groups_projects.select_for_project_authorization, + + # Projects shared with groups the user is a member of + user.groups.joins(:shared_projects).select_for_project_authorization + ] + + union = Gitlab::SQL::Union.new(relations) + + ProjectAuthorization. + unscoped. + select_from_union(union) + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index b7fef5dd068..e4d2a992470 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -2,39 +2,6 @@ module Gitlab module Regex extend self - # The namespace regex is used in Javascript to validate usernames in the "Register" form. However, Javascript - # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`. - # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to - # allow non-regex validatiions, etc), `NAMESPACE_REGEX_STR_JS` serves as a Javascript-compatible version of - # `NAMESPACE_REGEX_STR`, with the negative lookbehind assertion removed. This means that the client-side validation - # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. - PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze - NAMESPACE_REGEX_STR_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze - NO_SUFFIX_REGEX_STR = '(?<!\.git|\.atom)'.freeze - NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR_JS})#{NO_SUFFIX_REGEX_STR}".freeze - PROJECT_REGEX_STR = "(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX_STR}".freeze - - # Same as NAMESPACE_REGEX_STR but allows `/` in the path. - # So `group/subgroup` will match this regex but not NAMESPACE_REGEX_STR - FULL_NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR}/)*#{NAMESPACE_REGEX_STR}".freeze - - def namespace_regex - @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze - end - - def full_namespace_regex - @full_namespace_regex ||= %r{\A#{FULL_NAMESPACE_REGEX_STR}\z} - end - - def namespace_route_regex - @namespace_route_regex ||= /#{NAMESPACE_REGEX_STR}/.freeze - end - - def namespace_regex_message - "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-' or end in '.', '.git' or '.atom'." \ - end - def namespace_name_regex @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze end @@ -52,23 +19,6 @@ module Gitlab "It must start with letter, digit, emoji or '_'." end - def project_path_regex - @project_path_regex ||= /\A#{PROJECT_REGEX_STR}\z/.freeze - end - - def project_route_regex - @project_route_regex ||= /#{PROJECT_REGEX_STR}/.freeze - end - - def project_git_route_regex - @project_route_git_regex ||= /#{PATH_REGEX_STR}\.git/.freeze - end - - def project_path_regex_message - "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-', end in '.git' or end in '.atom'" \ - end - def file_name_regex @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze end @@ -77,36 +27,8 @@ module Gitlab "can contain only letters, digits, '_', '-', '@', '+' and '.'." end - def archive_formats_regex - # |zip|tar| tar.gz | tar.bz2 | - @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze - end - - def git_reference_regex - # Valid git ref regex, see: - # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html - - @git_reference_regex ||= %r{ - (?! - (?# doesn't begins with) - \/| (?# rule #6) - (?# doesn't contain) - .*(?: - [\/.]\.| (?# rule #1,3) - \/\/| (?# rule #6) - @\{| (?# rule #8) - \\ (?# rule #9) - ) - ) - [^\000-\040\177~^:?*\[]+ (?# rule #4-5) - (?# doesn't end with) - (?<!\.lock) (?# rule #1) - (?<![\/.]) (?# rule #6-7) - }x.freeze - end - def container_registry_reference_regex - git_reference_regex + Gitlab::PathRegex.git_reference_regex end ## diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb new file mode 100644 index 00000000000..5b1b03820a3 --- /dev/null +++ b/lib/gitlab/sql/recursive_cte.rb @@ -0,0 +1,62 @@ +module Gitlab + module SQL + # Class for easily building recursive CTE statements. + # + # Example: + # + # cte = RecursiveCTE.new(:my_cte_name) + # ns = Arel::Table.new(:namespaces) + # + # cte << Namespace. + # where(ns[:parent_id].eq(some_namespace_id)) + # + # cte << Namespace. + # from([ns, cte.table]). + # where(ns[:parent_id].eq(cte.table[:id])) + # + # Namespace.with. + # recursive(cte.to_arel). + # from(cte.alias_to(ns)) + class RecursiveCTE + attr_reader :table + + # name - The name of the CTE as a String or Symbol. + def initialize(name) + @table = Arel::Table.new(name) + @queries = [] + end + + # Adds a query to the body of the CTE. + # + # relation - The relation object to add to the body of the CTE. + def <<(relation) + @queries << relation + end + + # Returns the Arel relation for this CTE. + def to_arel + sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries).to_sql) + + Arel::Nodes::As.new(table, Arel::Nodes::Grouping.new(sql)) + end + + # Returns an "AS" statement that aliases the CTE name as the given table + # name. This allows one to trick ActiveRecord into thinking it's selecting + # from an actual table, when in reality it's selecting from a CTE. + # + # alias_table - The Arel table to use as the alias. + def alias_to(alias_table) + Arel::Nodes::As.new(table, alias_table) + end + + # Applies the CTE to the given relation, returning a new one that will + # query from it. + def apply_to(relation) + relation.except(:where). + with. + recursive(to_arel). + from(alias_to(relation.model.arel_table)) + end + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 9ce13feb79a..c81dc7e30d0 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -18,12 +18,6 @@ module Gitlab false end - def self.http_credentials_for_user(user) - return {} unless user.respond_to?(:username) - - { user: user.username } - end - def initialize(url, credentials: nil) @url = Addressable::URI.parse(url.strip) @credentials = credentials diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index fe37e4da94f..18d8b4f4744 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -31,8 +31,7 @@ module Gitlab feature_enabled = case action.to_s when 'git_receive_pack' - # Disabled for now, see https://gitlab.com/gitlab-org/gitaly/issues/172 - false + Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) when 'git_upload_pack' Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) when 'info_refs' diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 6e351365de0..c5f93336346 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -48,7 +48,7 @@ gitlab_pages_pid_path="$pid_path/gitlab-pages.pid" gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" gitlab_pages_log="$app_root/log/gitlab-pages.log" shell_path="/bin/bash" -gitaly_enabled=false +gitaly_enabled=true gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd) gitaly_pid_path="$pid_path/gitaly.pid" gitaly_log="$app_root/log/gitaly.log" diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 9472c3c992f..295c79fccfc 100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -86,5 +86,7 @@ mail_room_pid_path="$pid_path/mail_room.pid" shell_path="/bin/bash" # This variable controls whether the init script starts/stops Gitaly -gitaly_enabled=false +gitaly_enabled=true +gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd) +gitaly_pid_path="$pid_path/gitaly.pid" gitaly_log="$app_root/log/gitaly.log" diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake index 95735f43802..ad1818ff1fa 100644 --- a/lib/tasks/tokens.rake +++ b/lib/tasks/tokens.rake @@ -11,6 +11,11 @@ namespace :tokens do reset_all_users_token(:reset_incoming_email_token!) end + desc "Reset all GitLab RSS tokens" + task reset_all_rss: :environment do + reset_all_users_token(:reset_rss_token!) + end + def reset_all_users_token(reset_token_method) TmpUser.find_in_batches do |batch| puts "Processing batch starting with user ID: #{batch.first.id}" @@ -35,4 +40,9 @@ class TmpUser < ActiveRecord::Base write_new_token(:incoming_email_token) save!(validate: false) end + + def reset_rss_token! + write_new_token(:rss_token) + save!(validate: false) + end end |