diff options
Diffstat (limited to 'lib')
119 files changed, 1787 insertions, 360 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 62ffebeacb0..073471b4c4d 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -78,6 +78,14 @@ module API rack_response({ 'message' => '404 Not found' }.to_json, 404) end + rescue_from UploadedFile::InvalidPathError do |e| + rack_response({ 'message' => e.message }.to_json, 400) + end + + rescue_from ObjectStorage::RemoteStoreError do |e| + rack_response({ 'message' => e.message }.to_json, 500) + end + # Retain 405 error rather than a 500 error for Grape 0.15.0+. # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes rescue_from Grape::Exceptions::MethodNotAllowed do |e| diff --git a/lib/api/badges.rb b/lib/api/badges.rb index 334948b2995..8ceffe9c5ef 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -127,6 +127,7 @@ module API end destroy_conditionally!(badge) + body false end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 38161d1f127..8aad320e376 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -72,7 +72,7 @@ module API class ProjectHook < Hook expose :project_id, :issues_events, :confidential_issues_events - expose :note_events, :pipeline_events, :wiki_page_events + expose :note_events, :confidential_note_events, :pipeline_events, :wiki_page_events expose :job_events end @@ -206,6 +206,7 @@ module API expose :request_access_enabled expose :only_allow_merge_if_all_discussions_are_resolved expose :printing_merge_request_link_enabled + expose :merge_method expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics @@ -405,6 +406,7 @@ module API class IssueBasic < ProjectEntity expose :closed_at + expose :closed_by, using: Entities::UserBasic expose :labels do |issue, options| # Avoids an N+1 query since labels are preloaded issue.labels.map(&:title).sort @@ -792,7 +794,7 @@ module API expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :confidential_issues_events expose :merge_requests_events, :tag_push_events, :note_events - expose :pipeline_events, :wiki_page_events + expose :confidential_note_events, :pipeline_events, :wiki_page_events expose :job_events # Expose serialized properties expose :properties do |service, options| @@ -926,7 +928,7 @@ module API end class Tag < Grape::Entity - expose :name, :message + expose :name, :message, :target expose :commit, using: Entities::Commit do |repo_tag, options| options[:project].repository.commit(repo_tag.dereferenced_target) diff --git a/lib/api/features.rb b/lib/api/features.rb index 9385c6ca174..11d848584d9 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -65,6 +65,13 @@ module API present feature, with: Entities::Feature, current_user: current_user end + + desc 'Remove the gate value for the given feature' + delete ':name' do + Feature.get(params[:name]).remove + + status 204 + end end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index e59e8a45908..61dab1dd5cb 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -83,12 +83,13 @@ module API end def available_labels_for(label_parent) - search_params = - if label_parent.is_a?(Project) - { project_id: label_parent.id } - else - { group_id: label_parent.id, only_group_labels: true } - end + search_params = { include_ancestor_groups: true } + + if label_parent.is_a?(Project) + search_params[:project_id] = label_parent.id + else + search_params.merge!(group_id: label_parent.id, only_group_labels: true) + end LabelsFinder.new(current_user, search_params).execute end @@ -388,28 +389,6 @@ module API # file helpers - def uploaded_file(field, uploads_path) - if params[field] - bad_request!("#{field} is not a file") unless params[field][:filename] - return params[field] - end - - return nil unless params["#{field}.path"] && params["#{field}.name"] - - # sanitize file paths - # this requires all paths to exist - required_attributes! %W(#{field}.path) - uploads_path = File.realpath(uploads_path) - file_path = File.realpath(params["#{field}.path"]) - bad_request!('Bad file path') unless file_path.start_with?(uploads_path) - - UploadedFile.new( - file_path, - params["#{field}.name"], - params["#{field}.type"] || 'application/octet-stream' - ) - end - def present_disk_file!(path, filename, content_type = 'application/octet-stream') filename ||= File.basename(path) header['Content-Disposition'] = "attachment; filename=#{filename}" @@ -489,8 +468,8 @@ module API header(*Gitlab::Workhorse.send_git_blob(repository, blob)) end - def send_git_archive(repository, ref:, format:) - header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format)) + def send_git_archive(repository, **kwargs) + header(*Gitlab::Workhorse.send_git_archive(repository, **kwargs)) end def send_artifacts_entry(build, entry) diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb new file mode 100644 index 00000000000..381d5e8968c --- /dev/null +++ b/lib/api/helpers/projects_helpers.rb @@ -0,0 +1,38 @@ +module API + module Helpers + module ProjectsHelpers + extend ActiveSupport::Concern + + included do + helpers do + params :optional_project_params_ce do + optional :description, type: String, desc: 'The description of the project' + optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' + optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' + optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' + optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' + optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' + optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' + optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' + optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' + optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' + optional :public_builds, type: Boolean, desc: 'Perform public builds' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' + optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + optional :tag_list, type: Array[String], desc: 'The list of tags for a project' + optional :avatar, type: File, desc: 'Avatar image for project' + optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' + optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' + end + + params :optional_project_params do + use :optional_project_params_ce + end + end + end + end + end +end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index f74b3b26802..88e7f46c92c 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -97,7 +97,7 @@ module API get ":id/issues" do group = find_group!(params[:id]) - issues = paginate(find_issues(group_id: group.id)) + issues = paginate(find_issues(group_id: group.id, include_subgroups: true)) options = { with: Entities::IssueBasic, diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index efc4a33ae1b..5ef4e9d530c 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -33,11 +33,28 @@ module API end params do optional :description, type: String, desc: 'Override the project description' + optional :upload, type: Hash do + optional :url, type: String, desc: 'The URL to upload the project' + optional :http_method, type: String, default: 'PUT', desc: 'HTTP method to upload the exported project' + end end post ':id/export' do project_export_params = declared_params(include_missing: false) + after_export_params = project_export_params.delete(:upload) || {} - user_project.add_export_job(current_user: current_user, params: project_export_params) + export_strategy = if after_export_params[:url].present? + params = after_export_params.slice(:url, :http_method).symbolize_keys + + Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(params) + end + + if export_strategy&.invalid? + render_validation_error!(export_strategy) + else + user_project.add_export_job(current_user: current_user, + after_export_strategy: export_strategy, + params: project_export_params) + end accepted! end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index f82241058e5..68921ae439b 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -14,6 +14,7 @@ module API optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" + optional :confidential_note_events, type: Boolean, desc: "Trigger hook on confidential note(comment) events" optional :job_events, type: Boolean, desc: "Trigger hook on job events" optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index a509c1f32c1..bc5152e539f 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -1,6 +1,7 @@ module API class ProjectImport < Grape::API include PaginationParams + include Helpers::ProjectsHelpers helpers do def import_params @@ -25,6 +26,12 @@ module API requires :path, type: String, desc: 'The new project path and name' requires :file, type: File, desc: 'The project export file to be imported' optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace." + optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it' + optional :override_params, + type: Hash, + desc: 'New project params to override values in the export' do + use :optional_project_params + end end desc 'Create a new project import' do detail 'This feature was introduced in GitLab 10.6.' @@ -44,10 +51,15 @@ module API project_params = { path: import_params[:path], namespace_id: namespace.id, - file: import_params[:file]['tempfile'] + file: import_params[:file]['tempfile'], + overwrite: import_params[:overwrite] } - project = ::Projects::GitlabProjectsImportService.new(current_user, project_params).execute + override_params = import_params.delete(:override_params) + + project = ::Projects::GitlabProjectsImportService.new( + current_user, project_params, override_params + ).execute render_api_error!(project.errors.full_messages&.first, 400) unless project.saved? diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 467bc78dad8..d0a4a23e074 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -4,36 +4,11 @@ module API class Projects < Grape::API include PaginationParams include Helpers::CustomAttributes + include Helpers::ProjectsHelpers before { authenticate_non_get! } helpers do - params :optional_params_ce do - optional :description, type: String, desc: 'The description of the project' - optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' - optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' - optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' - optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' - optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' - optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' - optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' - optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' - optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' - optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' - optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' - optional :public_builds, type: Boolean, desc: 'Perform public builds' - optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' - optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' - optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' - optional :tag_list, type: Array[String], desc: 'The list of tags for a project' - optional :avatar, type: File, desc: 'Avatar image for project' - optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' - end - - params :optional_params do - use :optional_params_ce - end - params :statistics_params do optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end @@ -143,7 +118,7 @@ module API optional :name, type: String, desc: 'The name of the project' optional :path, type: String, desc: 'The path of the repository' at_least_one_of :name, :path - use :optional_params + use :optional_project_params use :create_params end post do @@ -171,7 +146,7 @@ module API requires :user_id, type: Integer, desc: 'The ID of a user' optional :path, type: String, desc: 'The path of the repository' optional :default_branch, type: String, desc: 'The default branch of the project' - use :optional_params + use :optional_project_params use :create_params end post "user/:user_id" do @@ -274,6 +249,7 @@ module API :issues_enabled, :lfs_enabled, :merge_requests_enabled, + :merge_method, :name, :only_allow_merge_if_all_discussions_are_resolved, :only_allow_merge_if_pipeline_succeeds, @@ -291,7 +267,7 @@ module API optional :default_branch, type: String, desc: 'The default branch of the project' optional :path, type: String, desc: 'The path of the repository' - use :optional_params + use :optional_project_params at_least_one_of(*at_least_one_of_ce) end put ':id' do diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 9638c53a1df..2396dc73f0e 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -88,7 +88,7 @@ module API end get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do begin - send_git_archive user_project.repository, ref: params[:sha], format: params[:format] + send_git_archive user_project.repository, ref: params[:sha], format: params[:format], append_sha: true rescue not_found!('File') end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 57c0a729535..60aeb69e10a 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -186,7 +186,7 @@ module API status 200 content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - Gitlab::Workhorse.artifact_upload_ok + JobArtifactUploader.workhorse_authorize end desc 'Upload artifacts for job' do @@ -201,13 +201,15 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :token, type: String, desc: %q(Job's authentication token) optional :expire_in, type: String, desc: %q(Specify when artifacts should expire) - optional :file, type: File, desc: %q(Artifact's file) optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) - optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file) + optional 'file.size', type: Integer, desc: %q(real size of file (generated by Workhorse)) + optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file (generated by Workhorse)) optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) + optional 'metadata.size', type: Integer, desc: %q(real size of metadata (generated by Workhorse)) + optional 'metadata.sha256', type: String, desc: %q(sha256 checksum of metadata (generated by Workhorse)) end post '/:id/artifacts' do not_allowed! unless Gitlab.config.artifacts.enabled @@ -216,21 +218,34 @@ module API job = authenticate_job! forbidden!('Job is not running!') unless job.running? - workhorse_upload_path = JobArtifactUploader.workhorse_upload_path - artifacts = uploaded_file(:file, workhorse_upload_path) - metadata = uploaded_file(:metadata, workhorse_upload_path) + artifacts = UploadedFile.from_params(params, :file, JobArtifactUploader.workhorse_local_upload_path) + metadata = UploadedFile.from_params(params, :metadata, JobArtifactUploader.workhorse_local_upload_path) bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size + bad_request!("Already uploaded") if job.job_artifacts_archive + expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in - job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, file_sha256: params['file.sha256'], expire_in: expire_in) - job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, expire_in: expire_in) if metadata - job.artifacts_expire_in = expire_in + job.build_job_artifacts_archive( + project: job.project, + file: artifacts, + file_type: :archive, + file_sha256: artifacts.sha256, + expire_in: expire_in) + + if metadata + job.build_job_artifacts_metadata( + project: job.project, + file: metadata, + file_type: :metadata, + file_sha256: metadata.sha256, + expire_in: expire_in) + end - if job.save + if job.update(artifacts_expire_in: expire_in) present job, with: Entities::JobRequest::Response else render_validation_error!(job) diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb index 5b54734bb45..f701d64e886 100644 --- a/lib/api/v3/repositories.rb +++ b/lib/api/v3/repositories.rb @@ -75,7 +75,7 @@ module API end get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do begin - send_git_archive user_project.repository, ref: params[:sha], format: params[:format] + send_git_archive user_project.repository, ref: params[:sha], format: params[:format], append_sha: true rescue not_found!('File') end diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 4383124d150..6a5a223a614 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -5,9 +5,5 @@ module Backup def initialize super('artifacts', JobArtifactUploader.root) end - - def create_files_dir - Dir.mkdir(app_files_dir, 0700) - end end end diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 635967f4bd4..f869916e199 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -5,9 +5,5 @@ module Backup def initialize super('builds', Settings.gitlab_ci.builds_path) end - - def create_files_dir - Dir.mkdir(app_files_dir, 0700) - end end end diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 287d591e88d..88cb7e7b5a4 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -1,7 +1,10 @@ require 'open3' +require_relative 'helper' module Backup class Files + include Backup::Helper + attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir def initialize(name, app_files_dir) @@ -35,15 +38,22 @@ module Backup def restore backup_existing_files_dir - create_files_dir - run_pipeline!([%w(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) + run_pipeline!([%w(gzip -cd), %W(tar --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball) end def backup_existing_files_dir - timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}") + timestamped_files_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}.#{Time.now.to_i}") if File.exist?(app_files_dir) - FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path)) + # Move all files in the existing repos directory except . and .. to + # repositories.old.<timestamp> directory + FileUtils.mkdir_p(timestamped_files_path, mode: 0700) + files = Dir.glob(File.join(app_files_dir, "*"), File::FNM_DOTMATCH) - [File.join(app_files_dir, "."), File.join(app_files_dir, "..")] + begin + FileUtils.mv(files, timestamped_files_path) + rescue Errno::EACCES + access_denied_error(app_files_dir) + end end end diff --git a/lib/backup/helper.rb b/lib/backup/helper.rb new file mode 100644 index 00000000000..a1ee0faefe9 --- /dev/null +++ b/lib/backup/helper.rb @@ -0,0 +1,17 @@ +module Backup + module Helper + def access_denied_error(path) + message = <<~EOS + + ### NOTICE ### + As part of restore, the task tried to move existing content from #{path}. + However, it seems that directory contains files/folders that are not owned + by the user #{Gitlab.config.gitlab.user}. To proceed, please move the files + or folders inside #{path} to a secure location so that #{path} is empty and + run restore task again. + + EOS + raise message + end + end +end diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb index 4153467fbee..4e234e50a7a 100644 --- a/lib/backup/lfs.rb +++ b/lib/backup/lfs.rb @@ -5,9 +5,5 @@ module Backup def initialize super('lfs', Settings.lfs.storage_path) end - - def create_files_dir - Dir.mkdir(app_files_dir, 0700) - end end end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb index 215ded93bfe..5830b209d6e 100644 --- a/lib/backup/pages.rb +++ b/lib/backup/pages.rb @@ -5,9 +5,5 @@ module Backup def initialize super('pages', Gitlab.config.pages.path) end - - def create_files_dir - Dir.mkdir(app_files_dir, 0700) - end end end diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb index 67fe0231087..91698669402 100644 --- a/lib/backup/registry.rb +++ b/lib/backup/registry.rb @@ -5,9 +5,5 @@ module Backup def initialize super('registry', Settings.registry.path) end - - def create_files_dir - Dir.mkdir(app_files_dir, 0700) - end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 88a7f2a4235..89e3f1d9076 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -1,8 +1,11 @@ require 'yaml' +require_relative 'helper' module Backup class Repository + include Backup::Helper # rubocop:disable Metrics/AbcSize + def dump prepare @@ -63,18 +66,27 @@ module Backup end end - def restore + def prepare_directories Gitlab.config.repositories.storages.each do |name, repository_storage| path = repository_storage.legacy_disk_path next unless File.exist?(path) - # Move repos dir to 'repositories.old' dir - bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) - FileUtils.mv(path, bk_repos_path) - # This is expected from gitlab:check - FileUtils.mkdir_p(path, mode: 02770) + # Move all files in the existing repos directory except . and .. to + # repositories.old.<timestamp> directory + bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) + FileUtils.mkdir_p(bk_repos_path, mode: 0700) + files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] + + begin + FileUtils.mv(files, bk_repos_path) + rescue Errno::EACCES + access_denied_error(path) + end end + end + def restore + prepare_directories Project.find_each(batch_size: 1000) do |project| progress.print " * #{display_repo_path(project)} ... " path_to_project_repo = path_to_repo(project) diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 35118375499..d46e2cd869d 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -5,9 +5,5 @@ module Backup def initialize super('uploads', Rails.root.join('public/uploads')) end - - def create_files_dir - Dir.mkdir(app_files_dir) - end end end diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index d8fb7705b2a..3f1e95d4cc0 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -4,7 +4,7 @@ module Banzai module CrossProjectReference # Given a cross-project reference string, get the Project record # - # Defaults to value of `context[:project]` if: + # Defaults to value of `context[:project]`, or `context[:group]` if: # * No reference is given OR # * Reference given doesn't exist # @@ -12,7 +12,7 @@ module Banzai # # Returns a Project, or nil if the reference can't be found def parent_from_ref(ref) - return context[:project] unless ref + return context[:project] || context[:group] unless ref Project.find_by_full_path(ref) end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index c9e3f8ce42b..a848154b2d4 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -171,7 +171,7 @@ module Banzai end if object - title = object_link_title(object) + title = object_link_title(object, matches) klass = reference_class(object_sym) data = data_attributes_for(link_content || match, parent, object, @@ -196,13 +196,15 @@ module Banzai end end - def data_attributes_for(text, project, object, link_content: false, link_reference: false) + def data_attributes_for(text, parent, object, link_content: false, link_reference: false) + object_parent_type = parent.is_a?(Group) ? :group : :project + data_attribute( - original: text, - link: link_content, - link_reference: link_reference, - project: project.id, - object_sym => object.id + original: text, + link: link_content, + link_reference: link_reference, + object_parent_type => parent.id, + object_sym => object.id ) end @@ -213,10 +215,14 @@ module Banzai extras << "comment #{$1}" end + extension = matches[:extension] if matches.names.include?("extension") + + extras << extension if extension + extras end - def object_link_title(object) + def object_link_title(object, matches) object.title end @@ -337,6 +343,12 @@ module Banzai def parent parent_type == :project ? project : group end + + def full_group_path(group_ref) + return current_parent_path unless group_ref + + group_ref + end end end end diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index 21bcb1c5ca8..99fa2d9d8fb 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -34,7 +34,7 @@ module Banzai range.to_param.merge(only_path: context[:only_path])) end - def object_link_title(range) + def object_link_title(range, matches) nil end end diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb new file mode 100644 index 00000000000..ef16df1f3ae --- /dev/null +++ b/lib/banzai/filter/commit_trailers_filter.rb @@ -0,0 +1,152 @@ +module Banzai + module Filter + # HTML filter that replaces users' names and emails in commit trailers + # with links to their GitLab accounts or mailto links to their mentioned + # emails. + # + # Commit trailers are special labels in the form of `*-by:` and fall on a + # single line, ex: + # + # Reported-By: John S. Doe <john.doe@foo.bar> + # + # More info about this can be found here: + # * https://git.wiki.kernel.org/index.php/CommitMessageConventions + class CommitTrailersFilter < HTML::Pipeline::Filter + include ActionView::Helpers::TagHelper + include ApplicationHelper + include AvatarsHelper + + TRAILER_REGEXP = /(?<label>[[:alpha:]-]+-by:)/i.freeze + AUTHOR_REGEXP = /(?<author_name>.+)/.freeze + # Devise.email_regexp wouldn't work here since its designed to match + # against strings that only contains email addresses; the \A and \z + # around the expression will only match if the string being matched + # contains just the email nothing else. + MAIL_REGEXP = /<(?<author_email>[^@\s]+@[^@\s]+)>/.freeze + FILTER_REGEXP = /(?<trailer>^\s*#{TRAILER_REGEXP}\s*#{AUTHOR_REGEXP}\s+#{MAIL_REGEXP}$)/mi.freeze + + def call + doc.xpath('descendant-or-self::text()').each do |node| + content = node.to_html + + next unless content.match(FILTER_REGEXP) + + html = trailer_filter(content) + + next if html == content + + node.replace(html) + end + + doc + end + + private + + # Replace trailer lines with links to GitLab users or mailto links to + # non GitLab users. + # + # text - String text to replace trailers in. + # + # Returns a String with all trailer lines replaced with links to GitLab + # users and mailto links to non GitLab users. All links have `data-trailer` + # and `data-user` attributes attached. + def trailer_filter(text) + text.gsub(FILTER_REGEXP) do |author_match| + label = $~[:label] + "#{label} #{parse_user($~[:author_name], $~[:author_email], label)}" + end + end + + # Find a GitLab user using the supplied email and generate + # a valid link to them, otherwise, generate a mailto link. + # + # name - String name used in the commit message for the user + # email - String email used in the commit message for the user + # trailer - String trailer used in the commit message + # + # Returns a String with a link to the user. + def parse_user(name, email, trailer) + link_to_user User.find_by_any_email(email), + name: name, + email: email, + trailer: trailer + end + + def urls + Gitlab::Routing.url_helpers + end + + def link_to_user(user, name:, email:, trailer:) + wrapper = link_wrapper(data: { + trailer: trailer, + user: user.try(:id) + }) + + avatar = user_avatar_without_link( + user: user, + user_email: email, + css_class: 'avatar-inline', + has_tooltip: false + ) + + link_href = user.nil? ? "mailto:#{email}" : urls.user_url(user) + + avatar_link = link_tag( + link_href, + content: avatar, + title: email + ) + + name_link = link_tag( + link_href, + content: name, + title: email + ) + + email_link = link_tag( + "mailto:#{email}", + content: email, + title: email + ) + + wrapper << "#{avatar_link}#{name_link} <#{email_link}>" + end + + def link_wrapper(data: {}) + data_attributes = data_attributes_from_hash(data) + + doc.document.create_element( + 'span', + data_attributes + ) + end + + def link_tag(url, title: "", content: "", data: {}) + data_attributes = data_attributes_from_hash(data) + + attributes = data_attributes.merge( + href: url, + title: title + ) + + link = doc.document.create_element('a', attributes) + + if content.html_safe? + link << content + else + link.content = content # make sure we escape content using nokogiri's #content= + end + + link + end + + def data_attributes_from_hash(data = {}) + data.reject! {|_, value| value.nil?} + data.map do |key, value| + [%(data-#{key.to_s.dasherize}), value] + end.to_h + end + end + end +end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index b82c6ca6393..e1261e7bbbe 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -11,7 +11,7 @@ module Banzai IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set def call - search_text_nodes(doc).each do |node| + doc.search(".//text()").each do |node| content = node.to_html next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index c2b42673376..f2e9a5a1116 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -57,7 +57,7 @@ module Banzai ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i.freeze def call - search_text_nodes(doc).each do |node| + doc.search(".//text()").each do |node| # A Gollum ToC tag is `[[_TOC_]]`, but due to MarkdownFilter running # before this one, it will be converted into `[[<em>TOC</em>]]`, so it # needs special-case handling diff --git a/lib/banzai/filter/inline_diff_filter.rb b/lib/banzai/filter/inline_diff_filter.rb index beb21b19ab3..73e82a4d7e3 100644 --- a/lib/banzai/filter/inline_diff_filter.rb +++ b/lib/banzai/filter/inline_diff_filter.rb @@ -4,7 +4,7 @@ module Banzai IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set def call - search_text_nodes(doc).each do |node| + doc.search(".//text()").each do |node| next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) content = node.to_html diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index d5360ad8f68..1cbada818fb 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -32,16 +32,25 @@ module Banzai end end - def find_label(project_ref, label_id, label_name) - project = parent_from_ref(project_ref) - return unless project + def find_label(parent_ref, label_id, label_name) + parent = parent_from_ref(parent_ref) + return unless parent label_params = label_params(label_id, label_name) - find_labels(project).find_by(label_params) + find_labels(parent).find_by(label_params) end - def find_labels(project) - LabelsFinder.new(nil, project_id: project.id).execute(skip_authorization: true) + def find_labels(parent) + params = if parent.is_a?(Group) + { group_id: parent.id, + include_ancestor_groups: true, + only_group_labels: true } + else + { project_id: parent.id, + include_ancestor_groups: true } + end + + LabelsFinder.new(nil, params).execute(skip_authorization: true) end # Parameters to pass to `Label.find_by` based on the given arguments @@ -59,25 +68,39 @@ module Banzai end end - def url_for_object(label, project) + def url_for_object(label, parent) h = Gitlab::Routing.url_helpers - h.project_issues_url(project, label_name: label.name, only_path: context[:only_path]) + + if parent.is_a?(Project) + h.project_issues_url(parent, label_name: label.name, only_path: context[:only_path]) + elsif context[:label_url_method] + h.public_send(context[:label_url_method], parent, label_name: label.name, only_path: context[:only_path]) # rubocop:disable GitlabSecurity/PublicSend + end end def object_link_text(object, matches) - project_path = full_project_path(matches[:namespace], matches[:project]) - project_from_ref = from_ref_cached(project_path) - reference = project_from_ref.to_human_reference(project) - label_suffix = " <i>in #{reference}</i>" if reference.present? + label_suffix = '' + + if project || full_path_ref?(matches) + project_path = full_project_path(matches[:namespace], matches[:project]) + parent_from_ref = from_ref_cached(project_path) + reference = parent_from_ref.to_human_reference(project || group) + + label_suffix = " <i>in #{reference}</i>" if reference.present? + end LabelsHelper.render_colored_label(object, label_suffix) end + def full_path_ref?(matches) + matches[:namespace] && matches[:project] + end + def unescape_html_entities(text) CGI.unescapeHTML(text.to_s) end - def object_link_title(object) + def object_link_title(object, matches) # use title of wrapped element instead nil end diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index b3cfa97d0e0..5cbdb01c130 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -17,10 +17,19 @@ module Banzai only_path: context[:only_path]) end + def object_link_title(object, matches) + object_link_commit_title(object, matches) || super + end + def object_link_text_extras(object, matches) extras = super + if commit_ref = object_link_commit_ref(object, matches) + return extras.unshift(commit_ref) + end + path = matches[:path] if matches.names.include?("path") + case path when '/diffs' extras.unshift "diffs" @@ -38,6 +47,36 @@ module Banzai .where(iid: ids.to_a) .includes(target_project: :namespace) end + + private + + def object_link_commit_title(object, matches) + object_link_commit(object, matches)&.title + end + + def object_link_commit_ref(object, matches) + object_link_commit(object, matches)&.short_id + end + + def object_link_commit(object, matches) + return unless matches.names.include?('query') && query = matches[:query] + + # Removes leading "?". CGI.parse expects "arg1&arg2&arg3" + params = CGI.parse(query.sub(/^\?/, '')) + + return unless commit_sha = params['commit_id']&.first + + if commit = find_commit_by_sha(object, commit_sha) + Commit.from_hash(commit.to_hash, object.project) + end + end + + def find_commit_by_sha(object, commit_sha) + @all_commits ||= {} + @all_commits[object.id] ||= object.all_commits + + @all_commits[object.id].find { |commit| commit.sha == commit_sha } + end end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 8ec696ce5fc..1a1d7dbeb3d 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -84,7 +84,7 @@ module Banzai end end - def object_link_title(object) + def object_link_title(object, matches) nil end end diff --git a/lib/banzai/pipeline/commit_description_pipeline.rb b/lib/banzai/pipeline/commit_description_pipeline.rb new file mode 100644 index 00000000000..607c2731ed3 --- /dev/null +++ b/lib/banzai/pipeline/commit_description_pipeline.rb @@ -0,0 +1,11 @@ +module Banzai + module Pipeline + class CommitDescriptionPipeline < SingleLinePipeline + def self.filters + @filters ||= super.concat FilterArray[ + Filter::CommitTrailersFilter, + ] + end + end + end +end diff --git a/lib/forever.rb b/lib/forever.rb new file mode 100644 index 00000000000..7df17912544 --- /dev/null +++ b/lib/forever.rb @@ -0,0 +1,13 @@ +class Forever + POSTGRESQL_DATE = DateTime.new(3000, 1, 1) + MYSQL_DATE = DateTime.new(2038, 01, 19) + + # MySQL timestamp has a range of '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' UTC + def self.date + if Gitlab::Database.postgresql? + POSTGRESQL_DATE + else + MYSQL_DATE + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 6af763faf10..2a44e11efb6 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -5,7 +5,7 @@ module Gitlab REGISTRY_SCOPES = [:read_registry].freeze # Scopes used for GitLab API access - API_SCOPES = [:api, :read_user, :sudo].freeze + API_SCOPES = [:api, :read_user, :sudo, :read_repository].freeze # Scopes used for OpenID Connect OPENID_SCOPES = [:openid].freeze @@ -26,6 +26,7 @@ module Gitlab lfs_token_check(login, password, project) || oauth_access_token_check(login, password) || personal_access_token_check(password) || + deploy_token_check(login, password) || user_with_password_for_git(login, password) || Gitlab::Auth::Result.new @@ -163,7 +164,8 @@ module Gitlab def abilities_for_scopes(scopes) abilities_by_scope = { api: full_authentication_abilities, - read_registry: [:read_container_image] + read_registry: [:read_container_image], + read_repository: [:download_code] } scopes.flat_map do |scope| @@ -171,6 +173,22 @@ module Gitlab end.uniq end + def deploy_token_check(login, password) + return unless password.present? + + token = + DeployToken.active.find_by(token: password) + + return unless token && login + return if login != token.username + + scopes = abilities_for_scopes(token.scopes) + + if valid_scoped_token?(token, available_scopes) + Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes) + end + end + def lfs_token_check(login, password, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 77c0ddc2d48..34286900e72 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -52,6 +52,8 @@ module Gitlab block_user(user, 'does not exist anymore') false end + rescue LDAPConnectionError + false end def adapter diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index caf2d18c668..82ff1e77e5c 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -2,6 +2,9 @@ module Gitlab module Auth module LDAP class Adapter + SEARCH_RETRY_FACTOR = [1, 1, 2, 3].freeze + MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size.freeze + attr_reader :provider, :ldap def self.open(provider, &block) @@ -16,7 +19,7 @@ module Gitlab def initialize(provider, ldap = nil) @provider = provider - @ldap = ldap || Net::LDAP.new(config.adapter_options) + @ldap = ldap || renew_connection_adapter end def config @@ -47,8 +50,10 @@ module Gitlab end def ldap_search(*args) + retries ||= 0 + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. - Timeout.timeout(config.timeout) do + Timeout.timeout(timeout_time(retries)) do results = ldap.search(*args) if results.nil? @@ -63,16 +68,26 @@ module Gitlab results end end - rescue Net::LDAP::Error => error - Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") - [] - rescue Timeout::Error - Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") - [] + rescue Net::LDAP::Error, Timeout::Error => error + retries += 1 + error_message = connection_error_message(error) + + Rails.logger.warn(error_message) + + if retries < MAX_SEARCH_RETRIES + renew_connection_adapter + retry + else + raise LDAPConnectionError, error_message + end end private + def timeout_time(retry_number) + SEARCH_RETRY_FACTOR[retry_number] * config.timeout + end + def user_options(fields, value, limit) options = { attributes: Gitlab::Auth::LDAP::Person.ldap_attributes(config), @@ -104,6 +119,18 @@ module Gitlab filter end end + + def connection_error_message(exception) + if exception.is_a?(Timeout::Error) + "LDAP search timed out after #{config.timeout} seconds" + else + "LDAP search raised exception #{exception.class}: #{exception.message}" + end + end + + def renew_connection_adapter + @ldap = Net::LDAP.new(config.adapter_options) + end end end end diff --git a/lib/gitlab/auth/ldap/ldap_connection_error.rb b/lib/gitlab/auth/ldap/ldap_connection_error.rb new file mode 100644 index 00000000000..ef0a695742b --- /dev/null +++ b/lib/gitlab/auth/ldap/ldap_connection_error.rb @@ -0,0 +1,7 @@ +module Gitlab + module Auth + module LDAP + LDAPConnectionError = Class.new(StandardError) + end + end +end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index b6a96081278..d0c6b0386ba 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -124,6 +124,9 @@ module Gitlab Gitlab::Auth::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || Gitlab::Auth::LDAP::Person.find_by_email(auth_hash.uid, adapter) || Gitlab::Auth::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + + rescue Gitlab::Auth::LDAP::LDAPConnectionError + nil end def ldap_config diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index fd5cbf76e47..a357538a885 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -96,7 +96,7 @@ module Gitlab commit_hash.merge( merge_request_diff_id: merge_request_diff.id, relative_order: index, - sha: sha_attribute.type_cast_for_database(sha) + sha: sha_attribute.serialize(sha) ) end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb new file mode 100644 index 00000000000..e5e8837221e --- /dev/null +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + # Ensures services which previously recieved all notes events continue + # to recieve confidential ones. + class SetConfidentialNoteEventsOnServices + class Service < ActiveRecord::Base + self.table_name = 'services' + + include ::EachBatch + + def self.services_to_update + where(confidential_note_events: nil, note_events: true) + end + end + + def perform(start_id, stop_id) + Service.services_to_update + .where(id: start_id..stop_id) + .update_all(confidential_note_events: true) + end + end + end +end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb new file mode 100644 index 00000000000..171c8ef21b7 --- /dev/null +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + # Ensures hooks which previously recieved all notes events continue + # to recieve confidential ones. + class SetConfidentialNoteEventsOnWebhooks + class WebHook < ActiveRecord::Base + self.table_name = 'web_hooks' + + include ::EachBatch + + def self.hooks_to_update + where(confidential_note_events: nil, note_events: true) + end + end + + def perform(start_id, stop_id) + WebHook.hooks_to_update + .where(id: start_id..stop_id) + .update_all(confidential_note_events: true) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index bffbcb86137..f3999e690fa 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -63,7 +63,7 @@ module Gitlab disk_path = project.wiki.disk_path import_url = project.import_url.sub(/\.git\z/, ".git/wiki") - gitlab_shell.import_repository(project.repository_storage_path, disk_path, import_url) + gitlab_shell.import_repository(project.repository_storage, disk_path, import_url) rescue StandardError => e errors << { type: :wiki, errors: e.message } end diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index f7276a380dc..f0e5773ec3c 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -15,8 +15,7 @@ module Gitlab return false unless new_lfs_pointers.present? - existing_count = @project.lfs_storage_project - .lfs_objects + existing_count = @project.all_lfs_objects .where(oid: new_lfs_pointers.map(&:lfs_oid)) .count diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb index b20d374288f..782f6c4c0af 100644 --- a/lib/gitlab/ci/build/policy/kubernetes.rb +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -9,7 +9,7 @@ module Gitlab end end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) pipeline.has_kubernetes_active? end end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index eadc0948d2f..4aa5dc89f47 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -7,7 +7,7 @@ module Gitlab @patterns = Array(refs) end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) @patterns.any? do |pattern| pattern, path = pattern.split('@', 2) diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index c317291f29d..f09ba42c074 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -15,7 +15,7 @@ module Gitlab @spec = spec end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) raise NotImplementedError end end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb new file mode 100644 index 00000000000..9d2a362b7d4 --- /dev/null +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -0,0 +1,24 @@ +module Gitlab + module Ci + module Build + module Policy + class Variables < Policy::Specification + def initialize(expressions) + @expressions = Array(expressions) + end + + def satisfied_by?(pipeline, seed) + variables = seed.to_resource.scoped_variables_hash + + statements = @expressions.map do |statement| + ::Gitlab::Ci::Pipeline::Expression::Statement + .new(statement, variables) + end + + statements.any?(&:truthful?) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 0027e9ec8c5..09e8e52b60f 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -25,15 +25,31 @@ module Gitlab include Entry::Validatable include Entry::Attributable - attributes :refs, :kubernetes + attributes :refs, :kubernetes, :variables validations do validates :config, presence: true - validates :config, allowed_keys: %i[refs kubernetes] + validates :config, allowed_keys: %i[refs kubernetes variables] + validate :variables_expressions_syntax with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true validates :kubernetes, allowed_values: %w[active] + validates :variables, array_of_strings: true + end + + def variables_expressions_syntax + return unless variables.is_a?(Array) + + statements = variables.map do |statement| + ::Gitlab::Ci::Pipeline::Expression::Statement.new(statement) + end + + statements.each do |statement| + unless statement.valid? + errors.add(:variables, "Invalid expression syntax") + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index b2b00c8cb4b..d299a5677de 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -17,8 +17,6 @@ module Gitlab # Populate pipeline with all stages and builds from pipeline seeds. # pipeline.stage_seeds.each do |stage| - stage.user = current_user - pipeline.stages << stage.to_resource stage.seeds.each do |build| diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 48bde213d44..346c92dc51e 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -4,7 +4,7 @@ module Gitlab module Expression module Lexeme class String < Lexeme::Value - PATTERN = /("(?<string>.+?)")|('(?<string>.+?)')/.freeze + PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze def initialize(value) @value = value diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index b781c15fd67..37643c8ef53 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -11,7 +11,7 @@ module Gitlab end def evaluate(variables = {}) - HashWithIndifferentAccess.new(variables).fetch(@name, nil) + variables.with_indifferent_access.fetch(@name, nil) end def self.build(string) diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 4f0e101b730..09a7c98464b 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -14,12 +14,9 @@ module Gitlab %w[variable] ].freeze - def initialize(statement, pipeline) + def initialize(statement, variables = {}) @lexer = Expression::Lexer.new(statement) - - @variables = pipeline.variables.map do |variable| - [variable.key, variable.value] - end + @variables = variables.with_indifferent_access end def parse_tree @@ -35,6 +32,16 @@ module Gitlab def evaluate parse_tree.evaluate(@variables.to_h) end + + def truthful? + evaluate.present? + end + + def valid? + parse_tree.is_a?(Lexeme::Base) + rescue StatementError + false + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 7cd7c864448..6980b0b7aff 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,21 +11,16 @@ module Gitlab @pipeline = pipeline @attributes = attributes - @only = attributes.delete(:only) - @except = attributes.delete(:except) - end - - def user=(current_user) - @attributes.merge!(user: current_user) + @only = Gitlab::Ci::Build::Policy + .fabricate(attributes.delete(:only)) + @except = Gitlab::Ci::Build::Policy + .fabricate(attributes.delete(:except)) end def included? strong_memoize(:inclusion) do - only_specs = Gitlab::Ci::Build::Policy.fabricate(@only) - except_specs = Gitlab::Ci::Build::Policy.fabricate(@except) - - only_specs.all? { |spec| spec.satisfied_by?(@pipeline) } && - except_specs.none? { |spec| spec.satisfied_by?(@pipeline) } + @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } && + @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } end end @@ -33,6 +28,7 @@ module Gitlab @attributes.merge( pipeline: @pipeline, project: @pipeline.project, + user: @pipeline.user, ref: @pipeline.ref, tag: @pipeline.tag, trigger_request: @pipeline.legacy_trigger, diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 1fcbdc1b15a..c101f30d6e8 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -17,10 +17,6 @@ module Gitlab end end - def user=(current_user) - @builds.each { |seed| seed.user = current_user } - end - def attributes { name: @attributes.fetch(:name), pipeline: @pipeline, diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 2d9166d6bdd..024047d4983 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -23,6 +23,10 @@ module Gitlab 'Cancel' end + def action_button_title + _('Cancel this job') + end + def self.matches?(build, user) build.cancelable? end diff --git a/lib/gitlab/ci/status/build/canceled.rb b/lib/gitlab/ci/status/build/canceled.rb new file mode 100644 index 00000000000..c83e2734a73 --- /dev/null +++ b/lib/gitlab/ci/status/build/canceled.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Canceled < Status::Extended + def illustration + { + image: 'illustrations/canceled-job_empty.svg', + size: 'svg-430', + title: _('This job has been canceled') + } + end + + def self.matches?(build, user) + build.canceled? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/created.rb b/lib/gitlab/ci/status/build/created.rb new file mode 100644 index 00000000000..5be8e9de425 --- /dev/null +++ b/lib/gitlab/ci/status/build/created.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Created < Status::Extended + def illustration + { + image: 'illustrations/job_not_triggered.svg', + size: 'svg-306', + title: _('This job has not been triggered yet'), + content: _('This job depends on upstream jobs that need to succeed in order for this job to be triggered') + } + end + + def self.matches?(build, user) + build.created? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/erased.rb b/lib/gitlab/ci/status/build/erased.rb new file mode 100644 index 00000000000..3a5113b16b6 --- /dev/null +++ b/lib/gitlab/ci/status/build/erased.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Erased < Status::Extended + def illustration + { + image: 'illustrations/skipped-job_empty.svg', + size: 'svg-430', + title: _('Job has been erased') + } + end + + def self.matches?(build, user) + build.erased? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index c852d607373..2b26ebb45a1 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -4,12 +4,20 @@ module Gitlab module Build class Factory < Status::Factory def self.extended_statuses - [[Status::Build::Cancelable, + [[Status::Build::Erased, + Status::Build::Manual, + Status::Build::Canceled, + Status::Build::Created, + Status::Build::Pending, + Status::Build::Skipped], + [Status::Build::Cancelable, Status::Build::Retryable], + [Status::Build::Failed], [Status::Build::FailedAllowed, Status::Build::Play, Status::Build::Stop], - [Status::Build::Action]] + [Status::Build::Action], + [Status::Build::Retried]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb new file mode 100644 index 00000000000..155f4fc1343 --- /dev/null +++ b/lib/gitlab/ci/status/build/failed.rb @@ -0,0 +1,40 @@ +module Gitlab + module Ci + module Status + module Build + class Failed < Status::Extended + REASONS = { + 'unknown_failure' => 'unknown failure', + 'script_failure' => 'script failure', + 'api_failure' => 'API failure', + 'stuck_or_timeout_failure' => 'stuck or timeout failure', + 'runner_system_failure' => 'runner system failure', + 'missing_dependency_failure' => 'missing dependency failure' + }.freeze + + def status_tooltip + base_message + end + + def badge_tooltip + base_message + end + + def self.matches?(build, user) + build.failed? + end + + private + + def base_message + "#{s_('CiStatusLabel|failed')} #{description}" + end + + def description + "<br> (#{REASONS[subject.failure_reason]})" + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index dc90f398c7e..ca0046fb1f7 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -4,7 +4,7 @@ module Gitlab module Build class FailedAllowed < Status::Extended def label - 'failed (allowed to fail)' + "failed #{allowed_to_fail_title}" end def icon @@ -15,9 +15,19 @@ module Gitlab 'failed_with_warnings' end + def status_tooltip + "#{@status.status_tooltip} #{allowed_to_fail_title}" + end + def self.matches?(build, user) build.failed? && build.allow_failure? end + + private + + def allowed_to_fail_title + "(allowed to fail)" + end end end end diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb new file mode 100644 index 00000000000..042da6392d3 --- /dev/null +++ b/lib/gitlab/ci/status/build/manual.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Manual < Status::Extended + def illustration + { + image: 'illustrations/manual_action.svg', + size: 'svg-394', + title: _('This job requires a manual action'), + content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + } + end + + def self.matches?(build, user) + build.playable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/pending.rb b/lib/gitlab/ci/status/build/pending.rb new file mode 100644 index 00000000000..9dd9a27ad57 --- /dev/null +++ b/lib/gitlab/ci/status/build/pending.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Pending < Status::Extended + def illustration + { + image: 'illustrations/pending_job_empty.svg', + size: 'svg-430', + title: _('This job has not started yet'), + content: _('This job is in pending state and is waiting to be picked by a runner') + } + end + + def self.matches?(build, user) + build.pending? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index b7b45466d3b..a8b9ebf0803 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -19,6 +19,10 @@ module Gitlab 'Play' end + def action_button_title + _('Trigger this manual action') + end + def action_path play_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/retried.rb b/lib/gitlab/ci/status/build/retried.rb new file mode 100644 index 00000000000..6e190e4ee3c --- /dev/null +++ b/lib/gitlab/ci/status/build/retried.rb @@ -0,0 +1,17 @@ +module Gitlab + module Ci + module Status + module Build + class Retried < Status::Extended + def status_tooltip + @status.status_tooltip + " (retried)" + end + + def self.matches?(build, user) + build.retried? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 44ffe783e50..5aeb8e51480 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -15,6 +15,10 @@ module Gitlab 'Retry' end + def action_button_title + _('Retry this job') + end + def action_path retry_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/skipped.rb b/lib/gitlab/ci/status/build/skipped.rb new file mode 100644 index 00000000000..3e678d0baee --- /dev/null +++ b/lib/gitlab/ci/status/build/skipped.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Skipped < Status::Extended + def illustration + { + image: 'illustrations/skipped-job_empty.svg', + size: 'svg-430', + title: _('This job has been skipped') + } + end + + def self.matches?(build, user) + build.skipped? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 46e730797e4..dea838bfa39 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -19,6 +19,10 @@ module Gitlab 'Stop' end + def action_button_title + _('Stop this environment') + end + def action_path play_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index d4fd83b93f8..9d6a2f51c11 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -22,6 +22,10 @@ module Gitlab raise NotImplementedError end + def illustration + raise NotImplementedError + end + def label raise NotImplementedError end @@ -57,6 +61,20 @@ module Gitlab def action_title raise NotImplementedError end + + def action_button_title + raise NotImplementedError + end + + # Hint that appears on all the pipeline graph tooltips and builds on the right sidebar in Job detail view + def status_tooltip + label + end + + # Hint that appears on the build badges + def badge_tooltip + subject.status + end end end end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index b3fe3ef1c4d..54894a46077 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :stream - delegate :close, :tell, :seek, :size, :path, :url, :truncate, to: :stream, allow_nil: true + delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true delegate :valid?, to: :stream, as: :present?, allow_nil: true @@ -25,6 +25,10 @@ module Gitlab self.path.present? end + def path + self.stream.path if self.stream.respond_to?(:path) + end + def limit(last_bytes = LIMIT_SIZE) if last_bytes < size stream.seek(-last_bytes, IO::SEEK_END) diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index 0deca55fe8f..ad30b3f427c 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -30,7 +30,13 @@ module Gitlab end def to_runner_variables - self.map(&:to_hash) + self.map(&:to_runner_variable) + end + + def to_hash + self.to_runner_variables + .map { |env| [env.fetch(:key), env.fetch(:value)] } + .to_h.with_indifferent_access end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 939912981e6..23ed71db8b0 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -17,7 +17,7 @@ module Gitlab end def ==(other) - to_hash == self.class.fabricate(other).to_hash + to_runner_variable == self.class.fabricate(other).to_runner_variable end ## @@ -25,7 +25,7 @@ module Gitlab # don't expose `file` attribute at all (stems from what the runner # expects). # - def to_hash + def to_runner_variable @variable.reject do |hash_key, hash_value| hash_key == :file && hash_value == false end diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb index 50fea1232af..f573368e572 100644 --- a/lib/gitlab/data_builder/note.rb +++ b/lib/gitlab/data_builder/note.rb @@ -9,6 +9,7 @@ module Gitlab # # data = { # object_kind: "note", + # event_type: "confidential_note", # user: { # name: String, # username: String, @@ -51,8 +52,11 @@ module Gitlab end def build_base_data(project, user, note) + event_type = note.confidential? ? 'confidential_note' : 'note' + base_data = { object_kind: "note", + event_type: event_type, user: user.hook_attrs, project_id: project.id, project: project.hook_attrs, diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 1634fe4e9cb..77079e5e72b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -860,7 +860,7 @@ into similar problems in the future (e.g. when new tables are created). # Each job is scheduled with a `delay_interval` in between. # If you use a small interval, then some jobs may run at the same time. # - # model_class - The table being iterated over + # model_class - The table or relation being iterated over # job_class_name - The background migration job class as a string # delay_interval - The duration between each job's scheduled time (must respond to `to_f`) # batch_size - The maximum number of rows per job diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index d9400e04b83..b2d8ee81977 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -1,12 +1,20 @@ module Gitlab module Database - BINARY_TYPE = if Gitlab::Database.postgresql? - # PostgreSQL defines its own class with slightly different - # behaviour from the default Binary type. - ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea - else - ActiveRecord::Type::Binary - end + BINARY_TYPE = + if Gitlab::Database.postgresql? + # PostgreSQL defines its own class with slightly different + # behaviour from the default Binary type. + ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea + else + # In Rails 5.0 `Type` has been moved from `ActiveRecord` to `ActiveModel` + # https://github.com/rails/rails/commit/9cc8c6f3730df3d94c81a55be9ee1b7b4ffd29f6#diff-f8ba7983a51d687976e115adcd95822b + # Remove this method and leave just `ActiveModel::Type::Binary` when removing Gitlab.rails5? code. + if Gitlab.rails5? + ActiveModel::Type::Binary + else + ActiveRecord::Type::Binary + end + end # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). # @@ -16,18 +24,39 @@ module Gitlab class ShaAttribute < BINARY_TYPE PACK_FORMAT = 'H*'.freeze - # Casts binary data to a SHA1 in hexadecimal. + # It is called from activerecord-4.2.10/lib/active_record internal methods. + # Remove this method when removing Gitlab.rails5? code. def type_cast_from_database(value) - value = super + unpack_sha(super) + end + + # It is called from activerecord-4.2.10/lib/active_record internal methods. + # Remove this method when removing Gitlab.rails5? code. + def type_cast_for_database(value) + serialize(value) + end + # It is called from activerecord-5.0.6/lib/active_record/attribute.rb + # Remove this method when removing Gitlab.rails5? code.. + def deserialize(value) + value = Gitlab.rails5? ? super : method(:type_cast_from_database).super_method.call(value) + + unpack_sha(value) + end + + # Rename this method to `deserialize(value)` removing Gitlab.rails5? code. + # Casts binary data to a SHA1 in hexadecimal. + def unpack_sha(value) + # Uncomment this line when removing Gitlab.rails5? code. + # value = super value ? value.unpack(PACK_FORMAT)[0] : nil end # Casts a SHA1 in hexadecimal to the proper binary format. - def type_cast_for_database(value) + def serialize(value) arg = value ? [value].pack(PACK_FORMAT) : nil - super(arg) + Gitlab.rails5? ? super(arg) : method(:type_cast_for_database).super_method.call(arg) end end end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 010b4be7b40..81e91ea0ab7 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,11 +1,14 @@ module Gitlab module Diff class InlineDiffMarker < Gitlab::StringRangeMarker + def initialize(line, rich_line = nil) + super(line, rich_line || line) + end + def mark(line_inline_diffs, mode: nil) - mark = super(line_inline_diffs) do |text, left:, right:| + super(line_inline_diffs) do |text, left:, right:| %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>} end - mark.html_safe end private diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index a616a80e8f5..05a60deb7d3 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -14,7 +14,7 @@ module Gitlab end def can_handle? - !incoming_email_token.nil? + !incoming_email_token.nil? && !incoming_email_token.include?("+") && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX) end def execute diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 93037ed8d90..0fb82441bf8 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -231,7 +231,8 @@ module Gitlab # relation to each other. The last 10 commits for a branch for example, # should go through .where def batch_by_oid(repo, oids) - repo.gitaly_migrate(:list_commits_by_oid) do |is_enabled| + repo.gitaly_migrate(:list_commits_by_oid, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled repo.gitaly_commit_client.list_commits_by_oid(oids) else diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 07b7e811a34..c3cb0264112 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -23,7 +23,7 @@ module Gitlab end rescue GRPC::FailedPrecondition => e raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) - rescue Rugged::OdbError, GRPC::BadStatus => e + rescue Rugged::ReferenceError, Rugged::OdbError, GRPC::BadStatus => e raise Gitlab::Git::CommandError.new(e) end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index dc0bc8518bc..099709620b3 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -4,20 +4,14 @@ module Gitlab include Gitlab::Git::Popen include Gitlab::Utils::StrongMemoize - ShardNameNotFoundError = Class.new(StandardError) - - # Absolute path to directory where repositories are stored. - # Example: /home/git/repositories - attr_reader :shard_path + # Name of shard where repositories are stored. + # Example: nfs-file06 + attr_reader :shard_name # Relative path is a directory name for repository with .git at the end. # Example: gitlab-org/gitlab-test.git attr_reader :repository_relative_path - # Absolute path to the repository. - # Example: /home/git/repositorities/gitlab-org/gitlab-test.git - attr_reader :repository_absolute_path - # This is the path at which the gitlab-shell hooks directory can be found. # It's essential for integration between git and GitLab proper. All new # repositories should have their hooks directory symlinked here. @@ -25,13 +19,12 @@ module Gitlab attr_reader :logger - def initialize(shard_path, repository_relative_path, global_hooks_path:, logger:) - @shard_path = shard_path + def initialize(shard_name, repository_relative_path, global_hooks_path:, logger:) + @shard_name = shard_name @repository_relative_path = repository_relative_path @logger = logger @global_hooks_path = global_hooks_path - @repository_absolute_path = File.join(shard_path, repository_relative_path) @output = StringIO.new end @@ -41,6 +34,22 @@ module Gitlab io.read end + # Absolute path to the repository. + # Example: /home/git/repositorities/gitlab-org/gitlab-test.git + # Probably will be removed when we fully migrate to Gitaly, part of + # https://gitlab.com/gitlab-org/gitaly/issues/1124. + def repository_absolute_path + strong_memoize(:repository_absolute_path) do + File.join(shard_path, repository_relative_path) + end + end + + def shard_path + strong_memoize(:shard_path) do + Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path + end + end + # Import project via git clone --bare # URL must be publicly cloneable def import_project(source, timeout) @@ -53,12 +62,12 @@ module Gitlab end end - def fork_repository(new_shard_path, new_repository_relative_path) + def fork_repository(new_shard_name, new_repository_relative_path) Gitlab::GitalyClient.migrate(:fork_repository) do |is_enabled| if is_enabled - gitaly_fork_repository(new_shard_path, new_repository_relative_path) + gitaly_fork_repository(new_shard_name, new_repository_relative_path) else - git_fork_repository(new_shard_path, new_repository_relative_path) + git_fork_repository(new_shard_name, new_repository_relative_path) end end end @@ -205,17 +214,6 @@ module Gitlab private - def shard_name - strong_memoize(:shard_name) do - shard_name_from_shard_path(shard_path) - end - end - - def shard_name_from_shard_path(shard_path) - Gitlab.config.repositories.storages.find { |_, info| info.legacy_disk_path == shard_path }&.first || - raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") - end - def git_import_repository(source, timeout) # Skip import if repo already exists return false if File.exist?(repository_absolute_path) @@ -252,8 +250,9 @@ module Gitlab false end - def git_fork_repository(new_shard_path, new_repository_relative_path) + def git_fork_repository(new_shard_name, new_repository_relative_path) from_path = repository_absolute_path + new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path to_path = File.join(new_shard_path, new_repository_relative_path) # The repository cannot already exist @@ -271,8 +270,8 @@ module Gitlab run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) end - def gitaly_fork_repository(new_shard_path, new_repository_relative_path) - target_repository = Gitlab::Git::Repository.new(shard_name_from_shard_path(new_shard_path), new_repository_relative_path, nil) + def gitaly_fork_repository(new_shard_name, new_repository_relative_path) + target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil) raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index 4a43b9b444d..4b505312f60 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -46,6 +46,8 @@ module Gitlab iterator = State.new @content.split("\n").each_with_object(iterator) do |text, iterator| + text.chomp! + next if text =~ /^\s*#/ if text =~ /\A\[submodule "(?<name>[^"]+)"\]\z/ @@ -55,7 +57,7 @@ module Gitlab next unless text =~ /\A\s*(?<key>\w+)\s*=\s*(?<value>.*)\z/ - value = $~[:value].chomp + value = $~[:value] iterator.set_attribute($~[:key], value) end end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 24f027d8da4..7c201c6169b 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -95,13 +95,13 @@ module Gitlab args = [ref, oldrev, newrev] stdout, stderr, status = Open3.capture3(env, path, *args, options) - [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] + [status.success?, Gitlab::Utils.nlbr(stderr.presence || stdout)] end def retrieve_error_message(stderr, stdout) err_message = stderr.read err_message = err_message.blank? ? stdout.read : err_message - err_message.gsub(/\R/, "<br>").html_safe + Gitlab::Utils.nlbr(err_message) end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e692c9ce342..f1b575bd872 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -23,6 +23,7 @@ module Gitlab SQUASH_WORKTREE_PREFIX = 'squash'.freeze GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout + EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -31,6 +32,7 @@ module Gitlab DeleteBranchError = Class.new(StandardError) CreateTreeError = Class.new(StandardError) TagExistsError = Class.new(StandardError) + ChecksumError = Class.new(StandardError) class << self # Unlike `new`, `create` takes the repository path @@ -96,7 +98,7 @@ module Gitlab storage_path = Gitlab.config.repositories.storages[@storage].legacy_disk_path @gitlab_projects = Gitlab::Git::GitlabProjects.new( - storage_path, + storage, relative_path, global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, logger: Rails.logger @@ -394,17 +396,24 @@ module Gitlab nil end - def archive_prefix(ref, sha) + def archive_prefix(ref, sha, append_sha:) + append_sha = (ref != sha) if append_sha.nil? + project_name = self.name.chomp('.git') - "#{project_name}-#{ref.tr('/', '-')}-#{sha}" + formatted_ref = ref.tr('/', '-') + + prefix_segments = [project_name, formatted_ref] + prefix_segments << sha if append_sha + + prefix_segments.join('-') end - def archive_metadata(ref, storage_path, format = "tar.gz") + def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) return {} if commit.nil? - prefix = archive_prefix(ref, commit.id) + prefix = archive_prefix(ref, commit.id, append_sha: append_sha) { 'RepoPath' => path, @@ -885,7 +894,8 @@ module Gitlab end def delete_refs(*ref_names) - gitaly_migrate(:delete_refs) do |is_enabled| + gitaly_migrate(:delete_refs, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_delete_refs(*ref_names) else @@ -1035,7 +1045,8 @@ module Gitlab end def license_short_name - gitaly_migrate(:license_short_name) do |is_enabled| + gitaly_migrate(:license_short_name, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_repository_client.license_short_name else @@ -1361,6 +1372,18 @@ module Gitlab raise CommandError.new(e) end + def clean_stale_repository_files + gitaly_migrate(:repository_cleanup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + gitaly_repository_client.cleanup if is_enabled && exists? + end + rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup + Rails.logger.error("Unable to clean repository on storage #{storage} with path #{path}: #{e.message}") + Gitlab::Metrics.counter( + :failed_repository_cleanup_total, + 'Number of failed repository cleanup events' + ).increment + end + def branch_names_contains_sha(sha) gitaly_migrate(:branch_names_contains_sha) do |is_enabled| if is_enabled @@ -1455,6 +1478,43 @@ module Gitlab run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"]) end + def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) + base_args = %w(worktree add --detach) + + # Note that we _don't_ want to test for `.present?` here: If the caller + # passes an non nil empty value it means it still wants sparse checkout + # but just isn't interested in any file, perhaps because it wants to + # checkout files in by a changeset but that changeset only adds files. + if sparse_checkout_files + # Create worktree without checking out + run_git!(base_args + ['--no-checkout', worktree_path], env: env) + worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp + + configure_sparse_checkout(worktree_git_path, sparse_checkout_files) + + # After sparse checkout configuration, checkout `branch` in worktree + run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) + else + # Create worktree and checkout `branch` in it + run_git!(base_args + [worktree_path, branch], env: env) + end + + yield + ensure + FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) + FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) + end + + def checksum + gitaly_migrate(:calculate_checksum) do |is_enabled| + if is_enabled + gitaly_repository_client.calculate_checksum + else + calculate_checksum_by_shelling_out + end + end + end + private def local_write_ref(ref_path, ref, old_ref: nil, shell: true) @@ -1541,33 +1601,6 @@ module Gitlab File.exist?(path) && !clean_stuck_worktree(path) end - def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) - base_args = %w(worktree add --detach) - - # Note that we _don't_ want to test for `.present?` here: If the caller - # passes an non nil empty value it means it still wants sparse checkout - # but just isn't interested in any file, perhaps because it wants to - # checkout files in by a changeset but that changeset only adds files. - if sparse_checkout_files - # Create worktree without checking out - run_git!(base_args + ['--no-checkout', worktree_path], env: env) - worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp - - configure_sparse_checkout(worktree_git_path, sparse_checkout_files) - - # After sparse checkout configuration, checkout `branch` in worktree - run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) - else - # Create worktree and checkout `branch` in it - run_git!(base_args + [worktree_path, branch], env: env) - end - - yield - ensure - FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) - FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) - end - def clean_stuck_worktree(path) return false unless File.mtime(path) < 15.minutes.ago @@ -2400,6 +2433,34 @@ module Gitlab def sha_from_ref(ref) rev_parse_target(ref).oid end + + def calculate_checksum_by_shelling_out + raise NoRepository unless exists? + + args = %W(--git-dir=#{path} show-ref --heads --tags) + output, status = run_git(args) + + if status.nil? || !status.zero? + # Empty repositories return with a non-zero status and an empty output. + return EMPTY_REPOSITORY_CHECKSUM if output&.empty? + + raise ChecksumError, output + end + + refs = output.split("\n") + + result = refs.inject(nil) do |checksum, ref| + value = Digest::SHA1.hexdigest(ref).hex + + if checksum.nil? + value + else + checksum ^ value + end + end + + result.to_s(16) + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index ed0644f6cf1..0d1ee73ca1a 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -29,9 +29,9 @@ module Gitlab PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type - def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil) + def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil) @actor = actor @project = project @protocol = protocol @@ -39,6 +39,7 @@ module Gitlab @namespace_path = namespace_path @project_path = project_path @redirected_path = redirected_path + @auth_result_type = auth_result_type end def check(cmd, changes) @@ -78,6 +79,12 @@ module Gitlab authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) end + def request_from_ci_build? + return false unless protocol == 'http' + + auth_result_type == :build || auth_result_type == :ci + end + def protocol_allowed? Gitlab::ProtocolAccess.allowed?(protocol) end @@ -93,6 +100,8 @@ module Gitlab end def check_protocol! + return if request_from_ci_build? + unless protocol_allowed? raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" end @@ -199,6 +208,7 @@ module Gitlab def check_download_access! passed = deploy_key? || + deploy_token? || user_can_download_code? || build_can_download_code? || guest_can_download_code? @@ -229,6 +239,11 @@ module Gitlab end def check_change_access!(changes) + # If there are worktrees with a HEAD pointing to a non-existent object, + # calls to `git rev-list --all` will fail in git 2.15+. This should also + # clear stale lock files. + project.repository.clean_stale_repository_files + changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied @@ -260,6 +275,14 @@ module Gitlab actor.is_a?(DeployKey) end + def deploy_token + actor if deploy_token? + end + + def deploy_token? + actor.is_a?(DeployToken) + end + def ci? actor == :ci end @@ -267,6 +290,8 @@ module Gitlab def can_read_project? if deploy_key? deploy_key.has_access_to?(project) + elsif deploy_token? + deploy_token.has_access_to?(project) elsif user user.can?(:read_project, project) elsif ci? diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index e1bc2f9ab61..6441065f5fe 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -19,6 +19,11 @@ module Gitlab response.exists end + def cleanup + request = Gitaly::CleanupRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :cleanup, request) + end + def garbage_collect(create_bitmap) request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) GitalyClient.call(@storage, :repository_service, :garbage_collect, request) @@ -257,6 +262,12 @@ module Gitlab response.license_short_name.presence end + + def calculate_checksum + request = Gitaly::CalculateChecksumRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request) + response.checksum.presence + end end end end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index b1b283e98b5..01168abde6c 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -56,9 +56,8 @@ module Gitlab def import_wiki_repository wiki_path = "#{project.disk_path}.wiki" - storage_path = project.repository_storage_path - gitlab_shell.import_repository(storage_path, wiki_path, wiki_url) + gitlab_shell.import_repository(project.repository_storage, wiki_path, wiki_url) true rescue Gitlab::Shell::Error => e diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb index 4febb0ab430..6ab36676127 100644 --- a/lib/gitlab/hook_data/issuable_builder.rb +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -11,7 +11,8 @@ module Gitlab def build(user: nil, changes: {}) hook_data = { - object_kind: issuable.class.name.underscore, + object_kind: object_kind, + event_type: event_type, user: user.hook_attrs, project: issuable.project.hook_attrs, object_attributes: issuable.hook_attrs, @@ -36,6 +37,18 @@ module Gitlab private + def object_kind + issuable.class.name.underscore + end + + def event_type + if issuable.try(:confidential?) + "confidential_#{object_kind}" + else + object_kind + end + end + def issuable_builder case issuable when Issue diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 96558872a37..9aca3b0fb26 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -4,6 +4,8 @@ # calling internal IP or services. module Gitlab class HTTP + BlockedUrlError = Class.new(StandardError) + include HTTParty # rubocop:disable Gitlab/HTTParty connection_adapter ProxyHTTPConnectionAdapter diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb new file mode 100644 index 00000000000..aef371d81eb --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -0,0 +1,83 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class BaseAfterExportStrategy + include ActiveModel::Validations + extend Forwardable + + StrategyError = Class.new(StandardError) + + AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action'.freeze + + private + + attr_reader :project, :current_user + + public + + def initialize(attributes = {}) + @options = OpenStruct.new(attributes) + + self.class.instance_eval do + def_delegators :@options, *attributes.keys + end + end + + def execute(current_user, project) + return unless project&.export_project_path + + @project = project + @current_user = current_user + + if invalid? + log_validation_errors + + return + end + + create_or_update_after_export_lock + strategy_execute + + true + rescue => e + project.import_export_shared.error(e) + false + ensure + delete_after_export_lock + end + + def to_json(options = {}) + @options.to_h.merge!(klass: self.class.name).to_json + end + + def self.lock_file_path(project) + return unless project&.export_path + + File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME) + end + + protected + + def strategy_execute + raise NotImplementedError + end + + private + + def create_or_update_after_export_lock + FileUtils.touch(self.class.lock_file_path(project)) + end + + def delete_after_export_lock + lock_file = self.class.lock_file_path(project) + + FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file) + end + + def log_validation_errors + errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb new file mode 100644 index 00000000000..4371a7eff56 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb @@ -0,0 +1,17 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class DownloadNotificationStrategy < BaseAfterExportStrategy + private + + def strategy_execute + notification_service.project_exported(project, current_user) + end + + def notification_service + @notification_service ||= NotificationService.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb new file mode 100644 index 00000000000..938664a95a1 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -0,0 +1,61 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class WebUploadStrategy < BaseAfterExportStrategy + PUT_METHOD = 'PUT'.freeze + POST_METHOD = 'POST'.freeze + INVALID_HTTP_METHOD = 'invalid. Only PUT and POST methods allowed.'.freeze + + validates :url, url: true + + validate do + unless [PUT_METHOD, POST_METHOD].include?(http_method.upcase) + errors.add(:http_method, INVALID_HTTP_METHOD) + end + end + + def initialize(url:, http_method: PUT_METHOD) + super + end + + protected + + def strategy_execute + handle_response_error(send_file) + + project.remove_exported_project_file + end + + def handle_response_error(response) + unless response.success? + error_code = response.dig('Error', 'Code') || response.code + error_message = response.dig('Error', 'Message') || response.message + + raise StrategyError.new("Error uploading the project. Code #{error_code}: #{error_message}") + end + end + + private + + def send_file + export_file = File.open(project.export_project_path) + + Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend + ensure + export_file.close if export_file + end + + def send_file_options(export_file) + { + body_stream: export_file, + headers: headers + } + end + + def headers + { 'Content-Length' => File.size(project.export_project_path).to_s } + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb new file mode 100644 index 00000000000..7eabcae2380 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategy_builder.rb @@ -0,0 +1,24 @@ +module Gitlab + module ImportExport + class AfterExportStrategyBuilder + StrategyNotFoundError = Class.new(StandardError) + + def self.build!(strategy_klass, attributes = {}) + return default_strategy.new unless strategy_klass + + attributes ||= {} + klass = strategy_klass.constantize rescue nil + + unless klass && klass < AfterExportStrategies::BaseAfterExportStrategy + raise StrategyNotFoundError.new("Strategy #{strategy_klass} not found") + end + + klass.new(**attributes.symbolize_keys) + end + + def self.default_strategy + AfterExportStrategies::DownloadNotificationStrategy + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 4bdd01f5e94..cd840bd5b01 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -105,6 +105,7 @@ excluded_attributes: - :last_repository_updated_at - :last_repository_check_at - :storage_version + - :description_html snippets: - :expired_at merge_request_diff: @@ -124,6 +125,8 @@ excluded_attributes: - :trace - :token - :when + - :artifacts_file + - :artifacts_metadata push_event_payload: - :event_id project_badges: @@ -144,8 +147,6 @@ methods: - :diff_head_sha - :source_branch_sha - :target_branch_sha - project: - - :description_html events: - :action push_event_payload: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index c38df9102eb..63cab07324a 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,6 +1,9 @@ module Gitlab module ImportExport class Importer + include Gitlab::Allowable + include Gitlab::Utils::StrongMemoize + def self.imports_repository? true end @@ -13,17 +16,24 @@ module Gitlab end def execute - if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore) + if import_file && check_version! && restorers.all?(&:restore) && overwrite_project project_tree.restored_project else raise Projects::ImportService::Error.new(@shared.errors.join(', ')) end - + rescue => e + raise Projects::ImportService::Error.new(e.message) + ensure remove_import_file end private + def restorers + [repo_restorer, wiki_restorer, project_tree, avatar_restorer, + uploads_restorer, lfs_restorer, statistics_restorer] + end + def import_file Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, shared: @shared) @@ -60,6 +70,14 @@ module Gitlab Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) end + def lfs_restorer + Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + + def statistics_restorer + Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + def path_with_namespace File.join(@project.namespace.full_path, @project.path) end @@ -75,6 +93,33 @@ module Gitlab def remove_import_file FileUtils.rm_rf(@archive_file) end + + def overwrite_project + project = project_tree.restored_project + + return unless can?(@current_user, :admin_namespace, project.namespace) + + if overwrite_project? + ::Projects::OverwriteProjectService.new(project, @current_user) + .execute(project_to_overwrite) + end + + true + end + + def original_path + @project.import_data&.data&.fetch('original_path', nil) + end + + def overwrite_project? + original_path.present? && project_to_overwrite.present? + end + + def project_to_overwrite + strong_memoize(:project_to_overwrite) do + Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}") + end + end end end end diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb new file mode 100644 index 00000000000..b28c3c161b7 --- /dev/null +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -0,0 +1,43 @@ +module Gitlab + module ImportExport + class LfsRestorer + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def restore + return true if lfs_file_paths.empty? + + lfs_file_paths.each do |file_path| + link_or_create_lfs_object!(file_path) + end + + true + rescue => e + @shared.error(e) + false + end + + private + + def link_or_create_lfs_object!(path) + size = File.size(path) + oid = LfsObject.calculate_oid(path) + + lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size) + lfs_object.file = File.open(path) unless lfs_object.file&.exists? + + @project.all_lfs_objects << lfs_object + end + + def lfs_file_paths + @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*") + end + + def lfs_storage_path + File.join(@shared.export_path, 'lfs-objects') + end + end + end +end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb new file mode 100644 index 00000000000..29410e2331c --- /dev/null +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -0,0 +1,55 @@ +module Gitlab + module ImportExport + class LfsSaver + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + @project.all_lfs_objects.each do |lfs_object| + save_lfs_object(lfs_object) + end + + true + rescue => e + @shared.error(e) + + false + end + + private + + def save_lfs_object(lfs_object) + if lfs_object.local_store? + copy_file_for_lfs_object(lfs_object) + else + download_file_for_lfs_object(lfs_object) + end + end + + def download_file_for_lfs_object(lfs_object) + destination = destination_path_for_object(lfs_object) + mkdir_p(File.dirname(destination)) + + File.open(destination, 'w') do |file| + IO.copy_stream(URI.parse(lfs_object.file.url).open, file) + end + end + + def copy_file_for_lfs_object(lfs_object) + copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) + end + + def destination_path_for_object(lfs_object) + File.join(lfs_export_path, lfs_object.oid) + end + + def lfs_export_path + File.join(@shared.export_path, 'lfs-objects') + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 8f5bb8f9597..d5590dde40f 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -77,24 +77,31 @@ module Gitlab end def default_relation_list - Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model| + reader.tree.reject do |model| model.is_a?(Hash) && model[:project_members] end end def restore_project - params = project_params - - if params[:description].present? - params[:description_html] = nil - end - - @project.update_columns(params) + @project.update_columns(project_params) @project end def project_params - @tree_hash.reject do |key, value| + @project_params ||= json_params.merge(override_params) + end + + def override_params + return {} unless params = @project.import_data&.data&.fetch('override_params', nil) + + @override_params ||= params.select do |key, _value| + Project.column_names.include?(key.to_s) && + !reader.project_tree[:except].include?(key.to_sym) + end + end + + def json_params + @json_params ||= @tree_hash.reject do |key, value| # return params that are not 1 to many or 1 to 1 relations value.respond_to?(:each) && !Project.column_names.include?(key) end @@ -181,6 +188,10 @@ module Gitlab relation_hash.merge(params) end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 791a54e1b69..598832fb2df 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -19,7 +19,7 @@ module Gitlab custom_attributes: 'ProjectCustomAttribute', project_badges: 'Badge' }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 3d3d998a6a3..6d7c36ce38b 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -22,7 +22,7 @@ module Gitlab def error(error) error_out(error.message, caller[0].dup) - @errors << error.message + add_error_message(error.message) # Debug: if error.backtrace @@ -32,6 +32,14 @@ module Gitlab end end + def add_error_message(error_message) + @errors << error_message + end + + def after_export_in_progress? + File.exist?(after_export_lock_file) + end + private def relative_path @@ -45,6 +53,10 @@ module Gitlab def error_out(message, caller) Rails.logger.error("Import/Export error raised on #{caller}: #{message}") end + + def after_export_lock_file + AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project) + end end end end diff --git a/lib/gitlab/import_export/statistics_restorer.rb b/lib/gitlab/import_export/statistics_restorer.rb new file mode 100644 index 00000000000..bcdd9c12c85 --- /dev/null +++ b/lib/gitlab/import_export/statistics_restorer.rb @@ -0,0 +1,17 @@ +module Gitlab + module ImportExport + class StatisticsRestorer + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def restore + @project.statistics.refresh! + rescue => e + @shared.error(e) + false + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 0526ef9eb13..7edd0ad2033 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -259,7 +259,7 @@ module Gitlab def import_wiki unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(project.repository_storage_path, wiki.disk_path, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage, wiki.disk_path, wiki.import_url) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb index db8bdde74b2..47b4af5d649 100644 --- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -4,6 +4,8 @@ require 'prometheus/client/rack/exporter' module Gitlab module Metrics class SidekiqMetricsExporter < Daemon + LOG_FILENAME = File.join(Rails.root, 'log', 'sidekiq_exporter.log') + def enabled? Gitlab::Metrics.metrics_folder_present? && settings.enabled end @@ -17,7 +19,13 @@ module Gitlab attr_reader :server def start_working - @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address) + logger = WEBrick::Log.new(LOG_FILENAME) + access_log = [ + [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] + ] + + @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address, + Logger: logger, AccessLog: access_log) server.mount "/", Rack::Handler::WEBrick, rack_app server.start end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index d4c54049b74..a5f5d719cc1 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -82,7 +82,7 @@ module Gitlab end def open_file(path, name) - ::UploadedFile.new(path, name || File.basename(path), 'application/octet-stream') + ::UploadedFile.new(path, filename: name || File.basename(path), content_type: 'application/octet-stream') end end diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 6c2b2036074..92a308a12dc 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -5,6 +5,7 @@ module Gitlab def self.enabled?(user = nil) return true if Rails.env.development? + return true if user&.admin? return false unless user && allowed_group_id allowed_user_ids.include?(user.id) diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index aad76e335af..f5879de1e94 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -79,7 +79,7 @@ module Gitlab def common_query_context(environment, timeframe_start:, timeframe_end:) base_query_context(timeframe_start, timeframe_end).merge({ ci_environment_slug: environment.slug, - kube_namespace: environment.project.deployment_platform&.actual_namespace || '', + kube_namespace: environment.deployment_platform&.actual_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} }) end diff --git a/lib/gitlab/proxy_http_connection_adapter.rb b/lib/gitlab/proxy_http_connection_adapter.rb index c70d6f4cd84..d682289b632 100644 --- a/lib/gitlab/proxy_http_connection_adapter.rb +++ b/lib/gitlab/proxy_http_connection_adapter.rb @@ -10,8 +10,12 @@ module Gitlab class ProxyHTTPConnectionAdapter < HTTParty::ConnectionAdapter def connection - if !allow_local_requests? && blocked_url? - raise URI::InvalidURIError + unless allow_local_requests? + begin + Gitlab::UrlBlocker.validate!(uri, allow_local_network: false) + rescue Gitlab::UrlBlocker::BlockedUrlError => e + raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" + end end super @@ -19,10 +23,6 @@ module Gitlab private - def blocked_url? - Gitlab::UrlBlocker.blocked_url?(uri, allow_private_networks: false) - end - def allow_local_requests? options.fetch(:allow_local_requests, allow_settings_local_requests?) end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index c8c15b9684a..67407b651a5 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -93,12 +93,12 @@ module Gitlab # Import repository # - # storage - project's storage path + # storage - project's storage name # name - project disk path # url - URL to import from # # Ex. - # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") + # import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) @@ -131,8 +131,7 @@ module Gitlab if is_enabled repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) else - storage_path = Gitlab.config.repositories.storages[repository.storage].legacy_disk_path - local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) + local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end end end @@ -156,13 +155,13 @@ module Gitlab end # Fork repository to new path - # forked_from_storage - forked-from project's storage path - # forked_from_disk_path - project disk path - # forked_to_storage - forked-to project's storage path - # forked_to_disk_path - forked project disk path + # forked_from_storage - forked-from project's storage name + # forked_from_disk_path - project disk relative path + # forked_to_storage - forked-to project's storage name + # forked_to_disk_path - forked project disk relative path # # Ex. - # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") + # fork_repository("nfs-file06", "gitlab/gitlab-ci", "nfs-file07", "new-namespace/gitlab-ci") # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) @@ -420,16 +419,16 @@ module Gitlab private - def gitlab_projects(shard_path, disk_path) + def gitlab_projects(shard_name, disk_path) Gitlab::Git::GitlabProjects.new( - shard_path, + shard_name, disk_path, global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, logger: Rails.logger ) end - def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) vars = { force: forced, tags: !no_tags, prune: prune } if ssh_auth&.ssh_import? @@ -442,7 +441,7 @@ module Gitlab end end - cmd = gitlab_projects(storage_path, repository_relative_path) + cmd = gitlab_projects(storage_name, repository_relative_path) success = cmd.fetch_remote(remote, git_timeout, vars) diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb new file mode 100644 index 00000000000..98f8222fd03 --- /dev/null +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -0,0 +1,21 @@ +module Gitlab + module SidekiqLogging + class JSONFormatter + def call(severity, timestamp, progname, data) + output = { + severity: severity, + time: timestamp.utc.iso8601(3) + } + + case data + when String + output[:message] = data + when Hash + output.merge!(data) + end + + output.to_json + "\n" + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb new file mode 100644 index 00000000000..9a89ae70b98 --- /dev/null +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -0,0 +1,96 @@ +module Gitlab + module SidekiqLogging + class StructuredLogger + START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze + DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze + + def call(job, queue) + started_at = current_time + base_payload = parse_job(job) + + Sidekiq.logger.info log_job_start(started_at, base_payload) + + yield + + Sidekiq.logger.info log_job_done(started_at, base_payload) + rescue => job_exception + Sidekiq.logger.warn log_job_done(started_at, base_payload, job_exception) + + raise + end + + private + + def base_message(payload) + "#{payload['class']} JID-#{payload['jid']}" + end + + def log_job_start(started_at, payload) + payload['message'] = "#{base_message(payload)}: start" + payload['job_status'] = 'start' + + payload + end + + def log_job_done(started_at, payload, job_exception = nil) + payload = payload.dup + payload['duration'] = elapsed(started_at) + payload['completed_at'] = Time.now.utc + + message = base_message(payload) + + if job_exception + payload['message'] = "#{message}: fail: #{payload['duration']} sec" + payload['job_status'] = 'fail' + payload['error_message'] = job_exception.message + payload['error'] = job_exception.class + payload['error_backtrace'] = backtrace_cleaner.clean(job_exception.backtrace) + else + payload['message'] = "#{message}: done: #{payload['duration']} sec" + payload['job_status'] = 'done' + end + + convert_to_iso8601(payload, DONE_TIMESTAMP_FIELDS) + + payload + end + + def parse_job(job) + job = job.dup + + # Add process id params + job['pid'] = ::Process.pid + + job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] + + convert_to_iso8601(job, START_TIMESTAMP_FIELDS) + + job + end + + def convert_to_iso8601(payload, keys) + keys.each do |key| + payload[key] = format_time(payload[key]) if payload[key] + end + end + + def elapsed(start) + (current_time - start).round(3) + end + + def current_time + Gitlab::Metrics::System.monotonic_time + end + + def backtrace_cleaner + @backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new + end + + def format_time(timestamp) + return timestamp if timestamp.is_a?(String) + + Time.at(timestamp).utc.iso8601(3) + end + end + end +end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 0f9f939e204..db97f65bd54 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -2,48 +2,84 @@ require 'resolv' module Gitlab class UrlBlocker - class << self - def blocked_url?(url, allow_private_networks: true, valid_ports: []) - return false if url.nil? + BlockedUrlError = Class.new(StandardError) - blocked_ips = ["127.0.0.1", "::1", "0.0.0.0"] - blocked_ips.concat(Socket.ip_address_list.map(&:ip_address)) + class << self + def validate!(url, allow_localhost: false, allow_local_network: true, valid_ports: []) + return true if url.nil? begin uri = Addressable::URI.parse(url) - # Allow imports from the GitLab instance itself but only from the configured ports - return false if internal?(uri) + rescue Addressable::URI::InvalidURIError + raise BlockedUrlError, "URI is invalid" + end - return true if blocked_port?(uri.port, valid_ports) - return true if blocked_user_or_hostname?(uri.user) - return true if blocked_user_or_hostname?(uri.hostname) + # Allow imports from the GitLab instance itself but only from the configured ports + return true if internal?(uri) - addrs_info = Addrinfo.getaddrinfo(uri.hostname, 80, nil, :STREAM) - server_ips = addrs_info.map(&:ip_address) + port = uri.port || uri.default_port + validate_port!(port, valid_ports) if valid_ports.any? + validate_user!(uri.user) + validate_hostname!(uri.hostname) - return true if (blocked_ips & server_ips).any? - return true if !allow_private_networks && private_network?(addrs_info) - rescue Addressable::URI::InvalidURIError - return true + begin + addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM) rescue SocketError - return false + return true end + validate_localhost!(addrs_info) unless allow_localhost + validate_local_network!(addrs_info) unless allow_local_network + + true + end + + def blocked_url?(*args) + validate!(*args) + false + rescue BlockedUrlError + true end private - def blocked_port?(port, valid_ports) - return false if port.blank? || valid_ports.blank? + def validate_port!(port, valid_ports) + return if port.blank? + # Only ports under 1024 are restricted + return if port >= 1024 + return if valid_ports.include?(port) - port < 1024 && !valid_ports.include?(port) + raise BlockedUrlError, "Only allowed ports are #{valid_ports.join(', ')}, and any over 1024" end - def blocked_user_or_hostname?(value) - return false if value.blank? + def validate_user!(value) + return if value.blank? + return if value =~ /\A\p{Alnum}/ - value !~ /\A\p{Alnum}/ + raise BlockedUrlError, "Username needs to start with an alphanumeric character" + end + + def validate_hostname!(value) + return if value.blank? + return if value =~ /\A\p{Alnum}/ + + raise BlockedUrlError, "Hostname needs to start with an alphanumeric character" + end + + def validate_localhost!(addrs_info) + local_ips = ["127.0.0.1", "::1", "0.0.0.0"] + local_ips.concat(Socket.ip_address_list.map(&:ip_address)) + + return if (local_ips & addrs_info.map(&:ip_address)).empty? + + raise BlockedUrlError, "Requests to localhost are not allowed" + end + + def validate_local_network!(addrs_info) + return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? } + + raise BlockedUrlError, "Requests to the local network are not allowed" end def internal?(uri) @@ -60,10 +96,6 @@ module Gitlab (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end - def private_network?(addrs_info) - addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? } - end - def config Gitlab.config end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 37d3512990e..8c0a4d55ea2 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -30,6 +30,7 @@ module Gitlab usage_data end + # rubocop:disable Metrics/AbcSize def system_usage_data { counts: { @@ -50,6 +51,12 @@ module Gitlab clusters: ::Clusters::Cluster.count, clusters_enabled: ::Clusters::Cluster.enabled.count, clusters_disabled: ::Clusters::Cluster.disabled.count, + clusters_platforms_gke: ::Clusters::Cluster.gcp_installed.enabled.count, + clusters_platforms_user: ::Clusters::Cluster.user_provided.enabled.count, + clusters_applications_helm: ::Clusters::Applications::Helm.installed.count, + clusters_applications_ingress: ::Clusters::Applications::Ingress.installed.count, + clusters_applications_prometheus: ::Clusters::Applications::Prometheus.installed.count, + clusters_applications_runner: ::Clusters::Applications::Runner.installed.count, in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index dc9391f32cf..b0a492eaa58 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -27,6 +27,11 @@ module Gitlab .gsub(/(\A-+|-+\z)/, '') end + # Converts newlines into HTML line break elements + def nlbr(str) + ActionView::Base.full_sanitizer.sanitize(str, tags: []).gsub(/\r?\n/, '<br>').html_safe + end + def remove_line_breaks(str) str.gsub(/\r?\n/, '') end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index b102812ec12..153cb2a8bb1 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -36,10 +36,6 @@ module Gitlab } end - def artifact_upload_ok - { TempPath: JobArtifactUploader.workhorse_upload_path } - end - def send_git_blob(repository, blob) params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { @@ -63,10 +59,10 @@ module Gitlab ] end - def send_git_archive(repository, ref:, format:) + def send_git_archive(repository, ref:, format:, append_sha:) format ||= 'tar.gz' format.downcase! - params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) + params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha) raise "Repository or ref not found" if params.empty? if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake index 7728c485e8d..6b22499a5c8 100644 --- a/lib/tasks/gitlab/two_factor.rake +++ b/lib/tasks/gitlab/two_factor.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :two_factor do desc "GitLab | Disable Two-factor authentication (2FA) for all users" - task disable_for_all_users: :environment do + task disable_for_all_users: :gitlab_environment do scope = User.with_two_factor count = scope.count diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index c26c3ccb3be..78e18992a8e 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -13,6 +13,7 @@ namespace :gitlab do def enqueue_batch(batch, index) job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, + @model_class, @mounted_as, @to_store) puts "Enqueued job ##{index}: #{job}" @@ -25,8 +26,8 @@ namespace :gitlab do Upload.class_eval { include EachBatch } unless Upload < EachBatch Upload - .where.not(store: @to_store) - .where(uploader: @uploader_class.to_s, + .where(store: [nil, ObjectStorage::Store::LOCAL], + uploader: @uploader_class.to_s, model_type: @model_class.base_class.sti_name) end end diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index 3e01f91d32c..b52af81fc16 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -4,8 +4,3 @@ desc "GitLab | Run all tests" task :test do Rake::Task["gitlab:test"].invoke end - -unless Rails.env.production? - desc "GitLab | Run all tests on CI with simplecov" - task test_ci: [:rubocop, :brakeman, :karma, :spinach, :spec] -end diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index 4a3c40f88eb..5dc85b2baea 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -1,8 +1,10 @@ require "tempfile" +require "tmpdir" require "fileutils" -# Taken from: Rack::Test::UploadedFile class UploadedFile + InvalidPathError = Class.new(StandardError) + # The filename, *not* including the path, of the "uploaded" file attr_reader :original_filename @@ -12,14 +14,46 @@ class UploadedFile # The content type of the "uploaded" file attr_accessor :content_type - def initialize(path, filename, content_type = "text/plain") - raise "#{path} file does not exist" unless ::File.exist?(path) + attr_reader :remote_id + attr_reader :sha256 + + def initialize(path, filename: nil, content_type: "application/octet-stream", sha256: nil, remote_id: nil) + raise InvalidPathError, "#{path} file does not exist" unless ::File.exist?(path) @content_type = content_type @original_filename = filename || ::File.basename(path) + @content_type = content_type + @sha256 = sha256 + @remote_id = remote_id @tempfile = File.new(path, 'rb') end + def self.from_params(params, field, upload_path) + unless params["#{field}.path"] + raise InvalidPathError, "file is invalid" if params["#{field}.remote_id"] + + return + end + + file_path = File.realpath(params["#{field}.path"]) + + unless self.allowed_path?(file_path, [upload_path, Dir.tmpdir].compact) + raise InvalidPathError, "insecure path used '#{file_path}'" + end + + UploadedFile.new(file_path, + filename: params["#{field}.name"], + content_type: params["#{field}.type"] || 'application/octet-stream', + sha256: params["#{field}.sha256"], + remote_id: params["#{field}.remote_id"]) + end + + def self.allowed_path?(file_path, paths) + paths.any? do |path| + File.exist?(path) && file_path.start_with?(File.realpath(path)) + end + end + def path @tempfile.path end |