diff options
| author | Frederic Van Espen <fes@escaux.com> | 2019-03-07 11:14:27 +0100 |
|---|---|---|
| committer | Frederic Van Espen <fes@escaux.com> | 2019-03-07 11:14:27 +0100 |
| commit | 31dfc31aaa227224152f200b9fb961404a08fa40 (patch) | |
| tree | 69f8e54ecf7a7205df5277ae997f0b1d8158835c /lib | |
| parent | 562a1fc87d0269ce5fb1561fea45f8d01f4889de (diff) | |
| parent | 5a75aa59dbafc8f0c25800f952df1e0aaa2d4dd5 (diff) | |
| download | gitlab-ce-31dfc31aaa227224152f200b9fb961404a08fa40.tar.gz | |
Merge branch 'master' into incremental-backups
Diffstat (limited to 'lib')
357 files changed, 6506 insertions, 1553 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index f1448da7403..bf8ddba6f0d 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -9,6 +9,7 @@ module API NO_SLASH_URL_PART_REGEX = %r{[^/]+} NAMESPACE_OR_PROJECT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze COMMIT_ENDPOINT_REQUIREMENTS = NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze + USER_REQUIREMENTS = { user_id: NO_SLASH_URL_PART_REGEX }.freeze insert_before Grape::Middleware::Error, GrapeLogging::Middleware::RequestLogger, @@ -100,6 +101,7 @@ module API mount ::API::CircuitBreakers mount ::API::Commits mount ::API::CommitStatuses + mount ::API::ContainerRegistry mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments @@ -107,9 +109,11 @@ module API mount ::API::Features mount ::API::Files mount ::API::GroupBoards + mount ::API::GroupLabels mount ::API::GroupMilestones mount ::API::Groups mount ::API::GroupVariables + mount ::API::ImportGithub mount ::API::Internal mount ::API::Issues mount ::API::JobArtifacts @@ -129,6 +133,7 @@ module API mount ::API::PagesDomains mount ::API::Pipelines mount ::API::PipelineSchedules + mount ::API::ProjectClusters mount ::API::ProjectExport mount ::API::ProjectImport mount ::API::ProjectHooks @@ -136,9 +141,12 @@ module API mount ::API::Projects mount ::API::ProjectSnapshots mount ::API::ProjectSnippets + mount ::API::ProjectStatistics mount ::API::ProjectTemplates mount ::API::ProtectedBranches mount ::API::ProtectedTags + mount ::API::Releases + mount ::API::Release::Links mount ::API::Repositories mount ::API::Runner mount ::API::Runners diff --git a/lib/api/branches.rb b/lib/api/branches.rb index e7e58ad0e66..07f529b01bb 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -34,11 +34,11 @@ module API repository = user_project.repository branches = BranchesFinder.new(repository, declared_params(include_missing: false)).execute - + branches = ::Kaminari.paginate_array(branches) merged_branch_names = repository.merged_branch_names(branches.map(&:name)) present( - paginate(::Kaminari.paginate_array(branches)), + paginate(branches), with: Entities::Branch, current_user: current_user, project: user_project, diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 9d23daafe95..65eb9bfb87e 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -99,6 +99,7 @@ module API optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' optional :stats, type: Boolean, default: true, desc: 'Include commit stats' + optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch`' end post ':id/repository/commits' do authorize_push_to_branch!(params[:branch]) @@ -318,10 +319,34 @@ module API use :pagination end get ':id/repository/commits/:sha/merge_requests', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + authorize! :read_merge_request, user_project + + commit = user_project.commit(params[:sha]) + not_found! 'Commit' unless commit + + commit_merge_requests = MergeRequestsFinder.new( + current_user, + project_id: user_project.id, + commit_sha: commit.sha + ).execute + + present paginate(commit_merge_requests), with: Entities::MergeRequestBasic + end + + desc "Get a commit's GPG signature" do + success Entities::CommitSignature + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end + get ':id/repository/commits/:sha/signature', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - present paginate(commit.merge_requests), with: Entities::MergeRequestBasic + signature = commit.signature + not_found! 'GPG Signature' unless signature + + present signature, with: Entities::CommitSignature end end end diff --git a/lib/api/container_registry.rb b/lib/api/container_registry.rb new file mode 100644 index 00000000000..e4493910196 --- /dev/null +++ b/lib/api/container_registry.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +module API + class ContainerRegistry < Grape::API + include PaginationParams + + REGISTRY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + tag_name: API::NO_SLASH_URL_PART_REGEX) + + before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) } + before { authorize_read_container_images! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get a project container repositories' do + detail 'This feature was introduced in GitLab 11.8.' + success Entities::ContainerRegistry::Repository + end + params do + use :pagination + end + get ':id/registry/repositories' do + repositories = user_project.container_repositories.ordered + + present paginate(repositories), with: Entities::ContainerRegistry::Repository + end + + desc 'Delete repository' do + detail 'This feature was introduced in GitLab 11.8.' + end + params do + requires :repository_id, type: Integer, desc: 'The ID of the repository' + end + delete ':id/registry/repositories/:repository_id', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + authorize_admin_container_image! + + DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) + + status :accepted + end + + desc 'Get a list of repositories tags' do + detail 'This feature was introduced in GitLab 11.8.' + success Entities::ContainerRegistry::Tag + end + params do + requires :repository_id, type: Integer, desc: 'The ID of the repository' + use :pagination + end + get ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + authorize_read_container_image! + + tags = Kaminari.paginate_array(repository.tags) + present paginate(tags), with: Entities::ContainerRegistry::Tag + end + + desc 'Delete repository tags (in bulk)' do + detail 'This feature was introduced in GitLab 11.8.' + end + params do + requires :repository_id, type: Integer, desc: 'The ID of the repository' + requires :name_regex, type: String, desc: 'The tag name regexp to delete, specify .* to delete all' + optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name' + optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month' + end + delete ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + authorize_admin_container_image! + + CleanupContainerRepositoryWorker.perform_async(current_user.id, repository.id, + declared_params.except(:repository_id)) # rubocop: disable CodeReuse/ActiveRecord + + status :accepted + end + + desc 'Get a details about repository tag' do + detail 'This feature was introduced in GitLab 11.8.' + success Entities::ContainerRegistry::TagDetails + end + params do + requires :repository_id, type: Integer, desc: 'The ID of the repository' + requires :tag_name, type: String, desc: 'The name of the tag' + end + get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + authorize_read_container_image! + validate_tag! + + present tag, with: Entities::ContainerRegistry::TagDetails + end + + desc 'Delete repository tag' do + detail 'This feature was introduced in GitLab 11.8.' + end + params do + requires :repository_id, type: Integer, desc: 'The ID of the repository' + requires :tag_name, type: String, desc: 'The name of the tag' + end + delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + authorize_destroy_container_image! + validate_tag! + + tag.delete + + status :ok + end + end + + helpers do + def authorize_read_container_images! + authorize! :read_container_image, user_project + end + + def authorize_read_container_image! + authorize! :read_container_image, repository + end + + def authorize_update_container_image! + authorize! :update_container_image, repository + end + + def authorize_destroy_container_image! + authorize! :admin_container_image, repository + end + + def authorize_admin_container_image! + authorize! :admin_container_image, repository + end + + def repository + @repository ||= user_project.container_repositories.find(params[:repository_id]) + end + + def tag + @tag ||= repository.tag(params[:tag_name]) + end + + def validate_tag! + not_found!('Tag') unless tag.valid? + end + end + end +end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 8706a971a1a..eb45df31ff9 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -33,7 +33,7 @@ module API success Entities::Deployment end params do - requires :deployment_id, type: Integer, desc: 'The deployment ID' + requires :deployment_id, type: Integer, desc: 'The deployment ID' end get ':id/deployments/:deployment_id' do authorize! :read_deployment, user_project diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 22403664c21..5176e9713c1 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -115,6 +115,9 @@ module API expose :group_name do |group_link, options| group_link.group.name end + expose :group_full_path do |group_link, options| + group_link.group.full_path + end expose :group_access, as: :group_access_level expose :expires_at end @@ -153,7 +156,7 @@ module API class BasicProjectDetails < ProjectIdentity include ::API::ProjectsRelationBuilder - expose :default_branch + expose :default_branch, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) } # Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770 expose :tag_list do |project| # project.tags.order(:name).pluck(:name) is the most suitable option @@ -187,7 +190,7 @@ module API expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes # rubocop: disable CodeReuse/ActiveRecord - def self.preload_relation(projects_relation, options = {}) + def self.preload_relation(projects_relation, options = {}) # Preloading tags, should be done with using only `:tags`, # as `:tags` are defined as: `has_many :tags, through: :taggings` # N+1 is solved then by using `subject.tags.map(&:name)` @@ -258,7 +261,7 @@ module API expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds, as: :public_jobs - expose :ci_config_path + expose :ci_config_path, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) } expose :shared_with_groups do |project, options| SharedGroup.represent(project.project_group_links, options) end @@ -267,17 +270,18 @@ module API expose :only_allow_merge_if_all_discussions_are_resolved expose :printing_merge_request_link_enabled expose :merge_method - - expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics + expose :statistics, using: 'API::Entities::ProjectStatistics', if: -> (project, options) { + options[:statistics] && Ability.allowed?(options[:current_user], :download_code, project) + } # rubocop: disable CodeReuse/ActiveRecord - def self.preload_relation(projects_relation, options = {}) + def self.preload_relation(projects_relation, options = {}) # Preloading tags, should be done with using only `:tags`, # as `:tags` are defined as: `has_many :tags, through: :taggings` # N+1 is solved then by using `subject.tags.map(&:name)` # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20555 super(projects_relation).preload(:group) - .preload(project_group_links: :group, + .preload(project_group_links: { group: :route }, fork_network: :root_project, fork_network_member: :forked_from_project, forked_from_project: [:route, :forks, :tags, namespace: :route]) @@ -297,6 +301,18 @@ module API expose :build_artifacts_size, as: :job_artifacts_size end + class ProjectDailyFetches < Grape::Entity + expose :fetch_count, as: :count + expose :date + end + + class ProjectDailyStatistics < Grape::Entity + expose :fetches do + expose :total_fetch_count, as: :total + expose :fetches, as: :days, using: ProjectDailyFetches + end + end + class Member < Grape::Entity expose :user, merge: true, using: UserBasic expose :access_level @@ -341,19 +357,23 @@ module API class GroupDetail < Group expose :projects, using: Entities::Project do |group, options| - GroupProjectsFinder.new( + projects = GroupProjectsFinder.new( group: group, current_user: options[:current_user], options: { only_owned: true } ).execute + + Entities::Project.prepare_relation(projects) end expose :shared_projects, using: Entities::Project do |group, options| - GroupProjectsFinder.new( + projects = GroupProjectsFinder.new( group: group, current_user: options[:current_user], options: { only_shared: true } ).execute + + Entities::Project.prepare_relation(projects) end end @@ -362,8 +382,9 @@ module API end class Commit < Grape::Entity - expose :id, :short_id, :title, :created_at + expose :id, :short_id, :created_at expose :parent_ids + expose :full_title, as: :title expose :safe_message, as: :message expose :author_name, :author_email, :authored_date expose :committer_name, :committer_email, :committed_date @@ -384,6 +405,13 @@ module API expose :project_id end + class CommitSignature < Grape::Entity + expose :gpg_key_id + expose :gpg_key_primary_keyid, :gpg_key_user_name, :gpg_key_user_email + expose :verification_status + expose :gpg_key_subkey_id + end + class BasicRef < Grape::Entity expose :type, :name end @@ -458,6 +486,12 @@ module API expose(:project_id) { |entity| entity&.project.try(:id) } expose :title, :description expose :state, :created_at, :updated_at + + # Avoids an N+1 query when metadata is included + def issuable_metadata(subject, options, method) + cached_subject = options.dig(:issuable_metadata, subject.id) + (cached_subject || subject).public_send(method) # rubocop: disable GitlabSecurity/PublicSend + end end class Diff < Grape::Entity @@ -503,39 +537,26 @@ module API class IssueBasic < ProjectEntity expose :closed_at expose :closed_by, using: Entities::UserBasic - expose :labels do |issue, options| + expose :labels do |issue| # Avoids an N+1 query since labels are preloaded issue.labels.map(&:title).sort end expose :milestone, using: Entities::Milestone expose :assignees, :author, using: Entities::UserBasic - expose :assignee, using: ::API::Entities::UserBasic do |issue, options| + expose :assignee, using: ::API::Entities::UserBasic do |issue| issue.assignees.first end - expose :user_notes_count - expose :upvotes do |issue, options| - if options[:issuable_metadata] - # Avoids an N+1 query when metadata is included - options[:issuable_metadata][issue.id].upvotes - else - issue.upvotes - end - end - expose :downvotes do |issue, options| - if options[:issuable_metadata] - # Avoids an N+1 query when metadata is included - options[:issuable_metadata][issue.id].downvotes - else - issue.downvotes - end - end + expose(:user_notes_count) { |issue, options| issuable_metadata(issue, options, :user_notes_count) } + expose(:merge_requests_count) { |issue, options| issuable_metadata(issue, options, :merge_requests_count) } + expose(:upvotes) { |issue, options| issuable_metadata(issue, options, :upvotes) } + expose(:downvotes) { |issue, options| issuable_metadata(issue, options, :downvotes) } expose :due_date expose :confidential expose :discussion_locked - expose :web_url do |issue, options| + expose :web_url do |issue| Gitlab::UrlBuilder.build(issue) end @@ -635,23 +656,12 @@ module API MarkupHelper.markdown_field(entity, :description) end expose :target_branch, :source_branch - expose :upvotes do |merge_request, options| - if options[:issuable_metadata] - options[:issuable_metadata][merge_request.id].upvotes - else - merge_request.upvotes - end - end - expose :downvotes do |merge_request, options| - if options[:issuable_metadata] - options[:issuable_metadata][merge_request.id].downvotes - else - merge_request.downvotes - end - end + expose(:user_notes_count) { |merge_request, options| issuable_metadata(merge_request, options, :user_notes_count) } + expose(:upvotes) { |merge_request, options| issuable_metadata(merge_request, options, :upvotes) } + expose(:downvotes) { |merge_request, options| issuable_metadata(merge_request, options, :downvotes) } expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id - expose :labels do |merge_request, options| + expose :labels do |merge_request| # Avoids an N+1 query since labels are preloaded merge_request.labels.map(&:title).sort end @@ -669,7 +679,6 @@ module API end expose :diff_head_sha, as: :sha expose :merge_commit_sha - expose :user_notes_count expose :discussion_locked expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch @@ -677,7 +686,7 @@ module API # Deprecated expose :allow_collaboration, as: :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? } - expose :web_url do |merge_request, options| + expose :web_url do |merge_request| Gitlab::UrlBuilder.build(merge_request) end @@ -724,6 +733,12 @@ module API def build_available?(options) options[:project]&.feature_available?(:builds, options[:current_user]) end + + expose :user do + expose :can_merge do |merge_request, options| + merge_request.can_be_merged_by?(options[:current_user]) + end + end end class MergeRequestChanges < MergeRequest @@ -961,7 +976,7 @@ module API if options[:group_members] options[:group_members].find { |member| member.source_id == project.namespace_id } else - project.group.group_member(options[:current_user]) + project.group.highest_group_member(options[:current_user]) end end end @@ -996,7 +1011,7 @@ module API end class LabelBasic < Grape::Entity - expose :id, :name, :color, :description + expose :id, :name, :color, :description, :text_color end class Label < LabelBasic @@ -1012,12 +1027,20 @@ module API label.open_merge_requests_count(options[:current_user]) end - expose :priority do |label, options| - label.priority(options[:project]) + expose :subscribed do |label, options| + label.subscribed?(options[:current_user], options[:parent]) end + end - expose :subscribed do |label, options| - label.subscribed?(options[:current_user], options[:project]) + class GroupLabel < Label + end + + class ProjectLabel < Label + expose :priority do |label, options| + label.priority(options[:parent]) + end + expose :is_project_label do |label, options| + label.is_a?(::ProjectLabel) end end @@ -1087,11 +1110,44 @@ module API expose :password_authentication_enabled_for_web, as: :signin_enabled end - class Release < Grape::Entity + # deprecated old Release representation + class TagRelease < Grape::Entity expose :tag, as: :tag_name expose :description end + module Releases + class Link < Grape::Entity + expose :id + expose :name + expose :url + expose :external?, as: :external + end + + class Source < Grape::Entity + expose :format + expose :url + end + end + + class Release < TagRelease + expose :name + expose :description_html do |entity| + MarkupHelper.markdown_field(entity, :description) + end + expose :created_at + expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? } + expose :commit, using: Entities::Commit + + expose :assets do + expose :assets_count, as: :count + expose :sources, using: Entities::Releases::Source + expose :links, using: Entities::Releases::Link do |release, options| + release.links.sorted + end + end + end + class Tag < Grape::Entity expose :name, :message, :target @@ -1100,7 +1156,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord - expose :release, using: Entities::Release do |repo_tag, options| + expose :release, using: Entities::TagRelease do |repo_tag, options| options[:project].releases.find_by(tag: repo_tag.name) end # rubocop: enable CodeReuse/ActiveRecord @@ -1185,8 +1241,11 @@ module API end class Trigger < Grape::Entity + include ::API::Helpers::Presentable + expose :id - expose :token, :description + expose :token + expose :description expose :created_at, :updated_at, :last_used expose :owner, using: Entities::UserBasic end @@ -1311,13 +1370,9 @@ module API class GitInfo < Grape::Entity expose :repo_url, :ref, :sha, :before_sha - expose :ref_type do |model| - if model.tag - 'tag' - else - 'branch' - end - end + expose :ref_type + expose :refspecs + expose :git_depth, as: :depth end class RunnerInfo < Grape::Entity @@ -1507,5 +1562,38 @@ module API expose :from_content expose :to_content end + + module Platform + class Kubernetes < Grape::Entity + expose :api_url + expose :namespace + expose :authorization_type + expose :ca_cert + end + end + + module Provider + class Gcp < Grape::Entity + expose :cluster_id + expose :status_name + expose :gcp_project_id + expose :zone + expose :machine_type + expose :num_nodes + expose :endpoint + end + end + + class Cluster < Grape::Entity + expose :id, :name, :created_at + expose :provider_type, :platform_type, :environment_scope, :cluster_type + expose :user, using: Entities::UserBasic + expose :platform_kubernetes, using: Entities::Platform::Kubernetes + expose :provider_gcp, using: Entities::Provider::Gcp + end + + class ClusterProject < Cluster + expose :project, using: Entities::BasicProjectDetails + end end end diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb new file mode 100644 index 00000000000..00833ca7480 --- /dev/null +++ b/lib/api/entities/container_registry.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module API + module Entities + module ContainerRegistry + class Repository < Grape::Entity + expose :id + expose :name + expose :path + expose :location + expose :created_at + end + + class Tag < Grape::Entity + expose :name + expose :path + expose :location + end + + class TagDetails < Tag + expose :revision + expose :short_revision + expose :digest + expose :created_at + expose :total_size + end + end + end +end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 633f24d3c9a..5b0f3b914cb 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -22,7 +22,7 @@ module API get ':id/environments' do authorize! :read_environment, user_project - present paginate(user_project.environments), with: Entities::Environment + present paginate(user_project.environments), with: Entities::Environment, current_user: current_user end desc 'Creates a new environment' do @@ -40,7 +40,7 @@ module API environment = user_project.environments.create(declared_params) if environment.persisted? - present environment, with: Entities::Environment + present environment, with: Entities::Environment, current_user: current_user else render_validation_error!(environment) end @@ -63,7 +63,7 @@ module API update_params = declared_params(include_missing: false).extract!(:name, :external_url) if environment.update(update_params) - present environment, with: Entities::Environment + present environment, with: Entities::Environment, current_user: current_user else render_validation_error!(environment) end @@ -74,7 +74,7 @@ module API success Entities::Environment end params do - requires :environment_id, type: Integer, desc: 'The environment ID' + requires :environment_id, type: Integer, desc: 'The environment ID' end delete ':id/environments/:environment_id' do authorize! :update_environment, user_project @@ -88,7 +88,7 @@ module API success Entities::Environment end params do - requires :environment_id, type: Integer, desc: 'The environment ID' + requires :environment_id, type: Integer, desc: 'The environment ID' end post ':id/environments/:environment_id/stop' do authorize! :read_environment, user_project @@ -99,7 +99,7 @@ module API environment.stop_with_action!(current_user) status 200 - present environment, with: Entities::Environment + present environment, with: Entities::Environment, current_user: current_user end end end diff --git a/lib/api/features.rb b/lib/api/features.rb index 1331248699f..4dc1834c644 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -16,15 +16,13 @@ module API end end - # rubocop: disable CodeReuse/ActiveRecord def gate_targets(params) - targets = [] - targets << Feature.group(params[:feature_group]) if params[:feature_group] - targets << UserFinder.new(params[:user]).find_by_username if params[:user] + Feature::Target.new(params).targets + end - targets + def gate_specified?(params) + Feature::Target.new(params).gate_specified? end - # rubocop: enable CodeReuse/ActiveRecord end resource :features do @@ -44,6 +42,8 @@ module API requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time' optional :feature_group, type: String, desc: 'A Feature group name' optional :user, type: String, desc: 'A GitLab username' + optional :group, type: String, desc: "A GitLab group's path, such as 'gitlab-org'" + optional :project, type: String, desc: 'A projects path, like gitlab-org/gitlab-ce' end post ':name' do feature = Feature.get(params[:name]) @@ -52,13 +52,13 @@ module API case value when true - if targets.present? + if gate_specified?(params) targets.each { |target| feature.enable(target) } else feature.enable end when false - if targets.present? + if gate_specified?(params) targets.each { |target| feature.disable(target) } else feature.disable diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb new file mode 100644 index 00000000000..0dbc5f45a68 --- /dev/null +++ b/lib/api/group_labels.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module API + class GroupLabels < Grape::API + include PaginationParams + helpers ::API::Helpers::LabelHelpers + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get all labels of the group' do + detail 'This feature was added in GitLab 11.8' + success Entities::GroupLabel + end + params do + use :pagination + end + get ':id/labels' do + get_labels(user_group, Entities::GroupLabel) + end + + desc 'Create a new label' do + detail 'This feature was added in GitLab 11.8' + success Entities::GroupLabel + end + params do + use :label_create_params + end + post ':id/labels' do + create_label(user_group, Entities::GroupLabel) + end + + desc 'Update an existing label. At least one optional parameter is required.' do + detail 'This feature was added in GitLab 11.8' + success Entities::GroupLabel + end + params do + requires :name, type: String, desc: 'The name of the label to be updated' + optional :new_name, type: String, desc: 'The new name of the label' + optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" + optional :description, type: String, desc: 'The new description of label' + at_least_one_of :new_name, :color, :description + end + put ':id/labels' do + update_label(user_group, Entities::GroupLabel) + end + + desc 'Delete an existing label' do + detail 'This feature was added in GitLab 11.8' + success Entities::GroupLabel + end + params do + requires :name, type: String, desc: 'The name of the label to be deleted' + end + delete ':id/labels' do + delete_label(user_group) + end + end + end +end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index c3eca713712..825fab62034 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -84,8 +84,8 @@ module API page || not_found!('Wiki Page') end - def available_labels_for(label_parent) - search_params = { include_ancestor_groups: true } + def available_labels_for(label_parent, include_ancestor_groups: true) + search_params = { include_ancestor_groups: include_ancestor_groups } if label_parent.is_a?(Project) search_params[:project_id] = label_parent.id @@ -170,13 +170,6 @@ module API end end - def find_project_label(id) - labels = available_labels_for(user_project) - label = labels.find_by_id(id) || labels.find_by_title(id) - - label || not_found!('Label') - end - # rubocop: disable CodeReuse/ActiveRecord def find_project_issue(iid) IssuesFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid) @@ -235,8 +228,8 @@ module API forbidden! unless current_user.admin? end - def authorize!(action, subject = :global) - forbidden! unless can?(current_user, action, subject) + def authorize!(action, subject = :global, reason = nil) + forbidden!(reason) unless can?(current_user, action, subject) end def authorize_push_project @@ -251,6 +244,10 @@ module API authorize! :read_build, user_project end + def authorize_destroy_artifacts! + authorize! :destroy_artifacts, user_project + end + def authorize_update_builds! authorize! :update_build, user_project end @@ -306,6 +303,12 @@ module API items.search(text) end + def order_options_with_tie_breaker + order_options = { params[:order_by] => params[:sort] } + order_options['id'] ||= 'desc' + order_options + end + # error helpers def forbidden!(reason = nil) @@ -400,7 +403,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord def reorder_projects(projects) - projects.reorder(params[:order_by] => params[:sort]) + projects.reorder(order_options_with_tie_breaker) end # rubocop: enable CodeReuse/ActiveRecord @@ -422,7 +425,7 @@ module API def present_disk_file!(path, filename, content_type = 'application/octet-stream') filename ||= File.basename(path) - header['Content-Disposition'] = "attachment; filename=#{filename}" + header['Content-Disposition'] = ::Gitlab::ContentDisposition.format(disposition: 'attachment', filename: filename) header['Content-Transfer-Encoding'] = 'binary' content_type content_type @@ -496,7 +499,11 @@ module API def send_git_blob(repository, blob) env['api.format'] = :txt content_type 'text/plain' - header['Content-Disposition'] = "attachment; filename=#{blob.name.inspect}" + header['Content-Disposition'] = ::Gitlab::ContentDisposition.format(disposition: 'inline', filename: blob.name) + + # Let Workhorse examine the content and determine the better content disposition + header[Gitlab::Workhorse::DETECT_HEADER] = "true" + header(*Gitlab::Workhorse.send_git_blob(repository, blob)) end @@ -512,7 +519,7 @@ module API # `request`. We workaround this by defining methods that returns the right # values. def define_params_for_grape_middleware - self.define_singleton_method(:request) { Rack::Request.new(env) } + self.define_singleton_method(:request) { ActionDispatch::Request.new(env) } self.define_singleton_method(:params) { request.params.symbolize_keys } end diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb index 1058f4e8a5e..c86eae6f2da 100644 --- a/lib/api/helpers/custom_validators.rb +++ b/lib/api/helpers/custom_validators.rb @@ -22,9 +22,22 @@ module API message: "should be an integer, 'None' or 'Any'" end end + + class ArrayNoneAny < Grape::Validations::Base + def validate_param!(attr_name, params) + value = params[attr_name] + + return if value.is_a?(Array) || + [IssuableFinder::FILTER_NONE, IssuableFinder::FILTER_ANY].include?(value.to_s.downcase) + + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], + message: "should be an array, 'None' or 'Any'" + end + end end end end Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence) Grape::Validations.register_validator(:integer_none_any, ::API::Helpers::CustomValidators::IntegerNoneAny) +Grape::Validations.register_validator(:array_none_any, ::API::Helpers::CustomValidators::ArrayNoneAny) diff --git a/lib/api/helpers/graphql_helpers.rb b/lib/api/helpers/graphql_helpers.rb new file mode 100644 index 00000000000..94010ab1bc2 --- /dev/null +++ b/lib/api/helpers/graphql_helpers.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module API + module Helpers + # GraphqlHelpers is used by the REST API when it is acting like a client + # against the graphql API. Helper code for the graphql server implementation + # should be in app/graphql/ or lib/gitlab/graphql/ + module GraphqlHelpers + def conditionally_graphql!(fallback:, query:, context: {}, transform: nil) + return fallback.call unless Feature.enabled?(:graphql) + + result = GitlabSchema.execute(query, context: context) + + if transform + transform.call(result) + else + result + end + end + end + end +end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 4eaaca96b49..fe78049af87 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -81,6 +81,14 @@ module API Gitlab::GlRepository.gl_repository(project, wiki?) end + def gl_project_path + if wiki? + project.wiki.full_path + else + project.full_path + end + end + # Return the repository depending on whether we want the wiki or the # regular repository def repository diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb new file mode 100644 index 00000000000..c11e7d614ab --- /dev/null +++ b/lib/api/helpers/label_helpers.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module API + module Helpers + module LabelHelpers + extend Grape::API::Helpers + + params :label_create_params do + requires :name, type: String, desc: 'The name of the label to be created' + requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" + optional :description, type: String, desc: 'The description of label to be created' + end + + def find_label(parent, id, include_ancestor_groups: true) + labels = available_labels_for(parent, include_ancestor_groups: include_ancestor_groups) + label = labels.find_by_id(id) || labels.find_by_title(id) + + label || not_found!('Label') + end + + def get_labels(parent, entity) + present paginate(available_labels_for(parent)), with: entity, current_user: current_user, parent: parent + end + + def create_label(parent, entity) + authorize! :admin_label, parent + + label = available_labels_for(parent).find_by_title(params[:name]) + conflict!('Label already exists') if label + + priority = params.delete(:priority) + label_params = declared_params(include_missing: false) + + label = + if parent.is_a?(Project) + ::Labels::CreateService.new(label_params).execute(project: parent) + else + ::Labels::CreateService.new(label_params).execute(group: parent) + end + + if label.persisted? + if parent.is_a?(Project) + label.prioritize!(parent, priority) if priority + end + + present label, with: entity, current_user: current_user, parent: parent + else + render_validation_error!(label) + end + end + + def update_label(parent, entity) + authorize! :admin_label, parent + + label = find_label(parent, params[:name], include_ancestor_groups: false) + update_priority = params.key?(:priority) + priority = params.delete(:priority) + + label = ::Labels::UpdateService.new(declared_params(include_missing: false)).execute(label) + render_validation_error!(label) unless label.valid? + + if parent.is_a?(Project) && update_priority + if priority.nil? + label.unprioritize!(parent) + else + label.prioritize!(parent, priority) + end + end + + present label, with: entity, current_user: current_user, parent: parent + end + + def delete_label(parent) + authorize! :admin_label, parent + + label = find_label(parent, params[:name], include_ancestor_groups: false) + + destroy_conditionally!(label) + end + end + end +end diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 216b2c45741..795dca5cf03 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -70,14 +70,7 @@ module API def find_noteable(parent, noteables_str, noteable_id) noteable = public_send("find_#{parent}_#{noteables_str.singularize}", noteable_id) # rubocop:disable GitlabSecurity/PublicSend - readable = - if noteable.is_a?(Commit) - # for commits there is not :read_commit policy, check if user - # has :read_note permission on the commit's project - can?(current_user, :read_note, user_project) - else - can?(current_user, noteable_read_ability_name(noteable), noteable) - end + readable = can?(current_user, noteable_read_ability_name(noteable), noteable) return not_found!(noteables_str) unless readable @@ -89,12 +82,11 @@ module API end def create_note(noteable, opts) - policy_object = noteable.is_a?(Commit) ? user_project : noteable - authorize!(:create_note, policy_object) + authorize!(:create_note, noteable) parent = noteable_parent(noteable) - opts.delete(:created_at) unless current_user.can?(:set_note_created_at, policy_object) + opts.delete(:created_at) unless current_user.can?(:set_note_created_at, noteable) opts[:updated_at] = opts[:created_at] if opts[:created_at] diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index d311cbb5f7e..94b58a64d26 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -13,6 +13,33 @@ module API strategy.new(self).paginate(relation) end + class Base + private + + def per_page + @per_page ||= params[:per_page] + end + + def base_request_uri + @base_request_uri ||= URI.parse(request.url).tap do |uri| + uri.host = Gitlab.config.gitlab.host + uri.port = nil + end + end + + def build_page_url(query_params:) + base_request_uri.tap do |uri| + uri.query = query_params + end.to_s + end + + def page_href(next_page_params = {}) + query_params = params.merge(**next_page_params, per_page: per_page).to_query + + build_page_url(query_params: query_params) + end + end + class KeysetPaginationInfo attr_reader :relation, :request_context @@ -85,7 +112,7 @@ module API end end - class KeysetPaginationStrategy + class KeysetPaginationStrategy < Base attr_reader :request_context delegate :params, :header, :request, to: :request_context @@ -122,7 +149,7 @@ module API def conditions(pagination) fields = pagination.fields - return nil if fields.empty? + return if fields.empty? placeholder = fields.map { '?' } @@ -141,12 +168,8 @@ module API ] end - def per_page - params[:per_page] - end - def add_default_pagination_headers - header 'X-Per-Page', per_page.to_s + header 'X-Per-Page', per_page.to_s end def add_navigation_links(next_page_params) @@ -154,22 +177,12 @@ module API header 'Link', link_for('next', next_page_params) end - def page_href(next_page_params) - request_url = request.url.split('?').first - request_params = params.dup - request_params[:per_page] = per_page - - request_params.merge!(next_page_params) if next_page_params - - "#{request_url}?#{request_params.to_query}" - end - def link_for(rel, next_page_params) %(<#{page_href(next_page_params)}>; rel="#{rel}") end end - class DefaultPaginationStrategy + class DefaultPaginationStrategy < Base attr_reader :request_context delegate :params, :header, :request, to: :request_context @@ -178,24 +191,33 @@ module API end def paginate(relation) - relation = add_default_order(relation) - - relation.page(params[:page]).per(params[:per_page]).tap do |data| + paginate_with_limit_optimization(add_default_order(relation)).tap do |data| add_pagination_headers(data) end end private - # rubocop: disable CodeReuse/ActiveRecord + def paginate_with_limit_optimization(relation) + pagination_data = relation.page(params[:page]).per(params[:per_page]) + return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) + return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit) + + limited_total_count = pagination_data.total_count_with_limit + if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT + pagination_data.without_count + else + pagination_data + end + end + def add_default_order(relation) if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? - relation = relation.order(:id) + relation = relation.order(:id) # rubocop: disable CodeReuse/ActiveRecord end relation end - # rubocop: enable CodeReuse/ActiveRecord def add_pagination_headers(paginated_data) header 'X-Per-Page', paginated_data.limit_value.to_s @@ -211,27 +233,13 @@ module API end def pagination_links(paginated_data) - request_url = request.url.split('?').first - request_params = params.clone - request_params[:per_page] = paginated_data.limit_value - - links = [] - - request_params[:page] = paginated_data.prev_page - links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") if request_params[:page] - - request_params[:page] = paginated_data.next_page - links << %(<#{request_url}?#{request_params.to_query}>; rel="next") if request_params[:page] - - request_params[:page] = 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="first") - - unless data_without_counts?(paginated_data) - request_params[:page] = total_pages(paginated_data) - links << %(<#{request_url}?#{request_params.to_query}>; rel="last") - end + [].tap do |links| + links << %(<#{page_href(page: paginated_data.prev_page)}>; rel="prev") if paginated_data.prev_page + links << %(<#{page_href(page: paginated_data.next_page)}>; rel="next") if paginated_data.next_page + links << %(<#{page_href(page: 1)}>; rel="first") - links.join(', ') + links << %(<#{page_href(page: total_pages(paginated_data))}>; rel="last") unless data_without_counts?(paginated_data) + end.join(', ') end def total_pages(paginated_data) diff --git a/lib/api/helpers/presentable.rb b/lib/api/helpers/presentable.rb new file mode 100644 index 00000000000..973c2132efe --- /dev/null +++ b/lib/api/helpers/presentable.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module API + module Helpers + ## + # This module makes it possible to use `app/presenters` with + # Grape Entities. It instantiates model presenter and passes + # options defined in the API endpoint to the presenter itself. + # + # present object, with: Entities::Something, + # current_user: current_user, + # another_option: 'my options' + # + # Example above will make `current_user` and `another_option` + # values available in the subclass of `Gitlab::View::Presenter` + # thorough a separate method in the presenter. + # + # The model class needs to have `::Presentable` module mixed in + # if you want to use `API::Helpers::Presentable`. + # + module Presentable + extend ActiveSupport::Concern + + def initialize(object, options = {}) + super(object.present(options), options) + end + end + end +end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 45d0343bc89..ff73a49d5e8 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -26,7 +26,7 @@ module API end def get_runner_ip - { ip_address: request.ip } + { ip_address: env["action_dispatch.remote_ip"].to_s || request.ip } end def current_runner diff --git a/lib/api/helpers/version.rb b/lib/api/helpers/version.rb new file mode 100644 index 00000000000..7f53094e90c --- /dev/null +++ b/lib/api/helpers/version.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module API + module Helpers + class Version + include Helpers::RelatedResourcesHelpers + + def initialize(version) + @version = version.to_s + + unless API.versions.include?(version) + raise ArgumentError, 'Unknown API version!' + end + end + + def root_path + File.join('/', API.prefix.to_s, @version) + end + + def root_url + @root_url ||= expose_url(root_path) + end + + def to_s + @version + end + end + end +end diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb new file mode 100644 index 00000000000..bb4e536cf57 --- /dev/null +++ b/lib/api/import_github.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module API + class ImportGithub < Grape::API + rescue_from Octokit::Unauthorized, with: :provider_unauthorized + + helpers do + def client + @client ||= Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], client_options) + end + + def access_params + { github_access_token: params[:personal_access_token] } + end + + def client_options + {} + end + + def provider + :github + end + end + + desc 'Import a GitHub project' do + detail 'This feature was introduced in GitLab 11.3.4.' + success Entities::ProjectEntity + end + params do + requires :personal_access_token, type: String, desc: 'GitHub personal access token' + requires :repo_id, type: Integer, desc: 'GitHub repository ID' + optional :new_name, type: String, desc: 'New repo name' + requires :target_namespace, type: String, desc: 'Namespace to import repo into' + end + post 'import/github' do + result = Import::GithubService.new(client, current_user, params).execute(access_params, provider) + + if result[:status] == :success + present ProjectSerializer.new.represent(result[:project]) + else + status result[:http_status] + { errors: result[:message] } + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index ae40b5f7557..70b32f7d758 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -77,6 +77,7 @@ module API when ::Gitlab::GitAccessResult::Success payload = { gl_repository: gl_repository, + gl_project_path: gl_project_path, gl_id: Gitlab::GlId.gl_id(user), gl_username: user&.username, git_config_options: [], @@ -117,13 +118,7 @@ module API raise ActiveRecord::RecordNotFound.new("No key_id or user_id passed!") end - token_handler = Gitlab::LfsToken.new(actor) - - { - username: token_handler.actor_name, - lfs_token: token_handler.token, - repository_http_path: project.http_url_to_repo - } + Gitlab::LfsToken.new(actor).authentication_payload(project.http_url_to_repo) end # rubocop: enable CodeReuse/ActiveRecord @@ -256,8 +251,9 @@ module API post '/post_receive' do status 200 + PostReceive.perform_async(params[:gl_repository], params[:identifier], - params[:changes]) + params[:changes], params[:push_options].to_a) broadcast_message = BroadcastMessage.current&.last&.message reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease diff --git a/lib/api/issues.rb b/lib/api/issues.rb index dac700482b4..d59d2f5a098 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -29,13 +29,12 @@ module API issues = IssuesFinder.new(current_user, args).execute .preload(:assignees, :labels, :notes, :timelogs, :project, :author, :closed_by) - - issues.reorder(args[:order_by] => args[:sort]) + issues.reorder(order_options_with_tie_breaker) end # rubocop: enable CodeReuse/ActiveRecord params :issues_params do - optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names' optional :milestone, type: String, desc: 'Milestone title' optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', desc: 'Return issues ordered by `created_at` or `updated_at` fields.' @@ -43,7 +42,8 @@ module API desc: 'Return issues sorted in `asc` or `desc` order.' optional :milestone, type: String, desc: 'Return issues for a specific milestone' optional :iids, type: Array[Integer], desc: 'The IID array of issues' - optional :search, type: String, desc: 'Search issues for text present in the title or description' + optional :search, type: String, desc: 'Search issues for text present in the title, description, or any combination of these' + optional :in, type: String, desc: '`title`, `description`, or a string joining them with comma' optional :created_after, type: DateTime, desc: 'Return issues created after the specified time' optional :created_before, type: DateTime, desc: 'Return issues created before the specified time' optional :updated_after, type: DateTime, desc: 'Return issues updated after the specified time' @@ -54,6 +54,7 @@ module API optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' + optional :confidential, type: Boolean, desc: 'Filter confidential or public issues' use :pagination use :issues_params_ee @@ -64,7 +65,7 @@ module API optional :assignee_ids, type: Array[Integer], desc: 'The array of user IDs to assign issue' optional :assignee_id, type: Integer, desc: '[Deprecated] The ID of a user to assign issue' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' - optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names' optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" @@ -294,7 +295,7 @@ module API end # rubocop: enable CodeReuse/ActiveRecord - desc 'List merge requests that are related to the issue' do + desc 'List merge requests that are related to the issue' do success Entities::MergeRequestBasic end params do @@ -303,22 +304,17 @@ module API get ':id/issues/:issue_iid/related_merge_requests' do issue = find_project_issue(params[:issue_iid]) - merge_request_iids = ::Issues::ReferencedMergeRequestsService.new(user_project, current_user) + merge_requests = ::Issues::ReferencedMergeRequestsService.new(user_project, current_user) .execute(issue) .flatten - .map(&:iid) - - merge_requests = - if merge_request_iids.present? - MergeRequestsFinder.new(current_user, project_id: user_project.id, iids: merge_request_iids).execute - else - MergeRequest.none - end - present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + present paginate(::Kaminari.paginate_array(merge_requests)), + with: Entities::MergeRequestBasic, + current_user: current_user, + project: user_project end - desc 'List merge requests closing issue' do + desc 'List merge requests closing issue' do success Entities::MergeRequestBasic end params do @@ -335,7 +331,7 @@ module API end # rubocop: enable CodeReuse/ActiveRecord - desc 'List participants for an issue' do + desc 'List participants for an issue' do success Entities::UserBasic end params do diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index a4068a200b3..e7fed55170e 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -23,17 +23,14 @@ module API requires :job, type: String, desc: 'The name for the job' end route_setting :authentication, job_token_allowed: true - # rubocop: disable CodeReuse/ActiveRecord get ':id/jobs/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do authorize_download_artifacts! - builds = user_project.latest_successful_builds_for(params[:ref_name]) - latest_build = builds.find_by!(name: params[:job]) + latest_build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) present_carrierwave_file!(latest_build.artifacts_file) end - # rubocop: enable CodeReuse/ActiveRecord desc 'Download a specific file from artifacts archive from a ref' do detail 'This feature was introduced in GitLab 11.5' @@ -48,7 +45,7 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - build = user_project.latest_successful_build_for(params[:job], params[:ref_name]) + build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) path = Gitlab::Ci::Build::Artifacts::Path .new(params[:artifact_path]) @@ -112,6 +109,22 @@ module API status 200 present build, with: Entities::Job end + + desc 'Delete the artifacts files from a job' do + detail 'This feature was introduced in GitLab 11.9' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + delete ':id/jobs/:job_id/artifacts' do + authorize_destroy_artifacts! + build = find_build!(params[:job_id]) + authorize!(:destroy_artifacts, build) + + build.erase_erasable_artifacts! + + status :no_content + end end end end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 80a5cbd6b19..59f0dbe8a9b 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -38,6 +38,8 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get ':id/jobs' do + authorize_read_builds! + builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) @@ -50,13 +52,16 @@ module API success Entities::Job end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' use :optional_scope use :pagination end # rubocop: disable CodeReuse/ActiveRecord get ':id/pipelines/:pipeline_id/jobs' do + authorize!(:read_pipeline, user_project) pipeline = user_project.ci_pipelines.find(params[:pipeline_id]) + authorize!(:read_build, pipeline) + builds = pipeline.builds builds = filter_builds(builds, params[:scope]) builds = builds.preload(:job_artifacts_archive, :job_artifacts, project: [:namespace]) diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 2e676b0aa6b..d729d3ee625 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -3,6 +3,7 @@ module API class Labels < Grape::API include PaginationParams + helpers ::API::Helpers::LabelHelpers before { authenticate! } @@ -11,98 +12,50 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all labels of the project' do - success Entities::Label + success Entities::ProjectLabel end params do use :pagination end get ':id/labels' do - present paginate(available_labels_for(user_project)), with: Entities::Label, current_user: current_user, project: user_project + get_labels(user_project, Entities::ProjectLabel) end desc 'Create a new label' do - success Entities::Label + success Entities::ProjectLabel end params do - requires :name, type: String, desc: 'The name of the label to be created' - requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" - optional :description, type: String, desc: 'The description of label to be created' + use :label_create_params optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true end - # rubocop: disable CodeReuse/ActiveRecord post ':id/labels' do - authorize! :admin_label, user_project - - label = available_labels_for(user_project).find_by(title: params[:name]) - conflict!('Label already exists') if label - - priority = params.delete(:priority) - label = ::Labels::CreateService.new(declared_params(include_missing: false)).execute(project: user_project) - - if label.valid? - label.prioritize!(user_project, priority) if priority - present label, with: Entities::Label, current_user: current_user, project: user_project - else - render_validation_error!(label) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - desc 'Delete an existing label' do - success Entities::Label - end - params do - requires :name, type: String, desc: 'The name of the label to be deleted' - end - # rubocop: disable CodeReuse/ActiveRecord - delete ':id/labels' do - authorize! :admin_label, user_project - - label = user_project.labels.find_by(title: params[:name]) - not_found!('Label') unless label - - destroy_conditionally!(label) + create_label(user_project, Entities::ProjectLabel) end - # rubocop: enable CodeReuse/ActiveRecord desc 'Update an existing label. At least one optional parameter is required.' do - success Entities::Label + success Entities::ProjectLabel end params do - requires :name, type: String, desc: 'The name of the label to be updated' + requires :name, type: String, desc: 'The name of the label to be updated' optional :new_name, type: String, desc: 'The new name of the label' optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" optional :description, type: String, desc: 'The new description of label' optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true at_least_one_of :new_name, :color, :description, :priority end - # rubocop: disable CodeReuse/ActiveRecord put ':id/labels' do - authorize! :admin_label, user_project - - label = user_project.labels.find_by(title: params[:name]) - not_found!('Label not found') unless label - - update_priority = params.key?(:priority) - priority = params.delete(:priority) - label_params = declared_params(include_missing: false) - # Rename new name to the actual label attribute name - label_params[:name] = label_params.delete(:new_name) if label_params.key?(:new_name) - - label = ::Labels::UpdateService.new(label_params).execute(label) - render_validation_error!(label) unless label.valid? - - if update_priority - if priority.nil? - label.unprioritize!(user_project) - else - label.prioritize!(user_project, priority) - end - end + update_label(user_project, Entities::ProjectLabel) + end - present label, with: Entities::Label, current_user: current_user, project: user_project + desc 'Delete an existing label' do + success Entities::ProjectLabel + end + params do + requires :name, type: String, desc: 'The name of the label to be deleted' + end + delete ':id/labels' do + delete_label(user_project) end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/lint.rb b/lib/api/lint.rb index 0342a4b6654..a7672021db0 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -8,7 +8,8 @@ module API requires :content, type: String, desc: 'Content of .gitlab-ci.yml' end post '/lint' do - error = Gitlab::Ci::YamlProcessor.validation_message(params[:content]) + error = Gitlab::Ci::YamlProcessor.validation_message(params[:content], + user: current_user) status 200 diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 8c1951cc535..6518ebbcff5 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -12,6 +12,9 @@ module API helpers do params :optional_params_ee do end + + params :optional_merge_requests_search_params do + end end def self.update_params_at_least_one_of @@ -38,7 +41,7 @@ module API args[:scope] = args[:scope].underscore if args[:scope] merge_requests = MergeRequestsFinder.new(current_user, args).execute - .reorder(args[:order_by] => args[:sort]) + .reorder(order_options_with_tie_breaker) merge_requests = paginate(merge_requests) .preload(:source_project, :target_project) @@ -95,7 +98,7 @@ module API optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return merge requests sorted in `asc` or `desc` order.' optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' - optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names' optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' optional :updated_after, type: DateTime, desc: 'Return merge requests updated after the specified time' @@ -109,8 +112,11 @@ module API optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' optional :source_branch, type: String, desc: 'Return merge requests with the given source branch' optional :target_branch, type: String, desc: 'Return merge requests with the given target branch' - optional :search, type: String, desc: 'Search merge requests for text present in the title or description' + optional :search, type: String, desc: 'Search merge requests for text present in the title, description, or any combination of these' + optional :in, type: String, desc: '`title`, `description`, or a string joining them with comma' optional :wip, type: String, values: %w[yes no], desc: 'Search merge requests for WIP in the title' + + use :optional_merge_requests_search_params use :pagination end end @@ -178,7 +184,7 @@ module API optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' - optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names' optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' optional :allow_collaboration, type: Boolean, desc: 'Allow commits from members who can merge to the target branch' optional :allow_maintainer_to_push, type: Boolean, as: :allow_collaboration, desc: '[deprecated] See allow_collaboration' @@ -342,6 +348,7 @@ module API end params do optional :merge_commit_message, type: String, desc: 'Custom merge commit message' + optional :squash_commit_message, type: String, desc: 'Custom squash commit message' optional :should_remove_source_branch, type: Boolean, desc: 'When true, the source branch will be deleted if possible' optional :merge_when_pipeline_succeeds, type: Boolean, @@ -367,10 +374,11 @@ module API merge_request.update(squash: params[:squash]) if params[:squash] - merge_params = { + merge_params = HashWithIndifferentAccess.new( commit_message: params[:merge_commit_message], + squash_commit_message: params[:squash_commit_message], should_remove_source_branch: params[:should_remove_source_branch] - } + ) if merge_when_pipeline_succeeds && merge_request.head_pipeline && merge_request.head_pipeline.active? ::MergeRequests::MergeWhenPipelineSucceedsService @@ -385,6 +393,31 @@ module API present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end + desc 'Merge a merge request to its default temporary merge ref path' + params do + optional :merge_commit_message, type: String, desc: 'Custom merge commit message' + end + put ':id/merge_requests/:merge_request_iid/merge_to_ref' do + merge_request = find_project_merge_request(params[:merge_request_iid]) + + authorize! :admin_merge_request, user_project + + merge_params = { + commit_message: params[:merge_commit_message] + } + + result = ::MergeRequests::MergeToRefService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) + + if result[:status] == :success + present result.slice(:commit_id), 200 + else + http_status = result[:http_status] || 400 + render_api_error!(result[:message], http_status) + end + end + desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do success Entities::MergeRequest end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 1bdf7aeb119..f7bd092ce50 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -39,7 +39,7 @@ module API # at the DB query level (which we cannot in that case), the current # page can have less elements than :per_page even if # there's more than one page. - raw_notes = noteable.notes.with_metadata.reorder(params[:order_by] => params[:sort]) + raw_notes = noteable.notes.with_metadata.reorder(order_options_with_tie_breaker) notes = # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 47b711917e2..c86b50d3736 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -32,7 +32,7 @@ module API success Entities::PipelineScheduleDetails end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end get ':id/pipeline_schedules/:pipeline_schedule_id' do present pipeline_schedule, with: Entities::PipelineScheduleDetails @@ -87,7 +87,7 @@ module API success Entities::PipelineScheduleDetails end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do authorize! :update_pipeline_schedule, pipeline_schedule @@ -103,7 +103,7 @@ module API success Entities::PipelineScheduleDetails end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end delete ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :admin_pipeline_schedule, pipeline_schedule diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 7a7b23d2bbb..ac8fe98e55e 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -42,7 +42,7 @@ module API success Entities::Pipeline end params do - requires :ref, type: String, desc: 'Reference' + requires :ref, type: String, desc: 'Reference' optional :variables, Array, desc: 'Array of variables available in the pipeline' end # rubocop: disable CodeReuse/ActiveRecord @@ -76,7 +76,7 @@ module API requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end get ':id/pipelines/:pipeline_id' do - authorize! :read_pipeline, user_project + authorize! :read_pipeline, pipeline present pipeline, with: Entities::Pipeline end @@ -101,10 +101,10 @@ module API success Entities::Pipeline end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end post ':id/pipelines/:pipeline_id/retry' do - authorize! :update_pipeline, user_project + authorize! :update_pipeline, pipeline pipeline.retry_failed(current_user) @@ -116,10 +116,10 @@ module API success Entities::Pipeline end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end post ':id/pipelines/:pipeline_id/cancel' do - authorize! :update_pipeline, user_project + authorize! :update_pipeline, pipeline pipeline.cancel_running diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb new file mode 100644 index 00000000000..c96261a7b57 --- /dev/null +++ b/lib/api/project_clusters.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +module API + class ProjectClusters < Grape::API + include PaginationParams + + before { authenticate! } + + # EE::API::ProjectClusters will + # override these methods + helpers do + params :create_params_ee do + end + + params :update_params_ee do + end + end + + params do + requires :id, type: String, desc: 'The ID of the project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get all clusters from the project' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Cluster + end + params do + use :pagination + end + get ':id/clusters' do + authorize! :read_cluster, user_project + + present paginate(clusters_for_current_user), with: Entities::Cluster + end + + desc 'Get specific cluster for the project' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::ClusterProject + end + params do + requires :cluster_id, type: Integer, desc: 'The cluster ID' + end + get ':id/clusters/:cluster_id' do + authorize! :read_cluster, cluster + + present cluster, with: Entities::ClusterProject + end + + desc 'Adds an existing cluster' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::ClusterProject + end + params do + requires :name, type: String, desc: 'Cluster name' + optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' + requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do + requires :api_url, type: String, allow_blank: false, desc: 'URL to access the Kubernetes API' + requires :token, type: String, desc: 'Token to authenticate against Kubernetes' + optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)' + optional :namespace, type: String, desc: 'Unique namespace related to Project' + optional :authorization_type, type: String, values: Clusters::Platforms::Kubernetes.authorization_types.keys, default: 'rbac', desc: 'Cluster authorization type, defaults to RBAC' + end + use :create_params_ee + end + post ':id/clusters/user' do + authorize! :add_cluster, user_project, 'Instance does not support multiple Kubernetes clusters' + + user_cluster = ::Clusters::CreateService + .new(current_user, create_cluster_user_params) + .execute + + if user_cluster.persisted? + present user_cluster, with: Entities::ClusterProject + else + render_validation_error!(user_cluster) + end + end + + desc 'Update an existing cluster' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::ClusterProject + end + params do + requires :cluster_id, type: Integer, desc: 'The cluster ID' + optional :name, type: String, desc: 'Cluster name' + optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do + optional :api_url, type: String, desc: 'URL to access the Kubernetes API' + optional :token, type: String, desc: 'Token to authenticate against Kubernetes' + optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)' + optional :namespace, type: String, desc: 'Unique namespace related to Project' + end + use :update_params_ee + end + put ':id/clusters/:cluster_id' do + authorize! :update_cluster, cluster + + update_service = Clusters::UpdateService.new(current_user, update_cluster_params) + + if update_service.execute(cluster) + present cluster, with: Entities::ClusterProject + else + render_validation_error!(cluster) + end + end + + desc 'Remove a cluster' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::ClusterProject + end + params do + requires :cluster_id, type: Integer, desc: 'The Cluster ID' + end + delete ':id/clusters/:cluster_id' do + authorize! :admin_cluster, cluster + + destroy_conditionally!(cluster) + end + end + + helpers do + def clusters_for_current_user + @clusters_for_current_user ||= ClustersFinder.new(user_project, current_user, :all).execute + end + + def cluster + @cluster ||= clusters_for_current_user.find(params[:cluster_id]) + end + + def create_cluster_user_params + declared_params.merge({ + provider_type: :user, + platform_type: :kubernetes, + clusterable: user_project + }) + end + + def update_cluster_params + declared_params(include_missing: false).without(:cluster_id) + end + end + end +end diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index da31bcb8dac..ca24742b7a3 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -98,6 +98,23 @@ module API milestone_issuables_for(user_project, :merge_request) end + + desc 'Promote a milestone to group milestone' do + detail 'This feature was introduced in GitLab 11.9' + end + post ':id/milestones/:milestone_id/promote' do + begin + authorize! :admin_milestone, user_project + authorize! :admin_milestone, user_project.group + + milestone = user_project.milestones.find(params[:milestone_id]) + Milestones::PromoteService.new(user_project, current_user).execute(milestone) + + status(200) + rescue Milestones::PromoteService::PromoteMilestoneError => error + render_api_error!(error.message, 400) + end + end end end end diff --git a/lib/api/project_statistics.rb b/lib/api/project_statistics.rb new file mode 100644 index 00000000000..2f73785f72d --- /dev/null +++ b/lib/api/project_statistics.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module API + class ProjectStatistics < Grape::API + before do + authenticate! + not_found! unless user_project.daily_statistics_enabled? + authorize! :daily_statistics, user_project + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get the list of project fetch statistics for the last 30 days' + get ":id/statistics" do + statistic_finder = ::Projects::DailyStatisticsFinder.new(user_project) + + present statistic_finder, with: Entities::ProjectDailyStatistics + end + end + end +end diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index d05ddad7466..119902a189c 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -36,7 +36,10 @@ module API optional :project, type: String, desc: 'The project name to use when expanding placeholders in the template. Only affects licenses' optional :fullname, type: String, desc: 'The full name of the copyright holder to use when expanding placeholders in the template. Only affects licenses' end - get ':id/templates/:type/:name', requirements: { name: /[\w\.-]+/ } do + # The regex is needed to ensure a period (e.g. agpl-3.0) + # isn't confused with a format type. We also need to allow encoded + # values (e.g. C%2B%2B for C++), so allow % and + as well. + get ':id/templates/:type/:name', requirements: { name: /[\w%.+-]+/ } do template = TemplateFinder .build(params[:type], user_project, name: params[:name]) .execute diff --git a/lib/api/projects.rb b/lib/api/projects.rb index f5d21d8923f..91501ba4d36 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -25,6 +25,9 @@ module API projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] projects = projects.with_statistics if params[:statistics] + lang = params[:with_programming_language] + projects = projects.with_programming_language(lang) if lang + projects end @@ -91,6 +94,7 @@ module API optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' + optional :with_programming_language, type: String, desc: 'Limit to repositories which use the given programming language' optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit by minimum access level of authenticated user' use :optional_filter_params_ee @@ -128,7 +132,7 @@ module API end end - resource :users, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :users, requirements: API::USER_REQUIREMENTS do desc 'Get a user projects' do success Entities::BasicProjectDetails end @@ -180,7 +184,8 @@ module API if project.saved? present project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, project) + user_can_admin_project: can?(current_user, :admin_project, project), + current_user: current_user else if project.errors[:limit_reached].present? error!(project.errors[:limit_reached], 403) @@ -213,7 +218,8 @@ module API if project.saved? present project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, project) + user_can_admin_project: can?(current_user, :admin_project, project), + current_user: current_user else render_validation_error!(project) end @@ -254,6 +260,8 @@ module API end params do optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' + optional :path, type: String, desc: 'The path that will be assigned to the fork' + optional :name, type: String, desc: 'The name that will be assigned to the fork' end post ':id/fork' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42284') @@ -275,7 +283,8 @@ module API conflict!(forked_project.errors.messages) else present forked_project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, forked_project) + user_can_admin_project: can?(current_user, :admin_project, forked_project), + current_user: current_user end end @@ -324,7 +333,8 @@ module API if result[:status] == :success present user_project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, user_project) + user_can_admin_project: can?(current_user, :admin_project, user_project), + current_user: current_user else render_validation_error!(user_project) end @@ -338,7 +348,7 @@ module API ::Projects::UpdateService.new(user_project, current_user, archived: true).execute - present user_project, with: Entities::Project + present user_project, with: Entities::Project, current_user: current_user end desc 'Unarchive a project' do @@ -349,7 +359,7 @@ module API ::Projects::UpdateService.new(@project, current_user, archived: false).execute - present user_project, with: Entities::Project + present user_project, with: Entities::Project, current_user: current_user end desc 'Star a project' do @@ -362,7 +372,7 @@ module API current_user.toggle_star(user_project) user_project.reload - present user_project, with: Entities::Project + present user_project, with: Entities::Project, current_user: current_user end end @@ -374,7 +384,7 @@ module API current_user.toggle_star(user_project) user_project.reload - present user_project, with: Entities::Project + present user_project, with: Entities::Project, current_user: current_user else not_modified! end @@ -382,7 +392,11 @@ module API desc 'Get languages in project repository' get ':id/languages' do - user_project.repository.languages.map { |language| language.values_at(:label, :value) }.to_h + if user_project.repository_languages.present? + user_project.repository_languages.map { |l| [l.name, l.share] }.to_h + else + user_project.repository.languages.map { |language| language.values_at(:label, :value) }.to_h + end end desc 'Remove a project' @@ -410,7 +424,7 @@ module API result = ::Projects::ForkService.new(fork_from_project, current_user).execute(user_project) if result - present user_project.reload, with: Entities::Project + present user_project.reload, with: Entities::Project, current_user: current_user else render_api_error!("Project already forked", 409) if user_project.forked? end @@ -432,27 +446,24 @@ module API end params do requires :group_id, type: Integer, desc: 'The ID of a group' - requires :group_access, type: Integer, values: Gitlab::Access.values, desc: 'The group access level' + requires :group_access, type: Integer, values: Gitlab::Access.values, as: :link_group_access, desc: 'The group access level' optional :expires_at, type: Date, desc: 'Share expiration date' end post ":id/share" do authorize! :admin_project, user_project group = Group.find_by_id(params[:group_id]) - unless group && can?(current_user, :read_group, group) - not_found!('Group') - end - unless user_project.allowed_to_share_with_group? break render_api_error!("The project sharing with group is disabled", 400) end - link = user_project.project_group_links.new(declared_params(include_missing: false)) + result = ::Projects::GroupLinks::CreateService.new(user_project, current_user, declared_params(include_missing: false)) + .execute(group) - if link.save - present link, with: Entities::ProjectGroupLink + if result[:status] == :success + present result[:link], with: Entities::ProjectGroupLink else - render_api_error!(link.errors.full_messages.first, 409) + render_api_error!(result[:message], result[:http_status]) end end @@ -475,7 +486,7 @@ module API requires :file, type: File, desc: 'The file to be uploaded' end post ":id/uploads" do - UploadService.new(user_project, params[:file]).execute + UploadService.new(user_project, params[:file]).execute.to_h end desc 'Get the users list of a project' do @@ -516,7 +527,7 @@ module API result = ::Projects::TransferService.new(user_project, current_user).execute(namespace) if result - present user_project, with: Entities::Project + present user_project, with: Entities::Project, current_user: current_user else render_api_error!("Failed to transfer project #{user_project.errors.messages}", 400) end diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb index 8edcfea7c93..263468c9aa6 100644 --- a/lib/api/projects_relation_builder.rb +++ b/lib/api/projects_relation_builder.rb @@ -11,7 +11,7 @@ module API projects_relation end - def preload_relation(projects_relation, options = {}) + def preload_relation(projects_relation, options = {}) projects_relation end diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb new file mode 100644 index 00000000000..5d1b40e3bff --- /dev/null +++ b/lib/api/release/links.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +module API + module Release + class Links < Grape::API + include PaginationParams + + RELEASE_ENDPOINT_REQUIREMETS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS + .merge(tag_name: API::NO_SLASH_URL_PART_REGEX) + + before { authorize! :read_release, user_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + end + resource 'releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do + resource :assets do + desc 'Get a list of links of a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Releases::Link + end + params do + use :pagination + end + get 'links' do + authorize! :read_release, release + + present paginate(release.links.sorted), with: Entities::Releases::Link + end + + desc 'Create a link of a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Releases::Link + end + params do + requires :name, type: String, desc: 'The name of the link' + requires :url, type: String, desc: 'The URL of the link' + end + post 'links' do + authorize! :create_release, release + + new_link = release.links.create(declared_params(include_missing: false)) + + if new_link.persisted? + present new_link, with: Entities::Releases::Link + else + render_api_error!(new_link.errors.messages, 400) + end + end + + params do + requires :link_id, type: String, desc: 'The id of the link' + end + resource 'links/:link_id' do + desc 'Get a link detail of a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Releases::Link + end + get do + authorize! :read_release, release + + present link, with: Entities::Releases::Link + end + + desc 'Update a link of a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Releases::Link + end + params do + optional :name, type: String, desc: 'The name of the link' + optional :url, type: String, desc: 'The URL of the link' + at_least_one_of :name, :url + end + put do + authorize! :update_release, release + + if link.update(declared_params(include_missing: false)) + present link, with: Entities::Releases::Link + else + render_api_error!(link.errors.messages, 400) + end + end + + desc 'Delete a link of a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Releases::Link + end + delete do + authorize! :destroy_release, release + + if link.destroy + present link, with: Entities::Releases::Link + else + render_api_error!(link.errors.messages, 400) + end + end + end + end + end + end + + helpers do + def release + @release ||= user_project.releases.find_by_tag!(params[:tag]) + end + + def link + @link ||= release.links.find(params[:link_id]) + end + end + end + end +end diff --git a/lib/api/releases.rb b/lib/api/releases.rb new file mode 100644 index 00000000000..cb85028f22c --- /dev/null +++ b/lib/api/releases.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +module API + class Releases < Grape::API + include PaginationParams + + RELEASE_ENDPOINT_REQUIREMETS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS + .merge(tag_name: API::NO_SLASH_URL_PART_REGEX) + + before { authorize_read_releases! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get a project releases' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + use :pagination + end + get ':id/releases' do + releases = ::ReleasesFinder.new(user_project, current_user).execute + + present paginate(releases), with: Entities::Release + end + + desc 'Get a single project release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + end + get ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do + authorize_read_release! + + present release, with: Entities::Release + end + + desc 'Create a new release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + requires :name, type: String, desc: 'The name of the release' + requires :description, type: String, desc: 'The release notes' + optional :ref, type: String, desc: 'The commit sha or branch name' + optional :assets, type: Hash do + optional :links, type: Array do + requires :name, type: String + requires :url, type: String + end + end + end + post ':id/releases' do + authorize_create_release! + + result = ::Releases::CreateService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + + desc 'Update a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + optional :name, type: String, desc: 'The name of the release' + optional :description, type: String, desc: 'Release notes with markdown support' + end + put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do + authorize_update_release! + + result = ::Releases::UpdateService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + + desc 'Delete a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + end + delete ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do + authorize_destroy_release! + + result = ::Releases::DestroyService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + end + + helpers do + def authorize_create_release! + authorize! :create_release, user_project + end + + def authorize_read_releases! + authorize! :read_release, user_project + end + + def authorize_read_release! + authorize! :read_release, release + end + + def authorize_update_release! + authorize! :update_release, release + end + + def authorize_destroy_release! + authorize! :destroy_release, release + end + + def release + @release ||= user_project.releases.find_by_tag(params[:tag]) + end + end + end +end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index f72b33605a7..f3fea463e7f 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -17,6 +17,7 @@ module API desc: 'The type of the runners to show' optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, desc: 'The status of the runners to show' + optional :tag_list, type: Array[String], desc: 'The tags of the runners to show' use :pagination end get do @@ -24,6 +25,7 @@ module API runners = filter_runners(runners, params[:scope], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + runners = runners.tagged_with(params[:tag_list]) if params[:tag_list] present paginate(runners), with: Entities::Runner end @@ -38,6 +40,7 @@ module API desc: 'The type of the runners to show' optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, desc: 'The status of the runners to show' + optional :tag_list, type: Array[String], desc: 'The tags of the runners to show' use :pagination end get 'all' do @@ -47,6 +50,7 @@ module API runners = filter_runners(runners, params[:scope]) runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + runners = runners.tagged_with(params[:tag_list]) if params[:tag_list] present paginate(runners), with: Entities::Runner end @@ -139,6 +143,7 @@ module API desc: 'The type of the runners to show' optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, desc: 'The status of the runners to show' + optional :tag_list, type: Array[String], desc: 'The tags of the runners to show' use :pagination end get ':id/runners' do @@ -146,6 +151,7 @@ module API runners = filter_runners(runners, params[:scope]) runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + runners = runners.tagged_with(params[:tag_list]) if params[:tag_list] present paginate(runners), with: Entities::Runner end diff --git a/lib/api/services.rb b/lib/api/services.rb index d60f0f5f08d..bda6be51553 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -368,46 +368,9 @@ module API name: :webhook, type: String, desc: 'The Hangouts Chat webhook. e.g. https://chat.googleapis.com/v1/spaces…' - } - ], - 'hipchat' => [ - { - required: true, - name: :token, - type: String, - desc: 'The room token' - }, - { - required: false, - name: :room, - type: String, - desc: 'The room name or ID' - }, - { - required: false, - name: :color, - type: String, - desc: 'The room color' }, - { - required: false, - name: :notify, - type: Boolean, - desc: 'Enable notifications' - }, - { - required: false, - name: :api_version, - type: String, - desc: 'Leave blank for default (v2)' - }, - { - required: false, - name: :server, - type: String, - desc: 'Leave blank for default. https://hipchat.example.com' - } - ], + CHAT_NOTIFICATION_EVENTS + ].flatten, 'irker' => [ { required: true, @@ -468,7 +431,7 @@ module API { required: false, name: :jira_issue_transition_id, - type: Integer, + type: String, desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' } ], @@ -629,6 +592,26 @@ module API desc: 'The description of the tracker' } ], + 'youtrack' => [ + { + required: true, + name: :project_url, + type: String, + desc: 'The project URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'The issues URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'The description of the tracker' + } + ], 'slack' => [ CHAT_NOTIFICATION_SETTINGS, CHAT_NOTIFICATION_FLAGS, @@ -691,7 +674,6 @@ module API ExternalWikiService, FlowdockService, HangoutsChatService, - HipchatService, IrkerService, JiraService, KubernetesService, @@ -703,6 +685,7 @@ module API PrometheusService, PushoverService, RedmineService, + YoutrackService, SlackService, MattermostService, MicrosoftTeamsService, @@ -763,7 +746,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authenticate! } before { authorize_admin_project } @@ -842,7 +825,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Trigger a slash command for #{service_slug}" do detail 'Added in GitLab 8.13' end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index f53ba0ab761..b16faffe335 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -35,7 +35,7 @@ module API end optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" - optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' + optional :default_branch_protection, type: Integer, values: Gitlab::Access.protection_values, desc: 'Determine if developers can push to master' optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' @@ -121,6 +121,7 @@ module API optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins' + optional :local_markdown_version, type: Integer, desc: "Local markdown version, increase this value when any cached markdown should be invalidated" ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 326d55afd0e..f8b37b33348 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -16,6 +16,10 @@ module API def public_snippets SnippetsFinder.new(current_user, scope: :are_public).execute end + + def snippets + SnippetsFinder.new(current_user).execute + end end desc 'Get a snippets list for authenticated user' do @@ -48,7 +52,10 @@ module API requires :id, type: Integer, desc: 'The ID of a snippet' end get ':id' do - snippet = snippets_for_current_user.find(params[:id]) + snippet = snippets.find_by_id(params[:id]) + + break not_found!('Snippet') unless snippet + present snippet, with: Entities::PersonalSnippet end @@ -94,9 +101,8 @@ module API desc: 'The visibility of the snippet' at_least_one_of :title, :file_name, :content, :visibility end - # rubocop: disable CodeReuse/ActiveRecord put ':id' do - snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + snippet = snippets_for_current_user.find_by_id(params.delete(:id)) break not_found!('Snippet') unless snippet authorize! :update_personal_snippet, snippet @@ -113,7 +119,6 @@ module API render_validation_error!(snippet) end end - # rubocop: enable CodeReuse/ActiveRecord desc 'Remove snippet' do detail 'This feature was introduced in GitLab 8.15.' @@ -122,16 +127,14 @@ module API params do requires :id, type: Integer, desc: 'The ID of a snippet' end - # rubocop: disable CodeReuse/ActiveRecord delete ':id' do - snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + snippet = snippets_for_current_user.find_by_id(params.delete(:id)) break not_found!('Snippet') unless snippet authorize! :destroy_personal_snippet, snippet destroy_conditionally!(snippet) end - # rubocop: enable CodeReuse/ActiveRecord desc 'Get a raw snippet' do detail 'This feature was introduced in GitLab 8.15.' @@ -139,9 +142,8 @@ module API params do requires :id, type: Integer, desc: 'The ID of a snippet' end - # rubocop: disable CodeReuse/ActiveRecord get ":id/raw" do - snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + snippet = snippets.find_by_id(params.delete(:id)) break not_found!('Snippet') unless snippet env['api.format'] = :txt @@ -149,7 +151,6 @@ module API header['Content-Disposition'] = 'attachment' present snippet.content end - # rubocop: enable CodeReuse/ActiveRecord desc 'Get the user agent details for a snippet' do success Entities::UserAgentDetail @@ -157,17 +158,15 @@ module API params do requires :id, type: Integer, desc: 'The ID of a snippet' end - # rubocop: disable CodeReuse/ActiveRecord get ":id/user_agent_detail" do authenticated_as_admin! - snippet = Snippet.find_by!(id: params[:id]) + snippet = Snippet.find_by_id!(params[:id]) break not_found!('UserAgentDetail') unless snippet.user_agent_detail present snippet.user_agent_detail, with: Entities::UserAgentDetail end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 74ad3c35a61..dfb54446ddf 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -2,51 +2,88 @@ module API class Subscriptions < Grape::API + helpers ::API::Helpers::LabelHelpers + before { authenticate! } - subscribable_types = { - 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, - 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) } - } + subscribables = [ + { + type: 'merge_requests', + entity: Entities::MergeRequest, + source: Project, + finder: ->(id) { find_merge_request_with_access(id, :update_merge_request) } + }, + { + type: 'issues', + entity: Entities::Issue, + source: Project, + finder: ->(id) { find_project_issue(id) } + }, + { + type: 'labels', + entity: Entities::ProjectLabel, + source: Project, + finder: ->(id) { find_label(user_project, id) } + }, + { + type: 'labels', + entity: Entities::GroupLabel, + source: Group, + finder: ->(id) { find_label(user_group, id) } + } + ] - params do - requires :id, type: String, desc: 'The ID of a project' - requires :subscribable_id, type: String, desc: 'The ID of a resource' - end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - subscribable_types.each do |type, finder| - type_singularized = type.singularize - entity_class = Entities.const_get(type_singularized.camelcase) + subscribables.each do |subscribable| + source_type = subscribable[:source].name.underscore + params do + requires :id, type: String, desc: "The #{source_type} ID" + requires :subscribable_id, type: String, desc: 'The ID of a resource' + end + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Subscribe to a resource' do - success entity_class + success subscribable[:entity] end - post ":id/#{type}/:subscribable_id/subscribe" do - resource = instance_exec(params[:subscribable_id], &finder) + post ":id/#{subscribable[:type]}/:subscribable_id/subscribe" do + parent = parent_resource(source_type) + resource = instance_exec(params[:subscribable_id], &subscribable[:finder]) - if resource.subscribed?(current_user, user_project) + if resource.subscribed?(current_user, parent) not_modified! else - resource.subscribe(current_user, user_project) - present resource, with: entity_class, current_user: current_user, project: user_project + resource.subscribe(current_user, parent) + present resource, with: subscribable[:entity], current_user: current_user, project: parent, parent: parent end end desc 'Unsubscribe from a resource' do - success entity_class + success subscribable[:entity] end - post ":id/#{type}/:subscribable_id/unsubscribe" do - resource = instance_exec(params[:subscribable_id], &finder) + post ":id/#{subscribable[:type]}/:subscribable_id/unsubscribe" do + parent = parent_resource(source_type) + resource = instance_exec(params[:subscribable_id], &subscribable[:finder]) - if !resource.subscribed?(current_user, user_project) + if !resource.subscribed?(current_user, parent) not_modified! else - resource.unsubscribe(current_user, user_project) - present resource, with: entity_class, current_user: current_user, project: user_project + resource.unsubscribe(current_user, parent) + present resource, with: subscribable[:entity], current_user: current_user, project: parent, parent: parent end end end end + + private + + helpers do + def parent_resource(source_type) + case source_type + when 'project' + user_project + else + nil + end + end + end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index b18eec7d796..f5359fd316c 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -20,12 +20,15 @@ module API desc: 'Return tags sorted in updated by `asc` or `desc` order.' optional :order_by, type: String, values: %w[name updated], default: 'updated', desc: 'Return tags ordered by `name` or `updated` fields.' + optional :search, type: String, desc: 'Return list of tags matching the search criteria' use :pagination end get ':id/repository/tags' do - tags = ::Kaminari.paginate_array(::TagsFinder.new(user_project.repository, sort: "#{params[:order_by]}_#{params[:sort]}").execute) + tags = ::TagsFinder.new(user_project.repository, + sort: "#{params[:order_by]}_#{params[:sort]}", + search: params[:search]).execute - present paginate(tags), with: Entities::Tag, project: user_project + present paginate(::Kaminari.paginate_array(tags)), with: Entities::Tag, project: user_project end desc 'Get a single repository tag' do @@ -42,21 +45,35 @@ module API end desc 'Create a new repository tag' do + detail 'This optional release_description parameter was deprecated in GitLab 11.7.' success Entities::Tag end params do requires :tag_name, type: String, desc: 'The name of the tag' requires :ref, type: String, desc: 'The commit sha or branch name' optional :message, type: String, desc: 'Specifying a message creates an annotated tag' - optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database' + optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database (deprecated in GitLab 11.7)' end post ':id/repository/tags' do authorize_push_project result = ::Tags::CreateService.new(user_project, current_user) - .execute(params[:tag_name], params[:ref], params[:message], params[:release_description]) + .execute(params[:tag_name], params[:ref], params[:message]) if result[:status] == :success + # Release creation with Tags API was deprecated in GitLab 11.7 + if params[:release_description].present? + release_create_params = { + tag: params[:tag_name], + name: params[:tag_name], # Name can be specified in new API + description: params[:release_description] + } + + ::Releases::CreateService + .new(user_project, current_user, release_create_params) + .execute + end + present result[:tag], with: Entities::Tag, project: user_project @@ -88,44 +105,72 @@ module API end desc 'Add a release note to a tag' do - success Entities::Release + detail 'This feature was deprecated in GitLab 11.7.' + success Entities::TagRelease end params do - requires :tag_name, type: String, desc: 'The name of the tag' + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag requires :description, type: String, desc: 'Release notes with markdown support' end post ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do - authorize_push_project + authorize_create_release! + + ## + # Legacy API does not support tag auto creation. + not_found!('Tag') unless user_project.repository.find_tag(params[:tag]) - result = CreateReleaseService.new(user_project, current_user) - .execute(params[:tag_name], params[:description]) + release_create_params = { + tag: params[:tag], + name: params[:tag], # Name can be specified in new API + description: params[:description] + } + + result = ::Releases::CreateService + .new(user_project, current_user, release_create_params) + .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::TagRelease else render_api_error!(result[:message], result[:http_status]) end end desc "Update a tag's release note" do - success Entities::Release + detail 'This feature was deprecated in GitLab 11.7.' + success Entities::TagRelease end params do - requires :tag_name, type: String, desc: 'The name of the tag' + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag requires :description, type: String, desc: 'Release notes with markdown support' end put ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do - authorize_push_project + authorize_update_release! - result = UpdateReleaseService.new(user_project, current_user) - .execute(params[:tag_name], params[:description]) + result = ::Releases::UpdateService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::TagRelease else render_api_error!(result[:message], result[:http_status]) end end end + + helpers do + def authorize_create_release! + authorize! :create_release, user_project + end + + def authorize_update_release! + authorize! :update_release, release + end + + def release + @release ||= user_project.releases.find_by_tag(params[:tag]) + end + end end end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index d2c8cf7c1aa..64ac8ece56c 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -14,7 +14,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do ISSUABLE_TYPES.each do |type, finder| type_id_str = "#{type.singularize}_iid".to_sym diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 3ce1529f259..8fc7c7361e1 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Trigger a GitLab project pipeline' do success Entities::Pipeline end @@ -51,7 +51,7 @@ module API triggers = user_project.triggers.includes(:trigger_requests) - present paginate(triggers), with: Entities::Trigger + present paginate(triggers), with: Entities::Trigger, current_user: current_user end # rubocop: enable CodeReuse/ActiveRecord @@ -59,7 +59,7 @@ module API success Entities::Trigger end params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' + requires :trigger_id, type: Integer, desc: 'The trigger ID' end get ':id/triggers/:trigger_id' do authenticate! @@ -68,14 +68,14 @@ module API trigger = user_project.triggers.find(params.delete(:trigger_id)) break not_found!('Trigger') unless trigger - present trigger, with: Entities::Trigger + present trigger, with: Entities::Trigger, current_user: current_user end desc 'Create a trigger' do success Entities::Trigger end params do - requires :description, type: String, desc: 'The trigger description' + requires :description, type: String, desc: 'The trigger description' end post ':id/triggers' do authenticate! @@ -85,7 +85,7 @@ module API declared_params(include_missing: false).merge(owner: current_user)) if trigger.valid? - present trigger, with: Entities::Trigger + present trigger, with: Entities::Trigger, current_user: current_user else render_validation_error!(trigger) end @@ -106,7 +106,7 @@ module API break not_found!('Trigger') unless trigger if trigger.update(declared_params(include_missing: false)) - present trigger, with: Entities::Trigger + present trigger, with: Entities::Trigger, current_user: current_user else render_validation_error!(trigger) end @@ -116,7 +116,7 @@ module API success Entities::Trigger end params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' + requires :trigger_id, type: Integer, desc: 'The trigger ID' end post ':id/triggers/:trigger_id/take_ownership' do authenticate! @@ -127,7 +127,7 @@ module API if trigger.update(owner: current_user) status :ok - present trigger, with: Entities::Trigger + present trigger, with: Entities::Trigger, current_user: current_user else render_validation_error!(trigger) end @@ -137,7 +137,7 @@ module API success Entities::Trigger end params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' + requires :trigger_id, type: Integer, desc: 'The trigger ID' end delete ':id/triggers/:trigger_id' do authenticate! diff --git a/lib/api/users.rb b/lib/api/users.rb index b41fce76df0..7d88880d412 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -26,7 +26,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord def reorder_users(users) if params[:order_by] && params[:sort] - users.reorder(params[:order_by] => params[:sort]) + users.reorder(order_options_with_tie_breaker) else users end @@ -133,10 +133,10 @@ module API desc "Get the status of a user" params do - requires :id_or_username, type: String, desc: 'The ID or username of the user' + requires :user_id, type: String, desc: 'The ID or username of the user' end - get ":id_or_username/status" do - user = find_user(params[:id_or_username]) + get ":user_id/status", requirements: API::USER_REQUIREMENTS do + user = find_user(params[:user_id]) not_found!('User') unless user && can?(current_user, :read_user, user) present user.status || {}, with: Entities::UserStatus diff --git a/lib/api/validations/types/labels_list.rb b/lib/api/validations/types/labels_list.rb new file mode 100644 index 00000000000..47cd83c29cf --- /dev/null +++ b/lib/api/validations/types/labels_list.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module API + module Validations + module Types + class LabelsList + def self.coerce + lambda do |value| + case value + when String + value.split(',').map(&:strip) + when Array + value.map { |v| v.to_s.split(',').map(&:strip) }.flatten + when LabelsList + value + else + [] + end + end + end + end + end + end +end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index f7cae2251c2..148deb86c4c 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -11,7 +11,7 @@ module API requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get project variables' do success Entities::Variable end diff --git a/lib/api/version.rb b/lib/api/version.rb index 74cd857f447..eca1b529094 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -2,13 +2,29 @@ module API class Version < Grape::API + helpers ::API::Helpers::GraphqlHelpers + before { authenticate! } + METADATA_QUERY = <<~EOF + { + metadata { + version + revision + } + } + EOF + desc 'Get the version information of the GitLab instance.' do detail 'This feature was introduced in GitLab 8.13.' end get '/version' do - { version: Gitlab::VERSION, revision: Gitlab.revision } + conditionally_graphql!( + query: METADATA_QUERY, + context: { current_user: current_user }, + transform: ->(result) { result.dig('data', 'metadata') }, + fallback: -> { { version: Gitlab::VERSION, revision: Gitlab.revision } } + ) end end end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 302b2797a34..994074ddc67 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -11,9 +11,7 @@ module API } end - params :wiki_page_params do - requires :content, type: String, desc: 'Content of a wiki page' - requires :title, type: String, desc: 'Title of a wiki page' + params :common_wiki_page_params do optional :format, type: String, values: ProjectWiki::MARKUPS.values.map(&:to_s), @@ -22,7 +20,9 @@ module API end end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + WIKI_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(slug: API::NO_SLASH_URL_PART_REGEX) + + resource :projects, requirements: WIKI_ENDPOINT_REQUIREMENTS do desc 'Get a list of wiki pages' do success Entities::WikiPageBasic end @@ -52,7 +52,9 @@ module API success Entities::WikiPage end params do - use :wiki_page_params + requires :title, type: String, desc: 'Title of a wiki page' + requires :content, type: String, desc: 'Content of a wiki page' + use :common_wiki_page_params end post ':id/wikis' do authorize! :create_wiki, user_project @@ -70,7 +72,10 @@ module API success Entities::WikiPage end params do - use :wiki_page_params + optional :title, type: String, desc: 'Title of a wiki page' + optional :content, type: String, desc: 'Content of a wiki page' + use :common_wiki_page_params + at_least_one_of :content, :title, :format end put ':id/wikis/:slug' do authorize! :create_wiki, user_project @@ -103,7 +108,7 @@ module API requires :file, type: ::API::Validations::Types::SafeFile, desc: 'The attachment file to be uploaded' optional :branch, type: String, desc: 'The name of the branch' end - post ":id/wikis/attachments", requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + post ":id/wikis/attachments" do authorize! :create_wiki, user_project result = ::Wikis::CreateAttachmentService.new(user_project, diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 427c65e2d91..098f2da6d88 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -71,8 +71,14 @@ module Backup end def run_pipeline!(cmd_list, options = {}) - status_list = Open3.pipeline(*cmd_list, options) - raise Backup::Error, 'Backup failed' unless status_list.compact.all?(&:success?) + err_r, err_w = IO.pipe + options[:err] = err_w + status = Open3.pipeline(*cmd_list, options) + err_w.close + return if status.compact.all?(&:success?) + + regex = /^g?tar: \.: Cannot mkdir: No such file or directory$/ + raise Backup::Error, 'Backup failed' unless err_r.read =~ regex end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 12121920c67..aeaf61cda39 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -50,6 +50,7 @@ module Backup if directory.files.create(key: remote_target, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, encryption: Gitlab.config.backup.upload.encryption, + encryption_key: Gitlab.config.backup.upload.encryption_key, storage_class: Gitlab.config.backup.upload.storage_class) progress.puts "done".color(:green) else @@ -195,7 +196,7 @@ module Backup if connection.service == ::Fog::Storage::Local connection.directories.create(key: remote_directory) else - connection.directories.get(remote_directory) + connection.directories.new(key: remote_directory) end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 184c7418e75..22ed1d8e7b4 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -93,7 +93,7 @@ module Backup progress.puts "Error: #{e}".color(:red) end else - restore_repo_success = gitlab_shell.create_repository(project.repository_storage, project.disk_path) + restore_repo_success = gitlab_shell.create_project_repository(project) end if restore_repo_success diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index deda4b1872e..086adf59d2b 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -8,6 +8,10 @@ module Banzai # # Based on HTML::Pipeline::AutolinkFilter # + # Note that our CommonMark parser, `commonmarker` (using the autolink extension) + # handles standard autolinking, like http/https. We detect additional + # schemes (smb, rdar, etc). + # # Context options: # :autolink - Boolean, skips all processing done by this filter when false # :link_attr - Hash of attributes for the generated links @@ -107,10 +111,17 @@ module Banzai end end - # match has come from node.to_html above, so we know it's encoded - # correctly. + # Since this came from a Text node, make sure the new href is encoded. + # `commonmarker` percent encodes the domains of links it handles, so + # do the same (instead of using `normalized_encode`). + begin + href_safe = Addressable::URI.encode(match).html_safe + rescue Addressable::URI::InvalidURIError + return uri.to_s + end + html_safe_match = match.html_safe - options = link_options.merge(href: html_safe_match) + options = link_options.merge(href: href_safe) content_tag(:a, html_safe_match, options) + dropped end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index c87948a30bf..fa1690f73ad 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/emoji.js module Banzai module Filter # HTML filter that replaces :emoji: and unicode with images. diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 2e6d742de27..61ee3eac216 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -4,18 +4,29 @@ module Banzai module Filter # HTML Filter to modify the attributes of external links class ExternalLinkFilter < HTML::Pipeline::Filter - SCHEMES = ['http', 'https', nil].freeze + SCHEMES = ['http', 'https', nil].freeze + RTLO = "\u202E".freeze + ENCODED_RTLO = '%E2%80%AE'.freeze def call links.each do |node| - uri = uri(node['href'].to_s) - next unless uri - - node.set_attribute('href', uri.to_s) + # URI.parse does stricter checking on the url than Addressable, + # such as on `mailto:` links. Since we've been using it, do an + # initial parse for validity and then use Addressable + # for IDN support, etc + uri = uri_strict(node['href'].to_s) + if uri + node.set_attribute('href', uri.to_s) + addressable_uri = addressable_uri(node['href']) + else + addressable_uri = nil + end - if SCHEMES.include?(uri.scheme) && external_url?(uri) - node.set_attribute('rel', 'nofollow noreferrer noopener') - node.set_attribute('target', '_blank') + unless internal_url?(addressable_uri) + punycode_autolink_node!(addressable_uri, node) + sanitize_link_text!(node) + add_malicious_tooltip!(addressable_uri, node) + add_nofollow!(addressable_uri, node) end end @@ -24,27 +35,85 @@ module Banzai private - def uri(href) + def uri_strict(href) URI.parse(href) rescue URI::Error nil end + def addressable_uri(href) + Addressable::URI.parse(href) + rescue Addressable::URI::InvalidURIError + nil + end + def links query = 'descendant-or-self::a[@href and not(@href = "")]' doc.xpath(query) end - def external_url?(uri) + def internal_url?(uri) + return false if uri.nil? # Relative URLs miss a hostname - return false unless uri.hostname + return true unless uri.hostname - uri.hostname != internal_url.hostname + uri.hostname == internal_url.hostname end def internal_url @internal_url ||= URI.parse(Gitlab.config.gitlab.url) end + + # Only replace an autolink with an IDN with it's punycode + # version if we need emailable links. Otherwise let it + # be shown normally and the tooltips will show the + # punycode version. + def punycode_autolink_node!(uri, node) + return unless uri + return unless context[:emailable_links] + + unencoded_uri_str = Addressable::URI.unencode(node['href']) + + if unencoded_uri_str == node.content && idn?(uri) + node.content = uri.normalize + end + end + + # escape any right-to-left (RTLO) characters in link text + def sanitize_link_text!(node) + node.inner_html = node.inner_html.gsub(RTLO, ENCODED_RTLO) + end + + # If the domain is an international domain name (IDN), + # let's expose with a tooltip in case it's intended + # to be malicious. This is particularly useful for links + # where the link text is not the same as the actual link. + # We will continue to show the unicode version of the domain + # in autolinked link text, which could contain emojis, etc. + # + # Also show the tooltip if the url contains the RTLO character, + # as this is an indicator of a malicious link + def add_malicious_tooltip!(uri, node) + if idn?(uri) || has_encoded_rtlo?(uri) + node.add_class('has-tooltip') + node.set_attribute('title', uri.normalize) + end + end + + def add_nofollow!(uri, node) + if SCHEMES.include?(uri&.scheme) + node.set_attribute('rel', 'nofollow noreferrer noopener') + node.set_attribute('target', '_blank') + end + end + + def idn?(uri) + uri&.normalized_host&.start_with?('xn--') + end + + def has_encoded_rtlo?(uri) + uri&.to_s&.include?(ENCODED_RTLO) + end end end end diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb new file mode 100644 index 00000000000..de133774dfa --- /dev/null +++ b/lib/banzai/filter/footnote_filter.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML Filter for footnotes + # + # Footnotes are supported in CommonMark. However we were stripping + # the ids during sanitization. Those are now allowed. + # + # Footnotes are numbered the same - the first one has `id=fn1`, the + # second is `id=fn2`, etc. In order to allow footnotes when rendering + # multiple markdown blocks on a page, we need to make each footnote + # reference unique. + # + # This filter adds a random number to each footnote (the same number + # can be used for a single render). So you get `id=fn1-4335` and `id=fn2-4335`. + # + class FootnoteFilter < HTML::Pipeline::Filter + INTEGER_PATTERN = /\A\d+\z/.freeze + FOOTNOTE_ID_PREFIX = 'fn'.freeze + FOOTNOTE_LINK_ID_PREFIX = 'fnref'.freeze + FOOTNOTE_LI_REFERENCE_PATTERN = /\A#{FOOTNOTE_ID_PREFIX}\d+\z/.freeze + FOOTNOTE_LINK_REFERENCE_PATTERN = /\A#{FOOTNOTE_LINK_ID_PREFIX}\d+\z/.freeze + FOOTNOTE_START_NUMBER = 1 + + def call + return doc unless first_footnote = doc.at_css("ol > li[id=#{fn_id(FOOTNOTE_START_NUMBER)}]") + + # Sanitization stripped off the section wrapper - add it back in + first_footnote.parent.wrap('<section class="footnotes">') + rand_suffix = "-#{random_number}" + modified_footnotes = {} + + doc.css('sup > a[id]').each do |link_node| + ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX) + footnote_node = doc.at_css("li[id=#{fn_id(ref_num)}]") + + if INTEGER_PATTERN.match?(ref_num) && (footnote_node || modified_footnotes[ref_num]) + link_node[:href] += rand_suffix + link_node[:id] += rand_suffix + + # Sanitization stripped off class - add it back in + link_node.parent.append_class('footnote-ref') + + unless modified_footnotes[ref_num] + footnote_node[:id] += rand_suffix + backref_node = footnote_node.at_css("a[href=\"##{fnref_id(ref_num)}\"]") + + if backref_node + backref_node[:href] += rand_suffix + backref_node.append_class('footnote-backref') + end + + modified_footnotes[ref_num] = true + end + end + end + + doc + end + + private + + def random_number + @random_number ||= rand(10000) + end + + def fn_id(num) + "#{FOOTNOTE_ID_PREFIX}#{num}" + end + + def fnref_id(num) + "#{FOOTNOTE_LINK_ID_PREFIX}#{num}" + end + end + end +end diff --git a/lib/banzai/filter/image_lazy_load_filter.rb b/lib/banzai/filter/image_lazy_load_filter.rb index afaee70f351..d8b9eb29cf5 100644 --- a/lib/banzai/filter/image_lazy_load_filter.rb +++ b/lib/banzai/filter/image_lazy_load_filter.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/image.js module Banzai module Filter # HTML filter that moves the value of image `src` attributes to `data-src` diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb index 884a94fb761..01237303c27 100644 --- a/lib/banzai/filter/image_link_filter.rb +++ b/lib/banzai/filter/image_link_filter.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/image.js module Banzai module Filter # HTML filter that wraps links around inline images. diff --git a/lib/banzai/filter/inline_diff_filter.rb b/lib/banzai/filter/inline_diff_filter.rb index e9ddc6e0e3d..5a1c0bee32d 100644 --- a/lib/banzai/filter/inline_diff_filter.rb +++ b/lib/banzai/filter/inline_diff_filter.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/marks/inline_diff.js module Banzai module Filter class InlineDiffFilter < HTML::Pipeline::Filter diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 04ec38209c7..f90a35952e5 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -29,7 +29,7 @@ module Banzai if label yield match, label.id, project, namespace, $~ else - match + escape_html_entities(match) end end end @@ -102,6 +102,10 @@ module Banzai CGI.unescapeHTML(text.to_s) end + def escape_html_entities(text) + CGI.escapeHTML(text.to_s) + end + def object_link_title(object, matches) # use title of wrapped element instead nil diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb index e52c0d15b31..d3af776db05 100644 --- a/lib/banzai/filter/markdown_engines/common_mark.rb +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -32,8 +32,13 @@ module Banzai :DEFAULT # default rendering system. Nothing special. ].freeze - def initialize - @renderer = Banzai::Renderer::CommonMark::HTML.new(options: RENDER_OPTIONS) + RENDER_OPTIONS_SOURCEPOS = RENDER_OPTIONS + [ + :SOURCEPOS # enable embedding of source position information + ].freeze + + def initialize(context) + @context = context + @renderer = Banzai::Renderer::CommonMark::HTML.new(options: render_options) end def render(text) @@ -41,6 +46,12 @@ module Banzai @renderer.render(doc) end + + private + + def render_options + @context[:no_sourcepos] ? RENDER_OPTIONS : RENDER_OPTIONS_SOURCEPOS + end end end end diff --git a/lib/banzai/filter/markdown_engines/redcarpet.rb b/lib/banzai/filter/markdown_engines/redcarpet.rb deleted file mode 100644 index ec150d041ff..00000000000 --- a/lib/banzai/filter/markdown_engines/redcarpet.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -# `Redcarpet` markdown engine for GitLab's Banzai markdown filter. -# This module is used in Banzai::Filter::MarkdownFilter. -# Used gem is `redcarpet` which is a ruby library for markdown processing. -# Homepage: https://github.com/vmg/redcarpet - -module Banzai - module Filter - module MarkdownEngines - class Redcarpet - OPTIONS = { - fenced_code_blocks: true, - footnotes: true, - lax_spacing: true, - no_intra_emphasis: true, - space_after_headers: true, - strikethrough: true, - superscript: true, - tables: true - }.freeze - - def initialize - html_renderer = Banzai::Renderer::Redcarpet::HTML.new - @renderer = ::Redcarpet::Markdown.new(html_renderer, OPTIONS) - end - - def render(text) - @renderer.render(text) - end - end - end - end -end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index cdf758472c1..242e39f5495 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -6,7 +6,7 @@ module Banzai def initialize(text, context = nil, result = nil) super(text, context, result) - @renderer = renderer(context[:markdown_engine]).new + @renderer = renderer(context[:markdown_engine]).new(context) @text = @text.delete("\r") end diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index 9d1bc3cf60c..8dd5a8979c8 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -2,6 +2,9 @@ require 'uri' +# Generated HTML is transformed back to GFM by: +# - app/assets/javascripts/behaviors/markdown/marks/math.js +# - app/assets/javascripts/behaviors/markdown/nodes/code_block.js module Banzai module Filter # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$. diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb index 7c8b165a330..f0adb83af8a 100644 --- a/lib/banzai/filter/mermaid_filter.rb +++ b/lib/banzai/filter/mermaid_filter.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/code_block.js module Banzai module Filter class MermaidFilter < HTML::Pipeline::Filter diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index c70c3f0c04e..fce042e8946 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -101,9 +101,9 @@ module Banzai def self_and_ancestors_ids(parent) if group_context?(parent) - parent.self_and_ancestors_ids + parent.self_and_ancestors.select(:id) elsif project_context?(parent) - parent.group&.self_and_ancestors_ids + parent.group&.self_and_ancestors&.select(:id) end end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index e5164e7f72a..42f9b3a689c 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/reference.js module Banzai module Filter # Base class for GitLab Flavored Markdown reference filters. diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 7acbc933adc..2745905c5ff 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -58,6 +58,8 @@ module Banzai path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') elsif project path_parts.unshift(relative_url_root, project.full_path) + else + path_parts.unshift(relative_url_root) end begin @@ -148,7 +150,10 @@ module Banzai end def uri_type(path) - @uri_types[path] ||= current_commit.uri_type(path) + # https://gitlab.com/gitlab-org/gitlab-ce/issues/58011 + Gitlab::GitalyClient.allow_n_plus_1_calls do + @uri_types[path] ||= current_commit.uri_type(path) + end end def current_commit diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 8ba09290e6d..a4a06eae7b7 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -8,8 +8,8 @@ module Banzai class SanitizationFilter < HTML::Pipeline::SanitizationFilter include Gitlab::Utils::StrongMemoize - UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze - TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/ + UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/.freeze def whitelist strong_memoize(:whitelist) do @@ -41,14 +41,16 @@ module Banzai whitelist[:elements].push('abbr') whitelist[:attributes]['abbr'] = %w(title) + # Allow the 'data-sourcepos' from CommonMark on all elements + whitelist[:attributes][:all].push('data-sourcepos') + # Disallow `name` attribute globally, allow on `a` whitelist[:attributes][:all].delete('name') whitelist[:attributes]['a'].push('name') - # Allow any protocol in `a` elements... + # Allow any protocol in `a` elements + # and then remove links with unsafe protocols whitelist[:protocols].delete('a') - - # ...but then remove links with unsafe protocols whitelist[:transformers].push(self.class.remove_unsafe_links) # Remove `rel` attribute from `a` elements @@ -57,6 +59,12 @@ module Banzai # Remove any `style` properties not required for table alignment whitelist[:transformers].push(self.class.remove_unsafe_table_style) + # Allow `id` in a and li elements for footnotes + # and remove any `id` properties not matching for footnotes + whitelist[:attributes]['a'].push('id') + whitelist[:attributes]['li'] = %w(id) + whitelist[:transformers].push(self.class.remove_non_footnote_ids) + whitelist end @@ -112,6 +120,20 @@ module Banzai end end end + + def remove_non_footnote_ids + lambda do |env| + node = env[:node] + + return unless node.name == 'a' || node.name == 'li' + return unless node.has_attribute?('id') + + return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN + return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN + + node.remove_attribute('id') + end + end end end end diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb index c6a3a763c23..50bf823929c 100644 --- a/lib/banzai/filter/spaced_link_filter.rb +++ b/lib/banzai/filter/spaced_link_filter.rb @@ -45,8 +45,6 @@ module Banzai ]).freeze def call - return doc if context[:markdown_engine] == :redcarpet - doc.xpath(TEXT_QUERY).each do |node| content = node.to_html @@ -73,7 +71,8 @@ module Banzai html = Banzai::Filter::MarkdownFilter.call(transform_markdown(match), context) # link is wrapped in a <p>, so strip that off - html.sub('<p>', '').chomp('</p>') + p_node = Nokogiri::HTML.fragment(html).at_css('p') + p_node ? p_node.children.to_html : html end def spaced_link_filter(text) diff --git a/lib/banzai/filter/suggestion_filter.rb b/lib/banzai/filter/suggestion_filter.rb index 307ea449140..9950db373d8 100644 --- a/lib/banzai/filter/suggestion_filter.rb +++ b/lib/banzai/filter/suggestion_filter.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/code_block.js module Banzai module Filter class SuggestionFilter < HTML::Pipeline::Filter diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 18e5e9185de..9ffde52b5f2 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true require 'rouge/plugins/common_mark' -require 'rouge/plugins/redcarpet' +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/code_block.js module Banzai module Filter # HTML Filter to highlight fenced code blocks diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index c6d1e028eaa..f2ae17b44fa 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/table_of_contents.js module Banzai module Filter # HTML filter that adds an anchor child element to all Headers in a diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index ef35a49edcb..c6b402575cb 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -2,6 +2,10 @@ require 'task_list/filter' +# Generated HTML is transformed back to GFM by: +# - app/assets/javascripts/behaviors/markdown/nodes/ordered_task_list.js +# - app/assets/javascripts/behaviors/markdown/nodes/task_list.js +# - app/assets/javascripts/behaviors/markdown/nodes/task_list_item.js module Banzai module Filter class TaskListFilter < TaskList::Filter diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb index 0fb59c914c3..0fff104cf91 100644 --- a/lib/banzai/filter/video_link_filter.rb +++ b/lib/banzai/filter/video_link_filter.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/video.js module Banzai module Filter # Find every image that isn't already wrapped in an `a` tag, and that has diff --git a/lib/banzai/pipeline/atom_pipeline.rb b/lib/banzai/pipeline/atom_pipeline.rb index 13a342351b6..c632910585d 100644 --- a/lib/banzai/pipeline/atom_pipeline.rb +++ b/lib/banzai/pipeline/atom_pipeline.rb @@ -6,7 +6,8 @@ module Banzai def self.transform_context(context) super(context).merge( only_path: false, - xhtml: true + xhtml: true, + no_sourcepos: true ) end end diff --git a/lib/banzai/pipeline/broadcast_message_pipeline.rb b/lib/banzai/pipeline/broadcast_message_pipeline.rb index a3d63e0aaf5..580b5b72474 100644 --- a/lib/banzai/pipeline/broadcast_message_pipeline.rb +++ b/lib/banzai/pipeline/broadcast_message_pipeline.rb @@ -14,6 +14,12 @@ module Banzai Filter::ExternalLinkFilter ] end + + def self.transform_context(context) + super(context).merge( + no_sourcepos: true + ) + end end end end diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb index 2c08581ce0d..13e6a990407 100644 --- a/lib/banzai/pipeline/email_pipeline.rb +++ b/lib/banzai/pipeline/email_pipeline.rb @@ -11,7 +11,9 @@ module Banzai def self.transform_context(context) super(context).merge( - only_path: false + only_path: false, + emailable_links: true, + no_sourcepos: true ) end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 5f13a6d6cde..30cafd11834 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -3,11 +3,11 @@ module Banzai module Pipeline class GfmPipeline < BasePipeline - # These filters convert GitLab Flavored Markdown (GFM) to HTML. - # The handlers defined in app/assets/javascripts/behaviors/markdown/copy_as_gfm.js - # consequently convert that same HTML to GFM to be copied to the clipboard. - # Every filter that generates HTML from GFM should have a handler in - # app/assets/javascripts/behaviors/markdown/copy_as_gfm.js, in reverse order. + # These filters transform GitLab Flavored Markdown (GFM) to HTML. + # The nodes and marks referenced in app/assets/javascripts/behaviors/markdown/editor_extensions.js + # consequently transform that same HTML to GFM to be copied to the clipboard. + # Every filter that generates HTML from GFM should have a node or mark in + # app/assets/javascripts/behaviors/markdown/editor_extensions.js. # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ @@ -30,6 +30,7 @@ module Banzai Filter::AutolinkFilter, Filter::ExternalLinkFilter, Filter::SuggestionFilter, + Filter::FootnoteFilter, *reference_filters, diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb index db79a22549c..ceba082cd4f 100644 --- a/lib/banzai/pipeline/markup_pipeline.rb +++ b/lib/banzai/pipeline/markup_pipeline.rb @@ -7,7 +7,8 @@ module Banzai @filters ||= FilterArray[ Filter::SanitizationFilter, Filter::ExternalLinkFilter, - Filter::PlantumlFilter + Filter::PlantumlFilter, + Filter::SyntaxHighlightFilter ] end end diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 61ff7b0bcce..72374207a8f 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -27,6 +27,12 @@ module Banzai Filter::CommitReferenceFilter ] end + + def self.transform_context(context) + super(context).merge( + no_sourcepos: true + ) + end end end end diff --git a/lib/banzai/renderer/redcarpet/html.rb b/lib/banzai/renderer/redcarpet/html.rb deleted file mode 100644 index 84931fdc784..00000000000 --- a/lib/banzai/renderer/redcarpet/html.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Renderer - module Redcarpet - class HTML < ::Redcarpet::Render::HTML - def block_code(code, lang) - lang_attr = lang ? %Q{ lang="#{lang}"} : '' - - "\n<pre>" \ - "<code#{lang_attr}>#{ERB::Util.html_escape(code)}</code>" \ - "</pre>" - end - end - end - end -end diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb index 83e8808db07..6a608058813 100644 --- a/lib/bitbucket_server/client.rb +++ b/lib/bitbucket_server/client.rb @@ -4,18 +4,6 @@ module BitbucketServer class Client attr_reader :connection - ServerError = Class.new(StandardError) - - SERVER_ERRORS = [SocketError, - OpenSSL::SSL::SSLError, - Errno::ECONNRESET, - Errno::ECONNREFUSED, - Errno::EHOSTUNREACH, - Net::OpenTimeout, - Net::ReadTimeout, - Gitlab::HTTP::BlockedUrlError, - BitbucketServer::Connection::ConnectionError].freeze - def initialize(options = {}) @connection = Connection.new(options) end @@ -64,8 +52,6 @@ module BitbucketServer def get_collection(path, type, page_offset: 0, limit: nil) paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type, page_offset: page_offset, limit: limit) BitbucketServer::Collection.new(paginator) - rescue *SERVER_ERRORS => e - raise ServerError, e end end end diff --git a/lib/bitbucket_server/collection.rb b/lib/bitbucket_server/collection.rb index 7e4b2277bbe..f549acbd87f 100644 --- a/lib/bitbucket_server/collection.rb +++ b/lib/bitbucket_server/collection.rb @@ -25,13 +25,13 @@ module BitbucketServer end def prev_page - return nil unless current_page > 1 + return unless current_page > 1 current_page - 1 end def next_page - return nil unless has_next_page? + return unless has_next_page? current_page + 1 end diff --git a/lib/bitbucket_server/connection.rb b/lib/bitbucket_server/connection.rb index 7efcdcf8619..fbd451efb23 100644 --- a/lib/bitbucket_server/connection.rb +++ b/lib/bitbucket_server/connection.rb @@ -7,6 +7,17 @@ module BitbucketServer DEFAULT_API_VERSION = '1.0' SEPARATOR = '/' + NETWORK_ERRORS = [ + SocketError, + OpenSSL::SSL::SSLError, + Errno::ECONNRESET, + Errno::ECONNREFUSED, + Errno::EHOSTUNREACH, + Net::OpenTimeout, + Net::ReadTimeout, + Gitlab::HTTP::BlockedUrlError + ].freeze + attr_reader :api_version, :base_uri, :username, :token ConnectionError = Class.new(StandardError) @@ -27,6 +38,8 @@ module BitbucketServer check_errors!(response) response.parsed_response + rescue *NETWORK_ERRORS => e + raise ConnectionError, e end def post(path, body) @@ -38,6 +51,8 @@ module BitbucketServer check_errors!(response) response.parsed_response + rescue *NETWORK_ERRORS => e + raise ConnectionError, e end # We need to support two different APIs for deletion: @@ -55,11 +70,14 @@ module BitbucketServer check_errors!(response) response.parsed_response + rescue *NETWORK_ERRORS => e + raise ConnectionError, e end private def check_errors!(response) + return if ActionDispatch::Response::NO_CONTENT_CODES.include?(response.code) raise ConnectionError, "Response is not valid JSON" unless response.parsed_response.is_a?(Hash) return if response.code >= 200 && response.code < 300 diff --git a/lib/bitbucket_server/paginator.rb b/lib/bitbucket_server/paginator.rb index aa5f84f44b3..9eda1c921b2 100644 --- a/lib/bitbucket_server/paginator.rb +++ b/lib/bitbucket_server/paginator.rb @@ -12,7 +12,7 @@ module BitbucketServer @url = url @page = nil @page_offset = page_offset - @limit = limit || PAGE_LENGTH + @limit = limit @total = 0 end @@ -34,6 +34,8 @@ module BitbucketServer attr_reader :connection, :page, :url, :type, :limit def over_limit? + return false unless @limit + @limit.positive? && @total >= @limit end @@ -42,11 +44,15 @@ module BitbucketServer end def starting_offset - [0, page_offset - 1].max * limit + [0, page_offset - 1].max * max_per_page + end + + def max_per_page + limit || PAGE_LENGTH end def fetch_next_page - parsed_response = connection.get(@url, start: next_offset, limit: @limit) + parsed_response = connection.get(@url, start: next_offset, limit: max_per_page) Page.new(parsed_response, type) end end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index eadfbf7bc01..d41490d2ebd 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -2,12 +2,13 @@ module Constraints class ProjectUrlConstrainer - def matches?(request) + def matches?(request, existence_check: true) namespace_path = request.params[:namespace_id] project_path = request.params[:project_id] || request.params[:id] full_path = [namespace_path, project_path].join('/') return false unless ProjectPathValidator.valid_path?(full_path) + return true unless existence_check # We intentionally allow SELECT(*) here so result of this query can be used # as cache for further Project.find_by_full_path calls within request diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 8633e764f90..ef41dc560c9 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -2,6 +2,8 @@ module ContainerRegistry class Tag + include Gitlab::Utils::StrongMemoize + attr_reader :repository, :name delegate :registry, :client, to: :repository @@ -15,6 +17,10 @@ module ContainerRegistry manifest.present? end + def latest? + name == "latest" + end + def v1? manifest && manifest['schemaVersion'] == 1 end @@ -24,7 +30,9 @@ module ContainerRegistry end def manifest - @manifest ||= client.repository_manifest(repository.path, name) + strong_memoize(:manifest) do + client.repository_manifest(repository.path, name) + end end def path @@ -42,36 +50,44 @@ module ContainerRegistry end def digest - @digest ||= client.repository_tag_digest(repository.path, name) + strong_memoize(:digest) do + client.repository_tag_digest(repository.path, name) + end end def config_blob - return @config_blob if defined?(@config_blob) return unless manifest && manifest['config'] - @config_blob = repository.blob(manifest['config']) + strong_memoize(:config_blob) do + repository.blob(manifest['config']) + end end def config - return unless config_blob + return unless config_blob&.data - @config ||= ContainerRegistry::Config.new(self, config_blob) if config_blob.data + strong_memoize(:config) do + ContainerRegistry::Config.new(self, config_blob) + end end def created_at return unless config - @created_at ||= DateTime.rfc3339(config['created']) + strong_memoize(:created_at) do + DateTime.rfc3339(config['created']) + end end def layers - return @layers if defined?(@layers) return unless manifest - layers = manifest['layers'] || manifest['fsLayers'] + strong_memoize(:layers) do + layers = manifest['layers'] || manifest['fsLayers'] - @layers = layers.map do |layer| - repository.blob(layer) + layers.map do |layer| + repository.blob(layer) + end end end diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb index f38f4f0a50f..964d35cde9e 100644 --- a/lib/declarative_policy/rule.rb +++ b/lib/declarative_policy/rule.rb @@ -84,7 +84,7 @@ module DeclarativePolicy # returns nil unless it's already cached def cached_pass?(context) condition = context.condition(@name) - return nil unless condition.cached? + return unless condition.cached? condition.pass? end @@ -124,7 +124,7 @@ module DeclarativePolicy def cached_pass?(context) condition = delegated_context(context).condition(@name) - return nil unless condition.cached? + return unless condition.cached? condition.pass? rescue MissingDelegate @@ -161,7 +161,7 @@ module DeclarativePolicy def cached_pass?(context) runner = context.runner(@ability) - return nil unless runner.cached? + return unless runner.cached? runner.pass? end diff --git a/lib/feature.rb b/lib/feature.rb index e048a443abc..749c861d740 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -102,4 +102,48 @@ class Feature expires_in: 1.hour) end end + + class Target + attr_reader :params + + def initialize(params) + @params = params + end + + def gate_specified? + %i(user project group feature_group).any? { |key| params.key?(key) } + end + + def targets + [feature_group, user, project, group].compact + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def feature_group + return unless params.key?(:feature_group) + + Feature.group(params[:feature_group]) + end + # rubocop: enable CodeReuse/ActiveRecord + + def user + return unless params.key?(:user) + + UserFinder.new(params[:user]).find_by_username! + end + + def project + return unless params.key?(:project) + + Project.find_by_full_path(params[:project]) + end + + def group + return unless params.key?(:group) + + Group.find_by_full_path(params[:group]) + end + end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 2ef54658a11..f42ca5a9cd6 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -7,6 +7,14 @@ module Gitlab Pathname.new(File.expand_path('..', __dir__)) end + def self.version_info + Gitlab::VersionInfo.parse(Gitlab::VERSION) + end + + def self.pre_release? + VERSION.include?('pre') + end + def self.config Settings end @@ -50,11 +58,15 @@ module Gitlab Rails.env.development? || org? || com? end - def self.pre_release? - VERSION.include?('pre') + def self.ee? + Object.const_defined?(:License) end - def self.version_info - Gitlab::VersionInfo.parse(Gitlab::VERSION) + def self.process_name + return 'sidekiq' if Sidekiq.server? + return 'console' if defined?(Rails::Console) + return 'test' if Rails.env.test? + + 'web' end end diff --git a/lib/gitlab/access/branch_protection.rb b/lib/gitlab/access/branch_protection.rb new file mode 100644 index 00000000000..f039e5c011f --- /dev/null +++ b/lib/gitlab/access/branch_protection.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Access + # A wrapper around Integer based branch protection levels. + # + # This wrapper can be used to work with branch protection levels without + # having to directly refer to the constants. For example, instead of this: + # + # if access_level == Gitlab::Access::PROTECTION_DEV_CAN_PUSH + # ... + # end + # + # You can write this instead: + # + # protection = BranchProtection.new(access_level) + # + # if protection.developer_can_push? + # ... + # end + class BranchProtection + attr_reader :level + + # @param [Integer] level The branch protection level as an Integer. + def initialize(level) + @level = level + end + + def any? + level != PROTECTION_NONE + end + + def developer_can_push? + level == PROTECTION_DEV_CAN_PUSH + end + + def developer_can_merge? + level == PROTECTION_DEV_CAN_MERGE + end + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 7aa02009aa0..b2ef04d23d7 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -12,6 +12,9 @@ module Gitlab # Scopes used for OpenID Connect OPENID_SCOPES = [:openid].freeze + # OpenID Connect profile scopes + PROFILE_SCOPES = [:profile, :email].freeze + # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [:api].freeze @@ -284,7 +287,7 @@ module Gitlab # Other available scopes def optional_scopes - available_scopes + OPENID_SCOPES - DEFAULT_SCOPES + available_scopes + OPENID_SCOPES + PROFILE_SCOPES - DEFAULT_SCOPES end def registry_scopes diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 42c657afe6a..15b9d5ad6e9 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -30,14 +30,7 @@ module Gitlab def users(fields, value, limit = nil) options = user_options(Array(fields), value, limit) - - entries = ldap_search(options).select do |entry| - entry.respond_to? config.uid - end - - entries.map do |entry| - Gitlab::Auth::LDAP::Person.new(entry, provider) - end + users_search(options) end def user(*args) @@ -90,6 +83,16 @@ module Gitlab SEARCH_RETRY_FACTOR[retry_number] * config.timeout end + def users_search(options) + entries = ldap_search(options).select do |entry| + entry.respond_to? config.uid + end + + entries.map do |entry| + Gitlab::Auth::LDAP::Person.new(entry, provider) + end + end + def user_options(fields, value, limit) options = { attributes: Gitlab::Auth::LDAP::Person.ldap_attributes(config), diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 7ceb96f502b..47d63eb53cf 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -75,7 +75,8 @@ module Gitlab encryption: options['encryption'], filter: omniauth_user_filter, name_proc: name_proc, - disable_verify_certificates: !options['verify_certificates'] + disable_verify_certificates: !options['verify_certificates'], + tls_options: tls_options ) if has_auth? @@ -85,9 +86,6 @@ module Gitlab ) end - opts[:ca_file] = options['ca_file'] if options['ca_file'].present? - opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? - opts end @@ -196,24 +194,28 @@ module Gitlab end def encryption_options - method = translate_method(options['encryption']) - return nil unless method + method = translate_method + return unless method { method: method, - tls_options: tls_options(method) + tls_options: tls_options } end - def translate_method(method_from_config) - NET_LDAP_ENCRYPTION_METHOD[method_from_config.to_sym] + def translate_method + NET_LDAP_ENCRYPTION_METHOD[options['encryption']&.to_sym] end - def tls_options(method) - return { verify_mode: OpenSSL::SSL::VERIFY_NONE } unless method + def tls_options + return @tls_options if defined?(@tls_options) + + method = translate_method + return unless method - opts = if options['verify_certificates'] - OpenSSL::SSL::SSLContext::DEFAULT_PARAMS + opts = if options['verify_certificates'] && method != 'plain' + # Dup so we don't accidentally overwrite the constant + OpenSSL::SSL::SSLContext::DEFAULT_PARAMS.dup else # It is important to explicitly set verify_mode for two reasons: # 1. The behavior of OpenSSL is undefined when verify_mode is not set. @@ -222,10 +224,35 @@ module Gitlab { verify_mode: OpenSSL::SSL::VERIFY_NONE } end - opts[:ca_file] = options['ca_file'] if options['ca_file'].present? - opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + opts.merge!(custom_tls_options) - opts + @tls_options = opts + end + + def custom_tls_options + return {} unless options['tls_options'] + + # Dup so we don't overwrite the original value + custom_options = options['tls_options'].dup.delete_if { |_, value| value.nil? || value.blank? } + custom_options.symbolize_keys! + + if custom_options[:cert] + begin + custom_options[:cert] = OpenSSL::X509::Certificate.new(custom_options[:cert]) + rescue OpenSSL::X509::CertificateError => e + Rails.logger.error "LDAP TLS Options 'cert' is invalid for provider #{provider}: #{e.message}" + end + end + + if custom_options[:key] + begin + custom_options[:key] = OpenSSL::PKey.read(custom_options[:key]) + rescue OpenSSL::PKey::PKeyError => e + Rails.logger.error "LDAP TLS Options 'key' is invalid for provider #{provider}: #{e.message}" + end + end + + custom_options end def auth_options diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index a0244a3cea1..13d67e0f871 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -98,9 +98,7 @@ module Gitlab private - def entry - @entry - end + attr_reader :entry def config @config ||= Gitlab::Auth::LDAP::Config.new(provider) @@ -114,7 +112,7 @@ module Gitlab attributes = Array(config.attributes[attribute.to_s]) selected_attr = attributes.find { |attr| entry.respond_to?(attr) } - return nil unless selected_attr + return unless selected_attr entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index a4e8a41b246..f38c5d57c44 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -46,7 +46,7 @@ module Gitlab gl_user.block if block_after_save - log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" + log.info "(#{provider}) saving user #{auth_hash.email} from login with admin => #{gl_user.admin}, extern_uid => #{auth_hash.uid}" gl_user rescue ActiveRecord::RecordInvalid => e log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}" diff --git a/lib/gitlab/auth/omniauth_identity_linker_base.rb b/lib/gitlab/auth/omniauth_identity_linker_base.rb index 253445570f2..c620fc5d6bd 100644 --- a/lib/gitlab/auth/omniauth_identity_linker_base.rb +++ b/lib/gitlab/auth/omniauth_identity_linker_base.rb @@ -12,7 +12,7 @@ module Gitlab end def link - save if identity.new_record? + save if unlinked? end def changed? @@ -35,6 +35,10 @@ module Gitlab @changed = identity.save end + def unlinked? + identity.new_record? + end + # rubocop: disable CodeReuse/ActiveRecord def identity @identity ||= current_user.identities diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb index 1af9fa40c3a..b0df9757bbd 100644 --- a/lib/gitlab/auth/saml/auth_hash.rb +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -10,11 +10,11 @@ module Gitlab def authn_context response_object = auth_hash.extra[:response_object] - return nil if response_object.blank? + return if response_object.blank? document = response_object.decrypted_document document ||= response_object.document - return nil if document.blank? + return if document.blank? extract_authn_context(document) end diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index d72befce571..5251e0fadf9 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -16,11 +16,18 @@ module Gitlab # re-raises the exception. # # steal_class - The name of the class for which to steal jobs. - def self.steal(steal_class) - enqueued = Sidekiq::Queue.new(self.queue) - scheduled = Sidekiq::ScheduledSet.new + def self.steal(steal_class, retry_dead_jobs: false) + queues = [ + Sidekiq::ScheduledSet.new, + Sidekiq::Queue.new(self.queue) + ] + + if retry_dead_jobs + queues << Sidekiq::RetrySet.new + queues << Sidekiq::DeadSet.new + end - [scheduled, enqueued].each do |queue| + queues.each do |queue| queue.each do |job| migration_class, migration_args = job.args @@ -51,6 +58,19 @@ module Gitlab migration_class_for(class_name).new.perform(*arguments) end + def self.exists?(migration_class) + enqueued = Sidekiq::Queue.new(self.queue) + scheduled = Sidekiq::ScheduledSet.new + + [enqueued, scheduled].each do |queue| + queue.each do |job| + return true if job.queue == self.queue && job.args.first == migration_class + end + end + + false + end + def self.migration_class_for(class_name) const_get(class_name) end diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index aaf520d70f6..c8d83cc1803 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -83,7 +83,7 @@ module Gitlab extend ActiveSupport::Concern def full_path - @full_path ||= build_full_path + route&.path || build_full_path end def build_full_path @@ -99,7 +99,12 @@ module Gitlab end end - # Namespace model. + # Route model + class Route < ActiveRecord::Base + belongs_to :source, inverse_of: :route, polymorphic: true + end + + # Namespace model class Namespace < ActiveRecord::Base self.table_name = 'namespaces' self.inheritance_column = nil @@ -108,6 +113,8 @@ module Gitlab belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces' + has_one :route, -> { where(source_type: 'Namespace') }, inverse_of: :source, foreign_key: :source_id + has_many :projects, inverse_of: :parent has_many :namespaces, inverse_of: :parent end @@ -134,6 +141,7 @@ module Gitlab belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects' + has_one :route, -> { where(source_type: 'Project') }, inverse_of: :source, foreign_key: :source_id has_one :project_repository, inverse_of: :project delegate :disk_path, to: :storage @@ -194,6 +202,8 @@ module Gitlab def project_repositories(start_id, stop_id) projects .without_project_repository + .includes(:route, parent: [:route]).references(:routes) + .includes(:parent).references(:namespaces) .where(id: start_id..stop_id) .map { |project| build_attributes_for_project(project) } .compact diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb index b9ad8267e37..173543b7c25 100644 --- a/lib/gitlab/background_migration/encrypt_columns.rb +++ b/lib/gitlab/background_migration/encrypt_columns.rb @@ -91,7 +91,8 @@ module Gitlab # No need to do anything if the plaintext is nil, or an encrypted # value already exists - return nil unless plaintext.present? && !ciphertext.present? + return unless plaintext.present? + return if ciphertext.present? # attr_encrypted will calculate and set the expected value for us instance.public_send("#{plain_column}=", plaintext) # rubocop:disable GitlabSecurity/PublicSend diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 38fecac1bfe..42fcaa87e66 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -24,7 +24,7 @@ module Gitlab def commit_title commit = commits.last - return nil unless commit && commit[:message] + return unless commit && commit[:message] index = commit[:message].index("\n") message = index ? commit[:message][0..index] : commit[:message] diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb index 0e5c7f092f2..6a29a632577 100644 --- a/lib/gitlab/background_migration/migrate_stage_status.rb +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -16,10 +16,10 @@ module Gitlab scope :running, -> { where(status: 'running') } scope :pending, -> { where(status: 'pending') } scope :success, -> { where(status: 'success') } - scope :failed, -> { where(status: 'failed') } - scope :canceled, -> { where(status: 'canceled') } - scope :skipped, -> { where(status: 'skipped') } - scope :manual, -> { where(status: 'manual') } + scope :failed, -> { where(status: 'failed') } + scope :canceled, -> { where(status: 'canceled') } + scope :skipped, -> { where(status: 'skipped') } + scope :manual, -> { where(status: 'manual') } scope :failed_but_allowed, -> do where(allow_failure: true, status: [:failed, :canceled]) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index 4a9a62aaeb5..a84f794bfae 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -127,7 +127,7 @@ module Gitlab full_path = matchd[1] project = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Project.find_by_full_path(full_path) - return nil unless project + return unless project project.id end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 3cd327f5109..144ba2ec031 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -108,7 +108,7 @@ module Gitlab end def find_or_create_groups - return nil unless group_path.present? + return unless group_path.present? log " * Using namespace: #{group_path}" diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index eaead41a720..441fdec8a1e 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -47,7 +47,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def find_user_id(username) - return nil unless username + return unless username return users[username] if users.key?(username) @@ -65,9 +65,9 @@ module Gitlab def import_wiki return if project.wiki.repository_exists? - disk_path = project.wiki.disk_path - import_url = project.import_url.sub(/\.git\z/, ".git/wiki") - gitlab_shell.import_repository(project.repository_storage, disk_path, import_url) + wiki = WikiFormatter.new(project) + + gitlab_shell.import_wiki_repository(project, wiki) rescue StandardError => e errors << { type: :wiki, errors: e.message } end diff --git a/lib/gitlab/bitbucket_import/wiki_formatter.rb b/lib/gitlab/bitbucket_import/wiki_formatter.rb new file mode 100644 index 00000000000..b8ff43b777b --- /dev/null +++ b/lib/gitlab/bitbucket_import/wiki_formatter.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + class WikiFormatter + attr_reader :project + + def initialize(project) + @project = project + end + + def disk_path + project.wiki.disk_path + end + + def full_path + project.wiki.full_path + end + + def import_url + project.import_url.sub(/\.git\z/, ".git/wiki") + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 28cfb46e2d4..4a789ae457f 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -65,7 +65,7 @@ module Gitlab end def find_user_id(email) - return nil unless email + return unless email return users[email] if users.key?(email) @@ -132,7 +132,7 @@ module Gitlab project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap, remote_name: REMOTE_NAME) log_info(stage: 'import_repository', message: 'finished import') - rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e + rescue Gitlab::Shell::Error => e log_error(stage: 'import_repository', message: 'failed import', error: e.message) # Expire cache to prevent scenarios such as: @@ -140,7 +140,7 @@ module Gitlab # 2. Retried import, repo is broken or not imported but +exists?+ still returns true project.repository.expire_content_cache if project.repository_exists? - raise e.message + raise end # Bitbucket Server keeps tracks of references for open pull requests in diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb index 488c1d85387..d3e15a79a8b 100644 --- a/lib/gitlab/blob_helper.rb +++ b/lib/gitlab/blob_helper.rb @@ -12,7 +12,7 @@ module Gitlab end def viewable? - !large? && text? + !large? && text_in_repo? end MEGABYTE = 1024 * 1024 @@ -21,7 +21,7 @@ module Gitlab size.to_i > MEGABYTE end - def binary? + def binary_in_repo? # Large blobs aren't even loaded into memory if data.nil? true @@ -40,8 +40,8 @@ module Gitlab end end - def text? - !binary? + def text_in_repo? + !binary_in_repo? end def image? @@ -113,7 +113,7 @@ module Gitlab def content_type # rubocop:disable Style/MultilineTernaryOperator # rubocop:disable Style/NestedTernaryOperator - @content_type ||= binary_mime_type? || binary? ? mime_type : + @content_type ||= binary_mime_type? || binary_in_repo? ? mime_type : (encoding ? "text/plain; charset=#{encoding.downcase}" : "text/plain") # rubocop:enable Style/NestedTernaryOperator # rubocop:enable Style/MultilineTernaryOperator diff --git a/lib/gitlab/chat.rb b/lib/gitlab/chat.rb new file mode 100644 index 00000000000..23d4fb36b66 --- /dev/null +++ b/lib/gitlab/chat.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module Chat + # Returns `true` if Chatops is available for the current instance. + def self.available? + ::Feature.enabled?(:chatops, default_enabled: true) + end + end +end diff --git a/lib/gitlab/chat/command.rb b/lib/gitlab/chat/command.rb new file mode 100644 index 00000000000..49b7dcf4bbe --- /dev/null +++ b/lib/gitlab/chat/command.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Gitlab + module Chat + # Class for scheduling chat pipelines. + # + # A Command takes care of creating a `Ci::Pipeline` with all the data + # necessary to execute a chat command. This includes data such as the chat + # data (e.g. the response URL) and any environment variables that should be + # exposed to the chat command. + class Command + include Utils::StrongMemoize + + attr_reader :project, :chat_name, :name, :arguments, :response_url, + :channel + + # project - The Project to schedule the command for. + # chat_name - The ChatName belonging to the user that scheduled the + # command. + # name - The name of the chat command to run. + # arguments - The arguments (as a String) to pass to the command. + # channel - The channel the message was sent from. + # response_url - The URL to send the response back to. + def initialize(project:, chat_name:, name:, arguments:, channel:, response_url:) + @project = project + @chat_name = chat_name + @name = name + @arguments = arguments + @channel = channel + @response_url = response_url + end + + # Tries to create a new pipeline. + # + # This method will return a pipeline that _may_ be persisted, or `nil` if + # the pipeline could not be created. + def try_create_pipeline + return unless valid? + + create_pipeline + end + + def create_pipeline + service = ::Ci::CreatePipelineService.new( + project, + chat_name.user, + ref: branch, + sha: commit, + chat_data: { + chat_name_id: chat_name.id, + command: name, + arguments: arguments, + response_url: response_url + } + ) + + service.execute(:chat) do |pipeline| + build_environment_variables(pipeline) + build_chat_data(pipeline) + end + end + + # pipeline - The `Ci::Pipeline` to create the environment variables for. + def build_environment_variables(pipeline) + pipeline.variables.build( + [{ key: 'CHAT_INPUT', value: arguments }, + { key: 'CHAT_CHANNEL', value: channel }] + ) + end + + # pipeline - The `Ci::Pipeline` to create the chat data for. + def build_chat_data(pipeline) + pipeline.build_chat_data( + chat_name_id: chat_name.id, + response_url: response_url + ) + end + + def valid? + branch && commit + end + + def branch + strong_memoize(:branch) { project.default_branch } + end + + def commit + strong_memoize(:commit) do + project.commit(branch)&.id if branch + end + end + end + end +end diff --git a/lib/gitlab/chat/output.rb b/lib/gitlab/chat/output.rb new file mode 100644 index 00000000000..411b1555a7d --- /dev/null +++ b/lib/gitlab/chat/output.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module Chat + # Class for gathering and formatting the output of a `Ci::Build`. + class Output + attr_reader :build + + MissingBuildSectionError = Class.new(StandardError) + + # The primary trace section to look for. + PRIMARY_SECTION = 'chat_reply' + + # The backup trace section in case the primary one could not be found. + FALLBACK_SECTION = 'build_script' + + # build - The `Ci::Build` to obtain the output from. + def initialize(build) + @build = build + end + + # Returns a `String` containing the output of the build. + # + # The output _does not_ include the command that was executed. + def to_s + offset, length = read_offset_and_length + + trace.read do |stream| + stream.seek(offset) + + output = stream + .stream + .read(length) + .force_encoding(Encoding.default_external) + + without_executed_command_line(output) + end + end + + # Returns the offset to seek to and the number of bytes to read relative + # to the offset. + def read_offset_and_length + section = find_build_trace_section(PRIMARY_SECTION) || + find_build_trace_section(FALLBACK_SECTION) + + unless section + raise( + MissingBuildSectionError, + "The build_script trace section could not be found for build #{build.id}" + ) + end + + length = section[:byte_end] - section[:byte_start] + + [section[:byte_start], length] + end + + # Removes the line containing the executed command from the build output. + # + # output - A `String` containing the output of a trace section. + def without_executed_command_line(output) + # If `output.split("\n")` produces an empty Array then the slicing that + # follows it will produce a nil. For example: + # + # "\n".split("\n") # => [] + # "\n".split("\n")[1..-1] # => nil + # + # To work around this we only "join" if we're given an Array. + if (converted = output.split("\n")[1..-1]) + converted.join("\n") + else + '' + end + end + + # Returns the trace section for the given name, or `nil` if the section + # could not be found. + # + # name - The name of the trace section to find. + def find_build_trace_section(name) + trace_sections.find { |s| s[:name] == name } + end + + def trace_sections + @trace_sections ||= trace.extract_sections + end + + def trace + @trace ||= build.trace + end + end + end +end diff --git a/lib/gitlab/chat/responder.rb b/lib/gitlab/chat/responder.rb new file mode 100644 index 00000000000..6267fbc20e2 --- /dev/null +++ b/lib/gitlab/chat/responder.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Chat + module Responder + # Returns an instance of the responder to use for generating chat + # responses. + # + # This method will return `nil` if no formatter is available for the given + # build. + # + # build - A `Ci::Build` that executed a chat command. + def self.responder_for(build) + service = build.pipeline.chat_data&.chat_name&.service + + if (responder = service.try(:chat_responder)) + responder.new(build) + end + end + end + end +end diff --git a/lib/gitlab/chat/responder/base.rb b/lib/gitlab/chat/responder/base.rb new file mode 100644 index 00000000000..f1ad0e36793 --- /dev/null +++ b/lib/gitlab/chat/responder/base.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Chat + module Responder + class Base + attr_reader :build + + # build - The `Ci::Build` that was executed. + def initialize(build) + @build = build + end + + def pipeline + build.pipeline + end + + def project + pipeline.project + end + + def success(*) + raise NotImplementedError, 'You must implement #success(output)' + end + + def failure + raise NotImplementedError, 'You must implement #failure' + end + + def send_response(output) + raise NotImplementedError, 'You must implement #send_response(output)' + end + + def scheduled_output + raise NotImplementedError, 'You must implement #scheduled_output' + end + end + end + end +end diff --git a/lib/gitlab/chat/responder/slack.rb b/lib/gitlab/chat/responder/slack.rb new file mode 100644 index 00000000000..0cf02c92a67 --- /dev/null +++ b/lib/gitlab/chat/responder/slack.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module Chat + module Responder + class Slack < Responder::Base + SUCCESS_COLOR = '#B3ED8E' + FAILURE_COLOR = '#FF5640' + RESPONSE_TYPE = :in_channel + + # Slack breaks messages apart if they're around 4 KB in size. We use a + # slightly smaller limit here to account for user mentions. + MESSAGE_SIZE_LIMIT = 3.5.kilobytes + + # Sends a response back to Slack + # + # output - The output to send back to Slack, as a Hash. + def send_response(output) + Gitlab::HTTP.post( + pipeline.chat_data.response_url, + { + headers: { Accept: 'application/json' }, + body: output.to_json + } + ) + end + + # Sends the output for a build that completed successfully. + # + # output - The output produced by the chat command. + def success(output) + return if output.empty? + + send_response( + text: message_text(limit_output(output)), + response_type: RESPONSE_TYPE + ) + end + + # Sends the output for a build that failed. + def failure + send_response( + text: message_text("<#{build_url}|Sorry, the build failed!>"), + response_type: RESPONSE_TYPE + ) + end + + # Returns the output to send back after a command has been scheduled. + def scheduled_output + # We return an empty message so that Slack still shows the input + # command, without polluting the channel with standard "The job has + # been scheduled" (or similar) responses. + { text: '' } + end + + private + + def limit_output(output) + if output.bytesize <= MESSAGE_SIZE_LIMIT + output + else + "<#{build_url}|The output is too large to be sent back directly!>" + end + end + + def mention_user + "<@#{pipeline.chat_data.chat_name.chat_id}>" + end + + def message_text(output) + "#{mention_user}: #{output}" + end + + def build_url + ::Gitlab::Routing.url_helpers.project_build_url(project, build) + end + end + end + end +end diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb index f8cda0382fe..09b17b5b76b 100644 --- a/lib/gitlab/checks/base_checker.rb +++ b/lib/gitlab/checks/base_checker.rb @@ -18,12 +18,16 @@ module Gitlab private + def creation? + Gitlab::Git.blank_ref?(oldrev) + end + def deletion? Gitlab::Git.blank_ref?(newrev) end def update? - !Gitlab::Git.blank_ref?(oldrev) && !deletion? + !creation? && !deletion? end def updated_from_web? @@ -33,6 +37,22 @@ module Gitlab def tag_exists? project.repository.tag_exists?(tag_name) end + + def validate_once(resource) + Gitlab::SafeRequestStore.fetch(cache_key_for_resource(resource)) do + yield(resource) + + true + end + end + + def cache_key_for_resource(resource) + "git_access:#{checker_cache_key}:#{resource.cache_key}" + end + + def checker_cache_key + self.class.name.demodulize.underscore + end end end end diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index d06b2df36f2..ad926739752 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -9,13 +9,17 @@ module Gitlab non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.', non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', - push_protected_branch: 'You are not allowed to push code to protected branches on this project.' + push_protected_branch: 'You are not allowed to push code to protected branches on this project.', + create_protected_branch: 'You are not allowed to create protected branches on this project.', + invalid_commit_create_protected_branch: 'You can only use an existing protected branch ref as the basis of a new protected branch.', + non_web_create_protected_branch: 'You can only create protected branches using the web interface and API.' }.freeze LOG_MESSAGES = { delete_default_branch_check: "Checking if default branch is being deleted...", protected_branch_checks: "Checking if you are force pushing to a protected branch...", protected_branch_push_checks: "Checking if you are allowed to push to the protected branch...", + protected_branch_creation_checks: "Checking if you are allowed to create a protected branch...", protected_branch_deletion_checks: "Checking if you are allowed to delete the protected branch..." }.freeze @@ -42,13 +46,33 @@ module Gitlab end end - if deletion? + if project.empty_repo? + protected_branch_push_checks + elsif creation? && protected_branch_creation_enabled? + protected_branch_creation_checks + elsif deletion? protected_branch_deletion_checks else protected_branch_push_checks end end + def protected_branch_creation_checks + logger.log_timed(LOG_MESSAGES[:protected_branch_creation_checks]) do + unless user_access.can_merge_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_branch] + end + + unless safe_commit_for_new_protected_branch? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:invalid_commit_create_protected_branch] + end + + unless updated_from_web? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_create_protected_branch] + end + end + end + def protected_branch_deletion_checks logger.log_timed(LOG_MESSAGES[:protected_branch_deletion_checks]) do unless user_access.can_delete_branch?(branch_name) @@ -98,6 +122,10 @@ module Gitlab Gitlab::Routing.url_helpers.project_project_members_url(project) end + def protected_branch_creation_enabled? + Feature.enabled?(:protected_branch_creation, project, default_enabled: true) + end + def matching_merge_request? Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? end @@ -105,6 +133,10 @@ module Gitlab def forced_push? Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev) end + + def safe_commit_for_new_protected_branch? + ProtectedBranch.any_protected?(project, project.repository.branch_names_contains_sha(newrev)) + end end end end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 7778d3068cc..8a57a3a6d9a 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader(*ATTRIBUTES) def initialize( - change, user_access:, project:, skip_authorization: false, + change, user_access:, project:, skip_lfs_integrity_check: false, protocol:, logger: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @@ -18,7 +18,6 @@ module Gitlab @tag_name = Gitlab::Git.tag_name(@ref) @user_access = user_access @project = project - @skip_authorization = skip_authorization @skip_lfs_integrity_check = skip_lfs_integrity_check @protocol = protocol @@ -27,8 +26,6 @@ module Gitlab end def exec - return true if skip_authorization - ref_level_checks # Check of commits should happen as the last step # given they're expensive in terms of performance diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index 8ee345ab45a..ea0d8c85a66 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -11,16 +11,20 @@ module Gitlab }.freeze def validate! - return if deletion? || newrev.nil? + return if deletion? return unless should_run_diff_validations? return if commits.empty? - return unless uses_raw_delta_validations? file_paths = [] - process_raw_deltas do |diff| - file_paths << (diff.new_path || diff.old_path) - validate_diff(diff) + process_commits do |commit| + validate_once(commit) do + commit.raw_deltas.each do |diff| + file_paths << (diff.new_path || diff.old_path) + + validate_diff(diff) + end + end end validate_file_paths(file_paths) @@ -28,17 +32,13 @@ module Gitlab private - def should_run_diff_validations? - validate_lfs_file_locks? - end - def validate_lfs_file_locks? strong_memoize(:validate_lfs_file_locks) do project.lfs_enabled? && project.any_lfs_file_locks? end end - def uses_raw_delta_validations? + def should_run_diff_validations? validations_for_diff.present? || path_validations.present? end @@ -59,16 +59,14 @@ module Gitlab validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] end - def process_raw_deltas + def process_commits logger.log_timed(LOG_MESSAGES[:diff_content_check]) do # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 ::Gitlab::GitalyClient.allow_n_plus_1_calls do commits.each do |commit| logger.check_timeout_reached - commit.raw_deltas.each do |diff| - yield(diff) - end + yield(commit) end end end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index e42684e679a..cc6a14d2d9a 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -7,6 +7,7 @@ module Gitlab ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'.freeze def validate! + return unless project.lfs_enabled? return if skip_lfs_integrity_check logger.log_timed(LOG_MESSAGE) do diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb index f3a52f09868..91f8d0bdbc8 100644 --- a/lib/gitlab/checks/push_check.rb +++ b/lib/gitlab/checks/push_check.rb @@ -6,7 +6,7 @@ module Gitlab def validate! logger.log_timed("Checking if you are allowed to push...") do unless can_push? - raise GitAccess::UnauthorizedError, 'You are not allowed to push code to this project.' + raise GitAccess::UnauthorizedError, GitAccess::ERROR_MESSAGES[:push_code] end end end @@ -15,7 +15,7 @@ module Gitlab def can_push? user_access.can_do_action?(:push_code) || - user_access.can_push_to_branch?(branch_name) + project.branch_allows_collaboration?(user_access.user, branch_name) end end end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 974b5ad6877..fba0de20ced 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -31,7 +31,7 @@ module Gitlab end class Converter - def on_0(_) reset() end + def on_0(_) reset end def on_1(_) enable(STYLE_SWITCHES[:bold]) end @@ -177,7 +177,7 @@ module Gitlab end end - close_open_tags() + close_open_tags OpenStruct.new( html: @out.force_encoding(Encoding.default_external), @@ -194,7 +194,7 @@ module Gitlab action = scanner[1] timestamp = scanner[2] section = scanner[3] - line = scanner.matched()[0...-5] # strips \r\033[0K + line = scanner.matched[0...-5] # strips \r\033[0K @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>} end @@ -209,10 +209,10 @@ module Gitlab # sequence gets stripped (including stuff like "delete last line") return unless indicator == '[' && terminator == 'm' - close_open_tags() + close_open_tags - if commands.empty?() - reset() + if commands.empty? + reset return end @@ -222,7 +222,7 @@ module Gitlab end def evaluate_command_stack(stack) - return unless command = stack.shift() + return unless command = stack.shift if self.respond_to?("on_#{command}", true) self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend @@ -313,7 +313,7 @@ module Gitlab def get_term_color_class(color_index, prefix) color_name = COLOR[color_index] - return nil if color_name.nil? + return if color_name.nil? get_color_class(["term", prefix, color_name]) end @@ -333,8 +333,8 @@ module Gitlab return unless command_stack.length >= 2 return unless command_stack[0] == "5" - command_stack.shift() # ignore the "5" command - color_index = command_stack.shift().to_i + command_stack.shift # ignore the "5" command + color_index = command_stack.shift.to_i return unless color_index >= 0 return unless color_index <= 255 diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 08dac756cc1..d45a044686e 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -103,7 +103,7 @@ module Gitlab def read_string(gz) string_size = read_uint32(gz) - return nil unless string_size + return unless string_size gz.read(string_size) end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index d0a80518ae8..80e69cdcc95 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -44,7 +44,7 @@ module Gitlab end def parent - return nil unless has_parent? + return unless has_parent? self.class.new(@path.to_s.chomp(basename), @entries) end diff --git a/lib/gitlab/ci/build/policy/changes.rb b/lib/gitlab/ci/build/policy/changes.rb index 1663c875426..9c705a1cd3e 100644 --- a/lib/gitlab/ci/build/policy/changes.rb +++ b/lib/gitlab/ci/build/policy/changes.rb @@ -10,7 +10,7 @@ module Gitlab end def satisfied_by?(pipeline, seed) - return true unless pipeline.branch_updated? + return true if pipeline.modified_paths.nil? pipeline.modified_paths.any? do |path| @globs.any? do |glob| diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 0e9bb5c94bb..df5f5ffc253 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -29,8 +29,8 @@ module Gitlab def matches_pattern?(pattern, pipeline) return true if pipeline.tag? && pattern == 'tags' return true if pipeline.branch? && pattern == 'branches' - return true if pipeline.source == pattern - return true if pipeline.source&.pluralize == pattern + return true if sanitized_source_name(pipeline) == pattern + return true if sanitized_source_name(pipeline)&.pluralize == pattern # patterns can be matched only when branch or tag is used # the pattern matching does not work for merge requests pipelines @@ -42,6 +42,10 @@ module Gitlab end end end + + def sanitized_source_name(pipeline) + @sanitized_source_name ||= pipeline&.source&.delete_suffix('_event') + end end end end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index d587c896712..7fcabc035ac 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -15,7 +15,6 @@ module Gitlab def from_commands(job) self.new(:script).tap do |step| step.script = job.options[:before_script].to_a + job.options[:script].to_a - step.script = job.commands.split("\n") if step.script.empty? step.timeout = job.metadata_timeout step.when = WHEN_ON_SUCCESS end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 6333799a491..15643fa03ac 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -8,9 +8,9 @@ module Gitlab class Config ConfigError = Class.new(StandardError) - def initialize(config, opts = {}) + def initialize(config, project: nil, sha: nil, user: nil) @config = Config::Extendable - .new(build_config(config, opts)) + .new(build_config(config, project: project, sha: sha, user: user)) .to_hash @global = Entry::Global.new(@config) @@ -70,20 +70,22 @@ module Gitlab private - def build_config(config, opts = {}) + def build_config(config, project:, sha:, user:) initial_config = Gitlab::Config::Loader::Yaml.new(config).load! - project = opts.fetch(:project, nil) if project - process_external_files(initial_config, project, opts) + process_external_files(initial_config, project: project, sha: sha, user: user) else initial_config end end - def process_external_files(config, project, opts) - sha = opts.fetch(:sha) { project.repository.root_ref_sha } - Config::External::Processor.new(config, project, sha).perform + def process_external_files(config, project:, sha:, user:) + Config::External::Processor.new(config, + project: project, + sha: sha || project.repository.root_ref_sha, + user: user, + expandset: Set.new).perform end end end diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb index 09ecb5fdb99..2b5a59c078e 100644 --- a/lib/gitlab/ci/config/entry/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -17,6 +17,9 @@ module Gitlab entry :image, Entry::Image, description: 'Docker image that will be used to execute jobs.' + entry :include, Entry::Includes, + description: 'List of external YAML files to include.' + entry :services, Entry::Services, description: 'Docker images that will be linked to the container.' diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb new file mode 100644 index 00000000000..f2f3dd84eda --- /dev/null +++ b/lib/gitlab/ci/config/entry/include.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a single include. + # + class Include < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + ALLOWED_KEYS = %i[local file remote template].freeze + + validations do + validates :config, hash_or_string: true + validates :config, allowed_keys: ALLOWED_KEYS + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/includes.rb b/lib/gitlab/ci/config/entry/includes.rb new file mode 100644 index 00000000000..82b2b1ccf4b --- /dev/null +++ b/lib/gitlab/ci/config/entry/includes.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a list of include. + # + class Includes < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: Array + end + + def self.aspects + super.append -> do + @config = Array.wrap(@config) + + @config.each_with_index do |config, i| + @entries[i] = ::Gitlab::Config::Entry::Factory.new(Entry::Include) + .value(config || {}) + .create! + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 50942fbdb40..290c9591b98 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -16,13 +16,6 @@ module Gitlab dependencies before_script after_script variables environment coverage retry parallel extends].freeze - DEFAULT_ONLY_POLICY = { - refs: %w(branches tags) - }.freeze - - DEFAULT_EXCEPT_POLICY = { - }.freeze - validations do validates :config, allowed_keys: ALLOWED_KEYS validates :config, presence: true @@ -73,7 +66,8 @@ module Gitlab description: 'Services that will be used to execute this job.' entry :only, Entry::Policy, - description: 'Refs policy this job will be executed for.' + description: 'Refs policy this job will be executed for.', + default: Entry::Policy::DEFAULT_ONLY entry :except, Entry::Policy, description: 'Refs policy this job will be executed for.' @@ -95,7 +89,7 @@ module Gitlab helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment, :coverage, :retry, + :artifacts, :environment, :coverage, :retry, :parallel attributes :script, :tags, :allow_failure, :when, :dependencies, @@ -121,10 +115,6 @@ module Gitlab @config.merge(to_hash.compact) end - def commands - (before_script_value.to_a + script_value.to_a).join("\n") - end - def manual_action? self.when == 'manual' end @@ -156,13 +146,12 @@ module Gitlab { name: name, before_script: before_script_value, script: script_value, - commands: commands, image: image_value, services: services_value, stage: stage_value, cache: cache_value, - only: DEFAULT_ONLY_POLICY.deep_merge(only_value.to_h), - except: DEFAULT_EXCEPT_POLICY.deep_merge(except_value.to_h), + only: only_value, + except: except_value, variables: variables_defined? ? variables_value : nil, environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 82b72e40404..9845c4af655 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -28,11 +28,15 @@ module Gitlab name.to_s.start_with?('.') end + def node_type(name) + hidden?(name) ? Entry::Hidden : Entry::Job + end + # rubocop: disable CodeReuse/ActiveRecord def compose!(deps = nil) super do @config.each do |name, config| - node = hidden?(name) ? Entry::Hidden : Entry::Job + node = node_type(name) factory = ::Gitlab::Config::Entry::Factory.new(node) .value(config || {}) diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 998da1f6837..adc3660d950 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -11,6 +11,8 @@ module Gitlab strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) } + DEFAULT_ONLY = { refs: %w[branches tags] }.freeze + class RefsPolicy < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable @@ -64,7 +66,8 @@ module Gitlab end end - def self.default + def value + default.to_h.deep_merge(subject.value.to_h) end end end diff --git a/lib/gitlab/ci/config/entry/retry.rb b/lib/gitlab/ci/config/entry/retry.rb index eaf8b38aa3c..e9cbcb31e21 100644 --- a/lib/gitlab/ci/config/entry/retry.rb +++ b/lib/gitlab/ci/config/entry/retry.rb @@ -82,9 +82,6 @@ module Gitlab 'retry config' end end - - def self.default - end end end end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 89d790ebfa6..c9d0c7cb568 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -14,7 +14,7 @@ module Gitlab validates :config, variables: true end - def self.default + def self.default(**) {} end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index ee4ea9bbb1d..2ffbb214a92 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -8,20 +8,26 @@ module Gitlab class Base include Gitlab::Utils::StrongMemoize - attr_reader :location, :opts, :errors + attr_reader :location, :params, :context, :errors YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze - def initialize(location, opts = {}) - @location = location - @opts = opts + Context = Struct.new(:project, :sha, :user, :expandset) + + def initialize(params, context) + @params = params + @context = context @errors = [] validate! end + def matching? + location.present? + end + def invalid_extension? - !::File.basename(location).match(YAML_WHITELIST_EXTENSION) + location.nil? || !::File.basename(location).match?(YAML_WHITELIST_EXTENSION) end def valid? @@ -37,13 +43,27 @@ module Gitlab end def to_hash - @hash ||= Gitlab::Config::Loader::Yaml.new(content).load! - rescue Gitlab::Config::Loader::FormatError - nil + expanded_content_hash end protected + def expanded_content_hash + return unless content_hash + + strong_memoize(:expanded_content_yaml) do + expand_includes(content_hash) + end + end + + def content_hash + strong_memoize(:content_yaml) do + Gitlab::Config::Loader::Yaml.new(content).load! + end + rescue Gitlab::Config::Loader::FormatError + nil + end + def validate! validate_location! validate_content! if errors.none? @@ -67,6 +87,14 @@ module Gitlab errors.push("Included file `#{location}` does not have valid YAML syntax!") end end + + def expand_includes(hash) + External::Processor.new(hash, **expand_context).perform + end + + def expand_context + { project: nil, sha: nil, user: nil, expandset: context.expandset } + end end end end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 2a256aff65c..229a06451e8 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -8,11 +8,8 @@ module Gitlab class Local < Base include Gitlab::Utils::StrongMemoize - attr_reader :project, :sha - - def initialize(location, opts = {}) - @project = opts.fetch(:project) - @sha = opts.fetch(:sha) + def initialize(params, context) + @location = params[:local] super end @@ -32,7 +29,14 @@ module Gitlab end def fetch_local_content - project.repository.blob_data_at(sha, location) + context.project.repository.blob_data_at(context.sha, location) + end + + def expand_context + super.merge( + project: context.project, + sha: context.sha, + user: context.user) end end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb new file mode 100644 index 00000000000..b828f77835c --- /dev/null +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module File + class Project < Base + include Gitlab::Utils::StrongMemoize + + attr_reader :project_name, :ref_name + + def initialize(params, context = {}) + @location = params[:file] + @project_name = params[:project] + @ref_name = params[:ref] || 'HEAD' + + super + end + + def matching? + super && project_name.present? + end + + def content + strong_memoize(:content) { fetch_local_content } + end + + private + + def validate_content! + if !can_access_local_content? + errors.push("Project `#{project_name}` not found or access denied!") + elsif sha.nil? + errors.push("Project `#{project_name}` reference `#{ref_name}` does not exist!") + elsif content.nil? + errors.push("Project `#{project_name}` file `#{location}` does not exist!") + elsif content.blank? + errors.push("Project `#{project_name}` file `#{location}` is empty!") + end + end + + def project + strong_memoize(:project) do + ::Project.find_by_full_path(project_name) + end + end + + def can_access_local_content? + Ability.allowed?(context.user, :download_code, project) + end + + def fetch_local_content + return unless can_access_local_content? + return unless sha + + project.repository.blob_data_at(sha, location) + rescue GRPC::NotFound, GRPC::Internal + nil + end + + def sha + strong_memoize(:sha) do + project.commit(ref_name).try(:sha) + end + end + + def expand_context + super.merge( + project: project, + sha: sha, + user: context.user) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index 86fa5ad8800..567a86c47e5 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -8,6 +8,12 @@ module Gitlab class Remote < Base include Gitlab::Utils::StrongMemoize + def initialize(params, context) + @location = params[:remote] + + super + end + def content strong_memoize(:content) { fetch_remote_content } end diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb new file mode 100644 index 00000000000..54f4cf74c4d --- /dev/null +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module File + class Template < Base + attr_reader :location, :project + + SUFFIX = '.gitlab-ci.yml'.freeze + + def initialize(params, context) + @location = params[:template] + + super + end + + def content + strong_memoize(:content) { fetch_template_content } + end + + private + + def validate_location! + super + + unless template_name_valid? + errors.push("Template file `#{location}` is not a valid location!") + end + end + + def template_name + return unless template_name_valid? + + location.first(-SUFFIX.length) + end + + def template_name_valid? + location.to_s.end_with?(SUFFIX) + end + + def fetch_template_content + Gitlab::Template::GitlabCiYmlTemplate.find(template_name, project)&.content + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index def3563e505..aff5c5b9651 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -5,25 +5,93 @@ module Gitlab class Config module External class Mapper - def initialize(values, project, sha) - @locations = Array(values.fetch(:include, [])) + include Gitlab::Utils::StrongMemoize + + MAX_INCLUDES = 50 + + FILE_CLASSES = [ + External::File::Remote, + External::File::Template, + External::File::Local, + External::File::Project + ].freeze + + Error = Class.new(StandardError) + AmbigiousSpecificationError = Class.new(Error) + DuplicateIncludesError = Class.new(Error) + TooManyIncludesError = Class.new(Error) + + def initialize(values, project:, sha:, user:, expandset:) + raise Error, 'Expanded needs to be `Set`' unless expandset.is_a?(Set) + + @locations = Array.wrap(values.fetch(:include, [])) @project = project @sha = sha + @user = user + @expandset = expandset end def process - locations.map { |location| build_external_file(location) } + return [] if locations.empty? + + locations + .compact + .map(&method(:normalize_location)) + .each(&method(:verify_duplicates!)) + .map(&method(:select_first_matching)) end private - attr_reader :locations, :project, :sha + attr_reader :locations, :project, :sha, :user, :expandset - def build_external_file(location) + # convert location if String to canonical form + def normalize_location(location) + if location.is_a?(String) + normalize_location_string(location) + else + location.deep_symbolize_keys + end + end + + def normalize_location_string(location) if ::Gitlab::UrlSanitizer.valid?(location) - External::File::Remote.new(location) + { remote: location } else - External::File::Local.new(location, project: project, sha: sha) + { local: location } + end + end + + def verify_duplicates!(location) + if expandset.count >= MAX_INCLUDES + raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" + end + + # We scope location to context, as this allows us to properly support + # relative incldues, and similarly looking relative in another project + # does not trigger duplicate error + scoped_location = location.merge( + context_project: project, + context_sha: sha) + + unless expandset.add?(scoped_location) + raise DuplicateIncludesError, "Include `#{location.to_json}` was already included!" + end + end + + def select_first_matching(location) + matching = FILE_CLASSES.map do |file_class| + file_class.new(location, context) + end.select(&:matching?) + + raise AmbigiousSpecificationError, "Include `#{location.to_json}` needs to match exactly one accessor!" unless matching.one? + + matching.first + end + + def context + strong_memoize(:context) do + External::File::Base::Context.new(project, sha, user, expandset) end end end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index eae0bdeb644..1dd2d42016a 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -7,10 +7,12 @@ module Gitlab class Processor IncludeError = Class.new(StandardError) - def initialize(values, project, sha) + def initialize(values, project:, sha:, user:, expandset:) @values = values - @external_files = External::Mapper.new(values, project, sha).process + @external_files = External::Mapper.new(values, project: project, sha: sha, user: user, expandset: expandset).process @content = {} + rescue External::Mapper::Error => e + raise IncludeError, e.message end def perform diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index b7743bd2090..191f5d09645 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -46,7 +46,8 @@ module Gitlab parallelized_job_names = @parallelized_jobs.keys.map(&:to_s) parallelized_config.each_with_object({}) do |(job_name, config), hash| if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any? - deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten + parallelized_deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten + deps = config[:dependencies] - intersection + parallelized_deps hash[job_name] = config.merge(dependencies: deps) else hash[job_name] = config diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index b1db9084662..94f4a4e36c9 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -35,7 +35,7 @@ module Gitlab # NOTE: # cron_timezone can only accept timezones listed in TZInfo::Timezone. # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted, - # because Rufus::Scheduler only supports TZInfo::Timezone. + # because Fugit::Cron only supports TZInfo::Timezone. # # For example, those codes have the same effect. # Time.zone = 'Pacific Time (US & Canada)' (ActiveSupport::TimeZone) @@ -47,10 +47,7 @@ module Gitlab # If you want to know more, please take a look # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/values/time_zone.rb def try_parse_cron(cron, cron_timezone) - cron_line = Rufus::Scheduler.parse("#{cron} #{cron_timezone}") - cron_line if cron_line.is_a?(Rufus::Scheduler::CronLine) - rescue - # noop + Fugit::Cron.parse("#{cron} #{cron_timezone}") end end end diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index d33d1edfe35..164a4634d84 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -12,12 +12,13 @@ module Gitlab ref: @command.ref, sha: @command.sha, before_sha: @command.before_sha, + source_sha: @command.source_sha, + target_sha: @command.target_sha, tag: @command.tag_exists?, trigger_requests: Array(@command.trigger_request), user: @command.current_user, pipeline_schedule: @command.schedule, merge_request: @command.merge_request, - protected: @command.protected_ref?, variables_attributes: Array(@command.variables_attributes) ) diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 100b9521412..7b77e86feae 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -7,10 +7,11 @@ module Gitlab module Chain Command = Struct.new( :source, :project, :current_user, - :origin_ref, :checkout_sha, :after_sha, :before_sha, + :origin_ref, :checkout_sha, :after_sha, :before_sha, :source_sha, :target_sha, :trigger_request, :schedule, :merge_request, :ignore_skip_ci, :save_incompleted, - :seeds_block, :variables_attributes + :seeds_block, :variables_attributes, :push_options, + :chat_data ) do include Gitlab::Utils::StrongMemoize @@ -54,7 +55,13 @@ module Gitlab def protected_ref? strong_memoize(:protected_ref) do - project.protected_for?(ref) + project.protected_for?(origin_ref) + end + end + + def ambiguous_ref? + strong_memoize(:ambiguous_ref) do + project.repository.ambiguous_ref?(origin_ref) end end end diff --git a/lib/gitlab/ci/pipeline/chain/limit/activity.rb b/lib/gitlab/ci/pipeline/chain/limit/activity.rb new file mode 100644 index 00000000000..fe7c8738cc0 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/limit/activity.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Limit + class Activity < Chain::Base + def perform! + # to be overriden in EE + end + + def break? + false # to be overriden in EE + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/limit/size.rb b/lib/gitlab/ci/pipeline/chain/limit/size.rb new file mode 100644 index 00000000000..b4d51437cd6 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/limit/size.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Limit + class Size < Chain::Base + def perform! + # to be overriden in EE + end + + def break? + false # to be overriden in EE + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 633d3cd4f6b..0405292a25b 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -13,6 +13,10 @@ module Gitlab # Allocate next IID. This operation must be outside of transactions of pipeline creations. pipeline.ensure_project_iid! + # Protect the pipeline. This is assigned in Populate instead of + # Build to prevent erroring out on ambiguous refs. + pipeline.protected = @command.protected_ref? + ## # Populate pipeline with block argument of CreatePipelineService#execute. # diff --git a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb new file mode 100644 index 00000000000..1e09b417311 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class RemoveUnwantedChatJobs < Chain::Base + def perform! + return unless pipeline.config_processor && pipeline.chat? + + # When scheduling a chat pipeline we only want to run the build + # that matches the chat command. + pipeline.config_processor.jobs.select! do |name, _| + name.to_s == command.chat_data[:command].to_s + end + end + + def break? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index b9707d2f8f5..79bbcc1ed1e 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -8,6 +8,7 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i + SKIP_PUSH_OPTION = 'ci.skip' def perform! if skipped? @@ -16,7 +17,7 @@ module Gitlab end def skipped? - !@command.ignore_skip_ci && commit_message_skips_ci? + !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?) end def break? @@ -32,6 +33,10 @@ module Gitlab !!(@pipeline.git_commit_message =~ SKIP_PATTERN) end end + + def push_option_skips_ci? + !!(@command.push_options&.include?(SKIP_PUSH_OPTION)) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb index d88851d8245..9c6c2bc8e25 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/repository.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -16,6 +16,10 @@ module Gitlab unless @command.sha return error('Commit not found') end + + if @command.ambiguous_ref? + return error('Ref is ambiguous') + end end def break? diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index ef738a93bfe..d8296940a04 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -38,9 +38,17 @@ module Gitlab ) end + def bridge? + @attributes.to_h.dig(:options, :trigger).present? + end + def to_resource strong_memoize(:resource) do - ::Ci::Build.new(attributes) + if bridge? + ::Ci::Bridge.new(attributes) + else + ::Ci::Build.new(attributes) + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 4775ff15581..9c15064756a 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -39,7 +39,13 @@ module Gitlab def to_resource strong_memoize(:stage) do ::Ci::Stage.new(attributes).tap do |stage| - seeds.each { |seed| stage.builds << seed.to_resource } + seeds.each do |seed| + if seed.bridge? + stage.bridges << seed.to_resource + else + stage.builds << seed.to_resource + end + end end end end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index c6cb620f7a0..4746195c618 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -18,7 +18,6 @@ module Gitlab end def details_path - raise NotImplementedError end end end diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb index 4169f5b3210..cd772819293 100644 --- a/lib/gitlab/ci/status/external/common.rb +++ b/lib/gitlab/ci/status/external/common.rb @@ -6,7 +6,7 @@ module Gitlab module External module Common def label - subject.description + subject.description.presence || super end def has_details? diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml new file mode 100644 index 00000000000..9c534b2b8e7 --- /dev/null +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -0,0 +1,121 @@ +# Read more about how to use this script on this blog post https://about.gitlab.com/2019/01/28/android-publishing-with-gitlab-and-fastlane/ +# You will also need to configure your build.gradle, Dockerfile, and fastlane configuration to make this work. +# If you are looking for a simpler template that does not publish, see the Android template. + +stages: + - environment + - build + - test + - internal + - alpha + - beta + - production + + +.updateContainerJob: + image: docker:stable + stage: environment + services: + - docker:dind + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG || true + - docker build --cache-from $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + +updateContainer: + extends: .updateContainerJob + only: + changes: + - Dockerfile + +ensureContainer: + extends: .updateContainerJob + allow_failure: true + before_script: + - "mkdir -p ~/.docker && echo '{\"experimental\": \"enabled\"}' > ~/.docker/config.json" + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + # Skip update container `script` if the container already exists + # via https://gitlab.com/gitlab-org/gitlab-ce/issues/26866#note_97609397 -> https://stackoverflow.com/a/52077071/796832 + - docker manifest inspect $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG > /dev/null && exit || true + + +.build_job: + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + stage: build + before_script: + # We store this binary file in a variable as hex with this command: `xxd -p android-app.jks` + # Then we convert the hex back to a binary file + - echo "$signing_jks_file_hex" | xxd -r -p - > android-signing-keystore.jks + - "export VERSION_CODE=$CI_PIPELINE_IID && echo $VERSION_CODE" + - "export VERSION_SHA=`echo ${CI_COMMIT_SHA:0:8}` && echo $VERSION_SHA" + after_script: + - rm -f android-signing-keystore.jks || true + artifacts: + paths: + - app/build/outputs + +buildDebug: + extends: .build_job + script: + - bundle exec fastlane buildDebug + +buildRelease: + extends: .build_job + script: + - bundle exec fastlane buildRelease + environment: + name: production + +testDebug: + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + stage: test + dependencies: + - buildDebug + script: + - bundle exec fastlane test + +publishInternal: + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + stage: internal + dependencies: + - buildRelease + when: manual + before_script: + - echo $google_play_service_account_api_key_json > ~/google_play_api_key.json + after_script: + - rm ~/google_play_api_key.json + script: + - bundle exec fastlane internal + +.promote_job: + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + when: manual + dependencies: [] + before_script: + - echo $google_play_service_account_api_key_json > ~/google_play_api_key.json + after_script: + - rm ~/google_play_api_key.json + +promoteAlpha: + extends: .promote_job + stage: alpha + script: + - bundle exec fastlane promote_internal_to_alpha + +promoteBeta: + extends: .promote_job + stage: beta + script: + - bundle exec fastlane promote_alpha_to_beta + +promoteProduction: + extends: .promote_job + stage: production + # We only allow production promotion on `master` because + # it has its own production scoped secret variables + only: + - master + script: + - bundle exec fastlane promote_beta_to_production +
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml index 6e138639b71..c169e3f7686 100644 --- a/lib/gitlab/ci/templates/Android.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -1,4 +1,6 @@ # Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny +# If you are interested in using Android with FastLane for publishing take a look at the Android-Fastlane template. + image: openjdk:8-jdk variables: diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index a9e361b0b32..6c99e20e7af 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -21,8 +21,8 @@ # # In order to deploy, you must have a Kubernetes cluster configured either # via a project integration, or via group/project variables. -# AUTO_DEVOPS_DOMAIN must also be set as a variable at the group or project -# level, or manually added below. +# KUBE_INGRESS_BASE_DOMAIN must also be set on the cluster settings, +# as a variable at the group or project level, or manually added below. # # Continuous deployment to production is enabled by default. # If you want to deploy to staging first, set STAGING_ENABLED environment variable. @@ -41,19 +41,22 @@ image: alpine:latest variables: - # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. - # AUTO_DEVOPS_DOMAIN: domain.example.com + # KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. + # KUBE_INGRESS_BASE_DOMAIN: domain.example.com POSTGRES_USER: user POSTGRES_PASSWORD: testing-password POSTGRES_ENABLED: "true" POSTGRES_DB: $CI_ENVIRONMENT_SLUG + POSTGRES_VERSION: 9.6.2 - KUBERNETES_VERSION: 1.10.9 - HELM_VERSION: 2.11.0 + KUBERNETES_VERSION: 1.11.7 + HELM_VERSION: 2.12.3 DOCKER_DRIVER: overlay2 + ROLLOUT_RESOURCE_TYPE: deployment + stages: - build - test @@ -71,14 +74,14 @@ stages: build: stage: build - image: docker:stable-git + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable" services: - - docker:stable-dind + - docker:stable-dind script: - - setup_docker - - build + - /build/build.sh only: - branches + - tags test: services: @@ -93,6 +96,7 @@ test: - /bin/herokuish buildpack test only: - branches + - tags except: variables: - $TEST_DISABLED @@ -110,13 +114,14 @@ code_quality: paths: [gl-code-quality-report.json] only: - branches + - tags except: variables: - $CODE_QUALITY_DISABLED license_management: stage: test - image: + image: name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" entrypoint: [""] allow_failure: true @@ -127,6 +132,7 @@ license_management: only: refs: - branches + - tags variables: - $GITLAB_FEATURES =~ /\blicense_management\b/ except: @@ -149,6 +155,7 @@ performance: only: refs: - branches + - tags kubernetes: active except: variables: @@ -169,6 +176,7 @@ sast: only: refs: - branches + - tags variables: - $GITLAB_FEATURES =~ /\bsast\b/ except: @@ -185,10 +193,12 @@ dependency_scanning: - setup_docker - dependency_scanning artifacts: - paths: [gl-dependency-scanning-report.json] + reports: + dependency_scanning: gl-dependency-scanning-report.json only: refs: - branches + - tags variables: - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ except: @@ -209,6 +219,7 @@ container_scanning: only: refs: - branches + - tags variables: - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ except: @@ -228,6 +239,7 @@ dast: only: refs: - branches + - tags kubernetes: active variables: - $GITLAB_FEATURES =~ /\bdast\b/ @@ -250,13 +262,14 @@ review: - persist_environment_url environment: name: review/$CI_COMMIT_REF_NAME - url: http://$CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN + url: http://$CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN on_stop: stop_review artifacts: paths: [environment_url.txt] only: refs: - branches + - tags kubernetes: active except: refs: @@ -280,6 +293,7 @@ stop_review: only: refs: - branches + - tags kubernetes: active except: refs: @@ -305,7 +319,7 @@ staging: - deploy environment: name: staging - url: http://$CI_PROJECT_PATH_SLUG-staging.$AUTO_DEVOPS_DOMAIN + url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN only: refs: - master @@ -329,7 +343,7 @@ canary: - deploy canary environment: name: production - url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN when: manual only: refs: @@ -353,7 +367,7 @@ canary: - persist_environment_url environment: name: production - url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN artifacts: paths: [environment_url.txt] @@ -402,7 +416,7 @@ production_manual: - persist_environment_url environment: name: production - url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN artifacts: paths: [environment_url.txt] @@ -485,9 +499,13 @@ rollout 100%: [[ "$TRACE" ]] && set -x auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} export DATABASE_URL=${DATABASE_URL-$auto_database_url} - export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG - export CI_APPLICATION_TAG=$CI_COMMIT_SHA - export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID} + if [[ -z "$CI_COMMIT_TAG" ]]; then + export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG + export CI_APPLICATION_TAG=$CI_COMMIT_SHA + else + export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE + export CI_APPLICATION_TAG=$CI_COMMIT_TAG + fi export TILLER_NAMESPACE=$KUBE_NAMESPACE # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') @@ -595,10 +613,55 @@ rollout 100%: fi } + # Extracts variables prefixed with K8S_SECRET_ + # and creates a Kubernetes secret. + # + # e.g. If we have the following environment variables: + # K8S_SECRET_A=value1 + # K8S_SECRET_B=multi\ word\ value + # + # Then we will create a secret with the following key-value pairs: + # data: + # A: dmFsdWUxCg== + # B: bXVsdGkgd29yZCB2YWx1ZQo= + function create_application_secret() { + track="${1-stable}" + export APPLICATION_SECRET_NAME=$(application_secret_name "$track") + + env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables + + kubectl create secret \ + -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \ + --from-env-file k8s_prefixed_variables -o yaml --dry-run | + kubectl replace -n "$KUBE_NAMESPACE" --force -f - + + export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1) + + rm k8s_prefixed_variables + } + + function deploy_name() { + name="$CI_ENVIRONMENT_SLUG" + track="${1-stable}" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + echo $name + } + + function application_secret_name() { + track="${1-stable}" + name=$(deploy_name "$track") + + echo "${name}-secret" + } + function deploy() { track="${1-stable}" percentage="${2:-100}" - name="$CI_ENVIRONMENT_SLUG" + name=$(deploy_name "$track") replicas="1" service_enabled="true" @@ -607,7 +670,6 @@ rollout 100%: # if track is different than stable, # re-use all attached resources if [[ "$track" != "stable" ]]; then - name="$name-$track" service_enabled="false" postgres_enabled="false" fi @@ -620,6 +682,16 @@ rollout 100%: secret_name='' fi + create_application_secret "$track" + + env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]') + eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS + if [ -n "$env_ADDITIONAL_HOSTS" ]; then + additional_hosts="{$env_ADDITIONAL_HOSTS}" + elif [ -n "$ADDITIONAL_HOSTS" ]; then + additional_hosts="{$ADDITIONAL_HOSTS}" + fi + if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then echo "Deploying first release with database initialization..." helm upgrade --install \ @@ -632,13 +704,18 @@ rollout 100%: --set image.secrets[0].name="$secret_name" \ --set application.track="$track" \ --set application.database_url="$DATABASE_URL" \ + --set application.secretName="$APPLICATION_SECRET_NAME" \ + --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ + --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \ --set service.url="$CI_ENVIRONMENT_URL" \ + --set service.additionalHosts="$additional_hosts" \ --set replicaCount="$replicas" \ --set postgresql.enabled="$postgres_enabled" \ --set postgresql.nameOverride="postgres" \ --set postgresql.postgresUser="$POSTGRES_USER" \ --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set postgresql.imageTag="$POSTGRES_VERSION" \ --set application.initializeCommand="$DB_INITIALIZE" \ --namespace="$KUBE_NAMESPACE" \ "$name" \ @@ -664,7 +741,11 @@ rollout 100%: --set image.secrets[0].name="$secret_name" \ --set application.track="$track" \ --set application.database_url="$DATABASE_URL" \ + --set application.secretName="$APPLICATION_SECRET_NAME" \ + --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ + --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \ --set service.url="$CI_ENVIRONMENT_URL" \ + --set service.additionalHosts="$additional_hosts" \ --set replicaCount="$replicas" \ --set postgresql.enabled="$postgres_enabled" \ --set postgresql.nameOverride="postgres" \ @@ -677,17 +758,13 @@ rollout 100%: chart/ fi - kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name" + kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name" } function scale() { track="${1-stable}" percentage="${2-100}" - name="$CI_ENVIRONMENT_SLUG" - - if [[ "$track" != "stable" ]]; then - name="$name-$track" - fi + name=$(deploy_name "$track") replicas=$(get_replicas "$track" "$percentage") @@ -748,7 +825,7 @@ rollout 100%: fi helm init --client-only - helm repo add gitlab https://charts.gitlab.io + helm repo add gitlab ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} if [[ ! -d "$auto_chart" ]]; then helm fetch ${auto_chart} --untar fi @@ -764,59 +841,28 @@ rollout 100%: kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" } - function check_kube_domain() { - if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then - echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set" - echo "You can do it in Auto DevOps project settings or defining a variable at group or project level" - echo "You can also manually add it in .gitlab-ci.yml" - false - else - true + + # Function to ensure backwards compatibility with AUTO_DEVOPS_DOMAIN + function ensure_kube_ingress_base_domain() { + if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ] && [ -n "$AUTO_DEVOPS_DOMAIN" ] ; then + export KUBE_INGRESS_BASE_DOMAIN=$AUTO_DEVOPS_DOMAIN fi } - function build() { - registry_login + function check_kube_domain() { + ensure_kube_ingress_base_domain - if [[ -f Dockerfile ]]; then - echo "Building Dockerfile-based application..." - docker build \ - --build-arg HTTP_PROXY="$HTTP_PROXY" \ - --build-arg http_proxy="$http_proxy" \ - --build-arg HTTPS_PROXY="$HTTPS_PROXY" \ - --build-arg https_proxy="$https_proxy" \ - --build-arg FTP_PROXY="$FTP_PROXY" \ - --build-arg ftp_proxy="$ftp_proxy" \ - --build-arg NO_PROXY="$NO_PROXY" \ - --build-arg no_proxy="$no_proxy" \ - -t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" . + if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ]; then + echo "In order to deploy or use Review Apps," + echo "AUTO_DEVOPS_DOMAIN or KUBE_INGRESS_BASE_DOMAIN variables must be set" + echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" + echo "or by defining a variable at group or project level." + echo "You can also manually add it in .gitlab-ci.yml" + echo "AUTO_DEVOPS_DOMAIN support will be dropped on 12.0" + false else - echo "Building Heroku-based application using gliderlabs/herokuish docker image..." - docker run -i \ - -e BUILDPACK_URL \ - -e HTTP_PROXY \ - -e http_proxy \ - -e HTTPS_PROXY \ - -e https_proxy \ - -e FTP_PROXY \ - -e ftp_proxy \ - -e NO_PROXY \ - -e no_proxy \ - --name="$CI_CONTAINER_NAME" -v "$(pwd):/tmp/app:ro" gliderlabs/herokuish /bin/herokuish buildpack build - docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" - docker rm "$CI_CONTAINER_NAME" >/dev/null - echo "" - - echo "Configuring $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG docker image..." - docker create --expose 5000 --env PORT=5000 --name="$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" /bin/herokuish procfile start web - docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" - docker rm "$CI_CONTAINER_NAME" >/dev/null - echo "" + true fi - - echo "Pushing to GitLab Container Registry..." - docker push "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" - echo "" } function initialize_tiller() { @@ -881,15 +927,14 @@ rollout 100%: function delete() { track="${1-stable}" - name="$CI_ENVIRONMENT_SLUG" - - if [[ "$track" != "stable" ]]; then - name="$name-$track" - fi + name=$(deploy_name "$track") if [[ -n "$(helm ls -q "^$name$")" ]]; then helm delete --purge "$name" fi + + secret_name=$(application_secret_name "$track") + kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name" } before_script: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 93cb31f48c0..0d12cbc6460 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -24,7 +24,6 @@ before_script: - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - - gem install bundler --no-ri --no-rdoc # Bundler is not installed with the image - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby # Optional - Delete if not using `rubocop` diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml new file mode 100644 index 00000000000..805df26b957 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -0,0 +1,44 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/project/merge_requests/dependency_scanning.html +# +# Configure the scanning tool through the environment variables. +# List of the variables: https://gitlab.com/gitlab-org/security-products/dependency-scanning#settings +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +stages: + - test + +dependency_scanning: + stage: test + image: docker:stable + variables: + DOCKER_DRIVER: overlay2 + allow_failure: true + services: + - docker:stable-dind + script: + - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} + - | + docker run \ + --env DS_ANALYZER_IMAGES \ + --env DS_ANALYZER_IMAGE_PREFIX \ + --env DS_ANALYZER_IMAGE_TAG \ + --env DS_DEFAULT_ANALYZERS \ + --env DEP_SCAN_DISABLE_REMOTE_CHECKS \ + --env DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ + --env DS_PULL_ANALYZER_IMAGE_TIMEOUT \ + --env DS_RUN_ANALYZER_TIMEOUT \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_VERSION" /code + artifacts: + reports: + dependency_scanning: gl-dependency-scanning-report.json + dependencies: [] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + except: + variables: + - $DEPENDENCY_SCANNING_DISABLED diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml new file mode 100644 index 00000000000..4f3d08d98fe --- /dev/null +++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml @@ -0,0 +1,41 @@ +# GitLab Serverless template + +image: alpine:latest + +stages: + - build + - deploy + +.serverless:build:image: + variables: + DOCKERFILE: "Dockerfile" + stage: build + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + only: + refs: + - master + script: + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/$DOCKERFILE --destination $CI_REGISTRY_IMAGE + +.serverless:deploy:image: + stage: deploy + image: gcr.io/triggermesh/tm@sha256:e3ee74db94d215bd297738d93577481f3e4db38013326c90d57f873df7ab41d5 + only: + refs: + - master + environment: development + script: + - echo "$CI_REGISTRY_IMAGE" + - tm -n "$KUBE_NAMESPACE" --config "$KUBECONFIG" deploy service "$CI_PROJECT_NAME" --from-image "$CI_REGISTRY_IMAGE" --wait + +.serverless:deploy:functions: + stage: deploy + environment: development + image: gcr.io/triggermesh/tm:v0.0.9 + script: + - tm -n "$KUBE_NAMESPACE" set registry-auth gitlab-registry --registry "$CI_REGISTRY" --username "$CI_REGISTRY_USER" --password "$CI_JOB_TOKEN" --push + - tm -n "$KUBE_NAMESPACE" set registry-auth gitlab-registry --registry "$CI_REGISTRY" --username "$CI_DEPLOY_USER" --password "$CI_DEPLOY_PASSWORD" --pull + - tm -n "$KUBE_NAMESPACE" deploy --wait diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index fc3d4ecdbba..25a32ba0f74 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -57,6 +57,7 @@ test_job: script: - '& "$env:NUNIT_PATH" ".\$env:TEST_FOLDER\Tests.dll"' # running NUnit tests artifacts: + when: always # save test results even when the task fails expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on paths: - '.\TestResult.xml' # saving NUnit results to copy to deploy folder diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml new file mode 100644 index 00000000000..245e6bec60a --- /dev/null +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -0,0 +1,28 @@ +# This is a very simple template that mainly relies on FastLane to build and distribute your app. +# Read more about how to use this template on the blog post https://about.gitlab.com/2019/03/06/ios-publishing-with-gitlab-and-fastlane/ +# You will also need fastlane and signing configuration for this to work, along with a MacOS runner. +# These details are provided in the blog post. + +# Note that when you're using the shell executor for MacOS builds, the +# build and tests run as the identity of the runner logged in user, directly on +# the build host. This is less secure than using container executors, so please +# take a look at our security implications documentation at +# https://docs.gitlab.com/runner/security/#usage-of-shell-executor for additional +# detail on what to keep in mind in this scenario. + +stages: + - build + +variables: + LC_ALL: "en_US.UTF-8" + LANG: "en_US.UTF-8" + GIT_STRATEGY: clone + +build: + stage: build + script: + - bundle install + - bundle exec fastlane build + artifacts: + paths: + - ./*.ipa diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 0f23b95ba15..e61fb50a303 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -46,7 +46,7 @@ module Gitlab stream.seek(offset, IO::SEEK_SET) stream.write(data) stream.truncate(offset + data.bytesize) - stream.flush() + stream.flush end def set(data) diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index a7b4e0348c2..f7bbb58df7e 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -17,6 +17,8 @@ module Gitlab end def concat(resources) + return self if resources.nil? + tap { resources.each { |variable| self.append(variable) } } end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index e3e4e62cc02..833aa75adb5 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -5,12 +5,12 @@ module Gitlab module Variables class Collection class Item - def initialize(key:, value:, public: true, file: false) + def initialize(key:, value:, public: true, file: false, masked: false) raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless value.is_a?(String) || value.nil? @variable = { - key: key, value: value, public: public, file: file + key: key, value: value, public: public, file: file, masked: masked } end @@ -27,9 +27,13 @@ module Gitlab # don't expose `file` attribute at all (stems from what the runner # expects). # + # If the `variable_masking` feature is enabled we expose the `masked` + # attribute, otherwise it's not exposed. + # def to_runner_variable @variable.reject do |hash_key, hash_value| - hash_key == :file && hash_value == false + (hash_key == :file && hash_value == false) || + (hash_key == :masked && !Feature.enabled?(:variable_masking)) end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 172926b8ab0..07ba6f83d47 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :cache, :stages, :jobs def initialize(config, opts = {}) - @ci_config = Gitlab::Ci::Config.new(config, opts) + @ci_config = Gitlab::Ci::Config.new(config, **opts) @config = @ci_config.to_hash unless @ci_config.valid? @@ -33,8 +33,7 @@ module Gitlab { stage_idx: @stages.index(job[:stage]), stage: job[:stage], - commands: job[:commands], - tag_list: job[:tags] || [], + tag_list: job[:tags], name: job[:name].to_s, allow_failure: job[:ignore], when: job[:when] || 'on_success', @@ -54,8 +53,9 @@ module Gitlab retry: job[:retry], parallel: job[:parallel], instance: job[:instance], - start_in: job[:start_in] - }.compact } + start_in: job[:start_in], + trigger: job[:trigger] + }.compact }.compact end def stage_builds_attributes(stage) diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb index eba1faacc3a..03298d960a4 100644 --- a/lib/gitlab/cleanup/remote_uploads.rb +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -67,7 +67,7 @@ module Gitlab end def remote_directory - connection.directories.get(configuration['remote_directory']) + connection.directories.new(key: configuration['remote_directory']) end def connection diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb index afdb60b2cd5..37ba16dba25 100644 --- a/lib/gitlab/config/entry/configurable.rb +++ b/lib/gitlab/config/entry/configurable.rb @@ -56,6 +56,7 @@ module Gitlab def entry(key, entry, metadata) factory = ::Gitlab::Config::Entry::Factory.new(entry) .with(description: metadata[:description]) + .with(default: metadata[:default]) (@nodes ||= {}).merge!(key.to_sym => factory) end diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb index 30d43c9f9a1..79f9ff32514 100644 --- a/lib/gitlab/config/entry/factory.rb +++ b/lib/gitlab/config/entry/factory.rb @@ -12,7 +12,7 @@ module Gitlab def initialize(entry) @entry = entry @metadata = {} - @attributes = {} + @attributes = { default: entry.default } end def value(value) @@ -21,12 +21,12 @@ module Gitlab end def metadata(metadata) - @metadata.merge!(metadata) + @metadata.merge!(metadata.compact) self end def with(attributes) - @attributes.merge!(attributes) + @attributes.merge!(attributes.compact) self end @@ -38,9 +38,7 @@ module Gitlab # See issue #18775. # if @value.nil? - Entry::Unspecified.new( - fabricate_unspecified - ) + Entry::Unspecified.new(fabricate_unspecified) else fabricate(@entry, @value) end @@ -53,10 +51,12 @@ module Gitlab # If entry has a default value we fabricate concrete node # with default value. # - if @entry.default.nil? + default = @attributes.fetch(:default) + + if default.nil? fabricate(Entry::Undefined) else - fabricate(@entry, @entry.default) + fabricate(@entry, default) end end @@ -64,6 +64,7 @@ module Gitlab entry.new(value, @metadata).tap do |node| node.key = @attributes[:key] node.parent = @attributes[:parent] + node.default = @attributes[:default] node.description = @attributes[:description] end end diff --git a/lib/gitlab/config/entry/node.rb b/lib/gitlab/config/entry/node.rb index 30357b2c95b..9999ab4ff95 100644 --- a/lib/gitlab/config/entry/node.rb +++ b/lib/gitlab/config/entry/node.rb @@ -10,7 +10,7 @@ module Gitlab InvalidError = Class.new(StandardError) attr_reader :config, :metadata - attr_accessor :key, :parent, :description + attr_accessor :key, :parent, :default, :description def initialize(config, **metadata) @config = config @@ -85,7 +85,7 @@ module Gitlab "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>" end - def self.default + def self.default(**) end def self.aspects diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb index 3e148fe2e91..5fbf7565e2a 100644 --- a/lib/gitlab/config/entry/simplifiable.rb +++ b/lib/gitlab/config/entry/simplifiable.rb @@ -6,6 +6,8 @@ module Gitlab class Simplifiable < SimpleDelegator EntryStrategy = Struct.new(:name, :condition) + attr_reader :subject + def initialize(config, **metadata) unless self.class.const_defined?(:UnknownStrategy) raise ArgumentError, 'UndefinedStrategy not available!' @@ -17,7 +19,7 @@ module Gitlab entry = self.class.entry_class(strategy) - super(entry.new(config, metadata)) + super(@subject = entry.new(config, metadata)) end def self.strategy(name, **opts) @@ -37,6 +39,9 @@ module Gitlab self::UnknownStrategy end end + + def self.default + end end end end diff --git a/lib/gitlab/content_disposition.rb b/lib/gitlab/content_disposition.rb new file mode 100644 index 00000000000..32207514ce5 --- /dev/null +++ b/lib/gitlab/content_disposition.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true +# This ports ActionDispatch::Http::ContentDisposition (https://github.com/rails/rails/pull/33829, +# which will be available in Rails 6. +module Gitlab + class ContentDisposition # :nodoc: + # Make sure we remove this patch starting with Rails 6.0. + if Rails.version.start_with?('6.0') + raise <<~MSG + Please remove this file and use `ActionDispatch::Http::ContentDisposition` instead. + MSG + end + + def self.format(disposition:, filename:) + new(disposition: disposition, filename: filename).to_s + end + + attr_reader :disposition, :filename + + def initialize(disposition:, filename:) + @disposition = disposition + @filename = filename + end + + # rubocop:disable Style/VariableInterpolation + TRADITIONAL_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ + + def ascii_filename + 'filename="' + percent_escape(::I18n.transliterate(filename), TRADITIONAL_ESCAPED_CHAR) + '"' + end + + RFC_5987_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ + # rubocop:enable Style/VariableInterpolation + + def utf8_filename + "filename*=UTF-8''" + percent_escape(filename, RFC_5987_ESCAPED_CHAR) + end + + def to_s + if filename + "#{disposition}; #{ascii_filename}; #{utf8_filename}" + else + "#{disposition}" + end + end + + private + + def percent_escape(string, pattern) + string.gsub(pattern) do |char| + char.bytes.map { |byte| "%%%02X" % byte }.join + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 5ed6427072a..f7d046600e8 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -49,6 +49,7 @@ module Gitlab Event.contributions.where(author_id: contributor.id) .where(created_at: date.beginning_of_day..date.end_of_day) .where(project_id: projects) + .with_associations end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 477f9101e98..552aad83dd4 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -7,10 +7,6 @@ module Gitlab Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end - def fake_application_settings(attributes = {}) - Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) - end - def clear_in_memory_application_settings! @in_memory_application_settings = nil end @@ -50,28 +46,21 @@ module Gitlab # and other callers from failing, use any loaded settings and return # defaults for missing columns. if ActiveRecord::Migrator.needs_migration? - return fake_application_settings(current_settings&.attributes) - end - - return current_settings if current_settings.present? - - with_fallback_to_fake_application_settings do - ::ApplicationSetting.create_from_defaults || in_memory_application_settings + db_attributes = current_settings&.attributes || {} + ::ApplicationSetting.build_from_defaults(db_attributes) + elsif current_settings.present? + current_settings + else + ::ApplicationSetting.create_from_defaults end end - def in_memory_application_settings - with_fallback_to_fake_application_settings do - @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults - end + def fake_application_settings(attributes = {}) + Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) end - def with_fallback_to_fake_application_settings(&block) - yield - rescue - # In case the application_settings table is not created yet, or if a new - # ApplicationSetting column is not yet migrated we fallback to a simple OpenStruct - fake_application_settings + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults end def connect_to_db? diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index db8ac3becea..aeca9d00156 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -40,11 +40,11 @@ module Gitlab end def first_time_reference_commit(event) - return nil unless event && merge_request_diff_commits + return unless event && merge_request_diff_commits commits = merge_request_diff_commits[event['id'].to_i] - return nil if commits.blank? + return if commits.blank? commits.find do |commit| next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb new file mode 100644 index 00000000000..d2b7ca015d4 --- /dev/null +++ b/lib/gitlab/danger/helper.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true +require 'net/http' +require 'json' + +require_relative 'teammate' + +module Gitlab + module Danger + module Helper + ROULETTE_DATA_URL = URI.parse('https://about.gitlab.com/roulette.json').freeze + + # Returns a list of all files that have been added, modified or renamed. + # `git.modified_files` might contain paths that already have been renamed, + # so we need to remove them from the list. + # + # Considering these changes: + # + # - A new_file.rb + # - D deleted_file.rb + # - M modified_file.rb + # - R renamed_file_before.rb -> renamed_file_after.rb + # + # it will return + # ``` + # [ 'new_file.rb', 'modified_file.rb', 'renamed_file_after.rb' ] + # ``` + # + # @return [Array<String>] + def all_changed_files + Set.new + .merge(git.added_files.to_a) + .merge(git.modified_files.to_a) + .merge(git.renamed_files.map { |x| x[:after] }) + .subtract(git.renamed_files.map { |x| x[:before] }) + .to_a + .sort + end + + def ee? + ENV['CI_PROJECT_NAME'] == 'gitlab-ee' || File.exist?('../../CHANGELOG-EE.md') + end + + def project_name + ee? ? 'gitlab-ee' : 'gitlab-ce' + end + + # Looks up the current list of GitLab team members and parses it into a + # useful form + # + # @return [Array<Teammate>] + def team + @team ||= + begin + rsp = Net::HTTP.get_response(ROULETTE_DATA_URL) + raise "Failed to read #{ROULETTE_DATA_URL}: #{rsp.code} #{rsp.message}" unless + rsp.is_a?(Net::HTTPSuccess) + + data = JSON.parse(rsp.body) + data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) } + rescue JSON::ParserError + raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}" + end + end + + # Like +team+, but only returns teammates in the current project, based on + # project_name. + # + # @return [Array<Teammate>] + def project_team + team.select { |member| member.in_project?(project_name) } + end + + # @return [Hash<String,Array<String>>] + def changes_by_category + all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| + hash[category_for_file(file)] << file + end + end + + # Determines the category a file is in, e.g., `:frontend` or `:backend` + # @return[Symbol] + def category_for_file(file) + _, category = CATEGORIES.find { |regexp, _| regexp.match?(file) } + + category || :unknown + end + + # Returns the GFM for a category label, making its best guess if it's not + # a category we know about. + # + # @return[String] + def label_for_category(category) + CATEGORY_LABELS.fetch(category, "~#{category}") + end + + CATEGORY_LABELS = { + docs: "~Documentation", + none: "", + qa: "~QA" + }.freeze + + # rubocop:disable Style/RegexpLiteral + CATEGORIES = { + %r{\Adoc/} => :docs, + %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, + + %r{\A(ee/)?app/(assets|views)/} => :frontend, + %r{\A(ee/)?public/} => :frontend, + %r{\A(ee/)?spec/(javascripts|frontend)/} => :frontend, + %r{\A(ee/)?vendor/assets/} => :frontend, + %r{\A(jest\.config\.js|package\.json|yarn\.lock)\z} => :frontend, + + %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, + %r{\A(ee/)?(bin|config|danger|generator_templates|lib|rubocop|scripts)/} => :backend, + %r{\A(ee/)?spec/(?!javascripts|frontend)[^/]+} => :backend, + %r{\A(ee/)?vendor/(?!assets)[^/]+} => :backend, + %r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend, + %r{\A(Dangerfile|Gemfile|Gemfile.lock|Procfile|Rakefile|\.gitlab-ci\.yml)\z} => :backend, + %r{\A[A-Z_]+_VERSION\z} => :backend, + + %r{\A(ee/)?db/} => :database, + %r{\A(ee/)?qa/} => :qa, + + # Files that don't fit into any category are marked with :none + %r{\A(ee/)?changelogs/} => :none, + %r{\Alocale/gitlab\.pot\z} => :none, + + # Fallbacks in case the above patterns miss anything + %r{\.rb\z} => :backend, + %r{\.(md|txt)\z} => :docs, + %r{\.js\z} => :frontend + }.freeze + # rubocop:enable Style/RegexpLiteral + end + end +end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb new file mode 100644 index 00000000000..4b822aa86c5 --- /dev/null +++ b/lib/gitlab/danger/teammate.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Danger + class Teammate + attr_reader :name, :username, :projects + + def initialize(options = {}) + @name = options['name'] + @username = options['username'] + @projects = options['projects'] + end + + def markdown_name + "[#{name}](https://gitlab.com/#{username}) (`@#{username}`)" + end + + def in_project?(name) + projects&.has_key?(name) + end + + # Traintainers also count as reviewers + def reviewer?(project, category) + capabilities(project) == "reviewer #{category}" || traintainer?(project, category) + end + + def traintainer?(project, category) + capabilities(project) == "trainee_maintainer #{category}" + end + + def maintainer?(project, category) + capabilities(project) == "maintainer #{category}" + end + + private + + def capabilities(project) + projects.fetch(project, '') + end + end + end +end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 9bf2f9291a8..ea08b5f7eae 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -31,7 +31,11 @@ module Gitlab } } ], - total_commits_count: 1 + total_commits_count: 1, + push_options: [ + "ci.skip", + "custom option" + ] }.freeze # Produce a hash of post-receive data @@ -52,10 +56,12 @@ module Gitlab # homepage: String, # }, # commits: Array, - # total_commits_count: Fixnum + # total_commits_count: Fixnum, + # push_options: Array # } # - def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil) + # rubocop:disable Metrics/ParameterLists + def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil, push_options: []) commits = Array(commits) # Total commits count @@ -87,12 +93,13 @@ module Gitlab user_id: user.id, user_name: user.name, user_username: user.username, - user_email: user.email, + user_email: user.public_email, user_avatar: user.avatar_url(only_path: false), project_id: project.id, project: project.hook_attrs, commits: commit_attrs, total_commits_count: commits_count, + push_options: push_options, # DEPRECATED repository: project.hook_attrs.slice(:name, :url, :description, :homepage, :git_http_url, :git_ssh_url, :visibility_level) diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index cf1cf054dbf..fedf6ca4fe1 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -36,7 +36,7 @@ module Gitlab def perform_count(model, estimate) # If we estimate 0, we may not have statistics at all. Don't use them. - return nil unless estimate && estimate > 0 + return unless estimate && estimate > 0 if estimate < EXACT_COUNT_THRESHOLD # The table is considered small, the assumption here is that diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index ac2efe598b4..ffad00fa7d7 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -4,6 +4,7 @@ module Gitlab module DependencyLinker class BaseLinker URL_REGEX = %r{https?://[^'" ]+}.freeze + GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/.freeze REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze class_attribute :file_type @@ -29,8 +30,25 @@ module Gitlab highlighted_lines.join.html_safe end + def external_url(name, external_ref) + return if external_ref =~ GIT_INVALID_URL_REGEX + + case external_ref + when /\A#{URL_REGEX}\z/ + external_ref + when /\A#{REPO_REGEX}\z/ + github_url(external_ref) + else + package_url(name) + end + end + private + def package_url(_name) + raise NotImplementedError + end + def link_dependencies raise NotImplementedError end diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb index 22d2bead891..4b8862b31ee 100644 --- a/lib/gitlab/dependency_linker/composer_json_linker.rb +++ b/lib/gitlab/dependency_linker/composer_json_linker.rb @@ -8,8 +8,8 @@ module Gitlab private def link_packages - link_packages_at_key("require", &method(:package_url)) - link_packages_at_key("require-dev", &method(:package_url)) + link_packages_at_key("require") + link_packages_at_key("require-dev") end def package_url(name) diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb index 8ab219c4962..c6e02248b0a 100644 --- a/lib/gitlab/dependency_linker/gemfile_linker.rb +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -3,8 +3,14 @@ module Gitlab module DependencyLinker class GemfileLinker < MethodLinker + class_attribute :package_keyword + + self.package_keyword = :gem self.file_type = :gemfile + GITHUB_REGEX = /(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/.freeze + GIT_REGEX = /(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/.freeze + private def link_dependencies @@ -14,21 +20,35 @@ module Gitlab def link_urls # Link `github: "user/repo"` to https://github.com/user/repo - link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url)) + link_regex(GITHUB_REGEX, &method(:github_url)) # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo - link_regex(/(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/, &:itself) + link_regex(GIT_REGEX, &:itself) # Link `source "https://rubygems.org"` to https://rubygems.org link_method_call('source', URL_REGEX, &:itself) end def link_packages - # Link `gem "package_name"` to https://rubygems.org/gems/package_name - link_method_call('gem') do |name| - "https://rubygems.org/gems/#{name}" + packages = parse_packages + + return if packages.blank? + + packages.each do |package| + link_method_call('gem', package.name) do + external_url(package.name, package.external_ref) + end end end + + def package_url(name) + "https://rubygems.org/gems/#{name}" + end + + def parse_packages + parser = Gitlab::DependencyLinker::Parser::Gemfile.new(plain_text) + parser.parse(keyword: self.class.package_keyword) + end end end end diff --git a/lib/gitlab/dependency_linker/gemspec_linker.rb b/lib/gitlab/dependency_linker/gemspec_linker.rb index b924ea86d89..94c2b375cf9 100644 --- a/lib/gitlab/dependency_linker/gemspec_linker.rb +++ b/lib/gitlab/dependency_linker/gemspec_linker.rb @@ -11,7 +11,7 @@ module Gitlab link_method_call('homepage', URL_REGEX, &:itself) link_method_call('license', &method(:license_url)) - link_method_call(%w[name add_dependency add_runtime_dependency add_development_dependency]) do |name| + link_method_call(%w[add_dependency add_runtime_dependency add_development_dependency]) do |name| "https://rubygems.org/gems/#{name}" end end diff --git a/lib/gitlab/dependency_linker/method_linker.rb b/lib/gitlab/dependency_linker/method_linker.rb index d4d85bb3390..33899a931c6 100644 --- a/lib/gitlab/dependency_linker/method_linker.rb +++ b/lib/gitlab/dependency_linker/method_linker.rb @@ -23,18 +23,22 @@ module Gitlab # link_method_call('name') # # Will link `package` in `self.name = "package"` def link_method_call(method_name, value = nil, &url_proc) + regex = method_call_regex(method_name, value) + + link_regex(regex, &url_proc) + end + + def method_call_regex(method_name, value = nil) method_name = regexp_for_value(method_name) value = regexp_for_value(value) - regex = %r{ + %r{ #{method_name} # Method name \s* # Whitespace [(=]? # Opening brace or equals sign \s* # Whitespace ['"](?<name>#{value})['"] # Package name in quotes }x - - link_regex(regex, &url_proc) end end end diff --git a/lib/gitlab/dependency_linker/package.rb b/lib/gitlab/dependency_linker/package.rb new file mode 100644 index 00000000000..8a509bbd562 --- /dev/null +++ b/lib/gitlab/dependency_linker/package.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module DependencyLinker + class Package + attr_reader :name, :git_ref, :github_ref + + def initialize(name, git_ref, github_ref) + @name = name + @git_ref = git_ref + @github_ref = github_ref + end + + def external_ref + @git_ref || @github_ref + end + end + end +end diff --git a/lib/gitlab/dependency_linker/package_json_linker.rb b/lib/gitlab/dependency_linker/package_json_linker.rb index 578e25f806a..6857f2a4fa2 100644 --- a/lib/gitlab/dependency_linker/package_json_linker.rb +++ b/lib/gitlab/dependency_linker/package_json_linker.rb @@ -8,7 +8,6 @@ module Gitlab private def link_dependencies - link_json('name', json["name"], &method(:package_url)) link_json('license', &method(:license_url)) link_json(%w[homepage url], URL_REGEX, &:itself) @@ -16,25 +15,19 @@ module Gitlab end def link_packages - link_packages_at_key("dependencies", &method(:package_url)) - link_packages_at_key("devDependencies", &method(:package_url)) + link_packages_at_key("dependencies") + link_packages_at_key("devDependencies") end - def link_packages_at_key(key, &url_proc) + def link_packages_at_key(key) dependencies = json[key] return unless dependencies dependencies.each do |name, version| - link_json(name, version, link: :key, &url_proc) - - link_json(name) do |value| - case value - when /\A#{URL_REGEX}\z/ - value - when /\A#{REPO_REGEX}\z/ - github_url(value) - end - end + external_url = external_url(name, version) + + link_json(name, version, link: :key) { external_url } + link_json(name) { external_url } end end diff --git a/lib/gitlab/dependency_linker/parser/gemfile.rb b/lib/gitlab/dependency_linker/parser/gemfile.rb new file mode 100644 index 00000000000..7f755375cea --- /dev/null +++ b/lib/gitlab/dependency_linker/parser/gemfile.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module DependencyLinker + module Parser + class Gemfile < MethodLinker + GIT_REGEX = Gitlab::DependencyLinker::GemfileLinker::GIT_REGEX + GITHUB_REGEX = Gitlab::DependencyLinker::GemfileLinker::GITHUB_REGEX + + def initialize(plain_text) + @plain_text = plain_text + end + + # Returns a list of Gitlab::DependencyLinker::Package + # + # keyword - The package definition keyword, e.g. `:gem` for + # Gemfile parsing, `:pod` for Podfile. + def parse(keyword:) + plain_lines.each_with_object([]) do |line, packages| + name = fetch(line, method_call_regex(keyword)) + + next unless name + + git_ref = fetch(line, GIT_REGEX) + github_ref = fetch(line, GITHUB_REGEX) + + packages << Gitlab::DependencyLinker::Package.new(name, git_ref, github_ref) + end + end + + private + + def fetch(line, regex, group: :name) + match = line.match(regex) + match[group] if match + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/podfile_linker.rb b/lib/gitlab/dependency_linker/podfile_linker.rb index def9b04cca9..a20d285da79 100644 --- a/lib/gitlab/dependency_linker/podfile_linker.rb +++ b/lib/gitlab/dependency_linker/podfile_linker.rb @@ -5,12 +5,21 @@ module Gitlab class PodfileLinker < GemfileLinker include Cocoapods + self.package_keyword = :pod self.file_type = :podfile private def link_packages - link_method_call('pod', &method(:package_url)) + packages = parse_packages + + return unless packages + + packages.each do |package| + link_method_call('pod', package.name) do + external_url(package.name, package.external_ref) + end + end end end end diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb index 6b1758c5a43..14abd3999c4 100644 --- a/lib/gitlab/dependency_linker/podspec_linker.rb +++ b/lib/gitlab/dependency_linker/podspec_linker.rb @@ -19,7 +19,7 @@ module Gitlab link_method_call('license', &method(:license_url)) link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url)) - link_method_call(%w[name dependency], &method(:package_url)) + link_method_call('dependency', &method(:package_url)) end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index b5fc8d364c8..eac9bb88eb6 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -3,7 +3,9 @@ module Gitlab module Diff class File - attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs + include Gitlab::Utils::StrongMemoize + + attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs, :unique_identifier delegate :new_file?, :deleted_file?, :renamed_file?, :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, @@ -22,12 +24,20 @@ module Gitlab DiffViewer::Image ].sort_by { |v| v.binary? ? 0 : 1 }.freeze - def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil, stats: nil) + def initialize( + diff, + repository:, + diff_refs: nil, + fallback_diff_refs: nil, + stats: nil, + unique_identifier: nil) + @diff = diff @stats = stats @repository = repository @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + @unique_identifier = unique_identifier @unfolded = false # Ensure items are collected in the the batch @@ -65,9 +75,17 @@ module Gitlab end def line_for_position(pos) - return nil unless pos.position_type == 'text' + return unless pos.position_type == 'text' - diff_lines.find { |line| line.old_line == pos.old_line && line.new_line == pos.new_line } + # This method is normally used to find which line the diff was + # commented on, and in this context, it's normally the raw diff persisted + # at `note_diff_files`, which is a fraction of the entire diff + # (it goes from the first line, to the commented line, or + # one line below). Therefore it's more performant to fetch + # from bottom to top instead of the other way around. + diff_lines + .reverse_each + .find { |line| line.old_line == pos.old_line && line.new_line == pos.new_line } end def position_for_line_code(code) @@ -166,6 +184,10 @@ module Gitlab @unfolded end + def highlight_loaded? + @highlighted_diff_lines.present? + end + def highlighted_diff_lines @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight @@ -212,12 +234,12 @@ module Gitlab repository.attributes(file_path).fetch('diff') { true } end - def binary? - has_binary_notice? || try_blobs(:binary?) + def binary_in_repo? + has_binary_notice? || try_blobs(:binary_in_repo?) end - def text? - !binary? + def text_in_repo? + !binary_in_repo? end def external_storage_error? @@ -259,12 +281,20 @@ module Gitlab valid_blobs.map(&:empty?).all? end - def raw_binary? - try_blobs(:raw_binary?) + def binary? + strong_memoize(:is_binary) do + try_blobs(:binary?) + end + end + + def text? + strong_memoize(:is_text) do + !binary? && !different_type? + end end - def raw_text? - !raw_binary? && !different_type? + def viewer + rich_viewer || simple_viewer end def simple_viewer @@ -347,19 +377,19 @@ module Gitlab return DiffViewer::NotDiffable unless diffable? if content_changed? - if raw_text? + if text? DiffViewer::Text else DiffViewer::NoPreview end elsif new_file? - if raw_text? + if text? DiffViewer::Text else DiffViewer::Added end elsif deleted_file? - if raw_text? + if text? DiffViewer::Text else DiffViewer::Deleted diff --git a/lib/gitlab/diff/lines_unfolder.rb b/lib/gitlab/diff/lines_unfolder.rb index 9306b7e16a2..6cf904b2b2a 100644 --- a/lib/gitlab/diff/lines_unfolder.rb +++ b/lib/gitlab/diff/lines_unfolder.rb @@ -158,9 +158,14 @@ module Gitlab from = comment_position - UNFOLD_CONTEXT_SIZE - # There's no line before the match if it's in the top-most - # position. - prev_line_number = line_before_unfold_position&.old_pos || 0 + prev_line_number = + if bottom? + last_line.old_pos + else + # There's no line before the match if it's in the top-most + # position. + line_before_unfold_position&.old_pos || 0 + end if from <= prev_line_number + 1 @generate_top_match_line = false diff --git a/lib/gitlab/discussions_diff/file_collection.rb b/lib/gitlab/discussions_diff/file_collection.rb new file mode 100644 index 00000000000..4ab7314f509 --- /dev/null +++ b/lib/gitlab/discussions_diff/file_collection.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module DiscussionsDiff + class FileCollection + include Gitlab::Utils::StrongMemoize + + def initialize(collection) + @collection = collection + end + + # Returns a Gitlab::Diff::File with the given ID (`unique_identifier` in + # Gitlab::Diff::File). + def find_by_id(id) + diff_files_indexed_by_id[id] + end + + # Writes cache and preloads highlighted diff lines for + # object IDs, in @collection. + # + # highlightable_ids - Diff file `Array` responding to ID. The ID will be used + # to generate the cache key. + # + # - Highlight cache is written just for uncached diff files + # - The cache content is not updated (there's no need to do so) + def load_highlight(highlightable_ids) + preload_highlighted_lines(highlightable_ids) + end + + private + + def preload_highlighted_lines(ids) + cached_content = read_cache(ids) + + uncached_ids = ids.select.each_with_index { |_, i| cached_content[i].nil? } + mapping = highlighted_lines_by_ids(uncached_ids) + + HighlightCache.write_multiple(mapping) + + diffs = diff_files_indexed_by_id.values_at(*ids) + + diffs.zip(cached_content).each do |diff, cached_lines| + next unless diff && cached_lines + + diff.highlighted_diff_lines = cached_lines + end + end + + def read_cache(ids) + HighlightCache.read_multiple(ids) + end + + def diff_files_indexed_by_id + strong_memoize(:diff_files_indexed_by_id) do + diff_files.index_by(&:unique_identifier) + end + end + + def diff_files + strong_memoize(:diff_files) do + @collection.map(&:raw_diff_file) + end + end + + # Processes the diff lines highlighting for diff files matching the given + # IDs. + # + # Returns a Hash with { id => [Array of Gitlab::Diff::line], ...] + def highlighted_lines_by_ids(ids) + diff_files_indexed_by_id.slice(*ids).each_with_object({}) do |(id, file), hash| + hash[id] = file.highlighted_diff_lines.map(&:to_hash) + end + end + end + end +end diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb new file mode 100644 index 00000000000..270cfb89488 --- /dev/null +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true +# +module Gitlab + module DiscussionsDiff + class HighlightCache + class << self + VERSION = 1 + EXPIRATION = 1.week + + # Sets multiple keys to a given value. The value + # is serialized as JSON. + # + # mapping - Write multiple cache values at once + def write_multiple(mapping) + Redis::Cache.with do |redis| + redis.multi do |multi| + mapping.each do |raw_key, value| + key = cache_key_for(raw_key) + + multi.set(key, value.to_json, ex: EXPIRATION) + end + end + end + end + + # Reads multiple cache keys at once. + # + # raw_keys - An Array of unique cache keys, without namespaces. + # + # It returns a list of deserialized diff lines. Ex.: + # [[Gitlab::Diff::Line, ...], [Gitlab::Diff::Line]] + def read_multiple(raw_keys) + return [] if raw_keys.empty? + + keys = raw_keys.map { |id| cache_key_for(id) } + + content = + Redis::Cache.with do |redis| + redis.mget(keys) + end + + content.map! do |lines| + next unless lines + + JSON.parse(lines).map! do |line| + line = line.with_indifferent_access + rich_text = line[:rich_text] + line[:rich_text] = rich_text&.html_safe + + Gitlab::Diff::Line.init_from_hash(line) + end + end + end + + def cache_key_for(raw_key) + "#{cache_key_prefix}:#{raw_key}" + end + + private + + def cache_key_prefix + "#{Redis::Cache::CACHE_NAMESPACE}:#{VERSION}:discussion-highlight" + end + end + end + end +end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 5d9ecd651a0..01fd261404b 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -7,7 +7,7 @@ module Gitlab CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') - IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb|locale/gitlab\.pot}i.freeze + IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md/i.freeze PLEASE_READ_THIS_BANNER = %Q{ ============================================================ ===================== PLEASE READ THIS ===================== diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index a826519b2dd..3323ce60158 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -23,8 +23,8 @@ module Gitlab content_type: attachment.content_type } - link = UploadService.new(project, file).execute - attachments << link if link + uploader = UploadService.new(project, file).execute + attachments << uploader.to_h if uploader ensure tmp.close! end diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 35bb49ad19a..f89d1d15010 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -6,12 +6,14 @@ module Gitlab class BaseHandler attr_reader :mail, :mail_key + HANDLER_ACTION_BASE_REGEX ||= /(?<project_slug>.+)-(?<project_id>\d+)/.freeze + def initialize(mail, mail_key) @mail = mail @mail_key = mail_key end - def can_execute? + def can_handle? raise NotImplementedError end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 69982efbbe6..78a3a9489ac 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -2,21 +2,33 @@ require 'gitlab/email/handler/base_handler' +# handles issue creation emails with these formats: +# incoming+gitlab-org-gitlab-ce-20-Author_Token12345678-issue@incoming.gitlab.com +# incoming+gitlab-org/gitlab-ce+Author_Token12345678@incoming.gitlab.com (legacy) module Gitlab module Email module Handler class CreateIssueHandler < BaseHandler include ReplyProcessing - attr_reader :project_path, :incoming_email_token + + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue\z/.freeze + HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+(?<incoming_email_token>.*)\z/.freeze def initialize(mail, mail_key) super(mail, mail_key) - @project_path, @incoming_email_token = - mail_key && mail_key.split('+', 2) + + if !mail_key&.include?('/') && (matched = HANDLER_REGEX.match(mail_key.to_s)) + @project_slug = matched[:project_slug] + @project_id = matched[:project_id]&.to_i + @incoming_email_token = matched[:incoming_email_token] + elsif matched = HANDLER_REGEX_LEGACY.match(mail_key.to_s) + @project_path = matched[:project_path] + @incoming_email_token = matched[:incoming_email_token] + end end def can_handle? - !incoming_email_token.nil? && !incoming_email_token.include?("+") && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX) + incoming_email_token && (project_id || can_handle_legacy_format?) end def execute @@ -36,10 +48,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def project - @project ||= Project.find_by_full_path(project_path) - end - private def create_issue @@ -50,6 +58,10 @@ module Gitlab description: message_including_reply ).execute end + + def can_handle_legacy_format? + project_path && !incoming_email_token.include?('+') && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY) + end end end end diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index 5772727e855..b3b5063f2ca 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -3,23 +3,33 @@ require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' +# handles merge request creation emails with these formats: +# incoming+gitlab-org-gitlab-ce-20-Author_Token12345678-merge-request@incoming.gitlab.com +# incoming+gitlab-org/gitlab-ce+merge-request+Author_Token12345678@incoming.gitlab.com (legacy) module Gitlab module Email module Handler class CreateMergeRequestHandler < BaseHandler include ReplyProcessing - attr_reader :project_path, :incoming_email_token + + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-merge-request\z/.freeze + HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+merge-request\+(?<incoming_email_token>.*)/.freeze def initialize(mail, mail_key) super(mail, mail_key) - if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s) - @project_path, @incoming_email_token = m.captures + if !mail_key&.include?('/') && (matched = HANDLER_REGEX.match(mail_key.to_s)) + @project_slug = matched[:project_slug] + @project_id = matched[:project_id]&.to_i + @incoming_email_token = matched[:incoming_email_token] + elsif matched = HANDLER_REGEX_LEGACY.match(mail_key.to_s) + @project_path = matched[:project_path] + @incoming_email_token = matched[:incoming_email_token] end end def can_handle? - @project_path && @incoming_email_token + incoming_email_token && (project_id || project_path) end def execute @@ -40,10 +50,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def project - @project ||= Project.find_by_full_path(project_path) - end - def metrics_params super.merge(includes_patches: patch_attachments.any?) end @@ -97,7 +103,7 @@ module Gitlab def remove_patch_attachments patch_attachments.each { |patch| mail.parts.delete(patch) } - # reset the message, so it needs to be reporocessed when the attachments + # reset the message, so it needs to be reprocessed when the attachments # have been modified @message = nil end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index c7c573595fa..b00af15364d 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -3,6 +3,8 @@ require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' +# handles note/reply creation emails with these formats: +# incoming+1234567890abcdef1234567890abcdef@incoming.gitlab.com module Gitlab module Email module Handler diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index ff6b2c729b2..d8f4be8ada1 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -6,13 +6,26 @@ module Gitlab module ReplyProcessing private + attr_reader :project_id, :project_slug, :project_path, :incoming_email_token + def author raise NotImplementedError end + # rubocop:disable Gitlab/ModuleWithInstanceVariables def project - raise NotImplementedError + return @project if instance_variable_defined?(:@project) + + if project_id + @project = Project.find_by_id(project_id) + @project = nil unless valid_project_slug?(@project) + else + @project = Project.find_by_full_path(project_path) + end + + @project end + # rubocop:enable Gitlab/ModuleWithInstanceVariables def message @message ||= process_message @@ -43,7 +56,7 @@ module Gitlab raise ProjectNotFound unless author.can?(:read_project, project) end - raise UserNotAuthorizedError unless author.can?(permission, project || noteable) + raise UserNotAuthorizedError unless author.can?(permission, try(:noteable) || project) end def verify_record!(record:, invalid_exception:, record_name:) @@ -58,6 +71,10 @@ module Gitlab raise invalid_exception, msg end + + def valid_project_slug?(found_project) + project_slug == found_project.full_path_slug + end end end end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index d2f617b868a..20e4c125626 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -2,14 +2,28 @@ require 'gitlab/email/handler/base_handler' +# handles unsubscribe emails with these formats: +# incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com +# incoming+1234567890abcdef1234567890abcdef+unsubscribe@incoming.gitlab.com (legacy) module Gitlab module Email module Handler class UnsubscribeHandler < BaseHandler delegate :project, to: :sent_notification, allow_nil: true + HANDLER_REGEX_FOR = -> (suffix) { /\A(?<reply_token>\w+)#{Regexp.escape(suffix)}\z/ }.freeze + HANDLER_REGEX = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX).freeze + HANDLER_REGEX_LEGACY = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY).freeze + + def initialize(mail, mail_key) + super(mail, mail_key) + + matched = HANDLER_REGEX.match(mail_key.to_s) || HANDLER_REGEX_LEGACY.match(mail_key.to_s) + @reply_token = matched[:reply_token] if matched + end + def can_handle? - mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ + reply_token.present? end def execute @@ -24,12 +38,10 @@ module Gitlab private - def sent_notification - @sent_notification ||= SentNotification.for(reply_key) - end + attr_reader :reply_token - def reply_key - mail_key.sub(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX, '') + def sent_notification + @sent_notification ||= SentNotification.for(reply_token) end end end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 2743f011ca6..dc44e9d7481 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -61,7 +61,7 @@ module Gitlab # Force encoding to UTF-8 on a Mail::Message or Mail::Part def fix_charset(object) - return nil if object.nil? + return if object.nil? if object.charset object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s diff --git a/lib/gitlab/error_tracking/error.rb b/lib/gitlab/error_tracking/error.rb new file mode 100644 index 00000000000..4af5192aa6a --- /dev/null +++ b/lib/gitlab/error_tracking/error.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class Error + include ActiveModel::Model + + attr_accessor :id, :title, :type, :user_count, :count, + :first_seen, :last_seen, :message, :culprit, + :external_url, :project_id, :project_name, :project_slug, + :short_id, :status, :frequency + end + end +end diff --git a/lib/gitlab/error_tracking/project.rb b/lib/gitlab/error_tracking/project.rb new file mode 100644 index 00000000000..93e81da5034 --- /dev/null +++ b/lib/gitlab/error_tracking/project.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class Project + include ActiveModel::Model + + ACCESSORS = [ + :id, :name, :status, :slug, :organization_name, + :organization_id, :organization_slug + ].freeze + + attr_accessor(*ACCESSORS) + end + end +end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 0341f930b9c..a11d6b66409 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -8,7 +8,7 @@ module Gitlab end def call(env) - request = Rack::Request.new(env) + request = ActionDispatch::Request.new(env) route = Gitlab::EtagCaching::Router.match(request.path_info) return @app.call(env) unless route diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 08e30214b46..0891f79198d 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -52,6 +52,14 @@ module Gitlab Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z), 'environments' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/import/github/realtime_changes\.json\z), + 'realtime_changes_import_github' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/import/gitea/realtime_changes\.json\z), + 'realtime_changes_import_gitea' ) ].freeze diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 431911d1eee..2c53f9b026d 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -239,7 +239,7 @@ module Gitlab res = ::Projects::DownloadService.new(project, link).execute - return nil if res.nil? + return if res.nil? res[:markdown] end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 08d7db49ad7..4d82acd9d87 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -93,7 +93,7 @@ module Gitlab end def markdown(text) - Banzai.render(text, project: @source_parent, no_original_data: true) + Banzai.render(text, project: @source_parent, no_original_data: true, no_sourcepos: true) end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index c4aac228b2f..44a62586a23 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -54,11 +54,11 @@ module Gitlab end def tag_ref?(ref) - ref.start_with?(TAG_REF_PREFIX) + ref =~ /^#{TAG_REF_PREFIX}.+/ end def branch_ref?(ref) - ref.start_with?(BRANCH_REF_PREFIX) + ref =~ /^#{BRANCH_REF_PREFIX}.+/ end def blank_ref?(ref) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 2d25389594e..259a2b7911a 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -100,7 +100,7 @@ module Gitlab @loaded_all_data = @loaded_size == size end - def binary? + def binary_in_repo? @binary.nil? ? super : @binary == true end @@ -174,7 +174,7 @@ module Gitlab private def has_lfs_version_key? - !empty? && text? && data.start_with?("version https://git-lfs.github.com/spec") + !empty? && text_in_repo? && data.start_with?("version https://git-lfs.github.com/spec") end end end diff --git a/lib/gitlab/git/bundle_file.rb b/lib/gitlab/git/bundle_file.rb new file mode 100644 index 00000000000..8384a436fcc --- /dev/null +++ b/lib/gitlab/git/bundle_file.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class BundleFile + # All git bundle files start with this string + # + # https://github.com/git/git/blob/v2.20.1/bundle.c#L15 + MAGIC = "# v2 git bundle\n" + + InvalidBundleError = Class.new(StandardError) + + attr_reader :filename + + def self.check!(filename) + new(filename).check! + end + + def initialize(filename) + @filename = filename + end + + def check! + data = File.open(filename, 'r') { |f| f.read(MAGIC.size) } + + raise InvalidBundleError, 'Invalid bundle file' unless data == MAGIC + end + end + end +end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 5863815ca85..491e4b47196 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -5,6 +5,7 @@ module Gitlab module Git class Commit include Gitlab::EncodingHelper + prepend Gitlab::Git::RuggedImpl::Commit extend Gitlab::Git::WrapsGitalyErrors attr_accessor :raw_commit, :head @@ -57,20 +58,24 @@ module Gitlab return commit_id if commit_id.is_a?(Gitlab::Git::Commit) # Some weird thing? - return nil unless commit_id.is_a?(String) + return unless commit_id.is_a?(String) # This saves us an RPC round trip. - return nil if commit_id.include?(':') + return if commit_id.include?(':') - commit = wrapped_gitaly_errors do - repo.gitaly_commit_client.find_commit(commit_id) - end + commit = find_commit(repo, commit_id) decorate(repo, commit) if commit rescue Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository, ArgumentError nil end + def find_commit(repo, commit_id) + wrapped_gitaly_errors do + repo.gitaly_commit_client.find_commit(commit_id) + end + end + # Get last commit for HEAD # # Ex. @@ -185,6 +190,10 @@ module Gitlab @repository = repository @head = head + init_commit(raw_commit) + end + + def init_commit(raw_commit) case raw_commit when Hash init_from_hash(raw_commit) @@ -400,3 +409,5 @@ module Gitlab end end end + +Gitlab::Git::Commit.singleton_class.prepend Gitlab::Git::RuggedImpl::Commit::ClassMethods diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb index 1c6242b444a..e93ca3e11f8 100644 --- a/lib/gitlab/git/object_pool.rb +++ b/lib/gitlab/git/object_pool.rb @@ -10,12 +10,13 @@ module Gitlab delegate :exists?, :size, to: :repository delegate :unlink_repository, :delete, to: :object_pool_service - attr_reader :storage, :relative_path, :source_repository + attr_reader :storage, :relative_path, :source_repository, :gl_project_path - def initialize(storage, relative_path, source_repository) + def initialize(storage, relative_path, source_repository, gl_project_path) @storage = storage @relative_path = relative_path @source_repository = source_repository + @gl_project_path = gl_project_path end def create @@ -31,12 +32,12 @@ module Gitlab end def to_gitaly_repository - Gitlab::GitalyClient::Util.repository(storage, relative_path, GL_REPOSITORY) + Gitlab::GitalyClient::Util.repository(storage, relative_path, GL_REPOSITORY, gl_project_path) end # Allows for reusing other RPCs by 'tricking' Gitaly to think its a repository def repository - @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY) + @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path) end private diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index eec91194949..47cfb483509 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -4,6 +4,7 @@ module Gitlab module Git class Ref include Gitlab::EncodingHelper + include Gitlab::Git::RuggedImpl::Ref # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 5bbedc9d5e3..7750978fb95 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -11,6 +11,7 @@ module Gitlab include Gitlab::Git::WrapsGitalyErrors include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize + include Gitlab::Git::RuggedImpl::Repository SEARCH_CONTEXT_LINES = 3 REV_LIST_COMMIT_LIMIT = 2_000 @@ -67,7 +68,7 @@ module Gitlab # Relative path of repo attr_reader :relative_path - attr_reader :storage, :gl_repository, :relative_path + attr_reader :storage, :gl_repository, :relative_path, :gl_project_path # This remote name has to be stable for all types of repositories that # can join an object pool. If it's structure ever changes, a migration @@ -78,10 +79,11 @@ module Gitlab # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. - def initialize(storage, relative_path, gl_repository) + def initialize(storage, relative_path, gl_repository, gl_project_path) @storage = storage @relative_path = relative_path @gl_repository = gl_repository + @gl_project_path = gl_project_path @name = @relative_path.split("/").last end @@ -274,7 +276,7 @@ module Gitlab # senddata response. def archive_file_path(storage_path, sha, name, format = "tar.gz") # Build file path - return nil unless name + return unless name extension = case format @@ -490,6 +492,13 @@ module Gitlab end end + # Return total diverging commits count + def diverging_commit_count(from, to, max_count:) + wrapped_gitaly_errors do + gitaly_commit_client.diverging_commit_count(from, to, max_count: max_count) + end + end + # Mimic the `git clean` command and recursively delete untracked files. # Valid keys that can be passed in the +options+ hash are: # @@ -548,6 +557,12 @@ module Gitlab tags.find { |tag| tag.name == name } end + def merge_to_ref(user, source_sha, branch, target_ref, message) + wrapped_gitaly_errors do + gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message) + end + end + def merge(user, source_sha, target_branch, message, &block) wrapped_gitaly_errors do gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) @@ -789,6 +804,11 @@ module Gitlab end def create_from_bundle(bundle_path) + # It's important to check that the linked-to file is actually a valid + # .bundle file as it is passed to `git clone`, which may otherwise + # interpret it as a pointer to another repository + ::Gitlab::Git::BundleFile.check!(bundle_path) + gitaly_repository_client.create_from_bundle(bundle_path) end @@ -833,17 +853,20 @@ module Gitlab true end + # rubocop:disable Metrics/ParameterLists def multi_action( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, - start_branch_name: nil, start_repository: self) + start_branch_name: nil, start_repository: self, + force: false) wrapped_gitaly_errors do gitaly_operation_client.user_commit_files(user, branch_name, message, actions, author_email, author_name, - start_branch_name, start_repository) + start_branch_name, start_repository, force) end end + # rubocop:enable Metrics/ParameterLists def write_config(full_path:) return unless full_path.present? @@ -867,7 +890,7 @@ module Gitlab end def gitaly_repository - Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) + Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository, @gl_project_path) end def gitaly_ref_client diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb new file mode 100644 index 00000000000..251802878c3 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Git + module RuggedImpl + module Commit + module ClassMethods + extend ::Gitlab::Utils::Override + + def rugged_find(repo, commit_id) + obj = repo.rev_parse_target(commit_id) + + obj.is_a?(::Rugged::Commit) ? obj : nil + rescue ::Rugged::Error + nil + end + + override :find_commit + def find_commit(repo, commit_id) + if Feature.enabled?(:rugged_find_commit) + rugged_find(repo, commit_id) + else + super + end + end + end + + extend ::Gitlab::Utils::Override + + override :init_commit + def init_commit(raw_commit) + case raw_commit + when ::Rugged::Commit + init_from_rugged(raw_commit) + else + super + end + end + + def init_from_rugged(commit) + author = commit.author + committer = commit.committer + + @raw_commit = commit + @id = commit.oid + @message = commit.message + @authored_date = author[:time] + @committed_date = committer[:time] + @author_name = author[:name] + @author_email = author[:email] + @committer_name = committer[:name] + @committer_email = committer[:email] + @parent_ids = commit.parents.map(&:oid) + end + end + end + end +end +# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/git/rugged_impl/ref.rb b/lib/gitlab/git/rugged_impl/ref.rb new file mode 100644 index 00000000000..b553e82dc47 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/ref.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +module Gitlab + module Git + module RuggedImpl + module Ref + def self.dereference_object(object) + object = object.target while object.is_a?(::Rugged::Tag::Annotation) + + object + end + end + end + end +end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb new file mode 100644 index 00000000000..135c47017b3 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Git + module RuggedImpl + module Repository + FEATURE_FLAGS = %i(rugged_find_commit).freeze + + def alternate_object_directories + relative_object_directories.map { |d| File.join(path, d) } + end + + ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY_RELATIVE + GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE + ].freeze + + def relative_object_directories + Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact + end + + def rugged + @rugged ||= ::Rugged::Repository.new(path, alternates: alternate_object_directories) + rescue ::Rugged::RepositoryError, ::Rugged::OSError + raise ::Gitlab::Git::Repository::NoRepository.new('no repository for such path') + end + + def cleanup + @rugged&.close + end + + # Return the object that +revspec+ points to. If +revspec+ is an + # annotated tag, then return the tag's target instead. + def rev_parse_target(revspec) + obj = rugged.rev_parse(revspec) + Ref.dereference_object(obj) + end + end + end + end +end +# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 51542bcaaa2..6fcea4e12b4 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -44,7 +44,7 @@ module Gitlab entry[:name] == path_arr[0] && entry[:type] == :tree end - return nil unless entry + return unless entry if path_arr.size > 1 path_arr.shift diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 802fa65dd63..010bd0e520c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -12,6 +12,10 @@ module Gitlab TimeoutError = Class.new(StandardError) ProjectMovedError = Class.new(NotFoundError) + # Use the magic string '_any' to indicate we do not know what the + # changes are. This is also what gitlab-shell does. + ANY = '_any' + ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', download: 'You are not allowed to download code from this project.', @@ -24,7 +28,8 @@ module Gitlab upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', read_only: 'The repository is temporarily read-only. Please try again later.', - cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." + cannot_push_to_read_only: "You can't push code to a read-only GitLab instance.", + push_code: 'You are not allowed to push code to this project.' }.freeze INTERNAL_TIMEOUT = 50.seconds.freeze @@ -199,7 +204,7 @@ module Gitlab def ensure_project_on_push!(cmd, changes) return if project || deploy_key? - return unless receive_pack?(cmd) && changes == '_any' && authentication_abilities.include?(:push_code) + return unless receive_pack?(cmd) && changes == ANY && authentication_abilities.include?(:push_code) namespace = Namespace.find_by_full_path(namespace_path) @@ -256,24 +261,34 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:upload] end - return if changes.blank? # Allow access this is needed for EE. - check_change_access! end def check_change_access! - # If there are worktrees with a HEAD pointing to a non-existent object, - # calls to `git rev-list --all` will fail in git 2.15+. This should also - # clear stale lock files. - project.repository.clean_stale_repository_files - - # Iterate over all changes to find if user allowed all of them to be applied - changes_list.each.with_index do |change, index| - first_change = index == 0 - - # If user does not have access to make at least one change, cancel all - # push by allowing the exception to bubble up - check_single_change_access(change, skip_lfs_integrity_check: !first_change) + # Deploy keys with write access can push anything + return if deploy_key? + + if changes == ANY + can_push = user_access.can_do_action?(:push_code) || + project.any_branch_allows_collaboration?(user_access.user) + + unless can_push + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] + end + else + # If there are worktrees with a HEAD pointing to a non-existent object, + # calls to `git rev-list --all` will fail in git 2.15+. This should also + # clear stale lock files. + project.repository.clean_stale_repository_files + + # Iterate over all changes to find if user allowed all of them to be applied + changes_list.each.with_index do |change, index| + first_change = index == 0 + + # If user does not have access to make at least one change, cancel all + # push by allowing the exception to bubble up + check_single_change_access(change, skip_lfs_integrity_check: !first_change) + end end end @@ -282,7 +297,6 @@ module Gitlab change, user_access: user_access, project: project, - skip_authorization: deploy_key?, skip_lfs_integrity_check: skip_lfs_integrity_check, protocol: protocol, logger: logger @@ -348,7 +362,7 @@ module Gitlab protected def changes_list - @changes_list ||= Gitlab::ChangesList.new(changes) + @changes_list ||= Gitlab::ChangesList.new(changes == ANY ? [] : changes) end def user diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 3f24001e4ee..0af91957fa8 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -15,7 +15,7 @@ module Gitlab authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) end - def check_single_change_access(change, _options = {}) + def check_change_access! unless user_access.can_do_action?(:create_wiki) raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index cf2329e489d..426436c2164 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -3,12 +3,13 @@ module Gitlab class GitPostReceive include Gitlab::Identifier - attr_reader :project, :identifier, :changes + attr_reader :project, :identifier, :changes, :push_options - def initialize(project, identifier, changes) + def initialize(project, identifier, changes, push_options) @project = project @identifier = identifier @changes = deserialize_changes(changes) + @push_options = push_options end def identify diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 8bf8a3b53cd..48c113a8b14 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -28,7 +28,7 @@ module Gitlab PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' - MAXIMUM_GITALY_CALLS = 35 + MAXIMUM_GITALY_CALLS = 30 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new @@ -52,11 +52,18 @@ module Gitlab klass = stub_class(name) addr = stub_address(storage) creds = stub_creds(storage) - klass.new(addr, creds) + klass.new(addr, creds, interceptors: interceptors) end end end + def self.interceptors + return [] unless Gitlab::Tracing.enabled? + + [Gitlab::Tracing::GRPCInterceptor.instance] + end + private_class_method :interceptors + def self.stub_cert_paths cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"] cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE @@ -126,7 +133,11 @@ module Gitlab end def self.address_metadata(storage) - Base64.strict_encode64(JSON.dump({ storage => { 'address' => address(storage), 'token' => token(storage) } })) + Base64.strict_encode64(JSON.dump(storage => connection_data(storage))) + end + + def self.connection_data(storage) + { 'address' => address(storage), 'token' => token(storage) } end # All Gitaly RPC call sites should use GitalyClient.call. This method @@ -153,8 +164,6 @@ module Gitlab kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend - rescue GRPC::Unavailable => ex - handle_grpc_unavailable!(ex) ensure duration = Gitlab::Metrics::System.monotonic_time - start @@ -167,27 +176,6 @@ module Gitlab add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc) end - def self.handle_grpc_unavailable!(ex) - status = ex.to_status - raise ex unless status.details == 'Endpoint read failed' - - # There is a bug in grpc 1.8.x that causes a client process to get stuck - # always raising '14:Endpoint read failed'. The only thing that we can - # do to recover is to restart the process. - # - # See https://gitlab.com/gitlab-org/gitaly/issues/1029 - - if Sidekiq.server? - raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s) - else - # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request. - Process.kill('QUIT', Process.pid) - end - - raise ex - end - private_class_method :handle_grpc_unavailable! - def self.current_transaction_labels Gitlab::Metrics::Transaction.current&.labels || {} end @@ -240,7 +228,7 @@ module Gitlab result end - SERVER_FEATURE_FLAGS = %w[].freeze + SERVER_FEATURE_FLAGS = %w[go-find-all-tags].freeze def self.server_feature_flags SERVER_FEATURE_FLAGS.map do |f| @@ -256,7 +244,9 @@ module Gitlab end def self.feature_enabled?(feature_name) - Feature.enabled?("gitaly_#{feature_name}") + Feature::FlipperFeature.table_exists? && Feature.enabled?("gitaly_#{feature_name}") + rescue ActiveRecord::NoDatabaseError + false end # Ensures that Gitaly is not being abuse through n+1 misuse etc @@ -396,13 +386,13 @@ module Gitlab # Returns the stacks that calls Gitaly the most times. Used for n+1 detection def self.max_stacks - return nil unless Gitlab::SafeRequestStore.active? + return unless Gitlab::SafeRequestStore.active? stack_counter = Gitlab::SafeRequestStore[:stack_counter] - return nil unless stack_counter + return unless stack_counter max = max_call_count - return nil if max.zero? + return if max.zero? stack_counter.select { |_, v| v == max }.keys end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 39547328210..6b8e58e6199 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -27,7 +27,7 @@ module Gitlab data << msg.data end - return nil if blob.oid.blank? + return if blob.oid.blank? data = data.join diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 4e46cb9f05c..ea12424eb4a 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -150,6 +150,17 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end + def diverging_commit_count(from, to, max_count:) + request = Gitaly::CountDivergingCommitsRequest.new( + repository: @gitaly_repo, + from: encode_binary(from), + to: encode_binary(to), + max_count: max_count + ) + response = GitalyClient.call(@repository.storage, :commit_service, :count_diverging_commits, request, timeout: GitalyClient.medium_timeout) + [response.left_count, response.right_count] + end + def list_last_commits_for_tree(revision, path, offset: 0, limit: 25) request = Gitaly::ListLastCommitsForTreeRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 22d2d149e65..2528208440e 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -62,7 +62,7 @@ module Gitlab end branch = response.branch - return nil unless branch + return unless branch target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit) @@ -100,6 +100,25 @@ module Gitlab end end + def user_merge_to_ref(user, source_sha, branch, target_ref, message) + request = Gitaly::UserMergeToRefRequest.new( + repository: @gitaly_repo, + source_sha: source_sha, + branch: encode_binary(branch), + target_ref: encode_binary(target_ref), + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + message: message + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::PreReceiveError, pre_receive_error + end + + response.commit_id + end + def user_merge_branch(user, source_sha, target_branch, message) request_enum = QueueEnumerator.new response_enum = GitalyClient.call( @@ -258,14 +277,14 @@ module Gitlab end end + # rubocop:disable Metrics/ParameterLists def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository) - + start_branch_name, start_repository, force = false) req_enum = Enumerator.new do |y| header = user_commit_files_request_header(user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository) + start_branch_name, start_repository, force) y.yield Gitaly::UserCommitFilesRequest.new(header: header) @@ -300,6 +319,7 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end + # rubocop:enable Metrics/ParameterLists def user_commit_patches(user, branch_name, patches) header = Gitaly::UserApplyPatchRequest::Header.new( @@ -363,9 +383,10 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end + # rubocop:disable Metrics/ParameterLists def user_commit_files_request_header( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository) + start_branch_name, start_repository, force) Gitaly::UserCommitFilesRequestHeader.new( repository: @gitaly_repo, @@ -375,9 +396,11 @@ module Gitlab commit_author_name: encode_binary(author_name), commit_author_email: encode_binary(author_email), start_branch_name: encode_binary(start_branch_name), - start_repository: start_repository.gitaly_repository + start_repository: start_repository.gitaly_repository, + force: force ) end + # rubocop:enable Metrics/ParameterLists def user_commit_files_action_header(action) Gitaly::UserCommitFilesActionHeader.new( diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 8a1abfbf874..a08bfd0e25b 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -326,11 +326,31 @@ module Gitlab def search_files_by_content(ref, query) request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) - GitalyClient.call(@storage, :repository_service, :search_files_by_content, request).flat_map(&:matches) + response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request) + + search_results_from_response(response) end private + def search_results_from_response(gitaly_response) + matches = [] + current_match = +"" + + gitaly_response.each do |message| + next if message.nil? + + current_match << message.match_data + + if message.end_of_match + matches << current_match + current_match = +"" + end + end + + matches + end + def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout) request = request_class.new(repository: @gitaly_repo) response = GitalyClient.call( diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 754cccb6b3f..78ef6bfc0ec 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -32,11 +32,19 @@ module Gitlab end def self.disk_access_denied? + return false if rugged_enabled? + !temporarily_allowed?(ALLOW_KEY) && GitalyClient.feature_enabled?(DISK_ACCESS_DENIED_FLAG) rescue false # Err on the side of caution, don't break gitlab for people end + def self.rugged_enabled? + Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.any? do |flag| + Feature.enabled?(flag) + end + end + def initialize(storage) raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless storage.has_key?('path') diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index dce5d6a8ad0..899921f76e4 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -4,7 +4,7 @@ module Gitlab module GitalyClient module Util class << self - def repository(repository_storage, relative_path, gl_repository) + def repository(repository_storage, relative_path, gl_repository, gl_project_path) git_env = Gitlab::Git::HookEnv.all(gl_repository) git_object_directory = git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence git_alternate_object_directories = Array.wrap(git_env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE']) @@ -14,14 +14,16 @@ module Gitlab relative_path: relative_path, gl_repository: gl_repository.to_s, git_object_directory: git_object_directory.to_s, - git_alternate_object_directories: git_alternate_object_directories + git_alternate_object_directories: git_alternate_object_directories, + gl_project_path: gl_project_path ) end def git_repository(gitaly_repository) Gitlab::Git::Repository.new(gitaly_repository.storage_name, gitaly_repository.relative_path, - gitaly_repository.gl_repository) + gitaly_repository.gl_repository, + gitaly_repository.gl_project_path) end end end diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb index da2f96b5c4b..147597289cf 100644 --- a/lib/gitlab/github_import/bulk_importing.rb +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -15,12 +15,10 @@ module Gitlab end # Bulk inserts the given rows into the database. - def bulk_insert(model, rows, batch_size: 100, pre_hook: nil) + def bulk_insert(model, rows, batch_size: 100) rows.each_slice(batch_size) do |slice| - pre_hook.call(slice) if pre_hook Gitlab::Database.bulk_insert(model.table_name, slice) end - rows end end end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index 4226eee85cc..656d46b6a7d 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -57,11 +57,7 @@ module Gitlab updated_at: issue.updated_at } - insert_and_return_id(attributes, project.issues).tap do |id| - # We use .insert_and_return_id which effectively disables all callbacks. - # Trigger iid logic here to make sure we track internal id values consistently. - project.issues.find(id).ensure_project_iid! - end + insert_and_return_id(attributes, project.issues) rescue ActiveRecord::InvalidForeignKey # It's possible the project has been deleted since scheduling this # job. In this case we'll just skip creating the issue. diff --git a/lib/gitlab/github_import/importer/lfs_object_importer.rb b/lib/gitlab/github_import/importer/lfs_object_importer.rb index a88c17aaf82..195383fd3e9 100644 --- a/lib/gitlab/github_import/importer/lfs_object_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_object_importer.rb @@ -13,10 +13,12 @@ module Gitlab @project = project end + def lfs_download_object + LfsDownloadObject.new(oid: lfs_object.oid, size: lfs_object.size, link: lfs_object.link) + end + def execute - Projects::LfsPointers::LfsDownloadService - .new(project) - .execute(lfs_object.oid, lfs_object.download_link) + Projects::LfsPointers::LfsDownloadService.new(project, lfs_download_object).execute end end end diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index 8d54b27374c..71ff7465d9b 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -19,20 +19,10 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def execute - # We insert records in bulk, by-passing any standard model callbacks. - # The pre_hook here makes sure we track internal ids consistently. - # Note this has to be called before performing an insert of a batch - # because we're outside a transaction scope here. - bulk_insert(Milestone, build_milestones, pre_hook: method(:track_greatest_iid)) + bulk_insert(Milestone, build_milestones) build_milestones_cache end - def track_greatest_iid(slice) - greatest_iid = slice.max { |e| e[:iid] }[:iid] - - InternalId.track_greatest(nil, { project: project }, :milestones, greatest_iid, ->(_) { project.milestones.maximum(:iid) }) - end - def build_milestones build_database_rows(each_milestone) end @@ -52,6 +42,7 @@ module Gitlab description: milestone.description, project_id: project.id, state: state_for(milestone), + due_date: milestone.due_on&.to_date, created_at: milestone.created_at, updated_at: milestone.updated_at } diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index ae7c4cf1b38..e294173f992 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -67,6 +67,36 @@ module Gitlab def insert_git_data(merge_request, already_exists) insert_or_replace_git_data(merge_request, pull_request.source_branch_sha, pull_request.target_branch_sha, already_exists) + # We need to create the branch after the merge request is + # populated to ensure the merge request is in the right state + # when the branch is created. + create_source_branch_if_not_exists(merge_request) + end + + # An imported merge request will not be mergeable unless the + # source branch exists. For pull requests from forks, the source + # branch will be in the form of + # "github/fork/{project-name}/{source_branch}". This branch will never + # exist, so we create it here. + # + # Note that we only create the branch if the merge request is still open. + # For projects that have many pull requests, we assume that if it's closed + # the branch has already been deleted. + def create_source_branch_if_not_exists(merge_request) + return unless merge_request.open? + + source_branch = pull_request.formatted_source_branch + + return if project.repository.branch_exists?(source_branch) + + project.repository.add_branch(merge_request.author, source_branch, pull_request.source_branch_sha) + rescue Gitlab::Git::CommandError => e + Gitlab::Sentry.track_acceptable_exception(e, + extra: { + source_branch: source_branch, + project_id: merge_request.project.id, + merge_request_id: merge_request.id + }) end end end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index bc3ea9e9226..e2dfb00dcc5 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -6,11 +6,12 @@ module Gitlab class RepositoryImporter include Gitlab::ShellAdapter - attr_reader :project, :client + attr_reader :project, :client, :wiki_formatter def initialize(project, client) @project = project @client = client + @wiki_formatter = ::Gitlab::LegacyGithubImport::WikiFormatter.new(project) end # Returns true if we should import the wiki for the project. @@ -57,9 +58,7 @@ module Gitlab end def import_wiki_repository - wiki_path = "#{project.disk_path}.wiki" - - gitlab_shell.import_repository(project.repository_storage, wiki_path, wiki_url) + gitlab_shell.import_wiki_repository(project, wiki_formatter) true rescue Gitlab::Shell::Error => e @@ -72,7 +71,7 @@ module Gitlab end def wiki_url - project.import_url.sub(/\.git\z/, '.wiki.git') + wiki_formatter.import_url end def update_clone_time diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb index debe0fa0baf..a4606173f49 100644 --- a/lib/gitlab/github_import/representation/lfs_object.rb +++ b/lib/gitlab/github_import/representation/lfs_object.rb @@ -9,11 +9,11 @@ module Gitlab attr_reader :attributes - expose_attribute :oid, :download_link + expose_attribute :oid, :link, :size # Builds a lfs_object def self.from_api_response(lfs_object) - new({ oid: lfs_object[0], download_link: lfs_object[1] }) + new({ oid: lfs_object.oid, link: lfs_object.link, size: lfs_object.size }) end # Builds a new lfs_object using a Hash that was built from a JSON payload. diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index 593b491a837..0ccc4bfaed3 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -76,10 +76,10 @@ module Gitlab # Returns a formatted source branch. # # For cross-project pull requests the branch name will be in the format - # `owner-name:branch-name`. + # `github/fork/owner-name/branch-name`. def formatted_source_branch if cross_project? && source_repository_owner - "#{source_repository_owner}:#{source_branch}" + "github/fork/#{source_repository_owner}/#{source_branch}" elsif source_branch == target_branch # Sometimes the source and target branch are the same, but GitLab # doesn't support this. This can happen when both the user and diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 15137140639..e00309e7946 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -8,10 +8,7 @@ module Gitlab def add_gon_variables gon.api_version = 'v4' - gon.default_avatar_url = - Gitlab::Utils.append_path( - Gitlab.config.gitlab.url, - ActionController::Base.helpers.image_path('no_avatar.png')) + gon.default_avatar_url = default_avatar_url gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size gon.asset_host = ActionController::Base.asset_host gon.webpack_public_path = webpack_public_path @@ -27,6 +24,8 @@ module Gitlab gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites') gon.test_env = Rails.env.test? gon.suggested_label_colors = LabelsHelper.suggested_colors + gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week + gon.ee = Gitlab.ee? if current_user gon.current_user_id = current_user.id @@ -50,5 +49,15 @@ module Gitlab # use this method to push multiple feature flags. gon.push({ features: { var_name => enabled } }, true) end + + def default_avatar_url + # We can't use ActionController::Base.helpers.image_url because it + # doesn't return an actual URL because request is nil for some reason. + # + # We also can't use Gitlab::Utils.append_path because the image path + # may be an absolute URL. + URI.join(Gitlab.config.gitlab.url, + ActionController::Base.helpers.image_path('no_avatar.png')).to_s + end end end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 4fbb87385c3..5ff415b6126 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -88,9 +88,10 @@ module Gitlab def create_cached_signature! using_keychain do |gpg_key| - signature = GpgSignature.new(attributes(gpg_key)) - signature.save! unless Gitlab::Database.read_only? - signature + attributes = attributes(gpg_key) + break GpgSignature.new(attributes) if Gitlab::Database.read_only? + + GpgSignature.safe_create!(attributes) end end diff --git a/lib/gitlab/graphql/authorize.rb b/lib/gitlab/graphql/authorize.rb index 5e48bf9043d..f62813db82c 100644 --- a/lib/gitlab/graphql/authorize.rb +++ b/lib/gitlab/graphql/authorize.rb @@ -10,21 +10,6 @@ module Gitlab def self.use(schema_definition) schema_definition.instrument(:field, Instrumentation.new) end - - def required_permissions - # If the `#authorize` call is used on multiple classes, we add the - # permissions specified on a subclass, to the ones that were specified - # on it's superclass. - @required_permissions ||= if self.respond_to?(:superclass) && superclass.respond_to?(:required_permissions) - superclass.required_permissions.dup - else - [] - end - end - - def authorize(*permissions) - required_permissions.concat(permissions) - end end end end diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index a56c4f6368d..b367a97105c 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -6,8 +6,21 @@ module Gitlab module AuthorizeResource extend ActiveSupport::Concern - included do - extend Gitlab::Graphql::Authorize + class_methods do + def required_permissions + # If the `#authorize` call is used on multiple classes, we add the + # permissions specified on a subclass, to the ones that were specified + # on it's superclass. + @required_permissions ||= if self.respond_to?(:superclass) && superclass.respond_to?(:required_permissions) + superclass.required_permissions.dup + else + [] + end + end + + def authorize(*permissions) + required_permissions.concat(permissions) + end end def find_object(*args) diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb index d638d2b43ee..593da8471dd 100644 --- a/lib/gitlab/graphql/authorize/instrumentation.rb +++ b/lib/gitlab/graphql/authorize/instrumentation.rb @@ -6,19 +6,15 @@ module Gitlab class Instrumentation # Replace the resolver for the field with one that will only return the # resolved object if the permissions check is successful. - # - # Collections are not supported. Apply permissions checks for those at the - # database level instead, to avoid loading superfluous data from the DB def instrument(_type, field) - field_definition = field.metadata[:type_class] - return field unless field_definition.respond_to?(:required_permissions) - return field if field_definition.required_permissions.empty? + required_permissions = Array.wrap(field.metadata[:authorize]) + return field if required_permissions.empty? old_resolver = field.resolve_proc new_resolver = -> (obj, args, ctx) do resolved_obj = old_resolver.call(obj, args, ctx) - checker = build_checker(ctx[:current_user], field_definition.required_permissions) + checker = build_checker(ctx[:current_user], required_permissions) if resolved_obj.respond_to?(:then) resolved_obj.then(&checker) @@ -35,10 +31,22 @@ module Gitlab private def build_checker(current_user, abilities) - proc do |obj| + lambda do |value| # Load the elements if they weren't loaded by BatchLoader yet - obj = obj.sync if obj.respond_to?(:sync) - obj if abilities.all? { |ability| Ability.allowed?(current_user, ability, obj) } + value = value.sync if value.respond_to?(:sync) + + check = lambda do |object| + abilities.all? do |ability| + Ability.allowed?(current_user, ability, object) + end + end + + case value + when Array + value.select(&check) + else + value if check.call(value) + end end end end diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index 1f29cf10cad..7046b4e2a43 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -11,21 +11,29 @@ module Gitlab # Schedule a range of projects to be bulk migrated with #bulk_migrate asynchronously # - # @param [Object] start first project id for the range - # @param [Object] finish last project id for the range - def bulk_schedule(start, finish) - StorageMigratorWorker.perform_async(start, finish) + # @param [Integer] start first project id for the range + # @param [Integer] finish last project id for the range + def bulk_schedule_migration(start:, finish:) + ::HashedStorage::MigratorWorker.perform_async(start, finish) + end + + # Schedule a range of projects to be bulk rolledback with #bulk_rollback asynchronously + # + # @param [Integer] start first project id for the range + # @param [Integer] finish last project id for the range + def bulk_schedule_rollback(start:, finish:) + ::HashedStorage::RollbackerWorker.perform_async(start, finish) end # Start migration of projects from specified range # - # Flagging a project to be migrated is a synchronous action, + # Flagging a project to be migrated is a synchronous action # but the migration runs through async jobs # - # @param [Object] start first project id for the range - # @param [Object] finish last project id for the range + # @param [Integer] start first project id for the range + # @param [Integer] finish last project id for the range # rubocop: disable CodeReuse/ActiveRecord - def bulk_migrate(start, finish) + def bulk_migrate(start:, finish:) projects = build_relation(start, finish) projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| @@ -34,9 +42,26 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # Flag a project to be migrated + # Start rollback of projects from specified range + # + # Flagging a project to be rolled back is a synchronous action + # but the rollback runs through async jobs + # + # @param [Integer] start first project id for the range + # @param [Integer] finish last project id for the range + # rubocop: disable CodeReuse/ActiveRecord + def bulk_rollback(start:, finish:) + projects = build_relation(start, finish) + + projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| + rollback(project) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # Flag a project to be migrated to Hashed Storage # - # @param [Object] project that will be migrated + # @param [Project] project that will be migrated def migrate(project) Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." @@ -45,6 +70,17 @@ module Gitlab Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end + # Flag a project to be rolled-back to Legacy Storage + # + # @param [Project] project that will be rolled-back + def rollback(project) + Rails.logger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..." + + project.rollback_to_legacy_storage! + rescue => err + Rails.logger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") + end + private # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 38f552fab03..87a31a37e3f 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -24,7 +24,7 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def self.project_id_batches(&block) + def self.project_id_batches_migration(&block) Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches ids = relation.pluck(:id) @@ -34,6 +34,16 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord + def self.project_id_batches_rollback(&block) + Project.with_storage_feature(:repository).in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches + ids = relation.pluck(:id) + + yield ids.min, ids.max + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord def self.legacy_attachments_relation Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) JOIN projects diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index a4e60bbd828..381f1dd4e55 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -39,7 +39,7 @@ module Gitlab private def custom_language - return nil unless @language + return unless @language Rouge::Lexer.find_fancy(@language) end diff --git a/lib/gitlab/i18n/metadata_entry.rb b/lib/gitlab/i18n/metadata_entry.rb index 3764e379681..4facd10bfc8 100644 --- a/lib/gitlab/i18n/metadata_entry.rb +++ b/lib/gitlab/i18n/metadata_entry.rb @@ -15,7 +15,7 @@ module Gitlab end def expected_forms - return nil unless plural_information + return unless plural_information plural_information['nplurals'].to_i end diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb index 9215067d973..fa3ff6c3f12 100644 --- a/lib/gitlab/import/merge_request_helpers.rb +++ b/lib/gitlab/import/merge_request_helpers.rb @@ -24,10 +24,6 @@ module Gitlab merge_request = project.merge_requests.reload.find(merge_request_id) - # We use .insert_and_return_id which effectively disables all callbacks. - # Trigger iid logic here to make sure we track internal id values consistently. - merge_request.ensure_target_project_iid! - [merge_request, false] end rescue ActiveRecord::InvalidForeignKey diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 3cd8ede830c..fa54fc17d95 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -28,7 +28,7 @@ project_tree: - notes: :author - releases: - :author + - :links - project_members: - :user - merge_requests: @@ -63,7 +63,6 @@ project_tree: - :triggers - :pipeline_schedules - :services - - :hooks - protected_branches: - :merge_access_levels - :push_access_levels @@ -74,6 +73,7 @@ project_tree: - :prometheus_metrics - :project_badges - :ci_cd_settings + - :error_tracking_setting # Only include the following attributes for the models specified. included_attributes: @@ -129,9 +129,13 @@ excluded_attributes: snippets: - :expired_at merge_request_diff: - - :st_diffs + - :external_diff + - :stored_externally + - :external_diff_store merge_request_diff_files: - :diff + - :external_diff_offset + - :external_diff_size issues: - :milestone_id merge_requests: @@ -148,6 +152,7 @@ excluded_attributes: - :when - :artifacts_file - :artifacts_metadata + - :commands push_event_payload: - :event_id project_badges: @@ -156,17 +161,15 @@ excluded_attributes: - :reference - :reference_html - :epic_id - hooks: - - :token - - :encrypted_token - - :encrypted_token_iv - - :encrypted_url - - :encrypted_url_iv runners: - :token - :token_encrypted services: - :template + error_tracking_setting: + - :encrypted_token + - :encrypted_token_iv + - :enabled methods: labels: diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index 477499e1688..b145f37c052 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -67,7 +67,7 @@ module Gitlab # +value+ existing model to be included in the hash # +parsed_hash+ the original hash def parse_hash(value) - return nil if already_contains_methods?(value) + return if already_contains_methods?(value) @attributes_finder.parse(value) do |hash| { include: hash_or_merge(value, hash) } diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 040a70d6775..deb2f59f05f 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -20,6 +20,17 @@ module Gitlab create_target_branch unless branch_exists?(@merge_request.target_branch) end + # The merge_request_diff associated with the current @merge_request might + # be invalid. Than means, when the @merge_request object is saved, the + # @merge_request.merge_request_diff won't. This can leave the merge request + # in an invalid state, because a merge request must have an associated + # merge request diff. + # In this change, if the associated merge request diff is invalid, we set + # it to nil. This change, in association with the after callback + # :ensure_merge_request_diff in the MergeRequest class, makes that + # when the merge request is going to be created and it doesn't have + # one, a default one will be generated. + @merge_request.merge_request_diff = nil unless @merge_request.merge_request_diff&.valid? @merge_request end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index a56ec65b9f1..51001750a6c 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -107,7 +107,7 @@ module Gitlab def project_params @project_params ||= begin - attrs = json_params.merge(override_params) + attrs = json_params.merge(override_params).merge(visibility_level) # Cleaning all imported and overridden params Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: attrs, @@ -127,6 +127,13 @@ module Gitlab end end + def visibility_level + level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level + level = @project.group.visibility_level if @project.group && level > @project.group.visibility_level + + { 'visibility_level' => level } + end + # Given a relation hash containing one or more models and its relationships, # loops through each model and each object from a model type and # and assigns its correspondent attributes hash from +tree_hash+ diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index a4902e2104f..61a1aa6da5a 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -23,7 +23,9 @@ module Gitlab custom_attributes: 'ProjectCustomAttribute', project_badges: 'Badge', metrics: 'MergeRequest::Metrics', - ci_cd_settings: 'ProjectCiCdSetting' }.freeze + ci_cd_settings: 'ProjectCiCdSetting', + error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', + links: 'Releases::Link' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze @@ -73,7 +75,7 @@ module Gitlab # the relation_hash, updating references with new object IDs, mapping users using # the "members_mapper" object, also updating notes if required. def create - return nil if unknown_service? + return if unknown_service? setup_models @@ -149,6 +151,7 @@ module Gitlab if BUILD_MODELS.include?(@relation_name) @relation_hash.delete('trace') # old export files have trace @relation_hash.delete('token') + @relation_hash.delete('commands') imported_object elsif @relation_name == :merge_requests diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index c13e6c1d83b..725c1101d70 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -8,6 +8,7 @@ module Gitlab def initialize(project) @project = project @errors = [] + @logger = Gitlab::Import::Logger.build end def active_export_count @@ -23,19 +24,16 @@ module Gitlab end def error(error) - error_out(error.message, caller[0].dup) - add_error_message(error.message) + log_error(message: error.message, caller: caller[0].dup) + log_debug(backtrace: error.backtrace&.join("\n")) + + Gitlab::Sentry.track_acceptable_exception(error, extra: log_base_data) - # Debug: - if error.backtrace - Rails.logger.error("Import/Export backtrace: #{error.backtrace.join("\n")}") - else - Rails.logger.error("No backtrace found") - end + add_error_message(error.message) end - def add_error_message(error_message) - @errors << error_message + def add_error_message(message) + @errors << filtered_error_message(message) end def after_export_in_progress? @@ -52,8 +50,25 @@ module Gitlab @project.disk_path end - def error_out(message, caller) - Rails.logger.error("Import/Export error raised on #{caller}: #{message}") + def log_error(details) + @logger.error(log_base_data.merge(details)) + end + + def log_debug(details) + @logger.debug(log_base_data.merge(details)) + end + + def log_base_data + { + importer: 'Import/Export', + import_jid: @project&.import_state&.jid, + project_id: @project&.id, + project_path: @project&.full_path + } + end + + def filtered_error_message(message) + Projects::ImportErrorFilter.filter_message(message) end def after_export_lock_file diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 474e9d45566..e232198150a 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -40,7 +40,7 @@ module Gitlab def add_upload(upload) uploader_context = FileUploader.extract_dynamic_path(upload).named_captures.symbolize_keys - UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute + UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute.to_h end def copy_project_uploads diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 20fc8226611..8b346f6d7d2 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -2,8 +2,9 @@ module Gitlab module IncomingEmail - UNSUBSCRIBE_SUFFIX = '+unsubscribe'.freeze - WILDCARD_PLACEHOLDER = '%{key}'.freeze + UNSUBSCRIBE_SUFFIX = '-unsubscribe'.freeze + UNSUBSCRIBE_SUFFIX_LEGACY = '+unsubscribe'.freeze + WILDCARD_PLACEHOLDER = '%{key}'.freeze class << self def enabled? @@ -22,6 +23,7 @@ module Gitlab config.address.sub(WILDCARD_PLACEHOLDER, key) end + # example: incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com def unsubscribe_address(key) config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") end @@ -55,7 +57,7 @@ module Gitlab def address_regex wildcard_address = config.address - return nil unless wildcard_address + return unless wildcard_address regex = Regexp.escape(wildcard_address) regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index 1adf83739ad..24daad638f4 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -71,7 +71,21 @@ module Gitlab end def parse_entry(raw, klass) - klass.new(raw) if valid_entry?(raw, klass) + return unless valid_entry?(raw, klass) + return klass.new(raw) unless klass.ancestors.include?(ActiveRecord::Base) + + # When the cached value is a persisted instance of ActiveRecord::Base in + # some cases a relation can return an empty collection becauses scope.none! + # is being applied on ActiveRecord::Associations::CollectionAssociation#scope + # when the new_record? method incorrectly returns false. + # + # See https://gitlab.com/gitlab-org/gitlab-ee/issues/9903#note_145329964 + attributes = klass.attributes_builder.build_from_database(raw, {}) + klass.allocate.init_with("attributes" => attributes, "new_record" => new_record?(raw, klass)) + end + + def new_record?(raw, klass) + raw.fetch(klass.primary_key, nil).blank? end def valid_entry?(raw, klass) diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 03d38ec78fd..42c4745ff98 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -3,8 +3,8 @@ module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.11.0'.freeze - KUBECTL_VERSION = '1.11.0'.freeze + HELM_VERSION = '2.12.3'.freeze + KUBECTL_VERSION = '1.11.7'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze SERVICE_ACCOUNT = 'tiller'.freeze CLUSTER_ROLE_BINDING = 'tiller-admin'.freeze diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index b9903e37f40..7dfd9ed4f35 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -20,14 +20,7 @@ module Gitlab kubeclient.create_pod(command.pod_resource) end - def update(command) - namespace.ensure_exists! - - update_config_map(command) - - delete_pod!(command.pod_name) - kubeclient.create_pod(command.pod_resource) - end + alias_method :update, :install ## # Returns Pod phase @@ -62,6 +55,8 @@ module Gitlab def create_config_map(command) command.config_map_resource.tap do |config_map_resource| + break unless config_map_resource + if config_map_exists?(config_map_resource) kubeclient.update_config_map(config_map_resource) else diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index a1ab5e048ac..f931248b747 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -42,8 +42,17 @@ module Gitlab 'helm repo update' if repository end + # Uses `helm upgrade --install` which means we can use this for both + # installation and uprade of applications def install_command - command = ['helm', 'install', chart] + install_command_flags + command = ['helm', 'upgrade', name, chart] + + install_flag + + reset_values_flag + + optional_tls_flags + + optional_version_flag + + rbac_create_flag + + namespace_flag + + value_flag command.shelljoin end @@ -56,17 +65,20 @@ module Gitlab postinstall.join("\n") if postinstall end - def install_command_flags - name_flag = ['--name', name] - namespace_flag = ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] - value_flag = ['-f', "/data/helm/#{name}/config/values.yaml"] + def install_flag + ['--install'] + end - name_flag + - optional_tls_flags + - optional_version_flag + - rbac_create_flag + - namespace_flag + - value_flag + def reset_values_flag + ['--reset-values'] + end + + def value_flag + ['-f', "/data/helm/#{name}/config/values.yaml"] + end + + def namespace_flag + ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] end def rbac_create_flag diff --git a/lib/gitlab/kubernetes/helm/upgrade_command.rb b/lib/gitlab/kubernetes/helm/upgrade_command.rb deleted file mode 100644 index 9daffc138b5..00000000000 --- a/lib/gitlab/kubernetes/helm/upgrade_command.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class UpgradeCommand - include BaseCommand - include ClientCommand - - attr_reader :name, :chart, :version, :repository, :files - - def initialize(name, chart:, files:, rbac:, version: nil, repository: nil) - @name = name - @chart = chart - @rbac = rbac - @version = version - @files = files - @repository = repository - end - - def generate_script - super + [ - init_command, - wait_for_tiller_command, - repository_command, - script_command - ].compact.join("\n") - end - - def rbac? - @rbac - end - - def pod_name - "upgrade-#{name}" - end - - private - - def script_command - upgrade_flags = "#{optional_version_flag}#{optional_tls_flags}" \ - " --reset-values" \ - " --install" \ - " --namespace #{::Gitlab::Kubernetes::Helm::NAMESPACE}" \ - " -f /data/helm/#{name}/config/values.yaml" - - "helm upgrade #{name} #{chart}#{upgrade_flags}" - end - - def optional_version_flag - " --version #{version}" if version - end - - def optional_tls_flags - return unless files.key?(:'ca.pem') - - " --tls" \ - " --tls-ca-cert #{files_dir}/ca.pem" \ - " --tls-cert #{files_dir}/cert.pem" \ - " --tls-key #{files_dir}/key.pem" - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index fe839940f74..de14df56555 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -76,9 +76,14 @@ module Gitlab attr_reader :api_prefix, :kubeclient_options + # We disable redirects through 'http_max_redirects: 0', + # so that KubeClient does not follow redirects and + # expose internal services. def initialize(api_prefix, **kubeclient_options) @api_prefix = api_prefix - @kubeclient_options = kubeclient_options + @kubeclient_options = kubeclient_options.merge(http_max_redirects: 0) + + validate_url! end def create_or_update_cluster_role_binding(resource) @@ -115,6 +120,12 @@ module Gitlab private + def validate_url! + return if Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? + + Gitlab::UrlBlocker.validate!(api_prefix, allow_local_network: false) + end + def cluster_role_binding_exists?(resource) get_cluster_role_binding(resource.metadata.name) rescue ::Kubeclient::ResourceNotFoundError diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index bc952147667..bbdd094e33b 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -68,7 +68,7 @@ module Gitlab end def user(login) - return nil unless login.present? + return unless login.present? return @users[login] if @users.key?(login) @users[login] = api.user(login) diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index c526d31a591..f3323c98af2 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -267,7 +267,7 @@ module Gitlab def import_wiki unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(project.repository_storage, wiki.disk_path, wiki.import_url) + gitlab_shell.import_wiki_repository(project, wiki) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index ec0e221b1ff..889e6aaa968 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -25,7 +25,7 @@ module Gitlab end def find_by_email - return nil unless email + return unless email User.find_by_any_email(email) .try(:id) @@ -33,7 +33,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def find_by_external_uid - return nil unless id + return unless id identities = ::Identity.arel_table diff --git a/lib/gitlab/legacy_github_import/wiki_formatter.rb b/lib/gitlab/legacy_github_import/wiki_formatter.rb index ea52be5ee0f..cf1e21ad1e1 100644 --- a/lib/gitlab/legacy_github_import/wiki_formatter.rb +++ b/lib/gitlab/legacy_github_import/wiki_formatter.rb @@ -13,6 +13,10 @@ module Gitlab project.wiki.disk_path end + def full_path + project.wiki.full_path + end + def import_url project.import_url.sub(/\.git\z/, ".wiki.git") end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index c09d3ebc7be..31e6fc9d8c7 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -30,8 +30,8 @@ module Gitlab end end - def token(expire_time: DEFAULT_EXPIRE_TIME) - HMACToken.new(actor).token(expire_time) + def token + HMACToken.new(actor).token(DEFAULT_EXPIRE_TIME) end def token_valid?(token_to_check) @@ -47,7 +47,16 @@ module Gitlab user? ? :lfs_token : :lfs_deploy_token end - private # rubocop:disable Lint/UselessAccessModifier + def authentication_payload(repository_http_path) + { + username: actor_name, + lfs_token: token, + repository_http_path: repository_http_path, + expires_in: DEFAULT_EXPIRE_TIME + } + end + + private # rubocop:disable Lint/UselessAccessModifier class HMACToken include LfsTokenHelper @@ -100,7 +109,7 @@ module Gitlab # class LegacyRedisDeviseToken TOKEN_LENGTH = 50 - DEFAULT_EXPIRY_TIME = 1800 * 1000 # 30 mins + DEFAULT_EXPIRY_TIME = 1800 * 1000 # 30 mins def initialize(actor) @actor = actor diff --git a/lib/gitlab/loop_helpers.rb b/lib/gitlab/loop_helpers.rb new file mode 100644 index 00000000000..3873156a3b0 --- /dev/null +++ b/lib/gitlab/loop_helpers.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module LoopHelpers + ## + # This helper method repeats the same task until it's expired. + # + # Note: ExpiredLoopError does not happen until the given block finished. + # Please do not use this method for heavy or asynchronous operations. + def loop_until(timeout: nil, limit: 1_000_000) + raise ArgumentError unless limit + + start = Time.now + + limit.times do + return true unless yield + + return false if timeout && (Time.now - start) > timeout + end + + false + end + end +end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 1359e973590..0b04340fbb5 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -147,9 +147,7 @@ module Gitlab # # See `Gitlab::Metrics::Transaction#add_event` for more details. def add_event(*args) - trans = current_transaction - - trans&.add_event(*args) + current_transaction&.add_event(*args) end # Returns the prefix to use for the name of a series. diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 651e241362c..ff3fffe7b95 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -19,7 +19,7 @@ module Gitlab # Returns the name of the series to use for storing method calls. def self.series - @series ||= "#{Metrics.series_prefix}method_calls" + @series ||= "#{::Gitlab::Metrics.series_prefix}method_calls" end # Instruments a class method. @@ -118,7 +118,7 @@ module Gitlab # mod - The module containing the method. # name - The name of the method to instrument. def self.instrument(type, mod, name) - return unless Metrics.enabled? + return unless ::Gitlab::Metrics.enabled? name = name.to_sym target = type == :instance ? mod : mod.singleton_class diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 85438011cb9..d0c63a862c2 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -65,7 +65,7 @@ module Gitlab # Returns true if the total runtime of this method exceeds the method call # threshold. def above_threshold? - real_time.in_milliseconds >= Metrics.method_call_threshold + real_time.in_milliseconds >= ::Gitlab::Metrics.method_call_threshold end end end diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index 447d03bfca4..cee601ff14c 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -58,11 +58,11 @@ module Gitlab def build_metric!(type, name, options) case type when :gauge - Gitlab::Metrics.gauge(name, options.docstring, options.base_labels, options.multiprocess_mode) + ::Gitlab::Metrics.gauge(name, options.docstring, options.base_labels, options.multiprocess_mode) when :counter - Gitlab::Metrics.counter(name, options.docstring, options.base_labels) + ::Gitlab::Metrics.counter(name, options.docstring, options.base_labels) when :histogram - Gitlab::Metrics.histogram(name, options.docstring, options.base_labels, options.buckets) + ::Gitlab::Metrics.histogram(name, options.docstring, options.base_labels, options.buckets) when :summary raise NotImplementedError, "summary metrics are not currently supported" else diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 74c956ab5af..26aa0910047 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -8,15 +8,15 @@ module Gitlab end def self.http_request_total - @http_request_total ||= Gitlab::Metrics.counter(:http_requests_total, 'Request count') + @http_request_total ||= ::Gitlab::Metrics.counter(:http_requests_total, 'Request count') end def self.rack_uncaught_errors_count - @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') + @rack_uncaught_errors_count ||= ::Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') end def self.http_request_duration_seconds - @http_request_duration_seconds ||= Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', + @http_request_duration_seconds ||= ::Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) end diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index c4c38b23a55..5138b37f83e 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -10,7 +10,7 @@ module Gitlab # statistics, etc. class InfluxSampler < BaseSampler # interval - The sampling interval in seconds. - def initialize(interval = Metrics.settings[:sample_interval]) + def initialize(interval = ::Gitlab::Metrics.settings[:sample_interval]) super(interval) @last_step = nil @@ -32,7 +32,7 @@ module Gitlab end def flush - Metrics.submit_metrics(@metrics.map(&:to_hash)) + ::Gitlab::Metrics.submit_metrics(@metrics.map(&:to_hash)) end def sample_memory_usage diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 232a58a7d69..18a69321905 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -24,14 +24,14 @@ module Gitlab def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) - metrics[:total_time] = Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + metrics[:sampler_duration] = ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) + metrics[:total_time] = ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) GC.stat.keys.each do |key| - metrics[key] = Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) + metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) - metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) + metrics[:memory_usage] = ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) + metrics[:file_descriptors] = ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) metrics end diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 4c4ec026823..bec64e864b3 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -9,11 +9,11 @@ module Gitlab end def unicorn_active_connections - @unicorn_active_connections ||= Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) + @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) end def unicorn_queued_connections - @unicorn_queued_connections ||= Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) end def enabled? @@ -23,13 +23,13 @@ module Gitlab def sample Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - unicorn_active_connections.set({ type: 'tcp', address: addr }, stats.active) - unicorn_queued_connections.set({ type: 'tcp', address: addr }, stats.queued) + unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active) + unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued) end Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - unicorn_active_connections.set({ type: 'unix', address: addr }, stats.active) - unicorn_queued_connections.set({ type: 'unix', address: addr }, stats.queued) + unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active) + unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued) end end diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb index 56e106b9612..71a5406815f 100644 --- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -9,7 +9,7 @@ module Gitlab LOG_FILENAME = File.join(Rails.root, 'log', 'sidekiq_exporter.log') def enabled? - Gitlab::Metrics.metrics_folder_present? && settings.enabled + ::Gitlab::Metrics.metrics_folder_present? && settings.enabled end def settings diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index f633e1a9d7c..01db507761b 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -64,7 +64,7 @@ module Gitlab end def metric_cache_operation_duration_seconds - @metric_cache_operation_duration_seconds ||= Gitlab::Metrics.histogram( + @metric_cache_operation_duration_seconds ||= ::Gitlab::Metrics.histogram( :gitlab_cache_operation_duration_seconds, 'Cache access time', Transaction::BASE_LABELS.merge({ action: nil }), @@ -73,7 +73,7 @@ module Gitlab end def metric_cache_misses_total - @metric_cache_misses_total ||= Gitlab::Metrics.counter( + @metric_cache_misses_total ||= ::Gitlab::Metrics.counter( :gitlab_cache_misses_total, 'Cache read miss', Transaction::BASE_LABELS diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 468d7cb56fc..e91803ecd62 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -64,7 +64,7 @@ module Gitlab end def add_metric(series, values, tags = {}) - @metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags) + @metrics << Metric.new("#{::Gitlab::Metrics.series_prefix}#{series}", values, tags) end # Tracks a business level event @@ -127,7 +127,7 @@ module Gitlab hash end - Metrics.submit_metrics(submit_hashes) + ::Gitlab::Metrics.submit_metrics(submit_hashes) end def labels diff --git a/lib/gitlab/middleware/basic_health_check.rb b/lib/gitlab/middleware/basic_health_check.rb index f2a03217098..acf8c301b8f 100644 --- a/lib/gitlab/middleware/basic_health_check.rb +++ b/lib/gitlab/middleware/basic_health_check.rb @@ -24,7 +24,7 @@ module Gitlab def call(env) return @app.call(env) unless env['PATH_INFO'] == HEALTH_PATH - request = Rack::Request.new(env) + request = ActionDispatch::Request.new(env) return OK_RESPONSE if client_ip_whitelisted?(request) diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index d1a87c3b3bb..f9efef38825 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -6,6 +6,7 @@ module Gitlab module Middleware class Go include ActionView::Helpers::TagHelper + include ActionController::HttpAuthentication::Basic PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze @@ -14,7 +15,7 @@ module Gitlab end def call(env) - request = Rack::Request.new(env) + request = ActionDispatch::Request.new(env) render_go_doc(request) || @app.call(env) end @@ -110,21 +111,23 @@ module Gitlab def project_for_paths(paths, request) project = Project.where_full_path_in(paths).first - return unless Ability.allowed?(current_user(request), :read_project, project) + return unless Ability.allowed?(current_user(request, project), :read_project, project) project end - def current_user(request) - authenticator = Gitlab::Auth::RequestAuthenticator.new(request) - user = authenticator.find_user_from_access_token || authenticator.find_user_from_warden + def current_user(request, project) + return unless has_basic_credentials?(request) - return unless user&.can?(:access_api) + login, password = user_name_and_password(request) + auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, ip: request.ip) + return unless auth_result.success? - # Right now, the `api` scope is the only one that should be able to determine private project existence. - return unless authenticator.valid_access_token?(scopes: [:api]) + return unless auth_result.actor&.can?(:access_git) - user + return unless auth_result.authentication_abilities.include?(:read_project) + + auth_result.actor end end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 84c2f0d5720..433151b80e7 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -32,7 +32,7 @@ module Gitlab class Handler def initialize(env, message) - @request = Rack::Request.new(env) + @request = ActionDispatch::Request.new(env) @rewritten_fields = message['rewritten_fields'] @open_files = [] end diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index 96c6a0a7d28..a147e165262 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -7,6 +7,8 @@ module Gitlab module Middleware class RailsQueueDuration + GITLAB_RAILS_QUEUE_DURATION_KEY = 'GITLAB_RAILS_QUEUE_DURATION' + def initialize(app) @app = app end @@ -19,6 +21,7 @@ module Gitlab duration = Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000 trans.set(:rails_queue_duration, duration) metric_rails_queue_duration_seconds.observe(trans.labels, duration / 1_000) + env[GITLAB_RAILS_QUEUE_DURATION_KEY] = duration.round(2) end @app.call(env) diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 89941a9efa0..817db12ac55 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -60,7 +60,7 @@ module Gitlab end def request - @env['rack.request'] ||= Rack::Request.new(@env) + @env['actionpack.request'] ||= ActionDispatch::Request.new(@env) end def last_visited_url @@ -71,12 +71,16 @@ module Gitlab @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} end + def relative_url + File.join('', Gitlab.config.gitlab.relative_url_root).chomp('/') + end + # Overridden in EE module def whitelisted_routes - grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route + grack_route? || internal_route? || lfs_route? || sidekiq_route? end - def grack_route + def grack_route? # Calling route_hash may be expensive. Only do it if we think there's a possible match return false unless request.path.end_with?('.git/git-upload-pack', '.git/git-receive-pack') @@ -84,7 +88,11 @@ module Gitlab WHITELISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end - def lfs_route + def internal_route? + ReadOnly.internal_routes.any? { |path| request.path.include?(path) } + end + + def lfs_route? # Calling route_hash may be expensive. Only do it if we think there's a possible match unless request.path.end_with?('/info/lfs/objects/batch', '/info/lfs/locks', '/info/lfs/locks/verify') || @@ -95,8 +103,8 @@ module Gitlab WHITELISTED_GIT_LFS_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end - def sidekiq_route - request.path.start_with?('/admin/sidekiq') + def sidekiq_route? + request.path.start_with?("#{relative_url}/admin/sidekiq") end end end diff --git a/lib/gitlab/pages_client.rb b/lib/gitlab/pages_client.rb index 3626e53f84c..d74fdba2241 100644 --- a/lib/gitlab/pages_client.rb +++ b/lib/gitlab/pages_client.rb @@ -103,7 +103,7 @@ module Gitlab end def write_token(new_token) - Tempfile.open(File.basename(token_path), File.dirname(token_path), encoding: 'ascii-8bit') do |f| + Tempfile.open(File.basename(token_path), File.dirname(token_path), encoding: 'ascii-8bit') do |f| f.write(new_token) f.close File.link(f.path, token_path) diff --git a/lib/gitlab/patch/sprockets_base_file_digest_key.rb b/lib/gitlab/patch/sprockets_base_file_digest_key.rb new file mode 100644 index 00000000000..1c472638145 --- /dev/null +++ b/lib/gitlab/patch/sprockets_base_file_digest_key.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# This monkey patch prevent cache ballooning when caching tmp/cache/assets/sprockets +# on the CI. See https://github.com/rails/sprockets/issues/563 and +# https://github.com/rails/sprockets/compare/3.x...jmreid:no-mtime-for-digest-key. +module Gitlab + module Patch + module SprocketsBaseFileDigestKey + def file_digest(path) + if stat = self.stat(path) + digest = self.stat_digest(path, stat) + integrity_uri = self.integrity_uri(digest) + + key = Sprockets::UnloadedAsset.new(path, self).file_digest_key(integrity_uri) + cache.fetch(key) do + digest + end + end + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index fa68dead80b..3c888be0710 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -125,7 +125,8 @@ module Gitlab # allow non-regex validations, etc), `NAMESPACE_FORMAT_REGEX_JS` serves as a Javascript-compatible version of # `NAMESPACE_FORMAT_REGEX`, with the negative lookbehind assertion removed. This means that the client-side validation # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. - PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze + PATH_START_CHAR = '[a-zA-Z0-9_\.]'.freeze + PATH_REGEX_STR = PATH_START_CHAR + '[a-zA-Z0-9_\-\.]*'.freeze NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 3bfd6ee892c..9b6ff602fcd 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -2,14 +2,12 @@ module Gitlab class ProjectTemplate - attr_reader :title, :name, :description, :preview + attr_reader :title, :name, :description, :preview, :logo - def initialize(name, title, description, preview) - @name, @title, @description, @preview = name, title, description, preview + def initialize(name, title, description, preview, logo = 'illustrations/gitlab_logo.svg') + @name, @title, @description, @preview, @logo = name, title, description, preview, logo end - alias_method :logo, :name - def file archive_path.open end @@ -27,9 +25,21 @@ module Gitlab end TEMPLATES_TABLE = [ - ProjectTemplate.new('rails', 'Ruby on Rails', 'Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/rails'), - ProjectTemplate.new('spring', 'Spring', 'Includes an MVC structure, mvnw and pom.xml to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/spring'), - ProjectTemplate.new('express', 'NodeJS Express', 'Includes an MVC structure to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/express') + ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'), + ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'), + ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'), + ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/logos/dotnet.svg'), + ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro'), + ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo'), + ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll'), + ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'), + ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook'), + ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo'), + ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfplainhtml', 'Netlify/Plain HTML', _('A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfplain-html', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfgitbook', 'Netlify/GitBook', _('A GitBook site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfgitbook', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg') ].freeze class << self diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 8f30cdee232..394556e8708 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -10,9 +10,15 @@ module Gitlab validates :name, :priority, :metrics, presence: true def self.common_metrics - ::PrometheusMetric.common.group_by(&:group_title).map do |name, metrics| - MetricGroup.new(name: name, priority: 0, metrics: metrics.map(&:to_query_metric)) + all_groups = ::PrometheusMetric.common.group_by(&:group_title).map do |name, metrics| + MetricGroup.new( + name: name, + priority: metrics.map(&:priority).max, + metrics: metrics.map(&:to_query_metric) + ) end + + all_groups.sort_by(&:priority).reverse end # EE only diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 259345b8a9a..e7bfcb16582 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -48,6 +48,8 @@ module Gitlab def execute(context, arg) return if noop? || !available?(context) + count_commands_executed_in(context) + execute_block(action_block, context, arg) end @@ -73,6 +75,13 @@ module Gitlab private + def count_commands_executed_in(context) + return unless context.respond_to?(:commands_executed_count=) + + context.commands_executed_count ||= 0 + context.commands_executed_count += 1 + end + def execute_block(block, context, arg) if arg.present? parsed = parse_params(arg, context) diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb index 6b0808f5304..56007574b1b 100644 --- a/lib/gitlab/repository_cache.rb +++ b/lib/gitlab/repository_cache.rb @@ -7,13 +7,13 @@ module Gitlab def initialize(repository, extra_namespace: nil, backend: Rails.cache) @repository = repository - @namespace = "project:#{repository.project.id}" + @namespace = "#{repository.full_path}:#{repository.project.id}" @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace @backend = backend end def cache_key(type) - "#{namespace}:#{type}" + "#{type}:#{namespace}" end def expire(key) diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index f8f8ec789ce..d9811e036d3 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -13,7 +13,7 @@ module Gitlab end def call(env) - req = Rack::Request.new(env) + req = ActionDispatch::Request.new(env) Gitlab::SafeRequestStore[:client_ip] = req.ip diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 84a51773276..8e2f16271eb 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -26,6 +26,19 @@ module Gitlab puts "\nOK".color(:green) end + def self.without_gitaly_timeout + # Remove Gitaly timeout + old_timeout = Gitlab::CurrentSettings.current_application_settings.gitaly_timeout_default + Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: 0) + # Otherwise we still see the default value when running seed_fu + ApplicationSetting.expire + + yield + ensure + Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: old_timeout) + ApplicationSetting.expire + end + def self.mute_notifications NotificationService.prepend(MuteNotifications) end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 46d01964eac..956c16117f5 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -52,14 +52,6 @@ module Gitlab end end - def self.program_context - if Sidekiq.server? - 'sidekiq' - else - 'rails' - end - end - def self.should_raise_for_dev? Rails.env.development? || Rails.env.test? end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index c6a6fb9b5ce..40b641b8317 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -64,27 +64,48 @@ module Gitlab end end + # Convenience methods for initializing a new repository with a Project model. + def create_project_repository(project) + create_repository(project.repository_storage, project.disk_path, project.full_path) + end + + def create_wiki_repository(project) + create_repository(project.repository_storage, project.wiki.disk_path, project.wiki.full_path) + end + # Init new repository # # storage - the shard key - # name - project disk path + # disk_path - project disk path + # gl_project_path - project name # # Ex. - # create_repository("default", "gitlab/gitlab-ci") + # create_repository("default", "path/to/gitlab-ci", "gitlab/gitlab-ci") # - def create_repository(storage, name) - relative_path = name.dup + def create_repository(storage, disk_path, gl_project_path) + relative_path = disk_path.dup relative_path << '.git' unless relative_path.end_with?('.git') - repository = Gitlab::Git::Repository.new(storage, relative_path, '') + # During creation of a repository, gl_repository may not be known + # because that depends on a yet-to-be assigned project ID in the + # database (e.g. project-1234), so for now it is blank. + repository = Gitlab::Git::Repository.new(storage, relative_path, '', gl_project_path) wrapped_gitaly_errors { repository.gitaly_repository_client.create_repository } true rescue => err # Once the Rugged codes gets removes this can be improved - Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}") + Rails.logger.error("Failed to add repository #{storage}/#{disk_path}: #{err}") false end + def import_wiki_repository(project, wiki_formatter) + import_repository(project.repository_storage, wiki_formatter.disk_path, wiki_formatter.import_url, project.wiki.full_path) + end + + def import_project_repository(project) + import_repository(project.repository_storage, project.disk_path, project.import_url, project.full_path) + end + # Import repository # # storage - project's storage name @@ -94,13 +115,13 @@ module Gitlab # Ex. # import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - def import_repository(storage, name, url) + def import_repository(storage, name, url, gl_project_path) if url.start_with?('.', '/') raise Error.new("don't use disk paths with import_repository: #{url.inspect}") end relative_path = "#{name}.git" - cmd = GitalyGitlabProjects.new(storage, relative_path) + cmd = GitalyGitlabProjects.new(storage, relative_path, gl_project_path) success = cmd.import_project(url, git_timeout) raise Error, cmd.output unless success @@ -125,18 +146,13 @@ module Gitlab end # Fork repository to new path - # forked_from_storage - forked-from project's storage name - # forked_from_disk_path - project disk relative path - # forked_to_storage - forked-to project's storage name - # forked_to_disk_path - forked project disk relative path - # - # Ex. - # fork_repository("nfs-file06", "gitlab/gitlab-ci", "nfs-file07", "new-namespace/gitlab-ci") - def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) - forked_from_relative_path = "#{forked_from_disk_path}.git" - fork_args = [forked_to_storage, "#{forked_to_disk_path}.git"] + # source_project - forked-from Project + # target_project - forked-to Project + def fork_repository(source_project, target_project) + forked_from_relative_path = "#{source_project.disk_path}.git" + fork_args = [target_project.repository_storage, "#{target_project.disk_path}.git", target_project.full_path] - GitalyGitlabProjects.new(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) + GitalyGitlabProjects.new(source_project.repository_storage, forked_from_relative_path, source_project.full_path).fork_repository(*fork_args) end # Removes a repository from file system, using rm_diretory which is an alias @@ -264,7 +280,10 @@ module Gitlab # add_namespace("default", "gitlab") # def add_namespace(storage, name) - Gitlab::GitalyClient::NamespaceService.new(storage).add(name) + # https://gitlab.com/gitlab-org/gitlab-ce/issues/58012 + Gitlab::GitalyClient.allow_n_plus_1_calls do + Gitlab::GitalyClient::NamespaceService.new(storage).add(name) + end rescue GRPC::InvalidArgument => e raise ArgumentError, e.message end @@ -289,10 +308,12 @@ module Gitlab # def mv_namespace(storage, old_name, new_name) Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name) - rescue GRPC::InvalidArgument + rescue GRPC::InvalidArgument => e + Gitlab::Sentry.track_acceptable_exception(e, extra: { old_name: old_name, new_name: new_name, storage: storage }) + false end - alias_method :mv_directory, :mv_namespace + alias_method :mv_directory, :mv_namespace # Note: ShellWorker uses this alias def url_to_repo(path) Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git" @@ -319,16 +340,16 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def hooks_path + File.join(gitlab_shell_path, 'hooks') + end + protected def gitlab_shell_path File.expand_path(Gitlab.config.gitlab_shell.path) end - def gitlab_shell_hooks_path - File.expand_path(Gitlab.config.gitlab_shell.hooks_path) - end - def gitlab_shell_user_home File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") end @@ -395,16 +416,17 @@ module Gitlab end class GitalyGitlabProjects - attr_reader :shard_name, :repository_relative_path, :output + attr_reader :shard_name, :repository_relative_path, :output, :gl_project_path - def initialize(shard_name, repository_relative_path) + def initialize(shard_name, repository_relative_path, gl_project_path) @shard_name = shard_name @repository_relative_path = repository_relative_path @output = '' + @gl_project_path = gl_project_path end def import_project(source, _timeout) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil, gl_project_path) Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) true @@ -413,9 +435,9 @@ module Gitlab false end - def fork_repository(new_shard_name, new_repository_relative_path) - target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + def fork_repository(new_shard_name, new_repository_relative_path, new_project_name) + target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil, new_project_name) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil, gl_project_path) Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) rescue GRPC::BadStatus => e diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index e86db8db3a1..fdc0d518c59 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -5,6 +5,7 @@ module Gitlab class StructuredLogger START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze + MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes def call(job, queue) started_at = current_time @@ -64,6 +65,7 @@ module Gitlab job['pid'] = ::Process.pid job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] + job['args'] = limited_job_args(job['args']) if job['args'] convert_to_iso8601(job, START_TIMESTAMP_FIELDS) @@ -93,6 +95,21 @@ module Gitlab Time.at(timestamp).utc.iso8601(3) end + + def limited_job_args(args) + return unless args.is_a?(Array) + + total_length = 0 + limited_args = args.take_while do |arg| + total_length += arg.to_json.length + + total_length <= MAXIMUM_JOB_ARGUMENTS_LENGTH + end + + limited_args.push('...') if total_length > MAXIMUM_JOB_ARGUMENTS_LENGTH + + limited_args + end end end end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb new file mode 100644 index 00000000000..ed2c7ee9a2d --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class MemoryKiller + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + # Wait 30 seconds for running jobs to finish during graceful shutdown + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # Create a mutex used to ensure there will be only one thread waiting to + # shut Sidekiq down + MUTEX = Mutex.new + + def call(worker, job, queue) + yield + + current_rss = get_rss + + return unless MAX_RSS > 0 && current_rss > MAX_RSS + + Thread.new do + # Return if another thread is already waiting to shut Sidekiq down + next unless MUTEX.try_lock + + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ + " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + + # Wait `GRACE_TIME` to give the memory intensive job time to finish. + # Then, tell Sidekiq to stop fetching new jobs. + wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') + + # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. + # Then, tell Sidekiq to gracefully shut down by giving jobs a few more + # moments to finish, killing and requeuing them if they didn't, and + # then terminating itself. Sidekiq will replicate the TERM to all its + # children if it can. + wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') + + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. + # Kill the whole pgroup, so we can be sure no children are left behind + wait_and_signal_pgroup(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') + end + end + + private + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) + return 0 unless status.zero? + + output.to_i + end + + # If this sidekiq process is pgroup leader, signal to the whole pgroup + def wait_and_signal_pgroup(time, signal, explanation) + return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid + + Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + sleep(time) + + Sidekiq.logger.warn "sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + Process.kill(signal, "-#{pid}") + end + + def wait_and_signal(time, signal, explanation) + Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + sleep(time) + + Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + Process.kill(signal, pid) + end + + def pid + Process.pid + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb deleted file mode 100644 index 19f3be83bce..00000000000 --- a/lib/gitlab/sidekiq_middleware/shutdown.rb +++ /dev/null @@ -1,135 +0,0 @@ -# frozen_string_literal: true - -require 'mutex_m' - -module Gitlab - module SidekiqMiddleware - class Shutdown - extend Mutex_m - - # Default the RSS limit to 0, meaning the MemoryKiller is disabled - MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i - # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i - # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i - - # This exception can be used to request that the middleware start shutting down Sidekiq - WantShutdown = Class.new(StandardError) - - ShutdownWithoutRaise = Class.new(WantShutdown) - private_constant :ShutdownWithoutRaise - - # For testing only, to avoid race conditions (?) in Rspec mocks. - attr_reader :trace - - # We store the shutdown thread in a class variable to ensure that there - # can be only one shutdown thread in the process. - def self.create_shutdown_thread - mu_synchronize do - break unless @shutdown_thread.nil? - - @shutdown_thread = Thread.new { yield } - end - end - - # For testing only: so we can wait for the shutdown thread to finish. - def self.shutdown_thread - mu_synchronize { @shutdown_thread } - end - - # For testing only: so that we can reset the global state before each test. - def self.clear_shutdown_thread - mu_synchronize { @shutdown_thread = nil } - end - - def initialize - @trace = Queue.new if Rails.env.test? - end - - def call(worker, job, queue) - shutdown_exception = nil - - begin - yield - check_rss! - rescue WantShutdown => ex - shutdown_exception = ex - end - - return unless shutdown_exception - - self.class.create_shutdown_thread do - do_shutdown(worker, job, shutdown_exception) - end - - raise shutdown_exception unless shutdown_exception.is_a?(ShutdownWithoutRaise) - end - - private - - def do_shutdown(worker, job, shutdown_exception) - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\ - "#{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" - - # Wait `GRACE_TIME` to give the memory intensive job time to finish. - # Then, tell Sidekiq to stop fetching new jobs. - wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') - - # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. - # Then, tell Sidekiq to gracefully shut down by giving jobs a few more - # moments to finish, killing and requeuing them if they didn't, and - # then terminating itself. - wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') - - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. - wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') - end - - def check_rss! - return unless MAX_RSS > 0 - - current_rss = get_rss - return unless current_rss > MAX_RSS - - raise ShutdownWithoutRaise.new("current RSS #{current_rss} exceeds maximum RSS #{MAX_RSS}") - end - - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status.zero? - - output.to_i - end - - def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - sleep(time) - - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - kill(signal, pid) - end - - def pid - Process.pid - end - - def sleep(time) - if Rails.env.test? - @trace << [:sleep, time] - else - Kernel.sleep(time) - end - end - - def kill(signal, pid) - if Rails.env.test? - @trace << [:kill, signal, pid] - else - Process.kill(signal, pid) - end - end - end - end -end diff --git a/lib/gitlab/sidekiq_signals.rb b/lib/gitlab/sidekiq_signals.rb new file mode 100644 index 00000000000..b704ee9a0a9 --- /dev/null +++ b/lib/gitlab/sidekiq_signals.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + # As a process group leader, we can ensure that children of sidekiq are killed + # at the same time as sidekiq itself, to stop long-lived children from being + # reparented to init and "escaping". To do this, we override the default + # handlers used by sidekiq for INT and TERM signals + module SidekiqSignals + REPLACE_SIGNALS = %w[INT TERM].freeze + + SIDEKIQ_CHANGED_MESSAGE = + "Intercepting signal handlers: #{REPLACE_SIGNALS.join(", ")} failed. " \ + "Sidekiq should have registered them, but appears not to have done so." + + def self.install!(sidekiq_handlers) + # This only works if we're process group leader + return unless Process.getpgrp == Process.pid + + raise SIDEKIQ_CHANGED_MESSAGE unless + REPLACE_SIGNALS == sidekiq_handlers.keys & REPLACE_SIGNALS + + REPLACE_SIGNALS.each do |signal| + old_handler = sidekiq_handlers[signal] + sidekiq_handlers[signal] = ->(cli) do + blindly_signal_pgroup!(signal) + old_handler.call(cli) + end + end + end + + # The process group leader can forward INT and TERM signals to the whole + # group. However, the forwarded signal is *also* received by the leader, + # which could lead to an infinite loop. We can avoid this by temporarily + # ignoring the forwarded signal. This may cause us to miss some repeated + # signals from outside the process group, but that isn't fatal. + def self.blindly_signal_pgroup!(signal) + old_trap = trap(signal, 'IGNORE') + Process.kill(signal, "-#{Process.getpgrp}") + trap(signal, old_trap) + end + end +end diff --git a/lib/gitlab/slash_commands/application_help.rb b/lib/gitlab/slash_commands/application_help.rb new file mode 100644 index 00000000000..0ea7554ba64 --- /dev/null +++ b/lib/gitlab/slash_commands/application_help.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + class ApplicationHelp < BaseCommand + def initialize(params) + @params = params + end + + def execute + Gitlab::SlashCommands::Presenters::Help.new(commands).present(trigger, params[:text]) + end + + private + + def trigger + "#{params[:command]} [project name or alias]" + end + + def commands + Gitlab::SlashCommands::Command.commands + end + end + end +end diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index 474c09b9c4d..7c963fcf38a 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -9,7 +9,8 @@ module Gitlab Gitlab::SlashCommands::IssueNew, Gitlab::SlashCommands::IssueSearch, Gitlab::SlashCommands::IssueMove, - Gitlab::SlashCommands::Deploy + Gitlab::SlashCommands::Deploy, + Gitlab::SlashCommands::Run ] end diff --git a/lib/gitlab/slash_commands/presenters/error.rb b/lib/gitlab/slash_commands/presenters/error.rb new file mode 100644 index 00000000000..442f8796338 --- /dev/null +++ b/lib/gitlab/slash_commands/presenters/error.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module Presenters + class Error < Presenters::Base + def initialize(message) + @message = message + end + + def message + ephemeral_response(text: @message) + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/run.rb b/lib/gitlab/slash_commands/presenters/run.rb new file mode 100644 index 00000000000..c4bbc231464 --- /dev/null +++ b/lib/gitlab/slash_commands/presenters/run.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module Presenters + class Run < Presenters::Base + # rubocop: disable CodeReuse/ActiveRecord + def present(pipeline) + build = pipeline.builds.take + + if build && (responder = Chat::Responder.responder_for(build)) + in_channel_response(responder.scheduled_output) + else + unsupported_chat_service + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def unsupported_chat_service + ephemeral_response(text: 'Sorry, this chat service is currently not supported by GitLab ChatOps.') + end + + def failed_to_schedule(command) + ephemeral_response( + text: 'The command could not be scheduled. Make sure that your ' \ + 'project has a .gitlab-ci.yml that defines a job with the ' \ + "name #{command.inspect}" + ) + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/run.rb b/lib/gitlab/slash_commands/run.rb new file mode 100644 index 00000000000..10a545e28ac --- /dev/null +++ b/lib/gitlab/slash_commands/run.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + # Slash command for triggering chatops jobs. + class Run < BaseCommand + def self.match(text) + /\Arun\s+(?<command>\S+)(\s+(?<arguments>.+))?\z/.match(text) + end + + def self.help_message + 'run <command> <arguments>' + end + + def self.available?(project) + Chat.available? && project.builds_enabled? + end + + def self.allowed?(project, user) + can?(user, :create_pipeline, project) + end + + def execute(match) + command = Chat::Command.new( + project: project, + chat_name: chat_name, + name: match[:command], + arguments: match[:arguments], + channel: params[:channel_id], + response_url: params[:response_url] + ) + + presenter = Gitlab::SlashCommands::Presenters::Run.new + pipeline = command.try_create_pipeline + + if pipeline&.persisted? + presenter.present(pipeline) + else + presenter.failed_to_schedule(command.name) + end + end + end + end +end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 92388262035..07d0acdbae9 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -33,7 +33,7 @@ module Gitlab # `LOWER(column) = query` instead of using `ILIKE`. def fuzzy_arel_match(column, query, lower_exact_match: false) query = query.squish - return nil unless query.present? + return unless query.present? words = select_fuzzy_words(query) diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb index ec1f00a3a91..e45ac5d4765 100644 --- a/lib/gitlab/sql/recursive_cte.rb +++ b/lib/gitlab/sql/recursive_cte.rb @@ -48,7 +48,7 @@ module Gitlab # # alias_table - The Arel table to use as the alias. def alias_to(alias_table) - Arel::Nodes::As.new(table, alias_table) + Arel::Nodes::As.new(table, Arel::Table.new(alias_table.name.tr('.', '_'))) end # Applies the CTE to the given relation, returning a new one that will diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index d24d5116167..f05592fc3a3 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -9,7 +9,7 @@ module Gitlab # # Example usage: # - # union = Gitlab::SQL::Union.new(user.personal_projects, user.projects) + # union = Gitlab::SQL::Union.new([user.personal_projects, user.projects]) # sql = union.to_sql # # Project.where("id IN (#{sql})") diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 224bb648d8f..8532845f3cb 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'rainbow/ext/string' -require 'gitlab/utils/strong_memoize' +require_dependency 'gitlab/utils/strong_memoize' # rubocop:disable Rails/Output module Gitlab @@ -13,6 +13,12 @@ module Gitlab extend self + def invoke_and_time_task(task) + start = Time.now + Rake::Task[task].invoke + puts "`#{task}` finished in #{Time.now - start} seconds" + end + # Ask if the user wants to continue # # Returns "yes" the user chose to continue diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index fbefb5f7f0e..3e2bb11c35f 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -28,11 +28,6 @@ module Gitlab def finder(project = nil) Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) end - - def dropdown_names(context) - categories = context == 'autodeploy' ? ['Auto deploy'] : %w(General Pages) - super().slice(*categories) - end end end end diff --git a/lib/gitlab/tracing.rb b/lib/gitlab/tracing.rb new file mode 100644 index 00000000000..29517591c51 --- /dev/null +++ b/lib/gitlab/tracing.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Tracing + # Only enable tracing when the `GITLAB_TRACING` env var is configured. Note that we avoid using ApplicationSettings since + # the same environment variable needs to be configured for Workhorse, Gitaly and any other components which + # emit tracing. Since other components may start before Rails, and may not have access to ApplicationSettings, + # an env var makes more sense. + def self.enabled? + connection_string.present? + end + + def self.connection_string + ENV['GITLAB_TRACING'] + end + + def self.tracing_url_template + ENV['GITLAB_TRACING_URL'] + end + + def self.tracing_url_enabled? + enabled? && tracing_url_template.present? + end + + # This will provide a link into the distributed tracing for the current trace, + # if it has been captured. + def self.tracing_url + return unless tracing_url_enabled? + + # Avoid using `format` since it can throw TypeErrors + # which we want to avoid on unsanitised env var input + tracing_url_template.to_s + .gsub(/\{\{\s*correlation_id\s*\}\}/, Gitlab::CorrelationId.current_id.to_s) + .gsub(/\{\{\s*service\s*\}\}/, Gitlab.process_name) + end + end +end diff --git a/lib/gitlab/tracing/common.rb b/lib/gitlab/tracing/common.rb new file mode 100644 index 00000000000..3a08ede8138 --- /dev/null +++ b/lib/gitlab/tracing/common.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'opentracing' + +module Gitlab + module Tracing + module Common + def tracer + OpenTracing.global_tracer + end + + # Convience method for running a block with a span + def in_tracing_span(operation_name:, tags:, child_of: nil) + scope = tracer.start_active_span( + operation_name, + child_of: child_of, + tags: tags + ) + span = scope.span + + # Add correlation details to the span if we have them + correlation_id = Gitlab::CorrelationId.current_id + if correlation_id + span.set_tag('correlation_id', correlation_id) + end + + begin + yield span + rescue => e + log_exception_on_span(span, e) + raise e + ensure + scope.close + end + end + + def postnotify_span(operation_name, start_time, end_time, tags: nil, child_of: nil, exception: nil) + span = OpenTracing.start_span(operation_name, start_time: start_time, tags: tags, child_of: child_of) + + log_exception_on_span(span, exception) if exception + + span.finish(end_time: end_time) + end + + def log_exception_on_span(span, exception) + span.set_tag('error', true) + span.log_kv(kv_tags_for_exception(exception)) + end + + def kv_tags_for_exception(exception) + case exception + when Exception + { + 'event': 'error', + 'error.kind': exception.class.to_s, + 'message': Gitlab::UrlSanitizer.sanitize(exception.message), + 'stack': exception.backtrace&.join("\n") + } + else + { + 'event': 'error', + 'error.kind': exception.class.to_s, + 'error.object': Gitlab::UrlSanitizer.sanitize(exception.to_s) + } + end + end + end + end +end diff --git a/lib/gitlab/tracing/factory.rb b/lib/gitlab/tracing/factory.rb new file mode 100644 index 00000000000..fc714164353 --- /dev/null +++ b/lib/gitlab/tracing/factory.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require "cgi" + +module Gitlab + module Tracing + class Factory + OPENTRACING_SCHEME = "opentracing" + + def self.create_tracer(service_name, connection_string) + return unless connection_string.present? + + begin + opentracing_details = parse_connection_string(connection_string) + driver_name = opentracing_details[:driver_name] + + case driver_name + when "jaeger" + JaegerFactory.create_tracer(service_name, opentracing_details[:options]) + else + raise "Unknown driver: #{driver_name}" + end + rescue => e + # Can't create the tracer? Warn and continue sans tracer + warn "Unable to instantiate tracer: #{e}" + nil + end + end + + def self.parse_connection_string(connection_string) + parsed = URI.parse(connection_string) + + unless valid_uri?(parsed) + raise "Invalid tracing connection string" + end + + { + driver_name: parsed.host, + options: parse_query(parsed.query) + } + end + private_class_method :parse_connection_string + + def self.parse_query(query) + return {} unless query + + CGI.parse(query).symbolize_keys.transform_values(&:first) + end + private_class_method :parse_query + + def self.valid_uri?(uri) + return false unless uri + + uri.scheme == OPENTRACING_SCHEME && + uri.host.to_s =~ /^[a-z0-9_]+$/ && + uri.path.empty? + end + private_class_method :valid_uri? + end + end +end diff --git a/lib/gitlab/tracing/grpc_interceptor.rb b/lib/gitlab/tracing/grpc_interceptor.rb new file mode 100644 index 00000000000..6c2aab73125 --- /dev/null +++ b/lib/gitlab/tracing/grpc_interceptor.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'opentracing' +require 'grpc' + +module Gitlab + module Tracing + class GRPCInterceptor < GRPC::ClientInterceptor + include Common + include Singleton + + def request_response(request:, call:, method:, metadata:) + wrap_with_tracing(method, 'unary', metadata) do + yield + end + end + + def client_streamer(requests:, call:, method:, metadata:) + wrap_with_tracing(method, 'client_stream', metadata) do + yield + end + end + + def server_streamer(request:, call:, method:, metadata:) + wrap_with_tracing(method, 'server_stream', metadata) do + yield + end + end + + def bidi_streamer(requests:, call:, method:, metadata:) + wrap_with_tracing(method, 'bidi_stream', metadata) do + yield + end + end + + private + + def wrap_with_tracing(method, grpc_type, metadata) + tags = { + 'component' => 'grpc', + 'span.kind' => 'client', + 'grpc.method' => method, + 'grpc.type' => grpc_type + } + + in_tracing_span(operation_name: "grpc:#{method}", tags: tags) do |span| + OpenTracing.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, metadata) + + yield + end + end + end + end +end diff --git a/lib/gitlab/tracing/jaeger_factory.rb b/lib/gitlab/tracing/jaeger_factory.rb new file mode 100644 index 00000000000..93520d5667b --- /dev/null +++ b/lib/gitlab/tracing/jaeger_factory.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'jaeger/client' + +module Gitlab + module Tracing + class JaegerFactory + # When the probabilistic sampler is used, by default 0.1% of requests will be traced + DEFAULT_PROBABILISTIC_RATE = 0.001 + + # The default port for the Jaeger agent UDP listener + DEFAULT_UDP_PORT = 6831 + + # Reduce this from default of 10 seconds as the Ruby jaeger + # client doesn't have overflow control, leading to very large + # messages which fail to send over UDP (max packet = 64k) + # Flush more often, with smaller packets + FLUSH_INTERVAL = 5 + + def self.create_tracer(service_name, options) + kwargs = { + service_name: service_name, + sampler: get_sampler(options[:sampler], options[:sampler_param]), + reporter: get_reporter(service_name, options[:http_endpoint], options[:udp_endpoint]) + }.compact + + extra_params = options.except(:sampler, :sampler_param, :http_endpoint, :udp_endpoint, :strict_parsing, :debug) # rubocop: disable CodeReuse/ActiveRecord + if extra_params.present? + message = "jaeger tracer: invalid option: #{extra_params.keys.join(", ")}" + + if options[:strict_parsing] + raise message + else + warn message + end + end + + Jaeger::Client.build(kwargs) + end + + def self.get_sampler(sampler_type, sampler_param) + case sampler_type + when "probabilistic" + sampler_rate = sampler_param ? sampler_param.to_f : DEFAULT_PROBABILISTIC_RATE + Jaeger::Samplers::Probabilistic.new(rate: sampler_rate) + when "const" + const_value = sampler_param == "1" + Jaeger::Samplers::Const.new(const_value) + else + nil + end + end + private_class_method :get_sampler + + def self.get_reporter(service_name, http_endpoint, udp_endpoint) + encoder = Jaeger::Encoders::ThriftEncoder.new(service_name: service_name) + + if http_endpoint.present? + sender = get_http_sender(encoder, http_endpoint) + elsif udp_endpoint.present? + sender = get_udp_sender(encoder, udp_endpoint) + else + return + end + + Jaeger::Reporters::RemoteReporter.new( + sender: sender, + flush_interval: FLUSH_INTERVAL + ) + end + private_class_method :get_reporter + + def self.get_http_sender(encoder, address) + Jaeger::HttpSender.new( + url: address, + encoder: encoder, + logger: Logger.new(STDOUT) + ) + end + private_class_method :get_http_sender + + def self.get_udp_sender(encoder, address) + pair = address.split(":", 2) + host = pair[0] + port = pair[1] ? pair[1].to_i : DEFAULT_UDP_PORT + + Jaeger::UdpSender.new( + host: host, + port: port, + encoder: encoder, + logger: Logger.new(STDOUT) + ) + end + private_class_method :get_udp_sender + end + end +end diff --git a/lib/gitlab/tracing/rack_middleware.rb b/lib/gitlab/tracing/rack_middleware.rb new file mode 100644 index 00000000000..e6a31293f7b --- /dev/null +++ b/lib/gitlab/tracing/rack_middleware.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'opentracing' + +module Gitlab + module Tracing + class RackMiddleware + include Common + + REQUEST_METHOD = 'REQUEST_METHOD' + + def initialize(app) + @app = app + end + + def call(env) + method = env[REQUEST_METHOD] + + context = tracer.extract(OpenTracing::FORMAT_RACK, env) + tags = { + 'component' => 'rack', + 'span.kind' => 'server', + 'http.method' => method, + 'http.url' => self.class.build_sanitized_url_from_env(env) + } + + in_tracing_span(operation_name: "http:#{method}", child_of: context, tags: tags) do |span| + @app.call(env).tap do |status_code, _headers, _body| + span.set_tag('http.status_code', status_code) + end + end + end + + # Generate a sanitized (safe) request URL from the rack environment + def self.build_sanitized_url_from_env(env) + request = ActionDispatch::Request.new(env) + + original_url = request.original_url + uri = URI.parse(original_url) + uri.query = request.filtered_parameters.to_query if uri.query.present? + + uri.to_s + end + end + end +end diff --git a/lib/gitlab/tracing/rails/action_view_subscriber.rb b/lib/gitlab/tracing/rails/action_view_subscriber.rb new file mode 100644 index 00000000000..88816e1fb32 --- /dev/null +++ b/lib/gitlab/tracing/rails/action_view_subscriber.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Tracing + module Rails + class ActionViewSubscriber + include RailsCommon + + COMPONENT_TAG = 'ActionView' + RENDER_TEMPLATE_NOTIFICATION_TOPIC = 'render_template.action_view' + RENDER_COLLECTION_NOTIFICATION_TOPIC = 'render_collection.action_view' + RENDER_PARTIAL_NOTIFICATION_TOPIC = 'render_partial.action_view' + + # Instruments Rails ActionView events for opentracing. + # Returns a lambda, which, when called will unsubscribe from the notifications + def self.instrument + subscriber = new + + subscriptions = [ + ActiveSupport::Notifications.subscribe(RENDER_TEMPLATE_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify_render_template(start, finish, payload) + end, + ActiveSupport::Notifications.subscribe(RENDER_COLLECTION_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify_render_collection(start, finish, payload) + end, + ActiveSupport::Notifications.subscribe(RENDER_PARTIAL_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify_render_partial(start, finish, payload) + end + ] + + create_unsubscriber subscriptions + end + + # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html + def notify_render_template(start, finish, payload) + generate_span_for_notification("render_template", start, finish, payload, tags_for_render_template(payload)) + end + + def notify_render_collection(start, finish, payload) + generate_span_for_notification("render_collection", start, finish, payload, tags_for_render_collection(payload)) + end + + def notify_render_partial(start, finish, payload) + generate_span_for_notification("render_partial", start, finish, payload, tags_for_render_partial(payload)) + end + + private + + def tags_for_render_template(payload) + { + 'component' => COMPONENT_TAG, + 'template.id' => payload[:identifier], + 'template.layout' => payload[:layout] + } + end + + def tags_for_render_collection(payload) + { + 'component' => COMPONENT_TAG, + 'template.id' => payload[:identifier], + 'template.count' => payload[:count] || 0, + 'template.cache.hits' => payload[:cache_hits] || 0 + } + end + + def tags_for_render_partial(payload) + { + 'component' => COMPONENT_TAG, + 'template.id' => payload[:identifier] + } + end + end + end + end +end diff --git a/lib/gitlab/tracing/rails/active_record_subscriber.rb b/lib/gitlab/tracing/rails/active_record_subscriber.rb new file mode 100644 index 00000000000..32f5658e57e --- /dev/null +++ b/lib/gitlab/tracing/rails/active_record_subscriber.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Tracing + module Rails + class ActiveRecordSubscriber + include RailsCommon + + ACTIVE_RECORD_NOTIFICATION_TOPIC = 'sql.active_record' + OPERATION_NAME_PREFIX = 'active_record:' + DEFAULT_OPERATION_NAME = 'sqlquery' + + # Instruments Rails ActiveRecord events for opentracing. + # Returns a lambda, which, when called will unsubscribe from the notifications + def self.instrument + subscriber = new + + subscription = ActiveSupport::Notifications.subscribe(ACTIVE_RECORD_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify(start, finish, payload) + end + + create_unsubscriber [subscription] + end + + # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html + def notify(start, finish, payload) + generate_span_for_notification(notification_name(payload), start, finish, payload, tags_for_notification(payload)) + end + + private + + def notification_name(payload) + OPERATION_NAME_PREFIX + (payload[:name].presence || DEFAULT_OPERATION_NAME) + end + + def tags_for_notification(payload) + { + 'component' => 'ActiveRecord', + 'span.kind' => 'client', + 'db.type' => 'sql', + 'db.connection_id' => payload[:connection_id], + 'db.cached' => payload[:cached] || false, + 'db.statement' => payload[:sql] + } + end + end + end + end +end diff --git a/lib/gitlab/tracing/rails/rails_common.rb b/lib/gitlab/tracing/rails/rails_common.rb new file mode 100644 index 00000000000..88e914f62f8 --- /dev/null +++ b/lib/gitlab/tracing/rails/rails_common.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Tracing + module Rails + module RailsCommon + extend ActiveSupport::Concern + include Gitlab::Tracing::Common + + class_methods do + def create_unsubscriber(subscriptions) + -> { subscriptions.each { |subscriber| ActiveSupport::Notifications.unsubscribe(subscriber) } } + end + end + + def generate_span_for_notification(operation_name, start, finish, payload, tags) + exception = payload[:exception] + + postnotify_span(operation_name, start, finish, tags: tags, exception: exception) + end + end + end + end +end diff --git a/lib/gitlab/tracing/sidekiq/client_middleware.rb b/lib/gitlab/tracing/sidekiq/client_middleware.rb new file mode 100644 index 00000000000..2b71c1ea21e --- /dev/null +++ b/lib/gitlab/tracing/sidekiq/client_middleware.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'opentracing' + +module Gitlab + module Tracing + module Sidekiq + class ClientMiddleware + include SidekiqCommon + + SPAN_KIND = 'client' + + def call(worker_class, job, queue, redis_pool) + in_tracing_span( + operation_name: "sidekiq:#{job['class']}", + tags: tags_from_job(job, SPAN_KIND)) do |span| + # Inject the details directly into the job + tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job) + + yield + end + end + end + end + end +end diff --git a/lib/gitlab/tracing/sidekiq/server_middleware.rb b/lib/gitlab/tracing/sidekiq/server_middleware.rb new file mode 100644 index 00000000000..5b43c4310e6 --- /dev/null +++ b/lib/gitlab/tracing/sidekiq/server_middleware.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'opentracing' + +module Gitlab + module Tracing + module Sidekiq + class ServerMiddleware + include SidekiqCommon + + SPAN_KIND = 'server' + + def call(worker, job, queue) + context = tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job) + + in_tracing_span( + operation_name: "sidekiq:#{job['class']}", + child_of: context, + tags: tags_from_job(job, SPAN_KIND)) do |span| + yield + end + end + end + end + end +end diff --git a/lib/gitlab/tracing/sidekiq/sidekiq_common.rb b/lib/gitlab/tracing/sidekiq/sidekiq_common.rb new file mode 100644 index 00000000000..a911a29d773 --- /dev/null +++ b/lib/gitlab/tracing/sidekiq/sidekiq_common.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Tracing + module Sidekiq + module SidekiqCommon + include Gitlab::Tracing::Common + + def tags_from_job(job, kind) + { + 'component' => 'sidekiq', + 'span.kind' => kind, + 'sidekiq.queue' => job['queue'], + 'sidekiq.jid' => job['jid'], + 'sidekiq.retry' => job['retry'].to_s, + 'sidekiq.args' => job['args']&.join(", ") + } + end + end + end + end +end diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index 453d78e2f7b..8518a13cd1c 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -95,7 +95,7 @@ module Gitlab end def cache_commit(commit) - return nil unless commit.present? + return unless commit.present? resolved_commits[commit.id] ||= commit end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 44c71f8431d..9b7b0db9525 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -8,16 +8,18 @@ module Gitlab BlockedUrlError = Class.new(StandardError) class << self - def validate!(url, ports: [], protocols: [], allow_localhost: false, allow_local_network: true, ascii_only: false, enforce_user: false) + def validate!(url, ports: [], protocols: [], allow_localhost: false, allow_local_network: true, ascii_only: false, enforce_user: false, enforce_sanitization: false) return true if url.nil? # Param url can be a string, URI or Addressable::URI uri = parse_url(url) + validate_html_tags!(uri) if enforce_sanitization + # Allow imports from the GitLab instance itself but only from the configured ports return true if internal?(uri) - port = uri.port || uri.default_port + port = get_port(uri) validate_protocol!(uri.scheme, protocols) validate_port!(port, ports) if ports.any? validate_user!(uri.user) if enforce_user @@ -50,6 +52,18 @@ module Gitlab private + def get_port(uri) + uri.port || uri.default_port + end + + def validate_html_tags!(uri) + uri_str = uri.to_s + sanitized_uri = ActionController::Base.helpers.sanitize(uri_str, tags: []) + if sanitized_uri != uri_str + raise BlockedUrlError, 'HTML/CSS/JS tags are not allowed' + end + end + def parse_url(url) raise Addressable::URI::InvalidURIError if multiline?(url) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 083c620267a..0101ccc046a 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -64,12 +64,12 @@ module Gitlab group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type), clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), - clusters_applications_helm: count(::Clusters::Applications::Helm.installed), - clusters_applications_ingress: count(::Clusters::Applications::Ingress.installed), - clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.installed), - clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.installed), - clusters_applications_runner: count(::Clusters::Applications::Runner.installed), - clusters_applications_knative: count(::Clusters::Applications::Knative.installed), + clusters_applications_helm: count(::Clusters::Applications::Helm.available), + clusters_applications_ingress: count(::Clusters::Applications::Ingress.available), + clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.available), + clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.available), + clusters_applications_runner: count(::Clusters::Applications::Runner.available), + clusters_applications_knative: count(::Clusters::Applications::Knative.available), in_review_folder: count(::Environment.in_review_folder), groups: count(Group), issues: count(Issue), @@ -81,6 +81,7 @@ module Gitlab pages_domains: count(PagesDomain), projects: count(Project), projects_imported_from_github: count(Project.where(import_type: 'github')), + projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), protected_branches: count(ProtectedBranch), releases: count(Release), remote_mirrors: count(RemoteMirror), @@ -89,8 +90,14 @@ module Gitlab todos: count(Todo), uploads: count(Upload), web_hooks: count(WebHook) - }.merge(services_usage).merge(approximate_counts) - } + } + .merge(services_usage) + .merge(approximate_counts) + }.tap do |data| + if Feature.enabled?(:group_overview_security_dashboard) + data[:counts][:user_preferences] = user_preferences_usage + end + end end # rubocop: enable CodeReuse/ActiveRecord @@ -158,6 +165,10 @@ module Gitlab } end + def user_preferences_usage + {} # augmented in EE + end + def count(relation, fallback: -1) relation.count rescue ActiveRecord::StatementInvalid diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 980a8014409..9ef23cf849f 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -118,8 +118,8 @@ module Gitlab protected_refs: project.protected_tags) end - request_cache def protected?(kind, project, ref) - kind.protected?(project, ref) + request_cache def protected?(kind, project, refs) + kind.protected?(project, refs) end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index a81cee0d6d2..99fa65e0e90 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -115,5 +115,15 @@ module Gitlab string_or_array.split(',').map(&:strip) end + + def deep_indifferent_access(data) + if data.is_a?(Array) + data.map(&method(:deep_indifferent_access)) + elsif data.is_a?(Hash) + data.with_indifferent_access + else + data + end + end end end diff --git a/lib/gitlab/utils/merge_hash.rb b/lib/gitlab/utils/merge_hash.rb index fc237861e2f..48ba13b8561 100644 --- a/lib/gitlab/utils/merge_hash.rb +++ b/lib/gitlab/utils/merge_hash.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_dependency 'gitlab/utils' + module Gitlab module Utils module MergeHash diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index c87e97d0213..f5299439fce 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_dependency 'gitlab/utils' + module Gitlab module Utils module Override diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index aa1f8e2fdda..3021a91dd83 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_dependency 'gitlab/utils' + module Gitlab module Utils module StrongMemoize diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb index 5303b3582ab..e9be6db50da 100644 --- a/lib/gitlab/wiki_file_finder.rb +++ b/lib/gitlab/wiki_file_finder.rb @@ -2,8 +2,6 @@ module Gitlab class WikiFileFinder < FileFinder - BATCH_SIZE = 100 - attr_reader :repository def initialize(project, ref) @@ -19,7 +17,7 @@ module Gitlab safe_query = Regexp.new(safe_query, Regexp::IGNORECASE) filenames = repository.ls_files(ref) - filenames.grep(safe_query).first(BATCH_SIZE) + filenames.grep(safe_query) end end end diff --git a/lib/json_web_token/hmac_token.rb b/lib/json_web_token/hmac_token.rb index ceb1b9c913f..ec0917ab49d 100644 --- a/lib/json_web_token/hmac_token.rb +++ b/lib/json_web_token/hmac_token.rb @@ -18,7 +18,7 @@ module JSONWebToken end def encoded - JWT.encode(payload, secret, JWT_ALGORITHM) + JWT.encode(payload, secret, JWT_ALGORITHM, { typ: 'JWT' }) end private diff --git a/lib/json_web_token/rsa_token.rb b/lib/json_web_token/rsa_token.rb index 160e1e506f1..bcce811cd28 100644 --- a/lib/json_web_token/rsa_token.rb +++ b/lib/json_web_token/rsa_token.rb @@ -11,7 +11,8 @@ module JSONWebToken def encoded headers = { - kid: kid + kid: kid, + typ: 'JWT' } JWT.encode(payload, key, 'RS256', headers) end diff --git a/lib/peek/views/tracing.rb b/lib/peek/views/tracing.rb new file mode 100644 index 00000000000..0de32a8fdda --- /dev/null +++ b/lib/peek/views/tracing.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Peek + module Views + class Tracing < View + def results + { + tracing_url: Gitlab::Tracing.tracing_url + } + end + end + end +end diff --git a/lib/safe_zip/entry.rb b/lib/safe_zip/entry.rb new file mode 100644 index 00000000000..664e2f52f91 --- /dev/null +++ b/lib/safe_zip/entry.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +module SafeZip + class Entry + attr_reader :zip_archive, :zip_entry + attr_reader :path, :params + + def initialize(zip_archive, zip_entry, params) + @zip_archive = zip_archive + @zip_entry = zip_entry + @params = params + @path = ::File.expand_path(zip_entry.name, params.extract_path) + end + + def path_dir + ::File.dirname(path) + end + + def real_path_dir + ::File.realpath(path_dir) + end + + def exist? + ::File.exist?(path) + end + + def extract + # do not extract if file is not part of target directory + return false unless matching_target_directory + + # do not overwrite existing file + raise SafeZip::Extract::AlreadyExistsError, "File already exists #{zip_entry.name}" if exist? + + create_path_dir + + if zip_entry.file? + extract_file + elsif zip_entry.directory? + extract_dir + elsif zip_entry.symlink? + extract_symlink + else + raise SafeZip::Extract::UnsupportedEntryError, "File #{zip_entry.name} cannot be extracted" + end + rescue SafeZip::Extract::Error + raise + rescue => e + raise SafeZip::Extract::ExtractError, e.message + end + + private + + def extract_file + zip_archive.extract(zip_entry, path) + end + + def extract_dir + FileUtils.mkdir(path) + end + + def extract_symlink + source_path = read_symlink + real_source_path = expand_symlink(source_path) + + # ensure that source path of symlink is within target directories + unless real_source_path.start_with?(matching_target_directory) + raise SafeZip::Extract::PermissionDeniedError, "Symlink cannot be created targeting: #{source_path}" + end + + ::File.symlink(source_path, path) + end + + def create_path_dir + # Create all directories, but ignore permissions + FileUtils.mkdir_p(path_dir) + + # disallow to make path dirs to point to another directories + unless path_dir == real_path_dir + raise SafeZip::Extract::PermissionDeniedError, "Directory of #{zip_entry.name} points to another directory" + end + end + + def matching_target_directory + params.matching_target_directory(path) + end + + def read_symlink + zip_archive.read(zip_entry) + end + + def expand_symlink(source_path) + ::File.realpath(source_path, path_dir) + rescue + raise SafeZip::Extract::SymlinkSourceDoesNotExistError, "Symlink source #{source_path} does not exist" + end + end +end diff --git a/lib/safe_zip/extract.rb b/lib/safe_zip/extract.rb new file mode 100644 index 00000000000..679c021c730 --- /dev/null +++ b/lib/safe_zip/extract.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module SafeZip + class Extract + Error = Class.new(StandardError) + PermissionDeniedError = Class.new(Error) + SymlinkSourceDoesNotExistError = Class.new(Error) + UnsupportedEntryError = Class.new(Error) + AlreadyExistsError = Class.new(Error) + NoMatchingError = Class.new(Error) + ExtractError = Class.new(Error) + + attr_reader :archive_path + + def initialize(archive_file) + @archive_path = archive_file + end + + def extract(opts = {}) + params = SafeZip::ExtractParams.new(**opts) + + if Feature.enabled?(:safezip_use_rubyzip, default_enabled: true) + extract_with_ruby_zip(params) + else + legacy_unsafe_extract_with_system_zip(params) + end + end + + private + + def extract_with_ruby_zip(params) + ::Zip::File.open(archive_path) do |zip_archive| + # Extract all files in the following order: + # 1. Directories first, + # 2. Files next, + # 3. Symlinks last (or anything else) + extracted = extract_all_entries(zip_archive, params, + zip_archive.lazy.select(&:directory?)) + + extracted += extract_all_entries(zip_archive, params, + zip_archive.lazy.select(&:file?)) + + extracted += extract_all_entries(zip_archive, params, + zip_archive.lazy.reject(&:directory?).reject(&:file?)) + + raise NoMatchingError, 'No entries extracted' unless extracted > 0 + end + end + + def extract_all_entries(zip_archive, params, entries) + entries.count do |zip_entry| + SafeZip::Entry.new(zip_archive, zip_entry, params) + .extract + end + end + + def legacy_unsafe_extract_with_system_zip(params) + # Requires UnZip at least 6.00 Info-ZIP. + # -n never overwrite existing files + args = %W(unzip -n -qq #{archive_path}) + + # We add * to end of directory, because we want to extract directory and all subdirectories + args += params.directories_wildcard + + # Target directory where we extract + args += %W(-d #{params.extract_path}) + + unless system(*args) + raise Error, 'archive failed to extract' + end + end + end +end diff --git a/lib/safe_zip/extract_params.rb b/lib/safe_zip/extract_params.rb new file mode 100644 index 00000000000..bd3b788bac9 --- /dev/null +++ b/lib/safe_zip/extract_params.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module SafeZip + class ExtractParams + include Gitlab::Utils::StrongMemoize + + attr_reader :directories, :extract_path + + def initialize(directories:, to:) + @directories = directories + @extract_path = ::File.realpath(to) + end + + def matching_target_directory(path) + target_directories.find do |directory| + path.start_with?(directory) + end + end + + def target_directories + strong_memoize(:target_directories) do + directories.map do |directory| + ::File.join(::File.expand_path(directory, extract_path), '') + end + end + end + + def directories_wildcard + strong_memoize(:directories_wildcard) do + directories.map do |directory| + ::File.join(directory, '*') + end + end + end + end +end diff --git a/lib/sentry/client.rb b/lib/sentry/client.rb new file mode 100644 index 00000000000..49ec196b103 --- /dev/null +++ b/lib/sentry/client.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module Sentry + class Client + Error = Class.new(StandardError) + SentryError = Class.new(StandardError) + + attr_accessor :url, :token + + def initialize(api_url, token) + @url = api_url + @token = token + end + + def list_issues(issue_status:, limit:) + issues = get_issues(issue_status: issue_status, limit: limit) + map_to_errors(issues) + end + + def list_projects + projects = get_projects + map_to_projects(projects) + rescue KeyError => e + raise Client::SentryError, "Sentry API response is missing keys. #{e.message}" + end + + private + + def request_params + { + headers: { + 'Authorization' => "Bearer #{@token}" + }, + follow_redirects: false + } + end + + def http_get(url, params = {}) + resp = Gitlab::HTTP.get(url, **request_params.merge(params)) + + handle_response(resp) + end + + def get_issues(issue_status:, limit:) + http_get(issues_api_url, query: { + query: "is:#{issue_status}", + limit: limit + }) + end + + def get_projects + http_get(projects_api_url) + end + + def handle_response(response) + unless response.code == 200 + raise Client::Error, "Sentry response status code: #{response.code}" + end + + response.as_json + end + + def projects_api_url + projects_url = URI(@url) + projects_url.path = '/api/0/projects/' + + projects_url + end + + def issues_api_url + issues_url = URI(@url + '/issues/') + issues_url.path.squeeze!('/') + + issues_url + end + + def map_to_errors(issues) + issues.map(&method(:map_to_error)) + end + + def map_to_projects(projects) + projects.map(&method(:map_to_project)) + end + + def issue_url(id) + issues_url = @url + "/issues/#{id}" + issues_url = ErrorTracking::ProjectErrorTrackingSetting.extract_sentry_external_url(issues_url) + + uri = URI(issues_url) + uri.path.squeeze!('/') + + uri.to_s + end + + def map_to_error(issue) + id = issue.fetch('id') + project = issue.fetch('project') + + count = issue.fetch('count', nil) + + frequency = issue.dig('stats', '24h') + message = issue.dig('metadata', 'value') + + external_url = issue_url(id) + + Gitlab::ErrorTracking::Error.new( + id: id, + first_seen: issue.fetch('firstSeen', nil), + last_seen: issue.fetch('lastSeen', nil), + title: issue.fetch('title', nil), + type: issue.fetch('type', nil), + user_count: issue.fetch('userCount', nil), + count: count, + message: message, + culprit: issue.fetch('culprit', nil), + external_url: external_url, + short_id: issue.fetch('shortId', nil), + status: issue.fetch('status', nil), + frequency: frequency, + project_id: project.fetch('id'), + project_name: project.fetch('name', nil), + project_slug: project.fetch('slug', nil) + ) + end + + def map_to_project(project) + organization = project.fetch('organization') + + Gitlab::ErrorTracking::Project.new( + id: project.fetch('id'), + name: project.fetch('name'), + slug: project.fetch('slug'), + status: project.dig('status'), + organization_name: organization.fetch('name'), + organization_id: organization.fetch('id'), + organization_slug: organization.fetch('slug') + ) + end + end +end diff --git a/lib/serializers/json.rb b/lib/serializers/json.rb new file mode 100644 index 00000000000..93cb192087a --- /dev/null +++ b/lib/serializers/json.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Serializers + # This serializer exports data as JSON, + # it is designed to be used with interwork compatibility between MySQL and PostgreSQL + # implementations, as used version of MySQL does not support native json type + # + # Secondly, the loader makes the resulting hash to have deep indifferent access + class JSON + class << self + def dump(obj) + # MySQL stores data as text + # look at ./config/initializers/ar_mysql_jsonb_support.rb + if Gitlab::Database.mysql? + obj = ActiveSupport::JSON.encode(obj) + end + + obj + end + + def load(data) + return if data.nil? + + # On MySQL we store data as text + # look at ./config/initializers/ar_mysql_jsonb_support.rb + if Gitlab::Database.mysql? + data = ActiveSupport::JSON.decode(data) + end + + Gitlab::Utils.deep_indifferent_access(data) + end + end + end +end diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb index 6cd53779bfd..a331f88873b 100644 --- a/lib/system_check/app/git_user_default_ssh_config_check.rb +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -53,7 +53,7 @@ module SystemCheck end def ssh_dir - return nil unless home_dir + return unless home_dir File.join(home_dir, '.ssh') end diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index e06245294c4..46aad8aa885 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -70,18 +70,14 @@ module SystemCheck # multiple reasons why a check can fail # # @param [String] reason to be displayed - def skip_reason=(reason) - @skip_reason = reason - end + attr_writer :skip_reason # Skip reason defined during runtime # # This value have precedence over the one defined in the subclass # # @return [String] the reason - def skip_reason - @skip_reason - end + attr_reader :skip_reason # Does the check support automatically repair routine? # diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 4beb94eeb8e..b1db4dc94a6 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -10,6 +10,7 @@ namespace :dev do desc "GitLab | Eager load application" task load: :environment do + Rails.configuration.eager_load = true Rails.application.eager_load! end end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 560a52053d8..c24207b134a 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -30,33 +30,28 @@ namespace :gemojione do # We don't have `node_modules` available in built versions of GitLab FileUtils.cp_r(Rails.root.join('node_modules', 'emoji-unicode-version', 'emoji-unicode-version-map.json'), File.join(Rails.root, 'fixtures', 'emojis')) - dir = Gemojione.images_path resultant_emoji_map = {} Gitlab::Emoji.emojis.each do |name, emoji_hash| # Ignore aliases unless Gitlab::Emoji.emojis_aliases.key?(name) - fpath = File.join(dir, "#{emoji_hash['unicode']}.png") - hash_digest = Digest::SHA256.file(fpath).hexdigest - category = emoji_hash['category'] if name == 'gay_pride_flag' category = 'flags' end entry = { - category: category, - moji: emoji_hash['moji'], - description: emoji_hash['description'], - unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name), - digest: hash_digest + c: category, + e: emoji_hash['moji'], + d: emoji_hash['description'], + u: Gitlab::Emoji.emoji_unicode_version(name) } resultant_emoji_map[name] = entry end end - out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + out = File.join(Rails.root, 'public', '-', 'emojis', '1', 'emojis.json') File.open(out, 'w') do |handle| handle.write(JSON.pretty_generate(resultant_emoji_map)) end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index a42f02a84fd..7a42e4e92a0 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -1,13 +1,17 @@ namespace :gitlab do namespace :assets do desc 'GitLab | Assets | Compile all frontend assets' - task compile: [ - 'yarn:check', - 'gettext:po_to_json', - 'rake:assets:precompile', - 'webpack:compile', - 'fix_urls' - ] + task :compile do + require_dependency 'gitlab/task_helpers' + + %w[ + yarn:check + gettext:po_to_json + rake:assets:precompile + webpack:compile + gitlab:assets:fix_urls + ].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task)) + end desc 'GitLab | Assets | Clean up old compiled frontend assets' task clean: ['rake:assets:clean'] diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index e96fbb64372..3977fc7ad8c 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -47,9 +47,9 @@ namespace :gitlab do # Drop all tables Load the schema to ensure we don't have any newer tables # hanging out from a failed upgrade - progress.puts 'Cleaning the database ... '.color(:blue) + puts_time 'Cleaning the database ... '.color(:blue) Rake::Task['gitlab:db:drop_tables'].invoke - progress.puts 'done'.color(:green) + puts_time 'done'.color(:green) Rake::Task['gitlab:backup:db:restore'].invoke rescue Gitlab::TaskAbortedByUserError puts "Quitting...".color(:red) @@ -61,7 +61,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') - Rake::Task["gitlab:backup:pages:restore"].invoke unless backup.skipped?('pages') + Rake::Task['gitlab:backup:pages:restore'].invoke unless backup.skipped?('pages') Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke @@ -72,165 +72,169 @@ namespace :gitlab do namespace :repo do task create: :gitlab_environment do - progress.puts "Dumping repositories ...".color(:blue) + puts_time "Dumping repositories ...".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("repositories") - progress.puts "[SKIPPED]".color(:cyan) + puts_time "[SKIPPED]".color(:cyan) else Backup::Repository.new(progress).dump - progress.puts "done".color(:green) + puts_time "done".color(:green) end end task restore: :gitlab_environment do - progress.puts "Restoring repositories ...".color(:blue) + puts_time "Restoring repositories ...".color(:blue) Backup::Repository.new(progress).restore - progress.puts "done".color(:green) + puts_time "done".color(:green) end end namespace :db do task create: :gitlab_environment do - progress.puts "Dumping database ... ".color(:blue) + puts_time "Dumping database ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("db") - progress.puts "[SKIPPED]".color(:cyan) + puts_time "[SKIPPED]".color(:cyan) else Backup::Database.new(progress).dump - progress.puts "done".color(:green) + puts_time "done".color(:green) end end task restore: :gitlab_environment do - progress.puts "Restoring database ... ".color(:blue) + puts_time "Restoring database ... ".color(:blue) Backup::Database.new(progress).restore - progress.puts "done".color(:green) + puts_time "done".color(:green) end end namespace :builds do task create: :gitlab_environment do - progress.puts "Dumping builds ... ".color(:blue) + puts_time "Dumping builds ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("builds") - progress.puts "[SKIPPED]".color(:cyan) + puts_time "[SKIPPED]".color(:cyan) else Backup::Builds.new(progress).dump - progress.puts "done".color(:green) + puts_time "done".color(:green) end end task restore: :gitlab_environment do - progress.puts "Restoring builds ... ".color(:blue) + puts_time "Restoring builds ... ".color(:blue) Backup::Builds.new(progress).restore - progress.puts "done".color(:green) + puts_time "done".color(:green) end end namespace :uploads do task create: :gitlab_environment do - progress.puts "Dumping uploads ... ".color(:blue) + puts_time "Dumping uploads ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("uploads") - progress.puts "[SKIPPED]".color(:cyan) + puts_time "[SKIPPED]".color(:cyan) else Backup::Uploads.new(progress).dump - progress.puts "done".color(:green) + puts_time "done".color(:green) end end task restore: :gitlab_environment do - progress.puts "Restoring uploads ... ".color(:blue) + puts_time "Restoring uploads ... ".color(:blue) Backup::Uploads.new(progress).restore - progress.puts "done".color(:green) + puts_time "done".color(:green) end end namespace :artifacts do task create: :gitlab_environment do - progress.puts "Dumping artifacts ... ".color(:blue) + puts_time "Dumping artifacts ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") - progress.puts "[SKIPPED]".color(:cyan) + puts_time "[SKIPPED]".color(:cyan) else Backup::Artifacts.new(progress).dump - progress.puts "done".color(:green) + puts_time "done".color(:green) end end task restore: :gitlab_environment do - progress.puts "Restoring artifacts ... ".color(:blue) + puts_time "Restoring artifacts ... ".color(:blue) Backup::Artifacts.new(progress).restore - progress.puts "done".color(:green) + puts_time "done".color(:green) end end namespace :pages do task create: :gitlab_environment do - progress.puts "Dumping pages ... ".color(:blue) + puts_time "Dumping pages ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("pages") - progress.puts "[SKIPPED]".color(:cyan) + puts_time "[SKIPPED]".color(:cyan) else Backup::Pages.new(progress).dump - progress.puts "done".color(:green) + puts_time "done".color(:green) end end task restore: :gitlab_environment do - progress.puts "Restoring pages ... ".color(:blue) + puts_time "Restoring pages ... ".color(:blue) Backup::Pages.new(progress).restore - progress.puts "done".color(:green) + puts_time "done".color(:green) end end namespace :lfs do task create: :gitlab_environment do - progress.puts "Dumping lfs objects ... ".color(:blue) + puts_time "Dumping lfs objects ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("lfs") - progress.puts "[SKIPPED]".color(:cyan) + puts_time "[SKIPPED]".color(:cyan) else Backup::Lfs.new(progress).dump - progress.puts "done".color(:green) + puts_time "done".color(:green) end end task restore: :gitlab_environment do - progress.puts "Restoring lfs objects ... ".color(:blue) + puts_time "Restoring lfs objects ... ".color(:blue) Backup::Lfs.new(progress).restore - progress.puts "done".color(:green) + puts_time "done".color(:green) end end namespace :registry do task create: :gitlab_environment do - progress.puts "Dumping container registry images ... ".color(:blue) + puts_time "Dumping container registry images ... ".color(:blue) if Gitlab.config.registry.enabled if ENV["SKIP"] && ENV["SKIP"].include?("registry") - progress.puts "[SKIPPED]".color(:cyan) + puts_time "[SKIPPED]".color(:cyan) else Backup::Registry.new(progress).dump - progress.puts "done".color(:green) + puts_time "done".color(:green) end else - progress.puts "[DISABLED]".color(:cyan) + puts_time "[DISABLED]".color(:cyan) end end task restore: :gitlab_environment do - progress.puts "Restoring container registry images ... ".color(:blue) + puts_time "Restoring container registry images ... ".color(:blue) if Gitlab.config.registry.enabled Backup::Registry.new(progress).restore - progress.puts "done".color(:green) + puts_time "done".color(:green) else - progress.puts "[DISABLED]".color(:cyan) + puts_time "[DISABLED]".color(:cyan) end end end + def puts_time(msg) + progress.puts "#{Time.now} -- #{msg}" + end + def progress if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake index 26cbf0740b6..c0d6cc8ca8e 100644 --- a/lib/tasks/gitlab/bulk_add_permission.rake +++ b/lib/tasks/gitlab/bulk_add_permission.rake @@ -14,7 +14,7 @@ namespace :gitlab do end desc "GitLab | Add a specific user to all projects (as a developer)" - task :user_to_projects, [:email] => :environment do |t, args| + task :user_to_projects, [:email] => :environment do |t, args| user = User.find_by(email: args.email) project_ids = Project.pluck(:id) puts "Importing #{user.email} users into #{project_ids.size} projects" @@ -22,7 +22,7 @@ namespace :gitlab do end desc "GitLab | Add all users to all groups (admin users are added as owners)" - task all_users_to_all_groups: :environment do |t, args| + task all_users_to_all_groups: :environment do |t, args| user_ids = User.where(admin: false).pluck(:id) admin_ids = User.where(admin: true).pluck(:id) groups = Group.all @@ -36,7 +36,7 @@ namespace :gitlab do end desc "GitLab | Add a specific user to all groups (as a developer)" - task :user_to_groups, [:email] => :environment do |t, args| + task :user_to_groups, [:email] => :environment do |t, args| user = User.find_by_email args.email groups = Group.all puts "Importing #{user.email} users into #{groups.size} groups" diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 74cd70c6e9f..b94b21775ee 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -29,10 +29,11 @@ namespace :gitlab do # If MySQL, turn off foreign key checks connection.execute('SET FOREIGN_KEY_CHECKS=0') if Gitlab::Database.mysql? - tables = connection.tables + tables = connection.data_sources + # Removes the entry from the array tables.delete 'schema_migrations' # Truncate schema_migrations to ensure migrations re-run - connection.execute('TRUNCATE schema_migrations') + connection.execute('TRUNCATE schema_migrations') if connection.data_source_exists? 'schema_migrations' # Drop tables with cascade to avoid dependent table errors # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html diff --git a/lib/tasks/gitlab/features.rake b/lib/tasks/gitlab/features.rake new file mode 100644 index 00000000000..d115961108e --- /dev/null +++ b/lib/tasks/gitlab/features.rake @@ -0,0 +1,24 @@ +namespace :gitlab do + namespace :features do + desc 'GitLab | Features | Enable direct Git access via Rugged for NFS' + task enable_rugged: :environment do + set_rugged_feature_flags(true) + puts 'All Rugged feature flags were enabled.' + end + + task disable_rugged: :environment do + set_rugged_feature_flags(false) + puts 'All Rugged feature flags were disabled.' + end + end + + def set_rugged_feature_flags(status) + Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.each do |flag| + if status + Feature.enable(flag) + else + Feature.disable(flag) + end + end + end +end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index e97d77d20e0..b8798fb3cfd 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -58,7 +58,7 @@ namespace :gitlab do puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab::Auth.omniauth_enabled? # check Gitolite version - gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.hooks_path}/../VERSION" + gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.path}/VERSION" if File.readable?(gitlab_shell_version_file) gitlab_shell_version = File.read(gitlab_shell_version_file) end @@ -72,7 +72,7 @@ namespace :gitlab do puts "- #{name}: \t#{repository_storage.legacy_disk_path}" end end - puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" + puts "GitLab Shell path:\t\t#{Gitlab.config.gitlab_shell.path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" end end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index f71e69987cb..e876b23d43f 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -25,6 +25,11 @@ namespace :gitlab do puts "" end + # In production, we might want to prevent ourselves from shooting + # ourselves in the foot, so let's only do this in a test or + # development environment. + terminate_all_connections unless Rails.env.production? + Rake::Task["db:reset"].invoke Rake::Task["add_limits_mysql"].invoke Rake::Task["setup_postgresql"].invoke @@ -33,4 +38,24 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end + + # If there are any clients connected to the DB, PostgreSQL won't let + # you drop the database. It's possible that Sidekiq, Unicorn, or + # some other client will be hanging onto a connection, preventing + # the DROP DATABASE from working. To workaround this problem, this + # method terminates all the connections so that a subsequent DROP + # will work. + def self.terminate_all_connections + return false unless Gitlab::Database.postgresql? + + cmd = <<~SQL + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE datname = current_database() + AND pid <> pg_backend_pid(); + SQL + + ActiveRecord::Base.connection.execute(cmd)&.result_status == PG::PGRES_TUPLES_OK + rescue ActiveRecord::NoDatabaseError + end end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index 09dc3aa9882..a2136ce1b92 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -36,8 +36,54 @@ namespace :gitlab do print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" - helper.project_id_batches do |start, finish| - storage_migrator.bulk_schedule(start, finish) + helper.project_id_batches_migration do |start, finish| + storage_migrator.bulk_schedule_migration(start: start, finish: finish) + + print '.' + end + + puts ' Done!' + end + + desc 'GitLab | Storage | Rollback existing projects to Legacy Storage' + task rollback_to_legacy: :environment do + if Gitlab::Database.read_only? + warn 'This task requires database write access. Exiting.' + + next + end + + storage_migrator = Gitlab::HashedStorage::Migrator.new + helper = Gitlab::HashedStorage::RakeHelper + + if helper.range_single_item? + project = Project.with_storage_feature(:repository).find_by(id: helper.range_from) + + unless project + warn "There are no projects that can be rolledback with ID=#{helper.range_from}" + + next + end + + puts "Enqueueing storage rollback of #{project.full_path} (ID=#{project.id})..." + storage_migrator.rollback(project) + + next + end + + hashed_projects_count = Project.with_storage_feature(:repository).count + + if hashed_projects_count == 0 + warn 'There are no projects that can have storage rolledback. Nothing to do!' + + next + end + + print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}" + + helper.project_id_batches_rollback do |start, finish| + puts "Start: #{start} FINISH: #{finish}" + storage_migrator.bulk_schedule_rollback(start: start, finish: finish) print '.' end diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake index 62a12174efa..53325d492d1 100644 --- a/lib/tasks/karma.rake +++ b/lib/tasks/karma.rake @@ -2,7 +2,7 @@ unless Rails.env.production? namespace :karma do desc 'GitLab | Karma | Generate fixtures for JavaScript tests' RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args| - args.with_defaults(pattern: 'spec/javascripts/fixtures/*.rb') + args.with_defaults(pattern: '{spec,ee/spec}/javascripts/fixtures/*.rb') ENV['NO_KNAPSACK'] = 'true' t.pattern = args[:pattern] t.rspec_opts = '--format documentation' |